diff options
686 files changed, 12236 insertions, 4470 deletions
diff --git a/.gitignore b/.gitignore index 4611f2aa98c..cc2aad047be 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,11 @@ tags WT_HOME/ WT_TEST/ +# Bench/workgen +/bench/workgen/_workgen.so +/bench/workgen/workgen/workgen.py +/bench/workgen/workgen_wrap.cxx + # Python /lang/python/_wiredtiger.so /lang/python/wiredtiger.py @@ -1,4 +1,4 @@ -Copyright (c) 2014-2016 MongoDB, Inc. +Copyright (c) 2014-2017 MongoDB, Inc. Copyright (c) 2008-2014 WiredTiger, Inc. All rights reserved. @@ -1,6 +1,30 @@ Ticket reference tags refer to tickets in the MongoDB JIRA tracking system: https://jira.mongodb.org +WiredTiger release 2.9.3, 2017-06-27 +------------------------------------ + +See the upgrading documentation for details of API and behavior changes. + +Significant changes: +* WT-2972 Add an interface allowing partial updates to existing values +* WT-3063 Add an interface allowing reservation of records for read-modify-write +* WT-3142 Add a workload generator application +* WT-3160 Improve eviction of internal pages from idle trees +* WT-3245 Avoid hangs on shutdown when a utility thread encounters an error +* WT-3258 Improve visibility into thread wait time due to pages exceeding memory_page_max +* WT-3263 Allow archive on restart/recovery if clean shutdown +* WT-3287 Review WiredTiger internal panic checks +* WT-3292 Review/cleanup full-barrier calls in WiredTiger +* WT-3296 LAS table fixes/improvements +* WT-3327 Checkpoints can hang if time runs backward +* WT-3345 Improve rwlock scaling +* WT-3373 Access violation due to a bug in internal page splitting +* WT-3379 Change when pages can be split to avoid excessively slowing some operations + +See JIRA changelog for a full listing: +https://jira.mongodb.org/browse/WT/fixforversion/18291 + WiredTiger release 2.9.2, 2017-05-25 ------------------------------------ @@ -1,6 +1,6 @@ -WiredTiger 2.9.2: (May 26, 2017) +WiredTiger 2.9.3: (June 26, 2017) -This is version 2.9.2 of WiredTiger. +This is version 2.9.3 of WiredTiger. WiredTiger release packages and documentation can be found at: @@ -8,7 +8,7 @@ WiredTiger release packages and documentation can be found at: The documentation for this specific release can be found at: - http://source.wiredtiger.com/2.9.2/index.html + http://source.wiredtiger.com/2.9.3/index.html The WiredTiger source code can be found at: diff --git a/RELEASE_INFO b/RELEASE_INFO index b7145aa2cb3..f18f6f67fc8 100644 --- a/RELEASE_INFO +++ b/RELEASE_INFO @@ -1,6 +1,6 @@ WIREDTIGER_VERSION_MAJOR=2 WIREDTIGER_VERSION_MINOR=9 -WIREDTIGER_VERSION_PATCH=2 +WIREDTIGER_VERSION_PATCH=3 WIREDTIGER_VERSION="$WIREDTIGER_VERSION_MAJOR.$WIREDTIGER_VERSION_MINOR.$WIREDTIGER_VERSION_PATCH" WIREDTIGER_RELEASE_DATE=`date "+%B %e, %Y"` diff --git a/SConstruct b/SConstruct index b397f662be7..2661807594d 100644 --- a/SConstruct +++ b/SConstruct @@ -67,15 +67,11 @@ var.Add('CPPPATH', 'C Preprocessor include path', [ ]) var.Add('CFLAGS', 'C Compiler Flags', [ - "/Z7", # Generate debugging symbols "/wd4090", # Ignore warning about mismatched const qualifiers "/wd4996", # Ignore deprecated functions "/W3", # Warning level 3 - #"/we4244", # Possible loss of data - "/we4013", # Error on undefined functions - #"/we4047", # Indirection differences in types - #"/we4024", # Differences in parameter types - #"/we4100", # Unreferenced local parameter + "/WX", # Warnings are fatal + "/Z7", # Generate debugging symbols "/TC", # Compile as C code #"/Od", # Disable optimization "/Ob1", # inline expansion @@ -338,6 +334,8 @@ if GetOption("lang-python"): "-nodefaultctor", "-nodefaultdtor", ]) + # Ignore warnings in swig-generated code. + pythonEnv['CFLAGS'].remove("/WX") swiglib = pythonEnv.SharedLibrary('_wiredtiger', [ 'lang\python\wiredtiger.i'], @@ -410,41 +408,37 @@ def builder_smoke_test(target, source, env): env.Append(BUILDERS={'SmokeTest' : Builder(action = builder_smoke_test)}) #Build the tests and setup the "scons test" target - testutil = env.Library('testutil', [ 'test/utility/misc.c', 'test/utility/parse_opts.c' ]) +env.Append(CPPPATH=["test/utility"]) -#Don't test bloom on Windows, its broken t = env.Program("t_bloom", "test/bloom/test_bloom.c", - LIBS=[wtlib, testutil] + wtlibs) -#env.Alias("check", env.SmokeTest(t)) + LIBS=[wtlib, shim, testutil] + wtlibs) Default(t) -#env.Program("t_checkpoint", - #["test/checkpoint/checkpointer.c", - #"test/checkpoint/test_checkpoint.c", - #"test/checkpoint/workers.c"], - #LIBS=[wtlib]) - -t = env.Program("t_huge", - "test/huge/huge.c", - LIBS=[wtlib] + wtlibs) +t = env.Program("t_checkpoint", + ["test/checkpoint/checkpointer.c", + "test/checkpoint/test_checkpoint.c", + "test/checkpoint/workers.c"], + LIBS=[wtlib, shim, testutil] + wtlibs) +Default(t) -#t = env.Program("t_recovery", -# "test/recovery/recovery.c", -# LIBS=[wtlib] + wtlibs) -#Default(t) +t = env.Program("t_cursor_order", + ["test/cursor_order/cursor_order.c", + "test/cursor_order/cursor_order_file.c", + "test/cursor_order/cursor_order_ops.c"], + LIBS=[wtlib, shim, testutil] + wtlibs) +Default(t) t = env.Program("t_fops", ["test/fops/file.c", "test/fops/fops.c", "test/fops/t.c"], LIBS=[wtlib, shim, testutil] + wtlibs) -env.Append(CPPPATH=["test/utility"]) Default(t) t = env.Program("t_format", @@ -459,19 +453,51 @@ t = env.Program("t_format", "test/format/t.c", "test/format/util.c", "test/format/wts.c"], - LIBS=[wtlib, shim, testutil] + wtlibs) + LIBS=[wtlib, shim, testutil] + wtlibs) Default(t) -#env.Program("t_thread", - #["test/thread/file.c", - #"test/thread/rw.c", - #"test/thread/stats.c", - #"test/thread/t.c"], - #LIBS=[wtlib]) +t = env.Program("t_huge", + "test/huge/huge.c", + LIBS=[wtlib, shim, testutil] + wtlibs) +Default(t) + +t = env.Program("t_manydbs", + "test/manydbs/manydbs.c", + LIBS=[wtlib, shim, testutil] + wtlibs) +Default(t) + +# t_readonly doesn't currently build/run. +#t = env.Program("t_readonly", +# "test/readonly/readonly.c", +# LIBS=[wtlib, shim, testutil] + wtlibs) +#Default(t) + +# t_random-abort doesn't currently build/run. +#t = env.Program("t_random-abort", +# "test/recovery/random-abort.c", +# LIBS=[wtlib, shim, testutil] + wtlibs) +#Default(t) + +# t_truncated-log doesn't currently build/run. +#t = env.Program("t_truncated-log", +# "test/recovery/truncated-log.c", +# LIBS=[wtlib, shim, testutil] + wtlibs) +#Default(t) + +# t_salvage-log doesn't currently build/run. +#t = env.Program("t_salvage", +# "test/salvage/salvage.c", +# LIBS=[wtlib, shim, testutil] + wtlibs) +#Default(t) -#env.Program("t_salvage", - #["test/salvage/salvage.c"], - #LIBS=[wtlib]) +# t_thread doesn't currently build/run. +#t = env.Program("t_thread", +# ["test/thread/file.c", +# "test/thread/rw.c", +# "test/thread/stats.c", +# "test/thread/t.c"], +# LIBS=[wtlib, shim, testutil] + wtlibs) +#Default(t) t = env.Program("wtperf", [ "bench/wtperf/config.c", diff --git a/api/leveldb/leveldb_wt.h b/api/leveldb/leveldb_wt.h index 351eb9f3dda..b167e03192e 100644 --- a/api/leveldb/leveldb_wt.h +++ b/api/leveldb/leveldb_wt.h @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/bench/workgen/Makefile.am b/bench/workgen/Makefile.am new file mode 100644 index 00000000000..61512d65319 --- /dev/null +++ b/bench/workgen/Makefile.am @@ -0,0 +1,30 @@ +AM_CPPFLAGS = -I$(top_builddir) +AM_CPPFLAGS += -I$(top_srcdir)/src/include +AM_CPPFLAGS +=-I$(top_srcdir)/test/utility + +PYSRC = $(top_srcdir)/bench/workgen +PYDIRS = -t $(abs_builddir) -I $(abs_top_srcdir):$(abs_top_builddir) -L $(abs_top_builddir)/.libs:$(abs_top_builddir)/bench/workgen/.libs +all-local: _workgen.so libworkgen.la +libworkgen_la_SOURCES = workgen.cxx workgen_func.c +noinst_LTLIBRARIES = libworkgen.la + +# We keep generated Python sources under bench/workgen. +$(PYSRC)/workgen_wrap.cxx: $(PYSRC)/workgen.h $(PYSRC)/workgen.swig + (cd $(PYSRC) && \ + $(SWIG) -c++ -python -threads -O -Wall -I$(abs_top_builddir) -outdir ./workgen workgen.swig) + +_workgen.so: $(top_builddir)/libwiredtiger.la $(PYSRC)/workgen_wrap.cxx libworkgen.la $(PYSRC)/workgen.h $(PYSRC)/workgen_time.h + (cd $(PYSRC) && \ + $(PYTHON) setup.py build_ext -f -b $(abs_builddir) $(PYDIRS)) + +install-exec-local: + (cd $(PYSRC) && \ + $(PYTHON) setup.py build_py -d $(abs_builddir)/build && \ + $(PYTHON) setup.py build_ext -f -b $(abs_builddir)/build $(PYDIRS) && \ + $(PYTHON) setup.py install_lib -b $(abs_builddir)/build --skip-build $(PYTHON_INSTALL_ARG)) + +# We build in different places for an install vs running from the tree: +# clean up both. Don't rely on "setup.py clean" -- everything that should +# be removed is created under the build directory. +clean-local: + rm -rf build _workgen.so workgen_wrap.o WT_TEST diff --git a/bench/workgen/runner/example_simple.py b/bench/workgen/runner/example_simple.py new file mode 100755 index 00000000000..626f7ca64a5 --- /dev/null +++ b/bench/workgen/runner/example_simple.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from runner import * +from wiredtiger import * +from workgen import * + +def show(tname): + print('') + print('<><><><> ' + tname + ' <><><><>') + c = s.open_cursor(tname, None) + for k,v in c: + print('key: ' + k) + print('value: ' + v) + print('<><><><><><><><><><><><>') + c.close() + +context = Context() +conn = wiredtiger_open("WT_TEST", "create,cache_size=1G") +s = conn.open_session() +tname = 'table:simple' +s.create(tname, 'key_format=S,value_format=S') + +ops = Operation(Operation.OP_INSERT, Table(tname), Key(Key.KEYGEN_APPEND, 10), Value(40)) +thread = Thread(ops) +workload = Workload(context, thread) +workload.run(conn) +show(tname) + +thread = Thread(ops * 5) +workload = Workload(context, thread) +workload.run(conn) +show(tname) diff --git a/bench/workgen/runner/example_txn.py b/bench/workgen/runner/example_txn.py new file mode 100644 index 00000000000..1b22dc10aba --- /dev/null +++ b/bench/workgen/runner/example_txn.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from runner import * +from wiredtiger import * +from workgen import * + +conn = wiredtiger_open("WT_TEST", "create,cache_size=500MB") +s = conn.open_session() +tname = "table:test" +s.create(tname, 'key_format=S,value_format=S') +table = Table(tname) +table.options.key_size = 20 +table.options.value_size = 100 + +context = Context() +op = Operation(Operation.OP_INSERT, table) +thread = Thread(op * 500000) +pop_workload = Workload(context, thread) +print('populate:') +pop_workload.run(conn) + +opread = Operation(Operation.OP_SEARCH, table) +opwrite = Operation(Operation.OP_INSERT, table) +treader = Thread(opread) +twriter = Thread(txn(opwrite * 2)) +workload = Workload(context, treader * 8 + twriter * 2) +workload.options.run_time = 10 +workload.options.report_interval = 5 +print('transactional write workload:') +workload.run(conn) diff --git a/bench/workgen/runner/insert_test.py b/bench/workgen/runner/insert_test.py new file mode 100644 index 00000000000..8380c6ba3eb --- /dev/null +++ b/bench/workgen/runner/insert_test.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from runner import * +from wiredtiger import * +from workgen import * + +def tablename(id): + return "table:test%06d" % id + +def show(tname): + print('') + print('<><><><> ' + tname + ' <><><><>') + c = s.open_cursor(tname, None) + for k,v in c: + print('key: ' + k) + print('value: ' + v) + print('<><><><><><><><><><><><>') + c.close() + +def expectException(expr): + gotit = False + try: + expr() + except BaseException as e: + print('got expected exception: ' + str(e)) + gotit = True + if not gotit: + raise Exception("missing expected exception") + +context = Context() +conn = wiredtiger_open("WT_TEST", "create,cache_size=1G") +s = conn.open_session() +tname0 = tablename(0) +tname1 = tablename(1) +s.create(tname0, 'key_format=S,value_format=S') +s.create(tname1, 'key_format=S,value_format=S') + +ops = Operation(Operation.OP_INSERT, Table(tname0), Key(Key.KEYGEN_APPEND, 10), Value(100)) +workload = Workload(context, Thread(ops)) + +print('RUN1') +workload.run(conn) +show(tname0) + +# The context has memory of how many keys are in all the tables. +# truncate goes behind context's back, but it doesn't matter for +# an insert-only test. +s.truncate(tname0, None, None) + +# Show how to 'multiply' operations +op = Operation(Operation.OP_INSERT, Table(tname0), Key(Key.KEYGEN_APPEND, 10), Value(100)) +op2 = Operation(Operation.OP_INSERT, Table(tname1), Key(Key.KEYGEN_APPEND, 20), Value(30)) +o = op2 * 10 +print 'op is: ' + str(op) +print 'multiplying op is: ' + str(o) +thread0 = Thread(o + op + op) +workload = Workload(context, thread0) +print('RUN2') +workload.run(conn) +show(tname0) +show(tname1) + +s.truncate(tname0, None, None) +s.truncate(tname1, None, None) + +# operations can be multiplied, added in any combination. +op += Operation(Operation.OP_INSERT, Table(tname0), Key(Key.KEYGEN_APPEND, 10), Value(10)) +op *= 2 +op += Operation(Operation.OP_INSERT, Table(tname0), Key(Key.KEYGEN_APPEND, 10), Value(10)) +thread0 = Thread(op * 10 + op2 * 20) +workload = Workload(context, thread0) +print('RUN3') +workload.run(conn) +show(tname0) +show(tname1) + +print('workload is ' + str(workload)) +print('thread0 is ' + str(thread0)) + +def assignit(k, n): + k._size = n + +expectException(lambda: Operation( + Operation.OP_INSERT, Table('foo'), Key(Key.KEYGEN_APPEND, 10))) +# we don't catch this exception here, but in Workload.run() +k = Key(Key.KEYGEN_APPEND, 1) +assignit(k, 30) +assignit(k, 1) # we don't catch this exception here, but in Workload.run() +op = Operation(Operation.OP_INSERT, Table(tname0), k, Value(10)) +workload = Workload(context, Thread(op)) +print('RUN4') +expectException(lambda: workload.run(conn)) + +print('HELP:') +print(workload.options.help()) diff --git a/bench/workgen/runner/multi_btree_heavy_stress.py b/bench/workgen/runner/multi_btree_heavy_stress.py new file mode 100644 index 00000000000..94dacfc4311 --- /dev/null +++ b/bench/workgen/runner/multi_btree_heavy_stress.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +# Drive a constant high workload through, even if WiredTiger isn't keeping +# up by dividing the workload across a lot of threads. This needs to be +# tuned to the particular machine so the workload is close to capacity in the +# steady state, but not overwhelming. +# +################ +# Note: as a proof of concept for workgen, this matches closely +# bench/wtperf/runner/multi-btree-read-heavy-stress.wtperf . +# Run time, #ops, #threads are ratcheted way down for testing. +# +from runner import * +from wiredtiger import * +from workgen import * + +def op_append(ops, op): + if ops == None: + ops = op + else: + ops += op + return ops + +def make_op(optype, table, key, value = None): + if value == None: + return Operation(optype, table, key) + else: + return Operation(optype, table, key, value) + +logkey = Key(Key.KEYGEN_APPEND, 8) ## should be 8 bytes format 'Q' +def operations(optype, tables, key, value = None, ops_per_txn = 0, logtable = None): + txn_list = [] + ops = None + nops = 0 + for table in tables: + ops = op_append(ops, make_op(optype, table, key, value)) + if logtable != None: + ops = op_append(ops, make_op(optype, logtable, logkey, value)) + nops += 1 + if ops_per_txn > 0 and nops % ops_per_txn == 0: + txn_list.append(txn(ops)) + ops = None + if ops_per_txn > 0: + if ops != None: + txn_list.append(txn(ops)) + ops = None + for t in txn_list: + ops = op_append(ops, t) + return ops + +context = Context() +## cache_size=20GB +conn_config="create,cache_size=1GB,session_max=1000,eviction=(threads_min=4,threads_max=8),log=(enabled=false),transaction_sync=(enabled=false),checkpoint_sync=true,checkpoint=(wait=60),statistics=(fast),statistics_log=(json,wait=1)" +table_config="allocation_size=4k,memory_page_max=10MB,prefix_compression=false,split_pct=90,leaf_page_max=32k,internal_page_max=16k,type=file,block_compressor=snappy" +conn_config += extensions_config(['compressors/snappy']) +conn = wiredtiger_open("WT_TEST", conn_config) +s = conn.open_session() + +tables = [] +for i in range(0, 8): + tname = "table:test" + str(i) + s.create(tname, 'key_format=S,value_format=S,' + table_config) + tables.append(Table(tname)) +tname = "table:log" +# TODO: use table_config for the log file? +s.create(tname, 'key_format=S,value_format=S,' + table_config) +logtable = Table(tname) + +##icount=200000000 / 8 +icount=20000 +ins_ops = operations(Operation.OP_INSERT, tables, Key(Key.KEYGEN_APPEND, 20), Value(500)) +thread = Thread(ins_ops * icount) +pop_workload = Workload(context, thread) +print('populate:') +pop_workload.run(conn) + +ins_ops = operations(Operation.OP_INSERT, tables, Key(Key.KEYGEN_APPEND, 20), Value(500), 0, logtable) +upd_ops = operations(Operation.OP_UPDATE, tables, Key(Key.KEYGEN_UNIFORM, 20), Value(500), 0, logtable) +read_ops = operations(Operation.OP_SEARCH, tables, Key(Key.KEYGEN_UNIFORM, 20), None, 3) + +ins_thread = Thread(ins_ops) +upd_thread = Thread(upd_ops) +read_thread = Thread(read_ops) +ins_thread.options.throttle = 250 +ins_thread.options.name = "Insert" +upd_thread.options.throttle = 250 +upd_thread.options.name = "Update" +read_thread.options.throttle = 1000 +read_thread.options.name = "Read" +##threads = [ins_thread] * 10 + [upd_thread] * 10 + [read_thread] * 80 +threads = ins_thread * 1 + upd_thread * 1 + read_thread * 2 +workload = Workload(context, threads) +##workload.options.run_time = 3600 +workload.options.run_time = 30 +workload.options.report_interval = 1 +workload.options.sample_interval = 5 +workload.options.sample_rate = 1 +print('heavy stress workload:') +workload.run(conn) + +latency_filename = conn.get_home() + '/latency.out' +print('for latency output, see: ' + latency_filename) +latency.workload_latency(workload, latency_filename) diff --git a/bench/workgen/runner/runner/__init__.py b/bench/workgen/runner/runner/__init__.py new file mode 100644 index 00000000000..ed21fffe8dc --- /dev/null +++ b/bench/workgen/runner/runner/__init__.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# runner/__init__.py +# Used as a first import by runners, does any common initialization. +from __future__ import print_function + +import os, shutil, sys +thisdir = os.path.dirname(os.path.abspath(__file__)) +workgen_src = os.path.dirname(os.path.dirname(thisdir)) +wt_dir = os.path.dirname(os.path.dirname(workgen_src)) +wt_builddir = os.path.join(wt_dir, 'build_posix') + +def _prepend_env_path(pathvar, s): + last = '' + try: + last = ':' + os.environ[pathvar] + except: + pass + os.environ[pathvar] = s + last + +# Initialize the python path so needed modules can be imported. +# If the path already works, don't change it. +try: + import wiredtiger +except: + # We'll try hard to make the importing work, we'd like to runners + # to be executable directly without having to set environment variables. + sys.path.insert(0, os.path.join(wt_dir, 'lang', 'python')) + sys.path.insert(0, os.path.join(wt_builddir, 'lang', 'python')) + try: + import wiredtiger + except: + # If the .libs directory is not in our library search path, + # we need to set it and retry. However, the dynamic link + # library has already cached its value, our only option is + # to restart the Python interpreter. + if '_workgen_init' not in os.environ: + os.environ['_workgen_init'] = 'true' + dotlibs = os.path.join(wt_builddir, '.libs') + _prepend_env_path('LD_LIBRARY_PATH', dotlibs) + _prepend_env_path('DYLD_LIBRARY_PATH', dotlibs) + py_args = sys.argv + py_args.insert(0, sys.executable) + try: + os.execv(sys.executable, py_args) + except Exception, exception: + print('re-exec failed: ' + str(exception), file=sys.stderr) + print(' exec(' + sys.executable + ', ' + str(py_args) + ')') + print('Try adding "' + dotlibs + '" to the', file=sys.stderr) + print('LD_LIBRARY_PATH environment variable before running ' + \ + 'this program again.', file=sys.stderr) + sys.exit(1) + +try: + import workgen +except: + sys.path.insert(0, os.path.join(workgen_src, 'workgen')) + sys.path.insert(0, os.path.join(wt_builddir, 'bench', 'workgen')) + import workgen + +# Clear out the WT_TEST directory. +shutil.rmtree('WT_TEST', True) +os.mkdir('WT_TEST') + +from .core import txn, extensions_config, op_group_transaction, op_log_like, op_multi_table +from .latency import workload_latency diff --git a/bench/workgen/runner/runner/core.py b/bench/workgen/runner/runner/core.py new file mode 100644 index 00000000000..2c8311c4ca7 --- /dev/null +++ b/bench/workgen/runner/runner/core.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# runner/core.py +# Core functions available to all runners +import glob, os +from workgen import Key, Operation, OpList, Table, Transaction, Value + +# txn -- +# Put the operation (and any suboperations) within a transaction. +def txn(op, config=None): + t = Transaction(config) + op._transaction = t + return op + +# Check for a local build that contains the wt utility. First check in +# current working directory, then in build_posix and finally in the disttop +# directory. This isn't ideal - if a user has multiple builds in a tree we +# could pick the wrong one. +def _wiredtiger_builddir(): + if os.path.isfile(os.path.join(os.getcwd(), 'wt')): + return os.getcwd() + + # The directory of this file should be within the distribution tree. + thisdir = os.path.dirname(os.path.abspath(__file__)) + wt_disttop = os.path.join(\ + thisdir, os.pardir, os.pardir, os.pardir, os.pardir) + if os.path.isfile(os.path.join(wt_disttop, 'wt')): + return wt_disttop + if os.path.isfile(os.path.join(wt_disttop, 'build_posix', 'wt')): + return os.path.join(wt_disttop, 'build_posix') + if os.path.isfile(os.path.join(wt_disttop, 'wt.exe')): + return wt_disttop + raise Exception('Unable to find useable WiredTiger build') + +# Return the wiredtiger_open extension argument for any needed shared library. +# Called with a list of extensions, e.g. +# [ 'compressors/snappy', 'encryptors/rotn=config_string' ] +def extensions_config(exts): + result = '' + extfiles = {} + errpfx = 'extensions_config' + builddir = _wiredtiger_builddir() + for ext in exts: + extconf = '' + if '=' in ext: + splits = ext.split('=', 1) + ext = splits[0] + extconf = '=' + splits[1] + splits = ext.split('/') + if len(splits) != 2: + raise Exception(errpfx + ": " + ext + + ": extension is not named <dir>/<name>") + libname = splits[1] + dirname = splits[0] + pat = os.path.join(builddir, 'ext', + dirname, libname, '.libs', 'libwiredtiger_*.so') + filenames = glob.glob(pat) + if len(filenames) == 0: + raise Exception(errpfx + + ": " + ext + + ": no extensions library found matching: " + pat) + elif len(filenames) > 1: + raise Exception(errpfx + ": " + ext + + ": multiple extensions libraries found matching: " + pat) + complete = '"' + filenames[0] + '"' + extconf + if ext in extfiles: + if extfiles[ext] != complete: + raise Exception(errpfx + + ": non-matching extension arguments in " + + str(exts)) + else: + extfiles[ext] = complete + if len(extfiles) != 0: + result = ',extensions=[' + ','.join(extfiles.values()) + ']' + return result + +def _op_multi_table_as_list(ops_arg, tables): + result = [] + if ops_arg._optype != Operation.OP_NONE: + for table in tables: + result.append(Operation(ops_arg._optype, table, ops_arg._key, ops_arg._value)) + else: + for op in ops._group: + result.extend(_op_multi_table_as_list(op, tables)) + return result + +# A convenient way to build a list of operations +def op_append(op1, op2): + if op1 == None: + op1 = op2 + else: + op1 += op2 + return op1 + +# Emulate wtperf's table_count option. Spread the given operations over +# a set of tables. +def op_multi_table(ops_arg, tables): + ops = None + for op in _op_multi_table_as_list(ops_arg, tables): + ops = op_append(ops, op) + return ops + +# should be 8 bytes format 'Q' +_logkey = Key(Key.KEYGEN_APPEND, 8) +def _op_log_op(op, log_table): + keysize = op._key._size + if keysize == 0: + keysize = op._table.options.key_size + valuesize = op._value._size + if valuesize == 0: + valuesize = op._table.options.value_size + v = Value(keysize + valuesize) + return Operation(Operation.OP_INSERT, log_table, _logkey, v) + +def _optype_is_write(optype): + return optype == Operation.OP_INSERT or optype == Operation.OP_UPDATE or \ + optype == Operation.OP_REMOVE + +# Emulate wtperf's log_like option. For all operations, add a second +# insert operation going to a log table. +def op_log_like(op, log_table, ops_per_txn): + if op._optype != Operation.OP_NONE: + if _optype_is_write(op._optype): + op += _op_log_op(op, log_table) + if ops_per_txn == 0: + op = txn(op) # txn for each action. + else: + oplist = [] + for op2 in op._group: + if op2._optype == Operation.OP_NONE: + oplist.append(op_log_like(op2, log_table)) + elif ops_per_txn == 0 and _optype_is_write(op2._optype): + op2 += _op_log_op(op2, log_table) + oplist.append(txn(op2)) # txn for each action. + else: + oplist.append(op2) + if _optype_is_write(op2._optype): + oplist.append(_op_log_op(op2, log_table)) + op._group = OpList(oplist) + return op + +def _op_transaction_list(oplist, txn_config): + result = None + for op in oplist: + result = op_append(result, op) + return txn(result, txn_config) + +# Emulate wtperf's ops_per_txn option. Create transactions around +# groups of operations of the indicated size. +def op_group_transaction(ops_arg, ops_per_txn, txn_config): + if ops_arg != Operation.OP_NONE: + return txn(ops_arg, txn_config) + if ops_arg._transaction != None: + raise Exception('nested transactions not supported') + if ops_arg._repeatgroup != None: + raise Exception('grouping transactions with multipliers not supported') + + oplist = [] + ops = None + nops = 0 + txgroup = [] + for op in ops_arg._group: + if op.optype == Operation.OP_NONE: + oplist.append(_op_transaction_list(txgroup, txn_config)) + txgroup = [] + oplist.append(op) + else: + txgroup.append(op) + if len(txgroup) >= ops_per_txn: + oplist.append(_op_transaction_list(txgroup, txn_config)) + txgroup = [] + if len(txgroup) > 0: + oplist.append(_op_transaction_list(txgroup, txn_config)) + ops_arg._group = OpList(oplist) + return ops_arg diff --git a/bench/workgen/runner/runner/latency.py b/bench/workgen/runner/runner/latency.py new file mode 100644 index 00000000000..8eaa10693a9 --- /dev/null +++ b/bench/workgen/runner/runner/latency.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# runner/latency.py +# Utility functions for showing latency statistics +from __future__ import print_function +import sys + +def _show_buckets(fh, title, mult, buckets, n): + shown = False + s = title + ': ' + for count in range(0, n): + val = buckets[count] + if val != 0: + if shown: + s += ',' + s += str(count*mult) + '=' + str(val) + shown = True + print(s, file=fh) + +def _latency_preprocess(arr, merge): + mx = 0 + cur = 0 + # SWIG arrays have a clunky interface + for i in range(0, arr.__len__()): + if i % merge == 0: + cur = 0 + cur += arr[i] + if cur > mx: + mx = cur + arr.height = mx + +def _latency_plot(box, ch, left, width, arr, merge, scale): + pos = 0 + for x in range(0, width): + t = 0 + for i in range(0, merge): + t += arr[pos] + pos += 1 + nch = scale * t + y = 0 + while nch > 0.0: + box[y][left + x] = ch + nch -= 1.0 + y += 1 + +def _latency_optype(fh, name, ch, t): + if t.ops == 0: + return + if t.latency_ops == 0: + print('**** ' + name + ' operations: ' + str(t.ops), file=fh) + return + print('**** ' + name + ' operations: ' + str(t.ops) + \ + ', latency operations: ' + str(t.latency_ops), file=fh) + print(' avg: ' + str(t.latency/t.latency_ops) + \ + ', min: ' + str(t.min_latency) + ', max: ' + str(t.max_latency), + file=fh) + us = t.us() + ms = t.ms() + sec = t.sec() + _latency_preprocess(us, 40) + _latency_preprocess(ms, 40) + _latency_preprocess(sec, 4) + max_height = max(us.height, ms.height, sec.height) + if max_height == 0: + return + height = 20 # 20 chars high + # a list of a list of characters + box = [list(' ' * 80) for x in range(height)] + scale = (1.0 / (max_height + 1)) * height + _latency_plot(box, ch, 0, 25, us, 40, scale) + _latency_plot(box, ch, 27, 25, ms, 40, scale) + _latency_plot(box, ch, 54, 25, sec, 4, scale) + box.reverse() + for line in box: + print(''.join(line), file=fh) + dash25 = '-' * 25 + print(' '.join([dash25] * 3), file=fh) + print(' 0 - 999 us (40/bucket) 1 - 999 ms (40/bucket) ' + \ + '1 - 99 sec (4/bucket)', file=fh) + print('', file=fh) + _show_buckets(fh, name + ' us', 1, us, 1000) + _show_buckets(fh, name + ' ms', 1000, ms, 1000) + _show_buckets(fh, name + ' sec', 1000000, sec, 100) + print('', file=fh) + +def workload_latency(workload, outfilename = None): + if outfilename: + fh = open(outfilename, 'w') + else: + fh = sys.stdout + _latency_optype(fh, 'insert', 'I', workload.stats.insert) + _latency_optype(fh, 'read', 'R', workload.stats.read) + _latency_optype(fh, 'remove', 'X', workload.stats.remove) + _latency_optype(fh, 'update', 'U', workload.stats.update) + _latency_optype(fh, 'truncate', 'T', workload.stats.truncate) + _latency_optype(fh, 'not found', 'N', workload.stats.not_found) diff --git a/bench/workgen/runner/small_btree.py b/bench/workgen/runner/small_btree.py new file mode 100644 index 00000000000..2bcc0188f30 --- /dev/null +++ b/bench/workgen/runner/small_btree.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from runner import * +from wiredtiger import * +from workgen import * + +context = Context() +conn = wiredtiger_open("WT_TEST", "create,cache_size=500MB") +s = conn.open_session() +tname = "file:test.wt" +s.create(tname, 'key_format=S,value_format=S') +table = Table(tname) +table.options.key_size = 20 +table.options.value_size = 100 + +op = Operation(Operation.OP_INSERT, table) +thread = Thread(op * 500000) +pop_workload = Workload(context, thread) +print('populate:') +pop_workload.run(conn) + +op = Operation(Operation.OP_SEARCH, table) +t = Thread(op) +workload = Workload(context, t * 8) +workload.options.run_time = 120 +workload.options.report_interval = 5 +print('read workload:') +workload.run(conn) diff --git a/bench/workgen/runner/workgen_stat.sh b/bench/workgen/runner/workgen_stat.sh new file mode 100755 index 00000000000..1739c29859e --- /dev/null +++ b/bench/workgen/runner/workgen_stat.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# +# workgen_stat.sh - combine JSON time series output from WT and workgen. +# +Usage() { + cat <<EOF +Usage: $0 [ options ] +Options: + -h <WT_home_directory> # set the WiredTiger home directory + -e <analyzer_name> # run analyzer on the combined files + -o <output_file> # output file for result + +At least one of '-t2' or '-o' must be selected. +EOF + exit 1 +} + +Filter() { + sed -e 's/"version" *: *"[^"]*",//' "$@" +} + +wthome=. +outfile= +analyze= + +while [ "$#" != 0 ]; do + arg="$1" + shift + case "$arg" in + -h ) + if [ $# = 0 ]; then + Usage + fi + wthome="$1" + shift + ;; + -o ) + if [ $# = 0 ]; then + Usage + fi + outfile="$1" + shift + ;; + -e ) + if [ $# = 0 ]; then + Usage + fi + analyze="$1" + shift + ;; + esac +done +if [ ! -d "$wthome" ]; then + echo "$wthome: WT home directory does not exist" + exit 1 +fi +if [ ! -f "$wthome/WiredTiger.wt" ]; then + echo "$wthome: directory is not a WiredTiger home directory" + exit 1 +fi +if [ "$outfile" = '' ]; then + if [ "$analyze" = false ]; then + Usage + fi + outfile="$wthome/stat_tmp.json" +fi +(cd $wthome; Filter WiredTigerStat.* sample.json) | sort > $outfile +if [ "$analyze" != '' ]; then + sysname=`uname -s` + if [ "$sysname" = Darwin ]; then + open -a "$analyze" "$outfile" + else + "$analyze" "$outfile" + fi +fi diff --git a/bench/workgen/setup.py b/bench/workgen/setup.py new file mode 100644 index 00000000000..9fb5fa7b73a --- /dev/null +++ b/bench/workgen/setup.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from __future__ import print_function +import re, os, sys +from distutils.core import setup, Extension + +# OS X hack: turn off the Universal binary support that is built into the +# Python build machinery, just build for the default CPU architecture. +if not 'ARCHFLAGS' in os.environ: + os.environ['ARCHFLAGS'] = '' + +# Suppress warnings building SWIG generated code +extra_cflags = [ '-w', '-Wno-sign-conversion', '-I../../src/include', \ + '-I../../test/utility'] + +dir = os.path.dirname(__file__) +abs_dir = os.path.dirname(os.path.abspath(__file__)) + +if abs_dir.endswith(os.sep + os.path.join('bench', 'workgen')): + wt_dir = os.path.dirname(os.path.dirname(abs_dir)) +else: + print(os.path.basename(__file__) + ": running from unknown dir", \ + file=sys.stderr) + sys.exit(1) + +build_dir = os.path.join(wt_dir, 'build_posix') + +# Read the version information from the RELEASE_INFO file +for l in open(os.path.join(dir, '..', '..', 'RELEASE_INFO')): + if re.match(r'WIREDTIGER_VERSION_(?:MAJOR|MINOR|PATCH)=', l): + exec(l) + +wt_ver = '%d.%d' % (WIREDTIGER_VERSION_MAJOR, WIREDTIGER_VERSION_MINOR) + +setup(name='workgen', version=wt_ver, + ext_modules=[Extension('_workgen', + [os.path.join(dir, 'workgen_wrap.cxx')], + libraries=['wiredtiger', 'pthread', 'workgen'], + extra_compile_args=extra_cflags, + )], + package_dir={'' : dir}, + packages=['workgen'], +) diff --git a/bench/workgen/workgen.cxx b/bench/workgen/workgen.cxx new file mode 100644 index 00000000000..880b8ca6467 --- /dev/null +++ b/bench/workgen/workgen.cxx @@ -0,0 +1,1651 @@ +/*- + * Public Domain 2014-2017 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#define __STDC_LIMIT_MACROS // needed to get UINT64_MAX in C++ +#include <iomanip> +#include <iostream> +#include <fstream> +#include <sstream> +#include "wiredtiger.h" +#include "workgen.h" +#include "workgen_int.h" +#include "workgen_time.h" +extern "C" { +// Include some specific WT files, as some files included by wt_internal.h +// have some C-ism's that don't work in C++. +#include <pthread.h> +#include <string.h> +#include <stdint.h> +#include <stdlib.h> +#include <unistd.h> +#include <errno.h> +#include <math.h> +#include "error.h" +#include "misc.h" +} + +#define LATENCY_US_BUCKETS 1000 +#define LATENCY_MS_BUCKETS 1000 +#define LATENCY_SEC_BUCKETS 100 + +#define THROTTLE_PER_SEC 20 // times per sec we will throttle + +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#define MAX(a, b) ((a) < (b) ? (b) : (a)) +#define TIMESPEC_DOUBLE(ts) ((double)(ts).tv_sec + ts.tv_nsec * 0.000000001) +#define PCT(n, total) ((total) == 0 ? 0 : ((n) * 100) / (total)) +#define OPS_PER_SEC(ops, ts) (int) ((ts) == 0 ? 0.0 : \ + (ops) / TIMESPEC_DOUBLE(ts)) + +// Get the value of a STL container, even if it is not present +#define CONTAINER_VALUE(container, idx, dfault) \ + (((container).count(idx) > 0) ? (container)[idx] : (dfault)) + +#define CROSS_USAGE(a, b) \ + (((a & USAGE_READ) != 0 && (b & USAGE_WRITE) != 0) || \ + ((a & USAGE_WRITE) != 0 && (b & USAGE_READ) != 0)) + +#define ASSERT(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "%s:%d: ASSERT failed: %s\n", \ + __FILE__, __LINE__, #cond); \ + abort(); \ + } \ + } while(0) + +#define THROW_ERRNO(e, args) \ + do { \ + std::stringstream __sstm; \ + __sstm << args; \ + WorkgenException __wge(e, __sstm.str().c_str()); \ + throw(__wge); \ + } while(0) + +#define THROW(args) THROW_ERRNO(0, args) + +#define VERBOSE(runner, args) \ + do { \ + if ((runner)._context->_verbose) \ + std::cout << args << std::endl; \ + } while(0) + +#define OP_HAS_VALUE(op) \ + ((op)->_optype == Operation::OP_INSERT || \ + (op)->_optype == Operation::OP_UPDATE) + +namespace workgen { + +// The number of contexts. Normally there is one context created, but it will +// be possible to use several eventually. More than one is not yet +// implemented, but we must at least guard against the caller creating more +// than one. +static uint32_t context_count = 0; + +static void *thread_runner_main(void *arg) { + ThreadRunner *runner = (ThreadRunner *)arg; + try { + runner->_errno = runner->run(); + } catch (WorkgenException &wge) { + runner->_exception = wge; + } + return (NULL); +} + +static void *monitor_main(void *arg) { + Monitor *monitor = (Monitor *)arg; + try { + monitor->_errno = monitor->run(); + } catch (WorkgenException &wge) { + monitor->_exception = wge; + } + return (NULL); +} + +// Exponentiate (like the pow function), except that it returns an exact +// integral 64 bit value, and if it overflows, returns the maximum possible +// value for the return type. +static uint64_t power64(int base, int exp) { + uint64_t last, result; + + result = 1; + for (int i = 0; i < exp; i++) { + last = result; + result *= base; + if (result < last) + return UINT64_MAX; + } + return result; +} + +OptionsList::OptionsList() : _option_map() {} +OptionsList::OptionsList(const OptionsList &other) : + _option_map(other._option_map) {} + +void OptionsList::add_option(const char *name, const std::string typestr, + const char *desc) { + TypeDescPair pair(typestr, desc); + _option_map[name] = pair; +} + +void OptionsList::add_int(const char *name, int default_value, + const char *desc) { + std::stringstream sstm; + sstm << "int, default=" << default_value; + add_option(name, sstm.str(), desc); +} + +void OptionsList::add_bool(const char *name, bool default_value, + const char *desc) { + std::stringstream sstm; + sstm << "boolean, default=" << (default_value ? "true" : "false"); + add_option(name, sstm.str(), desc); +} + +void OptionsList::add_double(const char *name, double default_value, + const char *desc) { + std::stringstream sstm; + sstm << "double, default=" << default_value; + add_option(name, sstm.str(), desc); +} + +void OptionsList::add_string(const char *name, + const std::string &default_value, const char *desc) { + std::stringstream sstm; + sstm << "string, default=\"" << default_value << "\""; + add_option(name, sstm.str(), desc); +} + +static void +pretty_print(const char *p, const char *indent, std::stringstream &sstm) +{ + const char *t; + + for (;; p = t + 1) { + if (strlen(p) <= 70) + break; + for (t = p + 70; t > p && *t != ' '; --t) + ; + if (t == p) /* No spaces? */ + break; + if (indent != NULL) + sstm << indent; + std::string line(p, (size_t)(t - p)); + sstm << line << std::endl; + } + if (*p != '\0') { + if (indent != NULL) + sstm << indent; + sstm << p << std::endl; + } +} + +std::string OptionsList::help() const { + std::stringstream sstm; + for (std::map<std::string, TypeDescPair>::const_iterator i = + _option_map.begin(); i != _option_map.end(); i++) { + sstm << i->first << " (" << i->second.first << ")" << std::endl; + pretty_print(i->second.second.c_str(), "\t", sstm); + } + return sstm.str(); +} + +std::string OptionsList::help_description(const char *option_name) const { + const std::string key(option_name); + if (_option_map.count(key) == 0) + return (std::string("")); + else + return (_option_map.find(key)->second.second); +} + +std::string OptionsList::help_type(const char *option_name) const { + const std::string key(option_name); + if (_option_map.count(key) == 0) + return std::string(""); + else + return (_option_map.find(key)->second.first); +} + +Context::Context() : _verbose(false), _internal(new ContextInternal()) {} +Context::~Context() { delete _internal; } +Context& Context::operator=(const Context &other) { + _verbose = other._verbose; + *_internal = *other._internal; + return (*this); +} + +ContextInternal::ContextInternal() : _tint(), _table_names(), + _recno(NULL), _recno_alloced(0), _tint_last(0), _context_count(0) { + uint32_t count; + if ((count = workgen_atomic_add32(&context_count, 1)) != 1) + THROW("multiple Contexts not supported"); + _context_count = count; +} + +ContextInternal::~ContextInternal() { + if (_recno != NULL) + delete _recno; +} + +int ContextInternal::create_all() { + if (_recno_alloced != _tint_last) { + // The array references are 1-based, we'll waste one entry. + uint64_t *new_recno = new uint64_t[_tint_last + 1]; + memcpy(new_recno, _recno, sizeof(uint64_t) * _recno_alloced); + memset(&new_recno[_recno_alloced], 0, + sizeof(uint64_t) * (_tint_last - _recno_alloced + 1)); + delete _recno; + _recno = new_recno; + _recno_alloced = _tint_last; + } + return (0); +} + +Monitor::Monitor(WorkloadRunner &wrunner) : + _errno(0), _exception(), _wrunner(wrunner), _stop(false), _handle(), + _out(NULL), _json(NULL) {} +Monitor::~Monitor() {} + +int Monitor::run() { + struct timespec t; + struct tm *tm, _tm; + char time_buf[64], version[100]; + Stats prev_totals; + WorkloadOptions *options = &_wrunner._workload->options; + uint64_t latency_max = (uint64_t)options->max_latency; + bool first; + + (*_out) << "#time," + << "totalsec," + << "read ops per second," + << "insert ops per second," + << "update ops per second," + << "checkpoints," + << "read average latency(uS)," + << "read minimum latency(uS)," + << "read maximum latency(uS)," + << "insert average latency(uS)," + << "insert min latency(uS)," + << "insert maximum latency(uS)," + << "update average latency(uS)," + << "update min latency(uS)," + << "update maximum latency(uS)" + << std::endl; + + first = true; + workgen_version(version, sizeof(version)); + Stats prev_interval; + while (!_stop) { + for (int i = 0; i < options->sample_interval && !_stop; i++) + sleep(1); + if (_stop) + break; + + workgen_epoch(&t); + tm = localtime_r(&t.tv_sec, &_tm); + (void)strftime(time_buf, sizeof(time_buf), "%b %d %H:%M:%S", tm); + + Stats new_totals(true); + for (std::vector<ThreadRunner>::iterator tr = + _wrunner._trunners.begin(); tr != _wrunner._trunners.end(); tr++) + new_totals.add(tr->_stats, true); + Stats interval(new_totals); + interval.subtract(prev_totals); + interval.smooth(prev_interval); + + int interval_secs = options->sample_interval; + uint64_t cur_reads = interval.read.ops / interval_secs; + uint64_t cur_inserts = interval.insert.ops / interval_secs; + uint64_t cur_updates = interval.update.ops / interval_secs; + + uint64_t totalsec = ts_sec(t - _wrunner._start); + (*_out) << time_buf + << "," << totalsec + << "," << cur_reads + << "," << cur_inserts + << "," << cur_updates + << "," << 'N' // checkpoint in progress + << "," << interval.read.average_latency() + << "," << interval.read.min_latency + << "," << interval.read.max_latency + << "," << interval.insert.average_latency() + << "," << interval.insert.min_latency + << "," << interval.insert.max_latency + << "," << interval.update.average_latency() + << "," << interval.update.min_latency + << "," << interval.update.max_latency + << std::endl; + + if (_json != NULL) { +#define WORKGEN_TIMESTAMP_JSON "%Y-%m-%dT%H:%M:%S.000Z" + (void)strftime(time_buf, sizeof(time_buf), + WORKGEN_TIMESTAMP_JSON, tm); + +#define TRACK_JSON(name, t) \ + "\"" << (name) << "\":{" \ + << "\"ops per sec\":" << ((t).ops / interval_secs) \ + << ",\"average latency\":" << (t).average_latency() \ + << ",\"min latency\":" << (t).min_latency \ + << ",\"max latency\":" << (t).max_latency \ + << "}" + + (*_json) << "{"; + if (first) { + (*_json) << "\"version\":\"" << version << "\","; + first = false; + } + (*_json) << "\"localTime\":\"" << time_buf + << "\",\"workgen\":{" + << TRACK_JSON("read", interval.read) << "," + << TRACK_JSON("insert", interval.insert) << "," + << TRACK_JSON("update", interval.update) + << "}}" << std::endl; + } + + uint64_t read_max = interval.read.max_latency; + uint64_t insert_max = interval.read.max_latency; + uint64_t update_max = interval.read.max_latency; + + if (latency_max != 0 && + (read_max > latency_max || insert_max > latency_max || + update_max > latency_max)) { + std::cerr << "WARNING: max latency exceeded:" + << " threshold " << latency_max + << " read max " << read_max + << " insert max " << insert_max + << " update max " << update_max << std::endl; + } + + prev_interval.assign(interval); + prev_totals.assign(new_totals); + } + return (0); +} + +ThreadRunner::ThreadRunner() : + _errno(0), _exception(), _thread(NULL), _context(NULL), _icontext(NULL), + _workload(NULL), _wrunner(NULL), _rand_state(NULL), + _throttle(NULL), _throttle_ops(0), _throttle_limit(0), + _in_transaction(false), _number(0), _stats(false), _table_usage(), + _cursors(NULL), _stop(false), _session(NULL), _keybuf(NULL), + _valuebuf(NULL), _repeat(false) { +} + +ThreadRunner::~ThreadRunner() { + free_all(); +} + +int ThreadRunner::create_all(WT_CONNECTION *conn) { + size_t keysize, valuesize; + + WT_RET(close_all()); + ASSERT(_session == NULL); + WT_RET(conn->open_session(conn, NULL, NULL, &_session)); + _table_usage.clear(); + _stats.track_latency(_workload->options.sample_interval > 0); + WT_RET(workgen_random_alloc(_session, &_rand_state)); + _throttle_ops = 0; + _throttle_limit = 0; + _in_transaction = 0; + keysize = 1; + valuesize = 1; + op_create_all(&_thread->_op, keysize, valuesize); + _keybuf = new char[keysize]; + _valuebuf = new char[valuesize]; + _keybuf[keysize - 1] = '\0'; + _valuebuf[valuesize - 1] = '\0'; + return (0); +} + +int ThreadRunner::open_all() { + typedef WT_CURSOR *WT_CURSOR_PTR; + if (_cursors != NULL) + delete _cursors; + _cursors = new WT_CURSOR_PTR[_icontext->_tint_last + 1]; + memset(_cursors, 0, sizeof (WT_CURSOR *) * (_icontext->_tint_last + 1)); + for (std::map<uint32_t, uint32_t>::iterator i = _table_usage.begin(); + i != _table_usage.end(); i++) { + uint32_t tindex = i->first; + const char *uri = _icontext->_table_names[tindex].c_str(); + WT_RET(_session->open_cursor(_session, uri, NULL, NULL, + &_cursors[tindex])); + } + return (0); +} + +int ThreadRunner::close_all() { + if (_throttle != NULL) { + delete _throttle; + _throttle = NULL; + } + if (_session != NULL) { + WT_RET(_session->close(_session, NULL)); + _session = NULL; + } + free_all(); + return (0); +} + +void ThreadRunner::free_all() { + if (_rand_state != NULL) { + workgen_random_free(_rand_state); + _rand_state = NULL; + } + if (_cursors != NULL) { + delete _cursors; + _cursors = NULL; + } + if (_keybuf != NULL) { + delete _keybuf; + _keybuf = NULL; + } + if (_valuebuf != NULL) { + delete _valuebuf; + _valuebuf = NULL; + } +} + +int ThreadRunner::cross_check(std::vector<ThreadRunner> &runners) { + std::map<uint32_t, uint32_t> usage; + + // Determine which tables have cross usage + for (std::vector<ThreadRunner>::iterator r = runners.begin(); + r != runners.end(); r++) { + for (std::map<uint32_t, uint32_t>::iterator i = r->_table_usage.begin(); + i != r->_table_usage.end(); i++) { + uint32_t tindex = i->first; + uint32_t thisusage = i->second; + uint32_t curusage = CONTAINER_VALUE(usage, tindex, 0); + if (CROSS_USAGE(curusage, thisusage)) + curusage |= USAGE_MIXED; + usage[tindex] = curusage; + } + } + for (std::map<uint32_t, uint32_t>::iterator i = usage.begin(); + i != usage.end(); i++) { + if ((i->second & USAGE_MIXED) != 0) { + for (std::vector<ThreadRunner>::iterator r = runners.begin(); + r != runners.end(); r++) { + r->_table_usage[i->first] |= USAGE_MIXED; + } + } + } + return (0); +} + +int ThreadRunner::run() { + WT_DECL_RET; + ThreadOptions *options = &_thread->options; + std::string name = options->name; + + VERBOSE(*this, "thread " << name << " running"); + if (options->throttle != 0) { + _throttle = new Throttle(*this, options->throttle, + options->throttle_burst); + } + for (int cnt = 0; !_stop && (_repeat || cnt < 1) && ret == 0; cnt++) + WT_ERR(op_run(&_thread->_op)); + +err: +#ifdef _DEBUG + { + std::string messages = this->get_debug(); + if (!messages.empty()) + std::cerr << "DEBUG (thread " << name << "): " + << messages << std::endl; + } +#endif + if (ret != 0) + std::cerr << "thread " << name << " failed err=" << ret << std::endl; + VERBOSE(*this, "thread " << name << "finished"); + return (ret); +} + +void ThreadRunner::get_static_counts(Stats &stats) { + _thread->_op.get_static_counts(stats, 1); +} + +void ThreadRunner::op_create_all(Operation *op, size_t &keysize, + size_t &valuesize) { + tint_t tint; + + op->size_check(); + if (op->_optype != Operation::OP_NONE) { + op->kv_compute_max(true); + if (OP_HAS_VALUE(op)) + op->kv_compute_max(false); + op->kv_size_buffer(true, keysize); + op->kv_size_buffer(false, valuesize); + + // Note: to support multiple contexts we'd need a generation + // count whenever we execute. + if (op->_table._internal->_context_count != 0 && + op->_table._internal->_context_count != _icontext->_context_count) + THROW("multiple Contexts not supported"); + if ((tint = op->_table._internal->_tint) == 0) { + std::string uri = op->_table._uri; + + // We are single threaded in this function, so do not have + // to worry about locking. + if (_icontext->_tint.count(uri) == 0) { + // TODO: don't use atomic add, it's overkill. + tint = workgen_atomic_add32(&_icontext->_tint_last, 1); + _icontext->_tint[uri] = tint; + _icontext->_table_names[tint] = uri; + } else + tint = _icontext->_tint[uri]; + op->_table._internal->_tint = tint; + } + uint32_t usage_flags = CONTAINER_VALUE(_table_usage, + op->_table._internal->_tint, 0); + if (op->_optype == Operation::OP_SEARCH) + usage_flags |= ThreadRunner::USAGE_READ; + else + usage_flags |= ThreadRunner::USAGE_WRITE; + _table_usage[op->_table._internal->_tint] = usage_flags; + } + if (op->_group != NULL) + for (std::vector<Operation>::iterator i = op->_group->begin(); + i != op->_group->end(); i++) + op_create_all(&*i, keysize, valuesize); +} + +uint64_t ThreadRunner::op_get_key_recno(Operation *op, tint_t tint) { + uint64_t recno_count; + uint32_t rand; + + recno_count = _icontext->_recno[tint]; + if (recno_count == 0) + // The file has no entries, returning 0 forces a WT_NOTFOUND return. + return (0); + rand = workgen_random(_rand_state); + return (rand % recno_count + 1); // recnos are one-based. +} + +int ThreadRunner::op_run(Operation *op) { + Track *track; + tint_t tint = op->_table._internal->_tint; + WT_CURSOR *cursor = _cursors[tint]; + WT_DECL_RET; + uint64_t recno; + bool measure_latency; + + recno = 0; + track = NULL; + if (_throttle != NULL) { + if (_throttle_ops >= _throttle_limit && !_in_transaction) { + WT_ERR(_throttle->throttle(_throttle_ops, + &_throttle_limit)); + _throttle_ops = 0; + } + if (op->_optype != Operation::OP_NONE) + ++_throttle_ops; + } + + // A potential race: thread1 is inserting, and increments + // Context->_recno[] for fileX.wt. thread2 is doing one of + // remove/search/update and grabs the new value of Context->_recno[] + // for fileX.wt. thread2 randomly chooses the highest recno (which + // has not yet been inserted by thread1), and when it accesses + // the record will get WT_NOTFOUND. It should be somewhat rare + // (and most likely when the threads are first beginning). Any + // WT_NOTFOUND returns are allowed and get their own statistic bumped. + switch (op->_optype) { + case Operation::OP_INSERT: + track = &_stats.insert; + recno = workgen_atomic_add64(&_icontext->_recno[tint], 1); + break; + case Operation::OP_REMOVE: + track = &_stats.remove; + recno = op_get_key_recno(op, tint); + break; + case Operation::OP_SEARCH: + track = &_stats.read; + recno = op_get_key_recno(op, tint); + break; + case Operation::OP_UPDATE: + track = &_stats.update; + recno = op_get_key_recno(op, tint); + break; + case Operation::OP_NONE: + recno = 0; + break; + } + + measure_latency = track != NULL && track->ops != 0 && + track->track_latency() && + (track->ops % _workload->options.sample_rate == 0); + + timespec start; + if (measure_latency) + workgen_epoch(&start); + + if (op->_transaction != NULL) { + if (_in_transaction) + THROW("nested transactions not supported"); + _session->begin_transaction(_session, + op->_transaction->_begin_config.c_str()); + _in_transaction = true; + } + if (op->_optype != Operation::OP_NONE) { + op->kv_gen(true, recno, _keybuf); + cursor->set_key(cursor, _keybuf); + if (OP_HAS_VALUE(op)) { + op->kv_gen(false, recno, _valuebuf); + cursor->set_value(cursor, _valuebuf); + } + switch (op->_optype) { + case Operation::OP_INSERT: + WT_ERR(cursor->insert(cursor)); + break; + case Operation::OP_REMOVE: + WT_ERR_NOTFOUND_OK(cursor->remove(cursor)); + break; + case Operation::OP_SEARCH: + ret = cursor->search(cursor); + break; + case Operation::OP_UPDATE: + WT_ERR_NOTFOUND_OK(cursor->update(cursor)); + break; + default: + ASSERT(false); + } + if (ret != 0) { + track = &_stats.not_found; + ret = 0; // WT_NOTFOUND allowed. + } + cursor->reset(cursor); + } + if (measure_latency) { + timespec stop; + workgen_epoch(&stop); + track->incr_with_latency(ts_us(stop - start)); + } else if (track != NULL) + track->incr(); + + if (op->_group != NULL) + for (int count = 0; !_stop && count < op->_repeatgroup; count++) + for (std::vector<Operation>::iterator i = op->_group->begin(); + i != op->_group->end(); i++) + WT_ERR(op_run(&*i)); +err: + if (op->_transaction != NULL) { + if (ret != 0 || op->_transaction->_rollback) + WT_TRET(_session->rollback_transaction(_session, NULL)); + else + ret = _session->commit_transaction(_session, + op->_transaction->_commit_config.c_str()); + _in_transaction = false; + } + return (ret); +} + +#ifdef _DEBUG +std::string ThreadRunner::get_debug() { + return (_debug_messages.str()); +} +#endif + +Throttle::Throttle(ThreadRunner &runner, double throttle, + double throttle_burst) : _runner(runner), _throttle(throttle), + _burst(throttle_burst), _next_div(), _ops_delta(0), _ops_prev(0), + _ops_per_div(0), _ms_per_div(0), _started(false) { + ts_clear(_next_div); + _ms_per_div = ceill(1000.0 / THROTTLE_PER_SEC); + _ops_per_div = ceill(_throttle / THROTTLE_PER_SEC); +} + +Throttle::~Throttle() {} + +// Given a random 32-bit value, return a float value equally distributed +// between -1.0 and 1.0. +static float rand_signed(uint32_t r) { + int sign = ((r & 0x1) == 0 ? 1 : -1); + return (((float)r * sign) / UINT32_MAX); +} + +// Each time throttle is called, we sleep and return a number of operations to +// perform next. To implement this we keep a time calculation in _next_div set +// initially to the current time + 1/THROTTLE_PER_SEC. Each call to throttle +// advances _next_div by 1/THROTTLE_PER_SEC, and if _next_div is in the future, +// we sleep for the difference between the _next_div and the current_time. We +// always return (Thread.options.throttle / THROTTLE_PER_SEC) as the number of +// operations. +// +// The only variation is that the amount of individual sleeps is modified by a +// random amount (which varies more widely as Thread.options.throttle_burst is +// greater). This has the effect of randomizing how much clumping happens, and +// ensures that multiple threads aren't executing in lock step. +// +int Throttle::throttle(uint64_t op_count, uint64_t *op_limit) { + uint64_t ops; + int64_t sleep_ms; + timespec now; + + workgen_epoch(&now); + DEBUG_CAPTURE(_runner, "throttle: ops=" << op_count); + if (!_started) { + _next_div = ts_add_ms(now, _ms_per_div); + _started = true; + } else { + _ops_delta += (op_count - _ops_prev); + if (now < _next_div) { + sleep_ms = ts_ms(_next_div - now); + sleep_ms += (_ms_per_div * _burst * + rand_signed(workgen_random(_runner._rand_state))); + if (sleep_ms > 0) { + DEBUG_CAPTURE(_runner, ", sleep=" << sleep_ms); + usleep((useconds_t)ms_to_us(sleep_ms)); + } + } + _next_div = ts_add_ms(_next_div, _ms_per_div); + } + ops = _ops_per_div; + if (_ops_delta < (int64_t)ops) { + ops -= _ops_delta; + _ops_delta = 0; + } else { + _ops_delta -= ops; + ops = 0; + } + *op_limit = ops; + _ops_prev = ops; + DEBUG_CAPTURE(_runner, ", return=" << ops << std::endl); + return (0); +} + +ThreadOptions::ThreadOptions() : name(), throttle(0.0), throttle_burst(1.0), + _options() { + _options.add_string("name", name, "name of the thread"); + _options.add_double("throttle", throttle, + "Limit to this number of operations per second"); + _options.add_double("throttle_burst", throttle_burst, + "Changes characteristic of throttling from smooth (0.0) " + "to having large bursts with lulls (10.0 or larger)"); +} +ThreadOptions::ThreadOptions(const ThreadOptions &other) : + name(other.name), throttle(other.throttle), + throttle_burst(other.throttle_burst), _options(other._options) {} +ThreadOptions::~ThreadOptions() {} + +void +ThreadListWrapper::extend(const ThreadListWrapper &other) { + for (std::vector<Thread>::const_iterator i = other._threads.begin(); + i != other._threads.end(); i++) + _threads.push_back(*i); +} + +void +ThreadListWrapper::append(const Thread &t) { + _threads.push_back(t); +} + +void +ThreadListWrapper::multiply(const int n) { + if (n == 0) { + _threads.clear(); + } else { + std::vector<Thread> copy(_threads); + for (int cnt = 1; cnt < n; cnt++) + extend(copy); + } +} + +Thread::Thread() : options(), _op() { +} + +Thread::Thread(const Operation &op) : options(), _op(op) { +} + +Thread::Thread(const Thread &other) : options(other.options), _op(other._op) { +} + +Thread::~Thread() { +} + +void Thread::describe(std::ostream &os) const { + os << "Thread: [" << std::endl; + _op.describe(os); os << std::endl; + os << "]"; +} + +Operation::Operation() : + _optype(OP_NONE), _table(), _key(), _value(), _transaction(NULL), + _group(NULL), _repeatgroup(0), + _keysize(0), _valuesize(0), _keymax(0), _valuemax(0) { +} + +Operation::Operation(OpType optype, Table table, Key key, Value value) : + _optype(optype), _table(table), _key(key), _value(value), + _transaction(NULL), _group(NULL), _repeatgroup(0), + _keysize(0), _valuesize(0), _keymax(0), _valuemax(0) { + size_check(); +} + +Operation::Operation(OpType optype, Table table, Key key) : + _optype(optype), _table(table), _key(key), _value(), _transaction(NULL), + _group(NULL), _repeatgroup(0), + _keysize(0), _valuesize(0), _keymax(0), _valuemax(0) { + size_check(); +} + +Operation::Operation(OpType optype, Table table) : + _optype(optype), _table(table), _key(), _value(), _transaction(NULL), + _group(NULL), _repeatgroup(0), + _keysize(0), _valuesize(0), _keymax(0), _valuemax(0) { + size_check(); +} + +Operation::Operation(const Operation &other) : + _optype(other._optype), _table(other._table), _key(other._key), + _value(other._value), _transaction(other._transaction), + _group(other._group), _repeatgroup(other._repeatgroup), + _keysize(other._keysize), _valuesize(other._valuesize), + _keymax(other._keymax), _valuemax(other._valuemax) { + // Creation and destruction of _group and _transaction is managed + // by Python. +} + +Operation::~Operation() { + // Creation and destruction of _group, _transaction is managed by Python. +} + +Operation& Operation::operator=(const Operation &other) { + _optype = other._optype; + _table = other._table; + _key = other._key; + _value = other._value; + _transaction = other._transaction; + _group = other._group; + _repeatgroup = other._repeatgroup; + _keysize = other._keysize; + _valuesize = other._valuesize; + _keymax = other._keymax; + _valuemax = other._valuemax; + return (*this); +} + +void Operation::describe(std::ostream &os) const { + os << "Operation: " << _optype; + if (_optype != OP_NONE) { + os << ", "; _table.describe(os); + os << ", "; _key.describe(os); + os << ", "; _value.describe(os); + } + if (_transaction != NULL) { + os << ", ["; _transaction->describe(os); os << "]"; + } + if (_group != NULL) { + os << ", group[" << _repeatgroup << "]: {"; + bool first = true; + for (std::vector<Operation>::const_iterator i = _group->begin(); + i != _group->end(); i++) { + if (!first) + os << "}, {"; + i->describe(os); + first = false; + } + os << "}"; + } +} + +void Operation::get_static_counts(Stats &stats, int multiplier) { + switch (_optype) { + case OP_NONE: + break; + case OP_INSERT: + stats.insert.ops += multiplier; + break; + case OP_REMOVE: + stats.remove.ops += multiplier; + break; + case OP_SEARCH: + stats.read.ops += multiplier; + break; + case OP_UPDATE: + stats.update.ops += multiplier; + break; + default: + ASSERT(false); + } + if (_group != NULL) + for (std::vector<Operation>::iterator i = _group->begin(); + i != _group->end(); i++) + i->get_static_counts(stats, multiplier * _repeatgroup); +} + +void Operation::kv_compute_max(bool iskey) { + uint64_t max; + int size; + + size = iskey ? _key._size : _value._size; + if (size == 0) + size = iskey ? _table.options.key_size : _table.options.value_size; + + if (iskey && size < 2) + THROW("Key.size too small for table '" << _table._uri << "'"); + if (!iskey && size < 1) + THROW("Value.size too small for table '" << _table._uri << "'"); + + if (size > 1) + max = power64(10, (size - 1)) - 1; + else + max = 0; + + if (iskey) { + _keysize = size; + _keymax = max; + } else { + _valuesize = size; + _valuemax = max; + } +} + +void Operation::kv_size_buffer(bool iskey, size_t &maxsize) const { + if (iskey) { + if ((size_t)_keysize > maxsize) + maxsize = _keysize; + } else { + if ((size_t)_valuesize > maxsize) + maxsize = _valuesize; + } +} + +void Operation::kv_gen(bool iskey, uint64_t n, char *result) const { + uint64_t max; + int size; + + size = iskey ? _keysize : _valuesize; + max = iskey ? _keymax : _valuemax; + if (n > max) + THROW((iskey ? "Key" : "Value") << " (" << n + << ") too large for size (" << size << ")"); + workgen_u64_to_string_zf(n, result, size); +} + +void Operation::size_check() const { + if (_optype != OP_NONE && _key._size == 0 && _table.options.key_size == 0) + THROW("operation requires a key size"); + if (OP_HAS_VALUE(this) && _value._size == 0 && + _table.options.value_size == 0) + THROW("operation requires a value size"); +} + +Track::Track(bool latency_tracking) : ops(0), latency_ops(0), latency(0), + min_latency(0), max_latency(0), us(NULL), ms(NULL), sec(NULL) { + track_latency(latency_tracking); +} + +Track::Track(const Track &other) : ops(other.ops), + latency_ops(other.latency_ops), latency(other.latency), + min_latency(other.min_latency), max_latency(other.max_latency), + us(NULL), ms(NULL), sec(NULL) { + if (other.us != NULL) { + us = new uint32_t[LATENCY_US_BUCKETS]; + ms = new uint32_t[LATENCY_MS_BUCKETS]; + sec = new uint32_t[LATENCY_SEC_BUCKETS]; + memcpy(us, other.us, sizeof(uint32_t) * LATENCY_US_BUCKETS); + memcpy(ms, other.ms, sizeof(uint32_t) * LATENCY_MS_BUCKETS); + memcpy(sec, other.sec, sizeof(uint32_t) * LATENCY_SEC_BUCKETS); + } +} + +Track::~Track() { + if (us != NULL) { + delete us; + delete ms; + delete sec; + } +} + +void Track::add(Track &other, bool reset) { + ops += other.ops; + latency_ops += other.latency_ops; + latency += other.latency; + + min_latency = MIN(min_latency, other.min_latency); + if (reset) + other.min_latency = 0; + max_latency = MAX(max_latency, other.max_latency); + if (reset) + other.max_latency = 0; + + if (us != NULL && other.us != NULL) { + for (int i = 0; i < LATENCY_US_BUCKETS; i++) + us[i] += other.us[i]; + for (int i = 0; i < LATENCY_MS_BUCKETS; i++) + ms[i] += other.ms[i]; + for (int i = 0; i < LATENCY_SEC_BUCKETS; i++) + sec[i] += other.sec[i]; + } +} + +void Track::assign(const Track &other) { + ops = other.ops; + latency_ops = other.latency_ops; + latency = other.latency; + min_latency = other.min_latency; + max_latency = other.max_latency; + + if (other.us == NULL && us != NULL) { + delete us; + delete ms; + delete sec; + us = NULL; + ms = NULL; + sec = NULL; + } + else if (other.us != NULL && us == NULL) { + us = new uint32_t[LATENCY_US_BUCKETS]; + ms = new uint32_t[LATENCY_MS_BUCKETS]; + sec = new uint32_t[LATENCY_SEC_BUCKETS]; + } + if (us != NULL) { + memcpy(us, other.us, sizeof(uint32_t) * LATENCY_US_BUCKETS); + memcpy(ms, other.ms, sizeof(uint32_t) * LATENCY_MS_BUCKETS); + memcpy(sec, other.sec, sizeof(uint32_t) * LATENCY_SEC_BUCKETS); + } +} + +uint64_t Track::average_latency() const { + if (latency_ops == 0) + return (0); + else + return (latency / latency_ops); +} + +void Track::clear() { + ops = 0; + latency_ops = 0; + latency = 0; + min_latency = 0; + max_latency = 0; + if (us != NULL) { + memset(us, 0, sizeof(uint32_t) * LATENCY_US_BUCKETS); + memset(ms, 0, sizeof(uint32_t) * LATENCY_MS_BUCKETS); + memset(sec, 0, sizeof(uint32_t) * LATENCY_SEC_BUCKETS); + } +} + +void Track::incr() { + ops++; +} + +void Track::incr_with_latency(uint64_t usecs) { + ASSERT(us != NULL); + + ops++; + latency_ops++; + latency += usecs; + if (usecs > max_latency) + max_latency = (uint32_t)usecs; + if (usecs < min_latency) + min_latency = (uint32_t)usecs; + + // Update a latency bucket. + // First buckets: usecs from 100us to 1000us at 100us each. + if (usecs < LATENCY_US_BUCKETS) + us[usecs]++; + + // Second buckets: milliseconds from 1ms to 1000ms, at 1ms each. + else if (usecs < ms_to_us(LATENCY_MS_BUCKETS)) + ms[us_to_ms(usecs)]++; + + // Third buckets are seconds from 1s to 100s, at 1s each. + else if (usecs < sec_to_us(LATENCY_SEC_BUCKETS)) + sec[us_to_sec(usecs)]++; + + // >100 seconds, accumulate in the biggest bucket. */ + else + sec[LATENCY_SEC_BUCKETS - 1]++; +} + +void Track::subtract(const Track &other) { + ops -= other.ops; + latency_ops -= other.latency_ops; + latency -= other.latency; + + // There's no sensible thing to be done for min/max_latency. + + if (us != NULL && other.us != NULL) { + for (int i = 0; i < LATENCY_US_BUCKETS; i++) + us[i] -= other.us[i]; + for (int i = 0; i < LATENCY_MS_BUCKETS; i++) + ms[i] -= other.ms[i]; + for (int i = 0; i < LATENCY_SEC_BUCKETS; i++) + sec[i] -= other.sec[i]; + } +} + +// If there are no entries in this Track, take them from +// a previous Track. Used to smooth graphs. We don't worry +// about latency buckets here. +void Track::smooth(const Track &other) { + if (latency_ops == 0) { + ops = other.ops; + latency = other.latency; + latency_ops = other.latency_ops; + min_latency = other.min_latency; + max_latency = other.max_latency; + } +} + +void Track::track_latency(bool newval) { + if (newval) { + if (us == NULL) { + us = new uint32_t[LATENCY_US_BUCKETS]; + ms = new uint32_t[LATENCY_MS_BUCKETS]; + sec = new uint32_t[LATENCY_SEC_BUCKETS]; + memset(us, 0, sizeof(uint32_t) * LATENCY_US_BUCKETS); + memset(ms, 0, sizeof(uint32_t) * LATENCY_MS_BUCKETS); + memset(sec, 0, sizeof(uint32_t) * LATENCY_SEC_BUCKETS); + } + } else { + if (us != NULL) { + delete us; + delete ms; + delete sec; + us = NULL; + ms = NULL; + sec = NULL; + } + } +} + +void Track::_get_us(long *result) { + if (us != NULL) { + for (int i = 0; i < LATENCY_US_BUCKETS; i++) + result[i] = (long)us[i]; + } else + memset(result, 0, sizeof(long) * LATENCY_US_BUCKETS); +} +void Track::_get_ms(long *result) { + if (ms != NULL) { + for (int i = 0; i < LATENCY_MS_BUCKETS; i++) + result[i] = (long)ms[i]; + } else + memset(result, 0, sizeof(long) * LATENCY_MS_BUCKETS); +} +void Track::_get_sec(long *result) { + if (sec != NULL) { + for (int i = 0; i < LATENCY_SEC_BUCKETS; i++) + result[i] = (long)sec[i]; + } else + memset(result, 0, sizeof(long) * LATENCY_SEC_BUCKETS); +} + +Stats::Stats(bool latency) : insert(latency), not_found(latency), + read(latency), remove(latency), update(latency), truncate(latency) { +} + +Stats::Stats(const Stats &other) : insert(other.insert), + not_found(other.not_found), read(other.read), remove(other.remove), + update(other.update), truncate(other.truncate) { +} + +Stats::~Stats() {} + +void Stats::add(Stats &other, bool reset) { + insert.add(other.insert, reset); + not_found.add(other.not_found, reset); + read.add(other.read, reset); + remove.add(other.remove, reset); + update.add(other.update, reset); + truncate.add(other.truncate, reset); +} + +void Stats::assign(const Stats &other) { + insert.assign(other.insert); + not_found.assign(other.not_found); + read.assign(other.read); + remove.assign(other.remove); + update.assign(other.update); + truncate.assign(other.truncate); +} + +void Stats::clear() { + insert.clear(); + not_found.clear(); + read.clear(); + remove.clear(); + update.clear(); + truncate.clear(); +} + +void Stats::describe(std::ostream &os) const { + os << "Stats: reads " << read.ops; + if (not_found.ops > 0) { + os << " (" << not_found.ops << " not found)"; + } + os << ", inserts " << insert.ops; + os << ", updates " << update.ops; + os << ", truncates " << truncate.ops; + os << ", removes " << remove.ops; +} + +void Stats::final_report(std::ostream &os, timespec &totalsecs) const { + uint64_t ops = 0; + ops += read.ops; + ops += not_found.ops; + ops += insert.ops; + ops += update.ops; + ops += truncate.ops; + ops += remove.ops; + +#define FINAL_OUTPUT(os, field, singular, ops, totalsecs) \ + os << "Executed " << field << " " #singular " operations (" \ + << PCT(field, ops) << "%) " << OPS_PER_SEC(field, totalsecs) \ + << " ops/sec" << std::endl + + FINAL_OUTPUT(os, read.ops, read, ops, totalsecs); + FINAL_OUTPUT(os, not_found.ops, not found, ops, totalsecs); + FINAL_OUTPUT(os, insert.ops, insert, ops, totalsecs); + FINAL_OUTPUT(os, update.ops, update, ops, totalsecs); + FINAL_OUTPUT(os, truncate.ops, truncate, ops, totalsecs); + FINAL_OUTPUT(os, remove.ops, remove, ops, totalsecs); +} + +void Stats::report(std::ostream &os) const { + os << read.ops << " reads"; + if (not_found.ops > 0) { + os << " (" << not_found.ops << " not found)"; + } + os << ", " << insert.ops << " inserts, "; + os << update.ops << " updates, "; + os << truncate.ops << " truncates, "; + os << remove.ops << " removes"; +} + +void Stats::smooth(const Stats &other) { + insert.smooth(other.insert); + not_found.smooth(other.not_found); + read.smooth(other.read); + remove.smooth(other.remove); + update.smooth(other.update); + truncate.smooth(other.truncate); +} + +void Stats::subtract(const Stats &other) { + insert.subtract(other.insert); + not_found.subtract(other.not_found); + read.subtract(other.read); + remove.subtract(other.remove); + update.subtract(other.update); + truncate.subtract(other.truncate); +} + +void Stats::track_latency(bool latency) { + insert.track_latency(latency); + not_found.track_latency(latency); + read.track_latency(latency); + remove.track_latency(latency); + update.track_latency(latency); + truncate.track_latency(latency); +} + +TableOptions::TableOptions() : key_size(0), value_size(0), _options() { + _options.add_int("key_size", key_size, + "default size of the key, unless overridden by Key.size"); + _options.add_int("value_size", value_size, + "default size of the value, unless overridden by Value.size"); +} +TableOptions::TableOptions(const TableOptions &other) : + key_size(other.key_size), value_size(other.value_size), + _options(other._options) {} +TableOptions::~TableOptions() {} + +Table::Table() : options(), _uri(), _internal(new TableInternal()) { +} +Table::Table(const char *uri) : options(), _uri(uri), + _internal(new TableInternal()) { +} +Table::Table(const Table &other) : options(other.options), _uri(other._uri), + _internal(new TableInternal(*other._internal)) { +} +Table::~Table() { delete _internal; } +Table& Table::operator=(const Table &other) { + options = other.options; + _uri = other._uri; + *_internal = *other._internal; + return (*this); +} + +void Table::describe(std::ostream &os) const { + os << "Table: " << _uri; +} + +TableInternal::TableInternal() : _tint(0), _context_count(0) {} +TableInternal::TableInternal(const TableInternal &other) : _tint(other._tint), + _context_count(other._context_count) {} +TableInternal::~TableInternal() {} + +WorkloadOptions::WorkloadOptions() : max_latency(0), + report_file("workload.stat"), report_interval(0), run_time(0), + sample_file("sample.json"), sample_interval(0), sample_rate(1), + _options() { + _options.add_int("max_latency", max_latency, + "prints warning if any latency measured exceeds this number of " + "milliseconds. Requires sample_interval to be configured."); + _options.add_int("report_interval", report_interval, + "output throughput information every interval seconds, 0 to disable"); + _options.add_string("report_file", report_file, + "file name for collecting run output, " + "including output from the report_interval option. " + "The file name is relative to the connection's home directory. " + "When set to the empty string, stdout is used."); + _options.add_int("run_time", run_time, "total workload seconds"); + _options.add_string("sample_file", sample_file, + "file name for collecting latency output in a JSON-like format, " + "enabled by the report_interval option. " + "The file name is relative to the connection's home directory. " + "When set to the empty string, no JSON is emitted."); + _options.add_int("sample_interval", sample_interval, + "performance logging every interval seconds, 0 to disable"); + _options.add_int("sample_rate", sample_rate, + "how often the latency of operations is measured. 1 for every operation, " + "2 for every second operation, 3 for every third operation etc."); +} + +WorkloadOptions::WorkloadOptions(const WorkloadOptions &other) : + max_latency(other.max_latency), report_interval(other.report_interval), + run_time(other.run_time), sample_interval(other.sample_interval), + sample_rate(other.sample_rate), _options(other._options) {} +WorkloadOptions::~WorkloadOptions() {} + +Workload::Workload(Context *context, const ThreadListWrapper &tlw) : + options(), stats(), _context(context), _threads(tlw._threads) { + if (context == NULL) + THROW("Workload contructor requires a Context"); +} + +Workload::Workload(Context *context, const Thread &thread) : + options(), stats(), _context(context), _threads() { + if (context == NULL) + THROW("Workload contructor requires a Context"); + _threads.push_back(thread); +} + +Workload::Workload(const Workload &other) : + options(other.options), stats(other.stats), _context(other._context), + _threads(other._threads) {} +Workload::~Workload() {} + +Workload& Workload::operator=(const Workload &other) { + options = other.options; + stats.assign(other.stats); + *_context = *other._context; + _threads = other._threads; + return (*this); +} + +int Workload::run(WT_CONNECTION *conn) { + WorkloadRunner runner(this); + + return (runner.run(conn)); +} + +WorkloadRunner::WorkloadRunner(Workload *workload) : + _workload(workload), _trunners(workload->_threads.size()), + _report_out(&std::cout), _start() { + ts_clear(_start); +} +WorkloadRunner::~WorkloadRunner() {} + +int WorkloadRunner::run(WT_CONNECTION *conn) { + WT_DECL_RET; + WorkloadOptions *options = &_workload->options; + std::ofstream report_out; + + _wt_home = conn->get_home(conn); + if (options->sample_interval > 0 && options->sample_rate <= 0) + THROW("Workload.options.sample_rate must be positive"); + if (!options->report_file.empty()) { + open_report_file(report_out, options->report_file.c_str(), + "Workload.options.report_file"); + _report_out = &report_out; + } + WT_ERR(create_all(conn, _workload->_context)); + WT_ERR(open_all()); + WT_ERR(ThreadRunner::cross_check(_trunners)); + WT_ERR(run_all()); + err: + //TODO: (void)close_all(); + _report_out = &std::cout; + return (ret); +} + +int WorkloadRunner::open_all() { + for (size_t i = 0; i < _trunners.size(); i++) { + WT_RET(_trunners[i].open_all()); + } + return (0); +} + +void WorkloadRunner::open_report_file(std::ofstream &of, const char *filename, + const char *desc) { + std::stringstream sstm; + + if (!_wt_home.empty()) + sstm << _wt_home << "/"; + sstm << filename; + of.open(sstm.str().c_str(), std::fstream::app); + if (!of) + THROW_ERRNO(errno, desc << ": \"" << sstm.str() + << "\" could not be opened"); +} + +int WorkloadRunner::create_all(WT_CONNECTION *conn, Context *context) { + for (size_t i = 0; i < _trunners.size(); i++) { + ThreadRunner *runner = &_trunners[i]; + std::stringstream sstm; + Thread *thread = &_workload->_threads[i]; + if (thread->options.name.empty()) { + sstm << "thread" << i; + thread->options.name = sstm.str(); + } + runner->_thread = thread; + runner->_context = context; + runner->_icontext = context->_internal; + runner->_workload = _workload; + runner->_wrunner = this; + runner->_number = (uint32_t)i; + // TODO: recover from partial failure here + WT_RET(runner->create_all(conn)); + } + WT_RET(context->_internal->create_all()); + return (0); +} + +int WorkloadRunner::close_all() { + for (size_t i = 0; i < _trunners.size(); i++) + _trunners[i].close_all(); + + return (0); +} + +void WorkloadRunner::get_stats(Stats *result) { + for (size_t i = 0; i < _trunners.size(); i++) + result->add(_trunners[i]._stats); +} + +void WorkloadRunner::report(time_t interval, time_t totalsecs, + Stats *prev_totals) { + std::ostream &out = *_report_out; + Stats new_totals(prev_totals->track_latency()); + + get_stats(&new_totals); + Stats diff(new_totals); + diff.subtract(*prev_totals); + prev_totals->assign(new_totals); + diff.report(out); + out << " in " << interval << " secs (" + << totalsecs << " total secs)" << std::endl; +} + +void WorkloadRunner::final_report(timespec &totalsecs) { + std::ostream &out = *_report_out; + Stats *stats = &_workload->stats; + + stats->clear(); + stats->track_latency(_workload->options.sample_interval > 0); + + get_stats(stats); + stats->final_report(out, totalsecs); + out << "Run completed: " << totalsecs << " seconds" << std::endl; +} + +int WorkloadRunner::run_all() { + void *status; + std::vector<pthread_t> thread_handles; + Stats counts(false); + WorkgenException *exception; + WorkloadOptions *options = &_workload->options; + Monitor monitor(*this); + std::ofstream monitor_out; + std::ofstream monitor_json; + std::ostream &out = *_report_out; + WT_DECL_RET; + + for (size_t i = 0; i < _trunners.size(); i++) + _trunners[i].get_static_counts(counts); + out << "Starting workload: " << _trunners.size() << " threads, "; + counts.report(out); + out << std::endl; + + workgen_epoch(&_start); + timespec end = _start + options->run_time; + timespec next_report = _start + options->report_interval; + + // Start all threads + if (options->sample_interval > 0) { + open_report_file(monitor_out, "monitor", "monitor output file"); + monitor._out = &monitor_out; + + if (!options->sample_file.empty()) { + open_report_file(monitor_json, options->sample_file.c_str(), + "sample JSON output file"); + monitor._json = &monitor_json; + } + + if ((ret = pthread_create(&monitor._handle, NULL, monitor_main, + &monitor)) != 0) { + std::cerr << "monitor thread failed err=" << ret << std::endl; + return (ret); + } + } + + for (size_t i = 0; i < _trunners.size(); i++) { + pthread_t thandle; + ThreadRunner *runner = &_trunners[i]; + runner->_stop = false; + runner->_repeat = (options->run_time != 0); + if ((ret = pthread_create(&thandle, NULL, thread_runner_main, + runner)) != 0) { + std::cerr << "pthread_create failed err=" << ret << std::endl; + std::cerr << "Stopping all threads." << std::endl; + for (size_t j = 0; j < thread_handles.size(); j++) { + _trunners[j]._stop = true; + (void)pthread_join(thread_handles[j], &status); + _trunners[j].close_all(); + } + return (ret); + } + thread_handles.push_back(thandle); + runner->_stats.clear(); + } + + // Let the test run, reporting as needed. + Stats curstats(false); + timespec now = _start; + while (now < end) { + timespec sleep_amt; + + sleep_amt = end - now; + if (next_report != 0) { + timespec next_diff = next_report - now; + if (next_diff < next_report) + sleep_amt = next_diff; + } + if (sleep_amt.tv_sec > 0) + sleep((unsigned int)sleep_amt.tv_sec); + else + usleep((useconds_t)((sleep_amt.tv_nsec + 999)/ 1000)); + + workgen_epoch(&now); + if (now >= next_report && now < end && options->report_interval != 0) { + report(options->report_interval, (now - _start).tv_sec, &curstats); + while (now >= next_report) + next_report += options->report_interval; + } + } + + // signal all threads to stop + if (options->run_time != 0) + for (size_t i = 0; i < _trunners.size(); i++) + _trunners[i]._stop = true; + if (options->sample_interval > 0) + monitor._stop = true; + + // wait for all threads + exception = NULL; + for (size_t i = 0; i < _trunners.size(); i++) { + WT_TRET(pthread_join(thread_handles[i], &status)); + if (_trunners[i]._errno != 0) + VERBOSE(_trunners[i], + "Thread " << i << " has errno " << _trunners[i]._errno); + WT_TRET(_trunners[i]._errno); + _trunners[i].close_all(); + if (exception == NULL && !_trunners[i]._exception._str.empty()) + exception = &_trunners[i]._exception; + } + if (options->sample_interval > 0) { + WT_TRET(pthread_join(monitor._handle, &status)); + if (monitor._errno != 0) + std::cerr << "Monitor thread has errno " << monitor._errno + << std::endl; + if (exception == NULL && !monitor._exception._str.empty()) + exception = &monitor._exception; + + monitor_out.close(); + if (!options->sample_file.empty()) + monitor_json.close(); + } + + // issue the final report + timespec finalsecs = now - _start; + final_report(finalsecs); + + if (ret != 0) + std::cerr << "run_all failed err=" << ret << std::endl; + (*_report_out) << std::endl; + if (exception != NULL) + throw *exception; + return (ret); +} + +}; diff --git a/bench/workgen/workgen.h b/bench/workgen/workgen.h new file mode 100644 index 00000000000..c7be8ee0035 --- /dev/null +++ b/bench/workgen/workgen.h @@ -0,0 +1,411 @@ +/*- + * Public Domain 2014-2017 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include <ostream> +#include <string> +#include <vector> +#include <map> + +namespace workgen { + +struct ContextInternal; +struct TableInternal; +struct Thread; +struct Transaction; + +#ifndef SWIG +struct OptionsList { + OptionsList(); + OptionsList(const OptionsList &other); + + void add_int(const char *name, int default_value, const char *desc); + void add_bool(const char *name, bool default_value, const char *desc); + void add_double(const char *name, double default_value, const char *desc); + void add_string(const char *name, const std::string &default_value, + const char *desc); + + std::string help() const; + std::string help_description(const char *option_name) const; + std::string help_type(const char *option_name) const; + +private: + void add_option(const char *name, const std::string typestr, + const char *desc); + typedef std::pair<std::string, std::string> TypeDescPair; + std::map<std::string, TypeDescPair> _option_map; +}; +#endif + +// These classes are all exposed to Python via SWIG. While they may contain +// data that is private to C++, such data must not prevent the objects from +// being shared. Tables, Keys, Values, Operations and Threads can be shared: a +// single Key object might appear in many operations; Operations may appear +// multiple times in a Thread or in different Threads; the same Thread may +// appear multiple times in a Workload list, etc. +// +// Certain kinds of state are allowed: A Table contains a unique pointer that +// is used within the internal part of the Context. Stats contain lots +// of state, but is made available after a Workload.run(). +// +// Python controls the lifetime of (nearly) all objects of these classes. +// The exception is Stat/Track objects, which are also created/used +// internally to calculate and show statistics during a run. +// +struct Track { + // Threads maintain the total thread operation and total latency they've + // experienced. + + uint64_t ops; // Total operations */ + uint64_t latency_ops; // Total ops sampled for latency + uint64_t latency; // Total latency */ + + // Minimum/maximum latency, shared with the monitor thread, that is, the + // monitor thread clears it so it's recalculated again for each period. + + uint32_t min_latency; // Minimum latency (uS) + uint32_t max_latency; // Maximum latency (uS) + + Track(bool latency_tracking = false); + Track(const Track &other); + ~Track(); + + void add(Track&, bool reset = false); + void assign(const Track&); + uint64_t average_latency() const; + void clear(); + void incr(); + void incr_with_latency(uint64_t usecs); + void smooth(const Track&); + void subtract(const Track&); + void track_latency(bool); + bool track_latency() const { return (us != NULL); } + + void _get_us(long *); + void _get_ms(long *); + void _get_sec(long *); + +private: + // Latency buckets. From python, accessed via methods us(), ms(), sec() + uint32_t *us; // < 1us ... 1000us + uint32_t *ms; // < 1ms ... 1000ms + uint32_t *sec; // < 1s 2s ... 100s + + Track & operator=(const Track &other); // use explicit assign method +}; + +struct Stats { + Track insert; + Track not_found; + Track read; + Track remove; + Track update; + Track truncate; + + Stats(bool latency = false); + Stats(const Stats &other); + ~Stats(); + + void add(Stats&, bool reset = false); + void assign(const Stats&); + void clear(); + void describe(std::ostream &os) const; +#ifndef SWIG + void final_report(std::ostream &os, timespec &totalsecs) const; + void report(std::ostream &os) const; +#endif + void smooth(const Stats&); + void subtract(const Stats&); + void track_latency(bool); + bool track_latency() const { return (insert.track_latency()); } + +private: + Stats & operator=(const Stats &other); // use explicit assign method +}; + +// A Context tracks the current record number for each uri, used +// for key generation. +// +struct Context { + bool _verbose; + ContextInternal *_internal; + + Context(); + ~Context(); + void describe(std::ostream &os) const { + os << "Context: verbose " << (_verbose ? "true" : "false"); + } + +#ifndef SWIG + Context& operator=(const Context &other); +#endif +}; + +// To prevent silent errors, this class is set up in Python so that new +// properties are prevented, only existing properties can be set. +// +struct TableOptions { + int key_size; + int value_size; + + TableOptions(); + TableOptions(const TableOptions &other); + ~TableOptions(); + + void describe(std::ostream &os) const { + os << "key_size " << key_size; + os << ", value_size " << value_size; + } + + std::string help() const { return _options.help(); } + std::string help_description(const char *option_name) const { + return _options.help_description(option_name); } + std::string help_type(const char *option_name) const { + return _options.help_type(option_name); } + +private: + OptionsList _options; +}; + +struct Table { + TableOptions options; + std::string _uri; + TableInternal *_internal; + + /* XXX select table from range */ + + Table(); + Table(const char *tablename); + Table(const Table &other); + ~Table(); + + void describe(std::ostream &os) const; + +#ifndef SWIG + Table& operator=(const Table &other); +#endif +}; + +struct Key { + typedef enum { + KEYGEN_AUTO, KEYGEN_APPEND, KEYGEN_PARETO, KEYGEN_UNIFORM } KeyType; + KeyType _keytype; + int _size; + + /* XXX specify more about key distribution */ + Key() : _keytype(KEYGEN_AUTO), _size(0) {} + Key(KeyType keytype, int size) : _keytype(keytype), _size(size) {} + Key(const Key &other) : _keytype(other._keytype), _size(other._size) {} + ~Key() {} + + void describe(std::ostream &os) const { + os << "Key: type " << _keytype << ", size " << _size; } +}; + +struct Value { + int _size; + + /* XXX specify how value is calculated */ + Value() : _size(0) {} + Value(int size) : _size(size) {} + Value(const Value &other) : _size(other._size) {} + ~Value() {} + + void describe(std::ostream &os) const { os << "Value: size " << _size; } +}; + +struct Operation { + enum OpType { + OP_NONE, OP_INSERT, OP_REMOVE, OP_SEARCH, OP_UPDATE }; + OpType _optype; + + Table _table; + Key _key; + Value _value; + Transaction *_transaction; + std::vector<Operation> *_group; + int _repeatgroup; + +#ifndef SWIG + int _keysize; // derived from Key._size and Table.options.key_size + int _valuesize; + uint64_t _keymax; + uint64_t _valuemax; +#endif + + Operation(); + Operation(OpType optype, Table table, Key key, Value value); + Operation(OpType optype, Table table, Key key); + Operation(OpType optype, Table table); + Operation(const Operation &other); + ~Operation(); + + void describe(std::ostream &os) const; +#ifndef SWIG + Operation& operator=(const Operation &other); + void get_static_counts(Stats &stats, int multiplier); + void kv_compute_max(bool); + void kv_gen(bool, uint64_t, char *) const; + void kv_size_buffer(bool iskey, size_t &size) const; + void size_check() const; +#endif +}; + +// To prevent silent errors, this class is set up in Python so that new +// properties are prevented, only existing properties can be set. +// +struct ThreadOptions { + std::string name; + double throttle; + double throttle_burst; + + ThreadOptions(); + ThreadOptions(const ThreadOptions &other); + ~ThreadOptions(); + + void describe(std::ostream &os) const { + os << "throttle " << throttle; + } + + std::string help() const { return _options.help(); } + std::string help_description(const char *option_name) const { + return _options.help_description(option_name); } + std::string help_type(const char *option_name) const { + return _options.help_type(option_name); } + +private: + OptionsList _options; +}; + +// This is a list of threads, which may be used in the Workload constructor. +// It participates with ThreadList defined on the SWIG/Python side and +// some Python operators added to Thread to allow Threads to be easily +// composed using '+' and multiplied (by integer counts) using '*'. +// Users of the workgen API in Python don't ever need to use +// ThreadListWrapper or ThreadList. +struct ThreadListWrapper { + std::vector<Thread> _threads; + + ThreadListWrapper() : _threads() {} + ThreadListWrapper(const ThreadListWrapper &other) : + _threads(other._threads) {} + ThreadListWrapper(const std::vector<Thread> &threads) : _threads(threads) {} + void extend(const ThreadListWrapper &); + void append(const Thread &); + void multiply(const int); +}; + +struct Thread { + ThreadOptions options; + Operation _op; + + Thread(); + Thread(const Operation &op); + Thread(const Thread &other); + ~Thread(); + + void describe(std::ostream &os) const; +}; + +struct Transaction { + bool _rollback; + std::string _begin_config; + std::string _commit_config; + + Transaction(const char *_config = NULL) : _rollback(false), + _begin_config(_config == NULL ? "" : _config), _commit_config() {} + + void describe(std::ostream &os) const { + os << "Transaction: "; + if (_rollback) + os << "(rollback) "; + os << "begin_config: " << _begin_config; + if (!_commit_config.empty()) + os << ", commit_config: " << _commit_config; + } +}; + +// To prevent silent errors, this class is set up in Python so that new +// properties are prevented, only existing properties can be set. +// +struct WorkloadOptions { + int max_latency; + std::string report_file; + int report_interval; + int run_time; + int sample_interval; + int sample_rate; + std::string sample_file; + + WorkloadOptions(); + WorkloadOptions(const WorkloadOptions &other); + ~WorkloadOptions(); + + void describe(std::ostream &os) const { + os << "run_time " << run_time; + os << ", report_interval " << report_interval; + } + + std::string help() const { return _options.help(); } + std::string help_description(const char *option_name) const { + return _options.help_description(option_name); } + std::string help_type(const char *option_name) const { + return _options.help_type(option_name); } + +private: + OptionsList _options; +}; + +struct Workload { + WorkloadOptions options; + Stats stats; + Context *_context; + std::vector<Thread> _threads; + + Workload(Context *context, const ThreadListWrapper &threadlist); + Workload(Context *context, const Thread &thread); + Workload(const Workload &other); + ~Workload(); + +#ifndef SWIG + Workload& operator=(const Workload &other); +#endif + + void describe(std::ostream &os) const { + os << "Workload: "; + _context->describe(os); + os << ", "; + options.describe(os); + os << ", [" << std::endl; + for (std::vector<Thread>::const_iterator i = _threads.begin(); i != _threads.end(); i++) { + os << " "; i->describe(os); os << std::endl; + } + os << "]"; + } + int run(WT_CONNECTION *conn); +}; + +}; diff --git a/bench/workgen/workgen.swig b/bench/workgen/workgen.swig new file mode 100644 index 00000000000..0f74942169c --- /dev/null +++ b/bench/workgen/workgen.swig @@ -0,0 +1,233 @@ +/*- + * Public Domain 2014-2017 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * workgen.swig + * The SWIG interface file defining the workgen python API. + */ + +%include "typemaps.i" +%include "std_vector.i" +%include "std_string.i" +%include "stdint.i" +%include "attribute.i" +%include "carrays.i" + +/* We only need to reference WiredTiger types. */ +%import "wiredtiger.h" + +%{ +#include <ostream> +#include <sstream> +#include <signal.h> +#include "wiredtiger.h" +#include "workgen.h" +#include "workgen_int.h" +%} + +%pythoncode %{ +import numbers +%} + +%exception { + try { + $action + } + catch (workgen::WorkgenException &wge) { + SWIG_exception_fail(SWIG_RuntimeError, wge._str.c_str()); + } +} + +/* + * Some functions are long running, turn off signal handling that was enabled + * by the Python interpreter. This means that a signal handler coded in Python + * won't work when spanning a call to one of these long running functions, but + * it's doubtful our test scripts need signals at all. This could be made to + * work, it's just not worth the trouble. + */ +%define InterruptableFunction(funcname) +%exception funcname { + try { + void (*savesig)(int) = signal(SIGINT, SIG_DFL); + $action + (void)signal(SIGINT, savesig); + } + catch (workgen::WorkgenException &wge) { + SWIG_exception_fail(SWIG_RuntimeError, wge._str.c_str()); + } +} +%enddef + +/* + * Define a __str__ function for all public workgen classes. + */ +%define WorkgenClass(classname) +%extend workgen::classname { + const std::string __str__() { + std::ostringstream out; + $self->describe(out); + return out.str(); + } +}; +%enddef + +/* + * To forestall errors, make it impossible to add new attributes to certain + * classes. This trick relies on the implementation of SWIG providing + * predictably named functions in the _workgen namespace to set attributes. + */ +%define WorkgenFrozenClass(classname) +%extend workgen::classname { +%pythoncode %{ + def __setattr__(self, attr, val): + if getattr(self, attr) == None: + raise AttributeError("'" + #classname + + "' object has no attribute '" + attr + "'") + f = _workgen.__dict__[#classname + '_' + attr + '_set'] + f(self, val) +%} +}; +%enddef + +InterruptableFunction(workgen::execute) +InterruptableFunction(workgen::Workload::run) + +%module workgen +/* Parse the header to generate wrappers. */ +%include "workgen.h" + +%template(OpList) std::vector<workgen::Operation>; +%template(ThreadList) std::vector<workgen::Thread>; +%array_class(uint32_t, uint32Array); +%array_class(long, longArray); + +WorkgenClass(Key) +WorkgenClass(Operation) +WorkgenClass(Stats) +WorkgenClass(Table) +WorkgenClass(TableOptions) +WorkgenClass(Thread) +WorkgenClass(ThreadOptions) +WorkgenClass(Transaction) +WorkgenClass(Value) +WorkgenClass(Workload) +WorkgenClass(WorkloadOptions) +WorkgenClass(Context) + +WorkgenFrozenClass(TableOptions) +WorkgenFrozenClass(ThreadOptions) +WorkgenFrozenClass(WorkloadOptions) + +%extend workgen::Operation { +%pythoncode %{ + def __mul__(self, other): + if not isinstance(other, numbers.Integral): + raise Exception('Operation.__mul__ requires an integral number') + op = Operation() + op._group = OpList([self]) + op._repeatgroup = other + return op + + __rmul__ = __mul__ + + def __add__(self, other): + if not isinstance(other, Operation): + raise Exception('Operation.__sum__ requires an Operation') + if self._group == None or self._repeatgroup != 1 or self._transaction != None: + op = Operation() + op._group = OpList([self, other]) + op._repeatgroup = 1 + return op + else: + self._group.append(other) + return self +%} +}; + +%extend workgen::Thread { +%pythoncode %{ + def __mul__(self, other): + if not isinstance(other, numbers.Integral): + raise Exception('Thread.__mul__ requires an integral number') + return ThreadListWrapper(ThreadList([self] * other)) + + __rmul__ = __mul__ + + def __add__(self, other): + if type(self) != type(other): + raise Exception('Thread.__sum__ requires an Thread') + return ThreadListWrapper(ThreadList([self, other])) +%} +}; + +%extend workgen::ThreadListWrapper { +%pythoncode %{ + def __mul__(self, other): + if not isinstance(other, numbers.Integral): + raise Exception('ThreadList.__mul__ requires an integral number') + tlw = ThreadListWrapper(self) + tlw.multiply(other) + return tlw + + __rmul__ = __mul__ + + def __add__(self, other): + tlw = ThreadListWrapper(self) + if isinstance(other, ThreadListWrapper): + tlw.extend(other) + elif isinstance(other, Thread): + tlw.append(other) + else: + raise Exception('ThreadList.__sum__ requires an Thread or ThreadList') + return tlw +%} +}; + +%extend workgen::Track { +%pythoncode %{ + def __longarray(self, size): + result = longArray(size) + result.__len__ = lambda: size + return result + + def us(self): + result = self.__longarray(1000) + self._get_us(result) + return result + + def ms(self): + result = self.__longarray(1000) + self._get_ms(result) + return result + + def sec(self): + result = self.__longarray(100) + self._get_sec(result) + return result +%} +}; diff --git a/bench/workgen/workgen/__init__.py b/bench/workgen/workgen/__init__.py new file mode 100644 index 00000000000..ff665bf9398 --- /dev/null +++ b/bench/workgen/workgen/__init__.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# __init__.py +# initialization for workgen module +# +import os, sys + +# After importing the SWIG-generated file, copy all symbols from from it +# to this module so they will appear in the workgen namespace. +me = sys.modules[__name__] +sys.path.append(os.path.dirname(__file__)) # needed for Python3 +import workgen, workgen_util +for module in workgen: + for name in dir(module): + value = getattr(module, name) + setattr(me, name, value) diff --git a/bench/workgen/workgen_func.c b/bench/workgen/workgen_func.c new file mode 100644 index 00000000000..5ce2146a8e4 --- /dev/null +++ b/bench/workgen/workgen_func.c @@ -0,0 +1,102 @@ +/*- + * Public Domain 2014-2017 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "wiredtiger.h" +#include "test_util.h" +#include "workgen_func.h" + +/* workgen_random_state is used as an opaque type handle. */ +typedef struct workgen_random_state { + WT_RAND_STATE state; +} workgen_random_state; + +/* + * These functions call their WiredTiger equivalents. + */ +uint32_t +workgen_atomic_add32(uint32_t *vp, uint32_t v) +{ + return (__wt_atomic_add32(vp, v)); +} + +uint64_t +workgen_atomic_add64(uint64_t *vp, uint64_t v) +{ + return (__wt_atomic_add64(vp, v)); +} + +void +workgen_epoch(struct timespec *tsp) +{ + __wt_epoch(NULL, tsp); +} + +uint32_t +workgen_random(workgen_random_state volatile * rnd_state) +{ + return (__wt_random(&rnd_state->state)); +} + +int +workgen_random_alloc(WT_SESSION *session, workgen_random_state **rnd_state) +{ + workgen_random_state *state; + + state = malloc(sizeof(workgen_random_state)); + if (state == NULL) { + *rnd_state = NULL; + return (ENOMEM); + } + __wt_random_init_seed((WT_SESSION_IMPL *)session, &state->state); + *rnd_state = state; + return (0); +} + +void +workgen_random_free(workgen_random_state *rnd_state) +{ + free(rnd_state); +} + +extern void +workgen_u64_to_string_zf(uint64_t n, char *buf, size_t len) +{ + u64_to_string_zf(n, buf, len); +} + +#define WORKGEN_VERSION_PREFIX "workgen-" +extern void +workgen_version(char *buf, size_t len) +{ + size_t prefix_len; + + prefix_len = strlen(WORKGEN_VERSION_PREFIX); + (void)strncpy(buf, WORKGEN_VERSION_PREFIX, len); + if (len > prefix_len) + (void)strncpy(&buf[prefix_len], WIREDTIGER_VERSION_STRING, + len - prefix_len); +} diff --git a/bench/workgen/workgen_func.h b/bench/workgen/workgen_func.h new file mode 100644 index 00000000000..ec7ecf0a504 --- /dev/null +++ b/bench/workgen/workgen_func.h @@ -0,0 +1,46 @@ +/*- + * Public Domain 2014-2017 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +struct workgen_random_state; + +extern uint32_t +workgen_atomic_add32(uint32_t *vp, uint32_t v); +extern uint64_t +workgen_atomic_add64(uint64_t *vp, uint64_t v); +extern void +workgen_epoch(struct timespec *tsp); +extern uint32_t +workgen_random(struct workgen_random_state volatile *rnd_state); +extern int +workgen_random_alloc(WT_SESSION *session, + struct workgen_random_state **rnd_state); +extern void +workgen_random_free(struct workgen_random_state *rnd_state); +extern void +workgen_u64_to_string_zf(uint64_t n, char *buf, size_t len); +extern void +workgen_version(char *buf, size_t len); diff --git a/bench/workgen/workgen_int.h b/bench/workgen/workgen_int.h new file mode 100644 index 00000000000..9283aea1d7b --- /dev/null +++ b/bench/workgen/workgen_int.h @@ -0,0 +1,206 @@ +/*- + * Public Domain 2014-2017 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include <ostream> +#include <string> +#include <vector> +#include <map> +#include <set> +#ifndef SWIG +extern "C" { +#include "workgen_func.h" +} +#endif + +namespace workgen { + +// A 'tint' or ('table integer') is a unique small value integer +// assigned to each table URI in use. Currently, we assign it once, +// and its value persists through the lifetime of the Context. +typedef uint32_t tint_t; + +struct ThreadRunner; +struct WorkloadRunner; + +// A exception generated by the workgen classes. Methods generally return an +// int errno, so this is useful primarily for notifying the caller about +// failures in constructors. +struct WorkgenException { + std::string _str; + WorkgenException() : _str() {} + WorkgenException(int err, const char *msg = NULL) : _str() { + if (err != 0) + _str += wiredtiger_strerror(err); + if (msg != NULL) { + if (!_str.empty()) + _str += ": "; + _str += msg; + } + } + WorkgenException(const WorkgenException &other) : _str(other._str) {} + ~WorkgenException() {} +}; + +struct Throttle { + ThreadRunner &_runner; + double _throttle; + double _burst; + timespec _next_div; + int64_t _ops_delta; + uint64_t _ops_prev; // previously returned value + uint64_t _ops_per_div; // statically calculated. + uint64_t _ms_per_div; // statically calculated. + bool _started; + + Throttle(ThreadRunner &runner, double throttle, double burst); + ~Throttle(); + + // Called with the number of operations since the last throttle. + // Sleeps for any needed amount and returns the number operations the + // caller should perform before the next call to throttle. + int throttle(uint64_t op_count, uint64_t *op_limit); +}; + +// There is one of these per Thread object. It exists for the duration of a +// call to Workload::run() method. +struct ThreadRunner { + int _errno; + WorkgenException _exception; + Thread *_thread; + Context *_context; + ContextInternal *_icontext; + Workload *_workload; + WorkloadRunner *_wrunner; + workgen_random_state *_rand_state; + Throttle *_throttle; + uint64_t _throttle_ops; + uint64_t _throttle_limit; + bool _in_transaction; + uint32_t _number; + Stats _stats; + + typedef enum { + USAGE_READ = 0x1, USAGE_WRITE = 0x2, USAGE_MIXED = 0x4 } Usage; + std::map<tint_t, uint32_t> _table_usage; // value is Usage + WT_CURSOR **_cursors; // indexed by tint_t + volatile bool _stop; + WT_SESSION *_session; + char *_keybuf; + char *_valuebuf; + bool _repeat; + + ThreadRunner(); + ~ThreadRunner(); + + void free_all(); + static int cross_check(std::vector<ThreadRunner> &runners); + + int close_all(); + int create_all(WT_CONNECTION *conn); + void get_static_counts(Stats &); + int open_all(); + int run(); + + void op_create_all(Operation *, size_t &keysize, size_t &valuesize); + uint64_t op_get_key_recno(Operation *, tint_t tint); + void op_get_static_counts(Operation *, Stats &, int); + int op_run(Operation *); + +#ifdef _DEBUG + std::stringstream _debug_messages; + std::string get_debug(); +#define DEBUG_CAPTURE(runner, expr) runner._debug_messages << expr +#else +#define DEBUG_CAPTURE(runner, expr) +#endif +}; + +struct Monitor { + int _errno; + WorkgenException _exception; + WorkloadRunner &_wrunner; + volatile bool _stop; + pthread_t _handle; + std::ostream *_out; + std::ostream *_json; + + Monitor(WorkloadRunner &wrunner); + ~Monitor(); + int run(); +}; + +struct ContextInternal { + std::map<std::string, tint_t> _tint; // maps uri -> tint_t + std::map<tint_t, std::string> _table_names; // reverse mapping + uint64_t *_recno; // # entries per tint_t + uint32_t _recno_alloced; // length of allocated _recno + tint_t _tint_last; // last tint allocated + // unique id per context, to work with multiple contexts, starts at 1. + uint32_t _context_count; + + ContextInternal(); + ~ContextInternal(); + int create_all(); +}; + +struct TableInternal { + tint_t _tint; + uint32_t _context_count; + + TableInternal(); + TableInternal(const TableInternal &other); + ~TableInternal(); +}; + +// An instance of this class only exists for the duration of one call to a +// Workload::run() method. +struct WorkloadRunner { + Workload *_workload; + std::vector<ThreadRunner> _trunners; + std::ostream *_report_out; + std::string _wt_home; + timespec _start; + + WorkloadRunner(Workload *); + ~WorkloadRunner(); + int run(WT_CONNECTION *conn); + +private: + int close_all(); + int create_all(WT_CONNECTION *conn, Context *context); + void final_report(timespec &); + void get_stats(Stats *stats); + int open_all(); + void open_report_file(std::ofstream &, const char *, const char *); + void report(time_t, time_t, Stats *stats); + int run_all(); + + WorkloadRunner(const WorkloadRunner &); // disallowed + WorkloadRunner& operator=(const WorkloadRunner &other); // disallowed +}; + +}; diff --git a/bench/workgen/workgen_time.h b/bench/workgen/workgen_time.h new file mode 100644 index 00000000000..f33eb64d9c9 --- /dev/null +++ b/bench/workgen/workgen_time.h @@ -0,0 +1,201 @@ +/*- + * Public Domain 2014-2017 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#define THOUSAND (1000ULL) +#define MILLION (1000000ULL) +#define BILLION (1000000000ULL) + +#define NSEC_PER_SEC BILLION +#define USEC_PER_SEC MILLION +#define MSEC_PER_SEC THOUSAND + +#define ns_to_ms(v) ((v) / MILLION) +#define ns_to_sec(v) ((v) / BILLION) +#define ns_to_us(v) ((v) / THOUSAND) + +#define us_to_ms(v) ((v) / THOUSAND) +#define us_to_ns(v) ((v) * THOUSAND) +#define us_to_sec(v) ((v) / MILLION) + +#define ms_to_ns(v) ((v) * MILLION) +#define ms_to_us(v) ((v) * THOUSAND) +#define ms_to_sec(v) ((v) / THOUSAND) + +#define sec_to_ns(v) ((v) * BILLION) +#define sec_to_us(v) ((v) * MILLION) +#define sec_to_ms(v) ((v) * THOUSAND) + +inline std::ostream& +operator<<(std::ostream &os, const timespec &ts) +{ + char oldfill; + std::streamsize oldwidth; + + os << ts.tv_sec << "."; + oldfill = os.fill('0'); + oldwidth = os.width(3); + os << (int)ns_to_ms(ts.tv_nsec); + os.fill(oldfill); + os.width(oldwidth); + return (os); +} + +inline timespec +operator-(const timespec &lhs, const timespec &rhs) +{ + timespec ts; + + if (lhs.tv_nsec < rhs.tv_nsec) { + ts.tv_sec = lhs.tv_sec - rhs.tv_sec - 1; + ts.tv_nsec = lhs.tv_nsec - rhs.tv_nsec + NSEC_PER_SEC; + } else { + ts.tv_sec = lhs.tv_sec - rhs.tv_sec; + ts.tv_nsec = lhs.tv_nsec - rhs.tv_nsec; + } + return (ts); +} + +inline timespec +operator+(const timespec &lhs, const int n) +{ + timespec ts = lhs; + ts.tv_sec += n; + return (ts); +} + +inline bool +operator<(const timespec &lhs, const timespec &rhs) +{ + if (lhs.tv_sec == rhs.tv_sec) + return (lhs.tv_nsec < rhs.tv_nsec); + else + return (lhs.tv_sec < rhs.tv_sec); +} + +inline bool +operator>(const timespec &lhs, const timespec &rhs) +{ + if (lhs.tv_sec == rhs.tv_sec) + return (lhs.tv_nsec > rhs.tv_nsec); + else + return (lhs.tv_sec > rhs.tv_sec); +} + +inline bool +operator>=(const timespec &lhs, const timespec &rhs) +{ + return (!(lhs < rhs)); +} + +inline bool +operator<=(const timespec &lhs, const timespec &rhs) +{ + return (!(lhs > rhs)); +} + +inline bool +operator==(const timespec &lhs, int n) +{ + return (lhs.tv_sec == n && lhs.tv_nsec == 0); +} + +inline bool +operator!=(const timespec &lhs, int n) +{ + return (lhs.tv_sec != n || lhs.tv_nsec != 0); +} + +inline timespec & +operator+=(timespec &lhs, const int n) +{ + lhs.tv_sec += n; + return (lhs); +} + +inline bool +operator==(const timespec &lhs, const timespec &rhs) +{ + return (lhs.tv_sec == rhs.tv_sec && lhs.tv_nsec == rhs.tv_nsec); +} + +inline timespec & +operator-=(timespec &lhs, const timespec &rhs) +{ + lhs.tv_sec -= rhs.tv_sec; + lhs.tv_nsec -= rhs.tv_nsec; + if (lhs.tv_nsec < 0) { + lhs.tv_nsec += NSEC_PER_SEC; + lhs.tv_sec -= 1; + } + return (lhs); +} + +inline timespec +ts_add_ms(const timespec &lhs, const uint64_t n) +{ + timespec ts; + + ts.tv_sec = lhs.tv_sec + ms_to_sec(n); + ts.tv_nsec = lhs.tv_nsec + ms_to_ns(n % THOUSAND); + while ((unsigned long)ts.tv_nsec > NSEC_PER_SEC) { + ts.tv_nsec -= NSEC_PER_SEC; + ts.tv_sec++; + } + return (ts); +} + +inline void +ts_assign(timespec &lhs, const timespec &rhs) +{ + lhs.tv_sec = rhs.tv_sec; + lhs.tv_nsec = rhs.tv_nsec; +} + +inline void +ts_clear(timespec &ts) +{ + ts.tv_sec = 0; + ts.tv_nsec = 0; +} + +inline uint64_t +ts_sec(const timespec &ts) +{ + return (ns_to_sec(ts.tv_nsec) + ts.tv_sec); +} + +inline uint64_t +ts_ms(const timespec &ts) +{ + return (ns_to_ms(ts.tv_nsec) + sec_to_ms(ts.tv_sec)); +} + +inline uint64_t +ts_us(const timespec &ts) +{ + return (ns_to_us(ts.tv_nsec) + sec_to_us(ts.tv_sec)); +} diff --git a/bench/workgen/wtperf.py b/bench/workgen/wtperf.py new file mode 100644 index 00000000000..3a196fe7b57 --- /dev/null +++ b/bench/workgen/wtperf.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +# wtperf.py +# A partial emulation of wtperf. Translates a .wtperf file into a Python +# script that uses the workgen module, and runs the script. Errors are +# issued for any .wtperf directives that are not known. +# See also the usage() function. +# +from __future__ import print_function +import os, sys, tempfile + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +class OptionValue: + def __init__(self, value, filename, linenum): + self.value = value + self.filename = filename + self.linenum = linenum + +class TranslateException(Exception): + pass + +class Options(object): + pass + +class Translator: + def __init__(self, filename, prefix, verbose): + self.filename = filename + self.prefix = prefix + self.verbose = verbose + self.linenum = 0 + self.opts = {} + self.used_opts = {} + self.has_error = False + + def error_file_line(self, fname, linenum, msg): + self.has_error = True + eprint(fname + ':' + str(linenum) + ': error: ' + msg) + + # Report an error and continue + def error(self, msg): + self.error_file_line(self.filename, self.linenum, msg) + + # Report an error and unwind the stack + def fatal_error(self, msg, errtype): + self.error(msg) + raise TranslateException(errtype) + + supported_opt_list = [ 'compression', 'conn_config', 'icount', + 'key_sz', 'log_like_table', + 'populate_ops_per_txn', 'populate_threads', + 'reopen_connection', + 'table_config', 'table_count', + 'threads', 'transaction_config', 'value_sz' ] + + def set_opt(self, optname, val): + if optname not in self.supported_opt_list: + self.error("unknown option: " + optname) + return + elif val[0] == '"' and val[-1] == '"': + v = val[1:-1] + elif val == 'true': + v = True + elif val == 'false': + v = False + elif val[0] == '(': + v = val # config string stored as is + else: + try: + v = int(val) # it might be an integer + except ValueError: + v = val # it's a string after all + self.opts[optname] = OptionValue(v, self.filename, self.linenum) + + def get_opt(self, optname, dfault): + if optname in self.opts: + ret = self.opts[optname] + self.filename = ret.filename + self.linenum = ret.linenum + self.used_opts[optname] = 1 + return ret.value + else: + return dfault + + def get_int_opt(self, optname, dfault): + return self.get_opt(optname, dfault) + 0 + + def get_boolean_opt(self, optname, dfault): + return not not self.get_opt(optname, dfault) + + # Split a string 'left_side=right_side' into two parts + def split_assign(self, s): + equalpos = s.find('=') + if equalpos < 0: + self.error("missing '=' for line: " + line) + return (None, None) + else: + return s.split('=', 1) + + # Split a config string honoring nesting e.g. + # "(abc=123,def=234,ghi=(hi=1,bye=2))" would return 3 items. + def split_config_parens(self, s): + if s[0:1] != '(': + import pdb + pdb.set_trace() + self.fatal_error('missing left paren', 'config parse error') + if s[-1:] != ')': + self.fatal_error('missing right paren', 'config parse error') + s = s[1:-1] + result = [] + level = 0 + cur = '' + for ch in s: + if ch == ',' and level == 0: + result.append(cur) + cur = '' + else: + cur += ch + if ch == '(': + level += 1 + elif ch == ')': + level -= 1 + if level < 0: + self.fatal_error('unbalanced paren', 'config parse error') + if level != 0: + self.fatal_error('unbalanced paren', 'config parse error') + if len(cur) != 0: + result.append(cur) + return result + + def assign_str(self, left, right): + return left + '=' + str(right) + '\n' + + def add_operation_str(self, count, opname, multi): + result = '' + tablename = 'tables[0]' if multi else 'table' + if count > 1: + result += str(count) + ' * ' + if count > 0: + result += 'Operation(Operation.' + opname + ', ' + \ + tablename + ') + \\\n' + result += ' ' + return result + + # Wtperf's throttle is based on the number of regular operations, + # not including log_like operations. Workgen counts all operations, + # it doesn't treat log operations any differently. Adjust the throttle + # number to account for the difference. + def calc_throttle(self, thread_opts, log_like_table): + throttle = thread_opts.throttle + if not log_like_table: + return (throttle, '') + modify = thread_opts.inserts + thread_opts.updates + regular = modify + thread_opts.reads + total = regular + modify + factor = (total + 0.0) / regular + new_throttle = int(throttle * factor) + if new_throttle == throttle: + comment = '' + else: + comment = '# wtperf throttle=' + str(throttle) + ' adjusted by ' + \ + str(factor) + ' to compensate for log_like operations.\n' + return (new_throttle, comment) + + def parse_threads(self, threads_config): + tdecls = '' + tlist = self.split_config_parens(threads_config) + table_count = self.get_int_opt('table_count', 1) + log_like_table = self.get_boolean_opt('log_like_table', False) + txn_config = self.get_opt('transaction_config', '') + if log_like_table: + tdecls += 'log_name = "table:log"\n' + tdecls += 's.create(log_name, "key_format=S,value_format=S," +' + \ + ' compress_table_config)\n' + tdecls += 'log_table = Table(log_name)\n\n' + thread_count = 0 + tnames = '' + multi = (table_count > 1) + for t in tlist: + thread_name = 'thread' + str(thread_count) + thread_count += 1 + + # For wtperf compatibility, we allow both 'insert/inserts' etc. + topts = Options() + topts.count = 1 + topts.insert = 0 + topts.inserts = 0 + topts.ops_per_txn = 0 + topts.read = 0 + topts.reads = 0 + topts.throttle = 0 + topts.update = 0 + topts.updates = 0 + + for o in self.split_config_parens(t): + (k, v) = self.split_assign(o) + if hasattr(topts, k): + try: + setattr(topts, k, int(v)) + except ValueError: + self.error('thread option ' + k + ': integer expected') + else: + self.error('unknown thread option: ' + k) + + topts.inserts += topts.insert; topts.insert = 0 + topts.updates += topts.update; topts.update = 0 + topts.reads += topts.read; topts.read = 0 + if topts.count == 0: + continue + + if topts.inserts + topts.reads + topts.updates == 0: + self.fatal_error('need read/insert/update/...', + 'thread config error') + tdecls += 'ops = ' + tdecls += self.add_operation_str(topts.inserts, 'OP_INSERT', multi) + tdecls += self.add_operation_str(topts.reads, 'OP_SEARCH', multi) + tdecls += self.add_operation_str(topts.updates, 'OP_UPDATE', multi) + tdecls = tdecls.rstrip(' \n\\+') + '\n' + if multi: + tdecls += 'ops = op_multi_table(ops, tables)\n' + if topts.ops_per_txn > 0: + tdecls += 'ops = op_group_transaction(ops, ' + \ + str(topts.ops_per_txn) + ', "' + txn_config + '")\n' + if log_like_table: + tdecls += 'ops = op_log_like(ops, log_table, ' + \ + str(topts.ops_per_txn) + ')\n' + tdecls += thread_name + ' = Thread(ops)\n' + if topts.throttle > 0: + (throttle, comment) = self.calc_throttle(topts, log_like_table) + tdecls += comment + tdecls += self.assign_str(thread_name + '.options.throttle', + throttle) + tdecls += '\n' + if topts.count > 1: + tnames += str(topts.count) + ' * ' + tnames += thread_name + ' + ' + + tnames = tnames.rstrip(' +') + return (tdecls, tnames) + + def translate(self): + try: + return self.translate_inner() + except TranslateException: + # An error has already been reported + return None + + def translate_inner(self): + workloadopts = '' + with open(self.filename) as fin: + for line in fin: + self.linenum += 1 + commentpos = line.find('#') + if commentpos >= 0: + line = line[0:commentpos] + line = line.strip() + if len(line) == 0: + continue + (key, val) = self.split_assign(line) + if key in [ 'max_latency', 'report_file', 'report_interval', + 'run_time', 'sample_interval', 'sample_rate' ]: + workloadopts += 'workload.options.' + key + '=' + val + '\n' + else: + self.set_opt(key, val) + + table_count = self.get_int_opt('table_count', 1) + conn_config = self.get_opt('conn_config', '') + table_config = self.get_opt('table_config', '') + key_sz = self.get_int_opt('key_sz', 20) + value_sz = self.get_int_opt('value_sz', 100) + reopen = self.get_boolean_opt('reopen_connection', False) + compression = self.get_opt('compression', '') + txn_config = self.get_opt('transaction_config', '') + + s = '#/usr/bin/env python\n' + s += '# generated from ' + self.filename + '\n' + s += self.prefix + s += 'from runner import *\n' + s += 'from wiredtiger import *\n' + s += 'from workgen import *\n' + s += '\n' + s += 'context = Context()\n' + s += 'conn_config = "' + conn_config + '"\n' + if compression != '': + s += 'conn_config += extensions_config(["compressors/' + \ + compression + '"])\n' + compression = 'block_compressor=' + compression + ',' + s += 'conn = wiredtiger_open("WT_TEST", "create," + conn_config)\n' + s += 's = conn.open_session()\n' + s += '\n' + s += 'wtperf_table_config = "key_format=S,value_format=S,type=lsm," +\\\n' + s += ' "exclusive=true,allocation_size=4kb," +\\\n' + s += ' "internal_page_max=64kb,leaf_page_max=4kb,split_pct=100,"\n' + s += 'compress_table_config = "' + compression + '"\n' + s += 'table_config = "' + table_config + '"\n' + if table_count == 1: + s += 'tname = "file:test.wt"\n' + s += 's.create(tname, wtperf_table_config +\\\n' + s += ' compress_table_config + table_config)\n' + s += 'table = Table(tname)\n' + s += 'table.options.key_size = ' + str(key_sz) + '\n' + s += 'table.options.value_size = ' + str(value_sz) + '\n' + else: + s += 'table_count = ' + str(table_count) + '\n' + s += 'tables = []\n' + s += 'for i in range(0, table_count):\n' + s += ' tname = "file:test" + str(i) + ".wt"\n' + s += ' s.create(tname, ' + \ + 'wtperf_table_config + ' + \ + 'compress_table_config + table_config)\n' + s += ' t = Table(tname)\n' + s += ' t.options.key_size = ' + str(key_sz) + '\n' + s += ' t.options.value_size = ' + str(value_sz) + '\n' + s += ' tables.append(t)\n' + s += '\n' + + icount = self.get_int_opt('icount', 0) + pop_thread = self.get_int_opt('populate_threads', 1) + pop_per_txn = self.get_int_opt('populate_ops_per_txn', 0) + if icount != 0: + if pop_thread == 0: + self.fatal_error('icount != 0 and populate_threads == 0: ' +\ + 'cannot populate entries with no threads') + elif pop_thread == 1: + mult = '' + else: + mult = str(pop_thread) + ' * ' + + # if there are multiple tables to be filled during populate, + # the icount is split between them all. + nops_per_thread = icount / (pop_thread * table_count) + if table_count == 1: + s += 'pop_ops = Operation(Operation.OP_INSERT, table)\n' + else: + s += 'pop_ops = Operation(Operation.OP_INSERT, tables[0])\n' + s += 'pop_ops = op_multi_table(pop_ops, tables)\n' + if pop_per_txn > 0: + s += 'pop_ops = op_group_transaction(pop_ops, ' + \ + str(pop_per_txn) + ', "' + txn_config + '")\n' + s += 'pop_thread = Thread(pop_ops * ' + str(nops_per_thread) + ')\n' + s += 'pop_workload = Workload(context, ' + mult + 'pop_thread)\n' + if self.verbose > 0: + s += 'print("populate:")\n' + s += 'pop_workload.run(conn)\n' + else: + if self.get_int_opt('populate_threads', 0) != 0: + self.error("populate_threads > 0, icount == 0") + + thread_config = self.get_opt('threads', '') + if thread_config != '': + (t_create, t_var) = self.parse_threads(thread_config) + s += '\n' + t_create + if reopen: + s += '\n# reopen the connection\n' + s += 'conn.close()\n' + s += 'conn = wiredtiger_open(' + \ + '"WT_TEST", "create," + conn_config)\n' + s += '\n' + s += 'workload = Workload(context, ' + t_var + ')\n' + s += workloadopts + if self.verbose > 0: + s += 'print("workload:")\n' + s += 'workload.run(conn)\n' + + for o in self.used_opts: + del self.opts[o] + if len(self.opts) != 0: + self.error('internal error, options not handled: ' + str(self.opts)) + return s + +def usage(): + eprint(( + 'Usage: python wtperf.py [ options ] file.wtperf ...\n' + '\n' + 'Options:\n' + ' --python Python output generated on stdout\n' + ' -v --verbose Verbose output\n' + '\n' + 'If --python is not specified, the resulting workload is run.')) + +verbose = 0 +py_out = False +workgen_dir = os.path.dirname(os.path.abspath(__file__)) +runner_dir = os.path.join(workgen_dir, 'runner') +prefix = ( + '# The next lines are unneeded if this script is in the runner directory.\n' + 'import sys\n' + 'sys.path.append("' + runner_dir + '")\n\n') + +exit_status = 0 +for arg in sys.argv[1:]: + if arg == '--python': + py_out = True + elif arg == '--verbose' or arg == '-v': + verbose += 1 + elif arg.endswith('.wtperf'): + translator = Translator(arg, prefix, verbose) + pysrc = translator.translate() + if translator.has_error: + exit_status = 1 + elif py_out: + print(pysrc) + else: + (outfd, tmpfile) = tempfile.mkstemp(suffix='.py') + os.write(outfd, pysrc) + os.close(outfd) + execfile(tmpfile) + os.remove(tmpfile) + else: + usage() + sys.exit(1) +sys.exit(exit_status) diff --git a/bench/wtperf/config.c b/bench/wtperf/config.c index e4eee66e4cb..c5a3dd40032 100644 --- a/bench/wtperf/config.c +++ b/bench/wtperf/config.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/bench/wtperf/config_opt.h b/bench/wtperf/config_opt.h index 3f1ab642227..68bcd3e45f1 100644 --- a/bench/wtperf/config_opt.h +++ b/bench/wtperf/config_opt.h @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/bench/wtperf/idle_table_cycle.c b/bench/wtperf/idle_table_cycle.c index 4387860cfb2..d0baa786ba9 100644 --- a/bench/wtperf/idle_table_cycle.c +++ b/bench/wtperf/idle_table_cycle.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -57,7 +57,7 @@ check_timing(WTPERF *wtperf, * Measure how long each step takes, and flag an error if it exceeds the * configured maximum. */ -static void * +static WT_THREAD_RET cycle_idle_tables(void *arg) { struct timespec start, stop; @@ -76,7 +76,7 @@ cycle_idle_tables(void *arg) wtperf->conn, NULL, opts->sess_config, &session)) != 0) { lprintf(wtperf, ret, 0, "Error opening a session on %s", wtperf->home); - return (NULL); + return (WT_THREAD_RET_VALUE); } for (cycle_count = 0; wtperf->idle_cycle_run; ++cycle_count) { @@ -96,10 +96,10 @@ cycle_idle_tables(void *arg) lprintf(wtperf, ret, 0, "Table create failed in cycle_idle_tables."); wtperf->error = true; - return (NULL); + return (WT_THREAD_RET_VALUE); } if (check_timing(wtperf, "create", start, &stop) != 0) - return (NULL); + return (WT_THREAD_RET_VALUE); start = stop; /* Open and close cursor. */ @@ -108,16 +108,16 @@ cycle_idle_tables(void *arg) lprintf(wtperf, ret, 0, "Cursor open failed in cycle_idle_tables."); wtperf->error = true; - return (NULL); + return (WT_THREAD_RET_VALUE); } if ((ret = cursor->close(cursor)) != 0) { lprintf(wtperf, ret, 0, "Cursor close failed in cycle_idle_tables."); wtperf->error = true; - return (NULL); + return (WT_THREAD_RET_VALUE); } if (check_timing(wtperf, "cursor", start, &stop) != 0) - return (NULL); + return (WT_THREAD_RET_VALUE); start = stop; #if 1 @@ -133,14 +133,14 @@ cycle_idle_tables(void *arg) lprintf(wtperf, ret, 0, "Table drop failed in cycle_idle_tables."); wtperf->error = true; - return (NULL); + return (WT_THREAD_RET_VALUE); } if (check_timing(wtperf, "drop", start, &stop) != 0) - return (NULL); + return (WT_THREAD_RET_VALUE); #endif } - return (NULL); + return (WT_THREAD_RET_VALUE); } /* @@ -150,47 +150,33 @@ cycle_idle_tables(void *arg) * structure. Should reshuffle the configuration structure so explicit static * initialization isn't necessary. */ -int -start_idle_table_cycle(WTPERF *wtperf, pthread_t *idle_table_cycle_thread) +void +start_idle_table_cycle(WTPERF *wtperf, wt_thread_t *idle_table_cycle_thread) { CONFIG_OPTS *opts; - pthread_t thread_id; - int ret; + wt_thread_t thread_id; opts = wtperf->opts; if (opts->idle_table_cycle == 0) - return (0); + return; wtperf->idle_cycle_run = true; - if ((ret = pthread_create( - &thread_id, NULL, cycle_idle_tables, wtperf)) != 0) { - lprintf(wtperf, - ret, 0, "Error creating idle table cycle thread."); - wtperf->idle_cycle_run = false; - return (ret); - } + testutil_check(__wt_thread_create( + NULL, &thread_id, cycle_idle_tables, wtperf)); *idle_table_cycle_thread = thread_id; - - return (0); } -int -stop_idle_table_cycle(WTPERF *wtperf, pthread_t idle_table_cycle_thread) +void +stop_idle_table_cycle(WTPERF *wtperf, wt_thread_t idle_table_cycle_thread) { CONFIG_OPTS *opts; - int ret; opts = wtperf->opts; if (opts->idle_table_cycle == 0 || !wtperf->idle_cycle_run) - return (0); + return; wtperf->idle_cycle_run = false; - if ((ret = pthread_join(idle_table_cycle_thread, NULL)) != 0) { - lprintf( - wtperf, ret, 0, "Error joining idle table cycle thread."); - return (ret); - } - return (0); + testutil_check(__wt_thread_join(NULL, idle_table_cycle_thread)); } diff --git a/bench/wtperf/misc.c b/bench/wtperf/misc.c index 0874794e01e..da48c600589 100644 --- a/bench/wtperf/misc.c +++ b/bench/wtperf/misc.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/bench/wtperf/runners/get_ckpt.py b/bench/wtperf/runners/get_ckpt.py index 03bbda7dab1..da188ad47d4 100755 --- a/bench/wtperf/runners/get_ckpt.py +++ b/bench/wtperf/runners/get_ckpt.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/bench/wtperf/track.c b/bench/wtperf/track.c index 86a26120a6a..13ca85aabfd 100644 --- a/bench/wtperf/track.c +++ b/bench/wtperf/track.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c index 6d79eebe8b2..a8d3f135280 100644 --- a/bench/wtperf/wtperf.c +++ b/bench/wtperf/wtperf.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -32,23 +32,23 @@ #define DEFAULT_HOME "WT_TEST" #define DEFAULT_MONITOR_DIR "WT_TEST" -static void *checkpoint_worker(void *); +static WT_THREAD_RET checkpoint_worker(void *); static int drop_all_tables(WTPERF *); static int execute_populate(WTPERF *); static int execute_workload(WTPERF *); static int find_table_count(WTPERF *); -static void *monitor(void *); -static void *populate_thread(void *); +static WT_THREAD_RET monitor(void *); +static WT_THREAD_RET populate_thread(void *); static void randomize_value(WTPERF_THREAD *, char *); static void recreate_dir(const char *); static int start_all_runs(WTPERF *); static int start_run(WTPERF *); -static int start_threads(WTPERF *, - WORKLOAD *, WTPERF_THREAD *, u_int, void *(*)(void *)); -static int stop_threads(WTPERF *, u_int, WTPERF_THREAD *); -static void *thread_run_wtperf(void *); +static void start_threads(WTPERF *, WORKLOAD *, + WTPERF_THREAD *, u_int, WT_THREAD_CALLBACK(*)(void *)); +static void stop_threads(u_int, WTPERF_THREAD *); +static WT_THREAD_RET thread_run_wtperf(void *); static void update_value_delta(WTPERF_THREAD *); -static void *worker(void *); +static WT_THREAD_RET worker(void *); static uint64_t wtperf_rand(WTPERF_THREAD *); static uint64_t wtperf_value_range(WTPERF *); @@ -312,7 +312,7 @@ op_name(uint8_t *op) /* NOTREACHED */ } -static void * +static WT_THREAD_RET worker_async(void *arg) { CONFIG_OPTS *opts; @@ -420,7 +420,7 @@ op_err: lprintf(wtperf, ret, 0, if (0) { err: wtperf->error = wtperf->stop = true; } - return (NULL); + return (WT_THREAD_RET_VALUE); } /* @@ -513,7 +513,7 @@ err: lprintf(wtperf, ret, 0, "Pre-workload traverse error"); return (ret); } -static void * +static WT_THREAD_RET worker(void *arg) { struct timespec start, stop; @@ -893,7 +893,7 @@ err: wtperf->error = wtperf->stop = true; } free(cursors); - return (NULL); + return (WT_THREAD_RET_VALUE); } /* @@ -1014,7 +1014,7 @@ run_mix_schedule(WTPERF *wtperf, WORKLOAD *workp) return (0); } -static void * +static WT_THREAD_RET populate_thread(void *arg) { struct timespec start, stop; @@ -1163,10 +1163,10 @@ err: wtperf->error = wtperf->stop = true; } free(cursors); - return (NULL); + return (WT_THREAD_RET_VALUE); } -static void * +static WT_THREAD_RET populate_async(void *arg) { struct timespec start, stop; @@ -1261,10 +1261,10 @@ populate_async(void *arg) if (0) { err: wtperf->error = wtperf->stop = true; } - return (NULL); + return (WT_THREAD_RET_VALUE); } -static void * +static WT_THREAD_RET monitor(void *arg) { struct timespec t; @@ -1426,10 +1426,10 @@ err: wtperf->error = wtperf->stop = true; (void)fclose(fp); free(path); - return (NULL); + return (WT_THREAD_RET_VALUE); } -static void * +static WT_THREAD_RET checkpoint_worker(void *arg) { CONFIG_OPTS *opts; @@ -1490,7 +1490,7 @@ checkpoint_worker(void *arg) err: wtperf->error = wtperf->stop = true; } - return (NULL); + return (WT_THREAD_RET_VALUE); } static int @@ -1498,15 +1498,15 @@ execute_populate(WTPERF *wtperf) { struct timespec start, stop; CONFIG_OPTS *opts; - WTPERF_THREAD *popth; WT_ASYNC_OP *asyncop; - pthread_t idle_table_cycle_thread; + WTPERF_THREAD *popth; + WT_THREAD_CALLBACK(*pfunc)(void *); size_t i; uint64_t last_ops, msecs, print_ops_sec; uint32_t interval, tables; + wt_thread_t idle_table_cycle_thread; double print_secs; int elapsed, ret; - void *(*pfunc)(void *); opts = wtperf->opts; @@ -1516,9 +1516,7 @@ execute_populate(WTPERF *wtperf) opts->populate_threads, opts->icount); /* Start cycling idle tables if configured. */ - if ((ret = - start_idle_table_cycle(wtperf, &idle_table_cycle_thread)) != 0) - return (ret); + start_idle_table_cycle(wtperf, &idle_table_cycle_thread); wtperf->insert_key = 0; @@ -1530,9 +1528,8 @@ execute_populate(WTPERF *wtperf) pfunc = populate_async; } else pfunc = populate_thread; - if ((ret = start_threads(wtperf, NULL, - wtperf->popthreads, opts->populate_threads, pfunc)) != 0) - return (ret); + start_threads(wtperf, NULL, + wtperf->popthreads, opts->populate_threads, pfunc); __wt_epoch(NULL, &start); for (elapsed = 0, interval = 0, last_ops = 0; @@ -1568,10 +1565,8 @@ execute_populate(WTPERF *wtperf) */ popth = wtperf->popthreads; wtperf->popthreads = NULL; - ret = stop_threads(wtperf, opts->populate_threads, popth); + stop_threads(opts->populate_threads, popth); free(popth); - if (ret != 0) - return (ret); /* Report if any worker threads didn't finish. */ if (wtperf->error) { @@ -1640,8 +1635,7 @@ execute_populate(WTPERF *wtperf) } /* Stop cycling idle tables. */ - if ((ret = stop_idle_table_cycle(wtperf, idle_table_cycle_thread)) != 0) - return (ret); + stop_idle_table_cycle(wtperf, idle_table_cycle_thread); return (0); } @@ -1701,13 +1695,13 @@ execute_workload(WTPERF *wtperf) WTPERF_THREAD *threads; WT_CONNECTION *conn; WT_SESSION **sessions; - pthread_t idle_table_cycle_thread; + WT_THREAD_CALLBACK(*pfunc)(void *); + wt_thread_t idle_table_cycle_thread; uint64_t last_ckpts, last_inserts, last_reads, last_truncates; uint64_t last_updates; uint32_t interval, run_ops, run_time; u_int i; - int ret, t_ret; - void *(*pfunc)(void *); + int ret; opts = wtperf->opts; @@ -1722,9 +1716,7 @@ execute_workload(WTPERF *wtperf) sessions = NULL; /* Start cycling idle tables. */ - if ((ret = - start_idle_table_cycle(wtperf, &idle_table_cycle_thread)) != 0) - return (ret); + start_idle_table_cycle(wtperf, &idle_table_cycle_thread); if (opts->warmup != 0) wtperf->in_warmup = true; @@ -1768,9 +1760,8 @@ execute_workload(WTPERF *wtperf) goto err; /* Start the workload's threads. */ - if ((ret = start_threads( - wtperf, workp, threads, (u_int)workp->threads, pfunc)) != 0) - goto err; + start_threads( + wtperf, workp, threads, (u_int)workp->threads, pfunc); threads += workp->threads; } @@ -1836,12 +1827,9 @@ execute_workload(WTPERF *wtperf) err: wtperf->stop = true; /* Stop cycling idle tables. */ - if ((ret = stop_idle_table_cycle(wtperf, idle_table_cycle_thread)) != 0) - return (ret); + stop_idle_table_cycle(wtperf, idle_table_cycle_thread); - if ((t_ret = stop_threads(wtperf, - (u_int)wtperf->workers_cnt, wtperf->workers)) != 0 && ret == 0) - ret = t_ret; + stop_threads((u_int)wtperf->workers_cnt, wtperf->workers); /* Drop tables if configured to and this isn't an error path */ if (ret == 0 && @@ -2163,9 +2151,9 @@ start_all_runs(WTPERF *wtperf) { CONFIG_OPTS *opts; WTPERF *next_wtperf, **wtperfs; - pthread_t *threads; size_t i, len; - int ret, t_ret; + wt_thread_t *threads; + int ret; opts = wtperf->opts; wtperfs = NULL; @@ -2178,7 +2166,7 @@ start_all_runs(WTPERF *wtperf) wtperfs = dcalloc(opts->database_count, sizeof(WTPERF *)); /* Allocate an array to hold our thread IDs. */ - threads = dcalloc(opts->database_count, sizeof(pthread_t)); + threads = dcalloc(opts->database_count, sizeof(*threads)); for (i = 0; i < opts->database_count; i++) { wtperf_copy(wtperf, &next_wtperf); @@ -2203,22 +2191,15 @@ start_all_runs(WTPERF *wtperf) strcmp(next_wtperf->home, next_wtperf->monitor_dir) != 0) recreate_dir(next_wtperf->monitor_dir); - if ((ret = pthread_create( - &threads[i], NULL, thread_run_wtperf, next_wtperf)) != 0) { - lprintf(wtperf, ret, 0, "Error creating thread"); - goto err; - } + testutil_check(__wt_thread_create(NULL, + &threads[i], thread_run_wtperf, next_wtperf)); } /* Wait for threads to finish. */ for (i = 0; i < opts->database_count; i++) - if ((t_ret = pthread_join(threads[i], NULL)) != 0) { - lprintf(wtperf, ret, 0, "Error joining thread"); - if (ret == 0) - ret = t_ret; - } + testutil_check(__wt_thread_join(NULL, threads[i])); -err: for (i = 0; i < opts->database_count && wtperfs[i] != NULL; i++) { + for (i = 0; i < opts->database_count && wtperfs[i] != NULL; i++) { wtperf_free(wtperfs[i]); free(wtperfs[i]); } @@ -2229,7 +2210,7 @@ err: for (i = 0; i < opts->database_count && wtperfs[i] != NULL; i++) { } /* Run an instance of wtperf for a given configuration. */ -static void * +static WT_THREAD_RET thread_run_wtperf(void *arg) { WTPERF *wtperf; @@ -2238,14 +2219,14 @@ thread_run_wtperf(void *arg) wtperf = (WTPERF *)arg; if ((ret = start_run(wtperf)) != 0) lprintf(wtperf, ret, 0, "Run failed for: %s.", wtperf->home); - return (NULL); + return (WT_THREAD_RET_VALUE); } static int start_run(WTPERF *wtperf) { CONFIG_OPTS *opts; - pthread_t monitor_thread; + wt_thread_t monitor_thread; uint64_t total_ops; uint32_t run_time; int monitor_created, ret, t_ret; @@ -2272,12 +2253,8 @@ start_run(WTPERF *wtperf) /* Start the monitor thread. */ if (opts->sample_interval != 0) { - if ((ret = pthread_create( - &monitor_thread, NULL, monitor, wtperf)) != 0) { - lprintf(wtperf, - ret, 0, "Error creating monitor thread."); - goto err; - } + testutil_check(__wt_thread_create( + NULL, &monitor_thread, monitor, wtperf)); monitor_created = 1; } @@ -2306,9 +2283,8 @@ start_run(WTPERF *wtperf) opts->checkpoint_threads); wtperf->ckptthreads = dcalloc( opts->checkpoint_threads, sizeof(WTPERF_THREAD)); - if (start_threads(wtperf, NULL, wtperf->ckptthreads, - opts->checkpoint_threads, checkpoint_worker) != 0) - goto err; + start_threads(wtperf, NULL, wtperf->ckptthreads, + opts->checkpoint_threads, checkpoint_worker); } if (opts->pre_load_data && (ret = pre_load_data(wtperf)) != 0) goto err; @@ -2362,16 +2338,10 @@ err: if (ret == 0) /* Notify the worker threads they are done. */ wtperf->stop = true; - if ((t_ret = stop_threads(wtperf, 1, wtperf->ckptthreads)) != 0) - if (ret == 0) - ret = t_ret; + stop_threads(1, wtperf->ckptthreads); - if (monitor_created != 0 && - (t_ret = pthread_join(monitor_thread, NULL)) != 0) { - lprintf(wtperf, ret, 0, "Error joining monitor thread."); - if (ret == 0) - ret = t_ret; - } + if (monitor_created != 0) + testutil_check(__wt_thread_join(NULL, monitor_thread)); if (wtperf->conn != NULL && opts->close_conn && (t_ret = wtperf->conn->close(wtperf->conn, NULL)) != 0) { @@ -2728,14 +2698,13 @@ err: wtperf_free(wtperf); return (ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE); } -static int -start_threads(WTPERF *wtperf, - WORKLOAD *workp, WTPERF_THREAD *base, u_int num, void *(*func)(void *)) +static void +start_threads(WTPERF *wtperf, WORKLOAD *workp, + WTPERF_THREAD *base, u_int num, WT_THREAD_CALLBACK(*func)(void *)) { CONFIG_OPTS *opts; WTPERF_THREAD *thread; u_int i; - int ret; opts = wtperf->opts; @@ -2779,29 +2748,20 @@ start_threads(WTPERF *wtperf, /* Start the threads. */ for (i = 0, thread = base; i < num; ++i, ++thread) - if ((ret = pthread_create( - &thread->handle, NULL, func, thread)) != 0) { - lprintf(wtperf, ret, 0, "Error creating thread"); - return (ret); - } - - return (0); + testutil_check(__wt_thread_create( + NULL, &thread->handle, func, thread)); } -static int -stop_threads(WTPERF *wtperf, u_int num, WTPERF_THREAD *threads) +static void +stop_threads(u_int num, WTPERF_THREAD *threads) { u_int i; - int ret; if (num == 0 || threads == NULL) - return (0); + return; for (i = 0; i < num; ++i, ++threads) { - if ((ret = pthread_join(threads->handle, NULL)) != 0) { - lprintf(wtperf, ret, 0, "Error joining thread"); - return (ret); - } + testutil_check(__wt_thread_join(NULL, threads->handle)); free(threads->key_buf); threads->key_buf = NULL; @@ -2815,7 +2775,6 @@ stop_threads(WTPERF *wtperf, u_int num, WTPERF_THREAD *threads) * being read by the monitor thread (among others). As a standalone * program, leaking memory isn't a concern, and it's simpler that way. */ - return (0); } static void diff --git a/bench/wtperf/wtperf.h b/bench/wtperf/wtperf.h index 3efb8ab700e..b17d082ddcf 100644 --- a/bench/wtperf/wtperf.h +++ b/bench/wtperf/wtperf.h @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -232,7 +232,7 @@ struct __wtperf_thread { /* Per-thread structure */ WT_RAND_STATE rnd; /* Random number generation state */ - pthread_t handle; /* Handle */ + wt_thread_t handle; /* Handle */ char *key_buf, *value_buf; /* Key/value memory */ @@ -269,8 +269,8 @@ int run_truncate( int setup_log_file(WTPERF *); void setup_throttle(WTPERF_THREAD *); int setup_truncate(WTPERF *, WTPERF_THREAD *, WT_SESSION *); -int start_idle_table_cycle(WTPERF *, pthread_t *); -int stop_idle_table_cycle(WTPERF *, pthread_t); +void start_idle_table_cycle(WTPERF *, wt_thread_t *); +void stop_idle_table_cycle(WTPERF *, wt_thread_t); void worker_throttle(WTPERF_THREAD *); uint64_t sum_ckpt_ops(WTPERF *); uint64_t sum_insert_ops(WTPERF *); diff --git a/bench/wtperf/wtperf_opt.i b/bench/wtperf/wtperf_opt.i index 90f70457407..b71d93b8cc7 100644 --- a/bench/wtperf/wtperf_opt.i +++ b/bench/wtperf/wtperf_opt.i @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/bench/wtperf/wtperf_throttle.c b/bench/wtperf/wtperf_throttle.c index d104a68175d..75dad09ed50 100644 --- a/bench/wtperf/wtperf_throttle.c +++ b/bench/wtperf/wtperf_throttle.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2015 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/bench/wtperf/wtperf_truncate.c b/bench/wtperf/wtperf_truncate.c index 3fbb740d2c8..5b794009afb 100644 --- a/bench/wtperf/wtperf_truncate.c +++ b/bench/wtperf/wtperf_truncate.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/build_posix/Make.subdirs b/build_posix/Make.subdirs index 4ecec37ca6c..ec928a9ead2 100644 --- a/build_posix/Make.subdirs +++ b/build_posix/Make.subdirs @@ -1,10 +1,11 @@ # List of sub-directories, used by makemake to create Makefile.am # # The format is: -# <dir> [<condition>] +# <dir> [<condition> ...] # # If the directory exists, it is added to AUTO_SUBDIRS. -# If a condition is included, the subdir is made conditional via AM_CONDITIONAL +# If condition(s) are included, the subdir is made conditional via +# AM_CONDITIONAL. All conditions must be true to include the directory. ext/collators/reverse ext/collators/revint ext/compressors/lz4 LZ4 @@ -45,4 +46,5 @@ test/syscall test/thread # Benchmark programs. +bench/workgen PYTHON HAVE_CXX bench/wtperf diff --git a/build_posix/aclocal/strict.m4 b/build_posix/aclocal/strict.m4 index 659867fa69e..8c15a22d575 100644 --- a/build_posix/aclocal/strict.m4 +++ b/build_posix/aclocal/strict.m4 @@ -41,7 +41,14 @@ AC_DEFUN([AM_GCC_WARNINGS], [ w="$w -Wno-error=inline" w="$w -Wno-error=unsafe-loop-optimizations" + # GCC 4.7 + # WiredTiger uses anonymous structures/unions, a C11 extension, + # turn off those warnings. + # GCC 6.X + # Additional warning messages. case "$1" in + [*4.7.[0-9]*]) # gcc4.7 + w="$w -Wno-c11-extensions";; [*6.[0-9].[0-9]*]) # gcc6.X w="$w -Wduplicated-cond" w="$w -Wmisleading-indentation";; diff --git a/build_posix/aclocal/version-set.m4 b/build_posix/aclocal/version-set.m4 index bba80baa176..07765503294 100644 --- a/build_posix/aclocal/version-set.m4 +++ b/build_posix/aclocal/version-set.m4 @@ -2,8 +2,8 @@ dnl build by dist/s_version VERSION_MAJOR=2 VERSION_MINOR=9 -VERSION_PATCH=2 -VERSION_STRING='"WiredTiger 2.9.2: (May 26, 2017)"' +VERSION_PATCH=3 +VERSION_STRING='"WiredTiger 2.9.3: (June 26, 2017)"' AC_SUBST(VERSION_MAJOR) AC_SUBST(VERSION_MINOR) diff --git a/build_posix/aclocal/version.m4 b/build_posix/aclocal/version.m4 index 29782a22f82..1126d7c147b 100644 --- a/build_posix/aclocal/version.m4 +++ b/build_posix/aclocal/version.m4 @@ -1,2 +1,2 @@ dnl WiredTiger product version for AC_INIT. Maintained by dist/s_version -2.9.2 +2.9.3 diff --git a/build_posix/configure.ac.in b/build_posix/configure.ac.in index 0fef587b4b8..4de12d5161e 100644 --- a/build_posix/configure.ac.in +++ b/build_posix/configure.ac.in @@ -25,6 +25,16 @@ AM_PROG_AS(as gas) define([AC_LIBTOOL_LANG_CXX_CONFIG], [:])dnl define([AC_LIBTOOL_LANG_F77_CONFIG], [:])dnl +# Check whether the C++ compiler works by linking a trivial program. +AC_CACHE_CHECK([whether the C++ compiler works], + [wt_cv_prog_cxx_works], + [AC_LANG_PUSH([C++]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([], [])], + [wt_cv_prog_cxx_works=yes], + [wt_cv_prog_cxx_works=no]) + AC_LANG_POP([C++])]) +AM_CONDITIONAL([HAVE_CXX], [test "$wt_cv_prog_cxx_works" = "yes"]) + LT_PREREQ(2.2.6) LT_INIT([pic-only]) AC_SUBST([LIBTOOL_DEPS]) diff --git a/build_posix/makemake b/build_posix/makemake index 506420b4aaf..73d6b6bcfb1 100755 --- a/build_posix/makemake +++ b/build_posix/makemake @@ -7,14 +7,24 @@ (sed -n '1,/BEGIN SUBDIRS/p' Make.base echo "SUBDIRS =" -sed -e 's/#.*$//' -e '/^$/d' Make.subdirs | while read dir cond ; do +sed -e 's/#.*$//' -e '/^$/d' Make.subdirs | while read dir conds ; do test -d ../$dir || continue - if test -n "$cond" ; then - cat <<END_CONDITIONAL + if test -n "$conds" ; then + # Multiple conditions are allowed, they will appear + # as nested 'if' statements. + for cond in $conds; do + cat <<END_CONDITIONAL if ${cond} +END_CONDITIONAL + done + cat <<END_CONDITIONAL SUBDIRS += $dir +END_CONDITIONAL + for cond in $conds; do + cat <<END_CONDITIONAL endif END_CONDITIONAL + done else echo "SUBDIRS += $dir" fi diff --git a/dist/api_data.py b/dist/api_data.py index 22600dd5e29..3297c68147a 100644 --- a/dist/api_data.py +++ b/dist/api_data.py @@ -529,6 +529,7 @@ connection_runtime_config = [ 'fileops', 'handleops', 'log', + 'lookaside_activity', 'lsm', 'lsm_manager', 'metadata', diff --git a/dist/api_err.py b/dist/api_err.py index bd379ac8d70..bfa4459d438 100644 --- a/dist/api_err.py +++ b/dist/api_err.py @@ -41,10 +41,10 @@ errors = [ WT_CURSOR::update or WT_CURSOR::remove.'''), Error('WT_PANIC', -31804, 'WiredTiger library panic', ''' - This error indicates an underlying problem that requires the - application exit and restart. The application can exit - immediately when \c WT_PANIC is returned from a WiredTiger - interface, no further WiredTiger calls are required.'''), + This error indicates an underlying problem that requires a database + restart. The application may exit immediately, no further WiredTiger + calls are required (and further calls will themselves immediately + fail).'''), Error('WT_RESTART', -31805, 'restart the operation (internal)', undoc=True), Error('WT_RUN_RECOVERY', -31806, @@ -112,8 +112,6 @@ tfile.write('''/* DO NOT EDIT: automatically built by dist/api_err.py. */ const char * __wt_wiredtiger_error(int error) { -\tconst char *p; - \t/* \t * Check for WiredTiger specific errors. \t */ @@ -125,14 +123,20 @@ for err in errors: tfile.write('\t\treturn ("' + err.name + ': ' + err.desc + '");\n') tfile.write('''\t} +\t/* Windows strerror doesn't support ENOTSUP. */ +\tif (error == ENOTSUP) +\t\treturn ("Operation not supported"); + \t/* -\t * POSIX errors are non-negative integers; check for 0 explicitly incase -\t * the underlying strerror doesn't handle 0, some historically didn't. +\t * Check for 0 in case the underlying strerror doesn't handle it, some +\t * historically didn't. \t */ \tif (error == 0) \t\treturn ("Successful return: 0"); -\tif (error > 0 && (p = strerror(error)) != NULL) -\t\treturn (p); + +\t/* POSIX errors are non-negative integers. */ +\tif (error > 0) +\t\treturn (strerror(error)); \treturn (NULL); } diff --git a/dist/filelist b/dist/filelist index 5a3348b940a..6b6e617c4b1 100644 --- a/dist/filelist +++ b/dist/filelist @@ -179,6 +179,7 @@ src/session/session_salvage.c src/support/cond_auto.c src/support/crypto.c src/support/err.c +src/support/generation.c src/support/global.c src/support/hash_city.c src/support/hash_fnv.c diff --git a/dist/flags.py b/dist/flags.py index 64b5d789e72..8edabd69648 100644 --- a/dist/flags.py +++ b/dist/flags.py @@ -32,7 +32,6 @@ flags = { 'READ_PREV', 'READ_RESTART_OK', 'READ_SKIP_INTL', - 'READ_SKIP_LEAF', 'READ_TRUNCATE', 'READ_WONT_NEED', ], @@ -68,6 +67,7 @@ flags = { 'VERB_FILEOPS', 'VERB_HANDLEOPS', 'VERB_LOG', + 'VERB_LOOKASIDE', 'VERB_LSM', 'VERB_LSM_MANAGER', 'VERB_METADATA', diff --git a/dist/package/wiredtiger.spec b/dist/package/wiredtiger.spec index aacdf327c98..9d9bdd3949c 100644 --- a/dist/package/wiredtiger.spec +++ b/dist/package/wiredtiger.spec @@ -1,5 +1,5 @@ Name: wiredtiger -Version: 2.9.2 +Version: 2.9.3 Release: 1%{?dist} Summary: WiredTiger data storage engine diff --git a/dist/s_c_test_create b/dist/s_c_test_create index f4f9eb3ac1f..1d379664e75 100755 --- a/dist/s_c_test_create +++ b/dist/s_c_test_create @@ -35,7 +35,7 @@ mkdir $CSUITE_DIRECTORY/$TEST_NAME (cat <<EOF /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/dist/s_copyright b/dist/s_copyright index 9ff6c20492e..4a93be73fb6 100755 --- a/dist/s_copyright +++ b/dist/s_copyright @@ -53,7 +53,7 @@ if [ $# -ne 0 ]; then exit 0 fi -trap 'rm -f $c1 $c2 $c3 $c4' 0 1 2 3 13 15 +trap 'rm -f $c1 $c2 $c3 $c4 $c5' 0 1 2 3 13 15 year=`date +%Y` @@ -117,18 +117,24 @@ fi -e '/api\/leveldb\/hyperleveldb\//d' \ -e '/api\/leveldb\/leveldb\//d' \ -e '/api\/leveldb\/rocksdb\//d' \ + -e '/checksum\/power8\//d' \ + -e '/checksum\/zseries\//d' \ -e '/\/3rdparty\//d' \ -e '/\/node_modules\//d' \ -e '/dist\/__/d' \ -e 's/^\.\///' | xargs $xp -n 1 -I{} sh dist/s_copyright {}) +# One-offs. +(cd .. && sh dist/s_copyright test/syscall/wt2336_base/base.run) + # A few special cases: LICENSE, documentation, wt utility, some of which have # more than one copyright notice in the file. For files that have only a single # copyright notice, we give it to MongoDB, from 2008 to now. string1="Copyright \(c\) 2014-$year MongoDB, Inc." string2="Copyright \(c\) 2008-$year MongoDB, Inc." string3="printf.*Copyright \(c\) 2008-$year MongoDB, Inc." +string4="Public Domain 2014-$year MongoDB, Inc." special_copyright() { cnt=`egrep "$3" ../$1 | wc -l` @@ -138,6 +144,7 @@ special_copyright() } special_copyright LICENSE 1 "$string1" +special_copyright dist/s_c_test_create 1 "$string4" special_copyright src/docs/build-javadoc.sh 1 "$string2" special_copyright src/docs/style/footer.html 2 "$string2" special_copyright src/utilities/util_cpyright.c 1 "$string3" diff --git a/dist/s_copyright.list b/dist/s_copyright.list index 4999d2a37a2..2ac63bcb159 100644 --- a/dist/s_copyright.list +++ b/dist/s_copyright.list @@ -1,4 +1,7 @@ +skip api/leveldb/leveldb_wt_config.h skip api/leveldb/leveldb_wt_config.in +skip bench/workgen/workgen/workgen.py +skip bench/workgen/workgen_wrap.cxx skip build_win/wiredtiger_config.h skip dist/api_config.py skip dist/api_data.py @@ -9,9 +12,11 @@ skip dist/flags.py skip dist/java_doc.py skip dist/log.py skip dist/log_data.py +skip dist/s_label_loop.py skip dist/stat.py skip dist/stat_data.py skip dist/style.py +skip dist/wtperf_config.py skip lang/java/java_doc.i skip lang/java/src/com/wiredtiger/db/AsyncOp.java skip lang/java/src/com/wiredtiger/db/AsyncOpType.java diff --git a/dist/s_define.list b/dist/s_define.list index 8911d888077..9f94132f584 100644 --- a/dist/s_define.list +++ b/dist/s_define.list @@ -58,6 +58,7 @@ WT_STAT_INCRV_BASE WT_STAT_WRITE WT_TIMEDIFF_US WT_TRET_ERROR_OK +WT_UPDATE_SIZE WT_WITH_LOCK_NOWAIT WT_WITH_LOCK_WAIT __F diff --git a/dist/s_prototypes b/dist/s_prototypes index d6228866f08..9675cd5a843 100755 --- a/dist/s_prototypes +++ b/dist/s_prototypes @@ -42,9 +42,6 @@ proto() -e '# Add the warn_unused_result attribute to any external' \ -e '# functions that return an int.' \ -e '/^extern int /s/$/ WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result))/' \ - -e '# Add the hidden attribute to any external functions without' \ - -e '# an explicit visibility.' \ - -e '/visibility/!s/$/ WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")))/' \ -e 's/$/;/' \ -e p < $1 } diff --git a/dist/s_stat b/dist/s_stat index 6aeeca6faa6..cf9303e5f95 100755 --- a/dist/s_stat +++ b/dist/s_stat @@ -25,15 +25,20 @@ cat << UNUSED_STAT_FIELDS lock_checkpoint_count lock_checkpoint_wait_application lock_checkpoint_wait_internal +lock_dhandle_read_count +lock_dhandle_wait_application +lock_dhandle_wait_internal +lock_dhandle_write_count lock_metadata_count lock_metadata_wait_application lock_metadata_wait_internal lock_schema_count lock_schema_wait_application lock_schema_wait_internal -lock_table_count +lock_table_read_count lock_table_wait_application lock_table_wait_internal +lock_table_write_count UNUSED_STAT_FIELDS echo "$search" diff --git a/dist/s_string.ok b/dist/s_string.ok index 1f7f7d9fd3a..4ddb64297f4 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -305,6 +305,7 @@ RMW RNG RPC RUNDIR +RWLOCK RXB Radu ReadFile @@ -344,6 +345,7 @@ Split's Stoica StoreLoad StoreStore +Su Syscall TAILQ TCMalloc @@ -353,6 +355,8 @@ TORTIOUS TSO TXN TXNC +ThreadList +ThreadListWrapper Timespec Timestamp TryCV @@ -515,11 +519,13 @@ change's changelog chdir checkfmt +checkkey checkpointed checkpointer checkpointing checksum checksums +checkvalue children's chk chmod @@ -731,6 +737,7 @@ fsyncLock fsyncs ftruncate func +fvisibility gcc gdb ge @@ -806,6 +813,7 @@ intl intnum intpack intptr +intr intrin inuse io @@ -864,7 +872,9 @@ llll llu loadtext localTime +localkey localtime +localvalue logf logmgr lognum @@ -938,7 +948,10 @@ nbits nchunks nclr nd +needkey +needvalue negint +nentries newbar newfile newuri @@ -957,6 +970,7 @@ noraw notfound notsup notused +novalue nowait nset nsnap @@ -1081,6 +1095,7 @@ rotN rotn rp rpc +ru run's runtime rwlock @@ -1186,6 +1201,7 @@ txnid txnmin txt typedef +typemaps uB uS ui @@ -1266,6 +1282,7 @@ whitespace wiredTiger wiredtiger workFactor +workgen wrapup writeable writelock diff --git a/dist/s_void b/dist/s_void index 249f043d029..d7f2c81a211 100755 --- a/dist/s_void +++ b/dist/s_void @@ -88,8 +88,8 @@ func_ok() -e '/int handle_progress$/d' \ -e '/int helium_cursor_reset$/d' \ -e '/int helium_session_verify$/d' \ - -e '/int index_compare_primary$/d' \ -e '/int index_compare_S$/d' \ + -e '/int index_compare_primary$/d' \ -e '/int index_compare_u$/d' \ -e '/int index_extractor_u$/d' \ -e '/int log_print_err$/d' \ @@ -103,7 +103,6 @@ func_ok() -e '/int nop_pre_size$/d' \ -e '/int nop_sizing$/d' \ -e '/int nop_terminate$/d' \ - -e '/int nop_terminate$/d' \ -e '/int os_errno$/d' \ -e '/int revint_terminate$/d' \ -e '/int rotn_error$/d' \ @@ -111,6 +110,7 @@ func_ok() -e '/int rotn_terminate$/d' \ -e '/int snappy_pre_size$/d' \ -e '/int snappy_terminate$/d' \ + -e '/int subtest_error_handler$/d' \ -e '/int uri2name$/d' \ -e '/int usage$/d' \ -e '/int util_err$/d' \ diff --git a/dist/s_whitespace b/dist/s_whitespace index 0de59bc5825..874074dfb50 100755 --- a/dist/s_whitespace +++ b/dist/s_whitespace @@ -8,6 +8,7 @@ trap 'rm -f $t' 0 1 2 3 13 15 # into a single line, discard trailing empty lines. whitespace() { + ! head $1 | grep -q 'automatically generated by SWIG' || return sed -e 's/[ ][ ]*$//' < $1 | \ cat -s | \ sed -e '${' -e '/^$/d' -e '}' > $t diff --git a/dist/stat_data.py b/dist/stat_data.py index ac79ffd029a..7b919848003 100644 --- a/dist/stat_data.py +++ b/dist/stat_data.py @@ -150,6 +150,7 @@ connection_stats = [ ConnStat('read_io', 'total read I/Os'), ConnStat('rwlock_read', 'pthread mutex shared lock read-lock calls'), ConnStat('rwlock_write', 'pthread mutex shared lock write-lock calls'), + ConnStat('time_travel', 'detected system time went backwards'), ConnStat('write_io', 'total write I/Os'), ########################################## @@ -203,9 +204,12 @@ connection_stats = [ CacheStat('cache_eviction_dirty', 'modified pages evicted'), CacheStat('cache_eviction_empty_score', 'eviction empty score', 'no_clear,no_scale'), CacheStat('cache_eviction_fail', 'pages selected for eviction unable to be evicted'), - CacheStat('cache_eviction_force', 'pages evicted because they exceeded the in-memory maximum'), - CacheStat('cache_eviction_force_delete', 'pages evicted because they had chains of deleted items'), - CacheStat('cache_eviction_force_fail', 'failed eviction of pages that exceeded the in-memory maximum'), + CacheStat('cache_eviction_force', 'pages evicted because they exceeded the in-memory maximum count'), + CacheStat('cache_eviction_force_time', 'pages evicted because they exceeded the in-memory maximum time (usecs)'), + CacheStat('cache_eviction_force_delete', 'pages evicted because they had chains of deleted items count'), + CacheStat('cache_eviction_force_delete_time', 'pages evicted because they had chains of deleted items time (usecs)'), + CacheStat('cache_eviction_force_fail', 'failed eviction of pages that exceeded the in-memory maximum count'), + CacheStat('cache_eviction_force_fail_time', 'failed eviction of pages that exceeded the in-memory maximum time (usecs)'), CacheStat('cache_eviction_force_retune', 'force re-tuning of eviction workers once in a while'), CacheStat('cache_eviction_get_ref', 'eviction calls to get a page'), CacheStat('cache_eviction_get_ref_empty', 'eviction calls to get a page found queue empty'), @@ -230,8 +234,8 @@ connection_stats = [ CacheStat('cache_eviction_walks_abandoned', 'eviction walks abandoned'), CacheStat('cache_eviction_walks_active', 'files with active eviction walks', 'no_clear,no_scale'), CacheStat('cache_eviction_walks_started', 'files with new eviction walks started'), - CacheStat('cache_eviction_worker_evicting', 'eviction worker thread evicting pages'), CacheStat('cache_eviction_worker_created', 'eviction worker thread created'), + CacheStat('cache_eviction_worker_evicting', 'eviction worker thread evicting pages'), CacheStat('cache_eviction_worker_removed', 'eviction worker thread removed'), CacheStat('cache_hazard_checks', 'hazard pointer check calls'), CacheStat('cache_hazard_max', 'hazard pointer maximum array length', 'max_aggregate,no_scale'), @@ -261,9 +265,11 @@ connection_stats = [ ########################################## CursorStat('cursor_create', 'cursor create calls'), CursorStat('cursor_insert', 'cursor insert calls'), + CursorStat('cursor_modify', 'cursor modify calls'), CursorStat('cursor_next', 'cursor next calls'), CursorStat('cursor_prev', 'cursor prev calls'), CursorStat('cursor_remove', 'cursor remove calls'), + CursorStat('cursor_reserve', 'cursor reserve calls'), CursorStat('cursor_reset', 'cursor reset calls'), CursorStat('cursor_restart', 'cursor restarted searches'), CursorStat('cursor_search', 'cursor search calls'), @@ -289,16 +295,20 @@ connection_stats = [ LockStat('lock_checkpoint_count', 'checkpoint lock acquisitions'), LockStat('lock_checkpoint_wait_application', 'checkpoint lock application thread wait time (usecs)'), LockStat('lock_checkpoint_wait_internal', 'checkpoint lock internal thread wait time (usecs)'), - LockStat('lock_handle_list_wait_eviction', 'handle-list lock eviction thread wait time (usecs)'), + LockStat('lock_dhandle_read_count', 'dhandle read lock acquisitions'), + LockStat('lock_dhandle_wait_application', 'dhandle lock application thread time waiting for the dhandle lock (usecs)'), + LockStat('lock_dhandle_wait_internal', 'dhandle lock internal thread time waiting for the dhandle lock (usecs)'), + LockStat('lock_dhandle_write_count', 'dhandle write lock acquisitions'), LockStat('lock_metadata_count', 'metadata lock acquisitions'), LockStat('lock_metadata_wait_application', 'metadata lock application thread wait time (usecs)'), LockStat('lock_metadata_wait_internal', 'metadata lock internal thread wait time (usecs)'), LockStat('lock_schema_count', 'schema lock acquisitions'), LockStat('lock_schema_wait_application', 'schema lock application thread wait time (usecs)'), LockStat('lock_schema_wait_internal', 'schema lock internal thread wait time (usecs)'), - LockStat('lock_table_count', 'table lock acquisitions'), + LockStat('lock_table_read_count', 'table read lock acquisitions'), LockStat('lock_table_wait_application', 'table lock application thread time waiting for the table lock (usecs)'), LockStat('lock_table_wait_internal', 'table lock internal thread time waiting for the table lock (usecs)'), + LockStat('lock_table_write_count', 'table write lock acquisitions'), ########################################## # Logging statistics @@ -324,16 +334,22 @@ connection_stats = [ LogStat('log_scan_records', 'records processed by log scan'), LogStat('log_scan_rereads', 'log scan records requiring two reads'), LogStat('log_scans', 'log scan operations'), - LogStat('log_slot_active_closed', 'consolidated slot join active slot closed'), - LogStat('log_slot_closes', 'consolidated slot closures'), + LogStat('log_slot_active_closed', 'slot join found active slot closed'), + LogStat('log_slot_close_race', 'slot close lost race'), + LogStat('log_slot_close_unbuf', 'slot close unbuffered waits'), + LogStat('log_slot_closes', 'slot closures'), LogStat('log_slot_coalesced', 'written slots coalesced'), LogStat('log_slot_consolidated', 'logging bytes consolidated', 'size'), - LogStat('log_slot_joins', 'consolidated slot joins'), - LogStat('log_slot_no_free_slots', 'consolidated slot transitions unable to find free slot'), - LogStat('log_slot_races', 'consolidated slot join races'), + LogStat('log_slot_immediate', 'slot join calls did not yield'), + LogStat('log_slot_no_free_slots', 'slot transitions unable to find free slot'), + LogStat('log_slot_races', 'slot join atomic update races'), LogStat('log_slot_switch_busy', 'busy returns attempting to switch slots'), - LogStat('log_slot_transitions', 'consolidated slot join transitions'), - LogStat('log_slot_unbuffered', 'consolidated slot unbuffered writes'), + LogStat('log_slot_unbuffered', 'slot unbuffered writes'), + LogStat('log_slot_yield', 'slot join calls yielded'), + LogStat('log_slot_yield_close', 'slot join calls found active slot closed'), + LogStat('log_slot_yield_duration', 'slot joins yield time (usecs)', 'no_clear,no_scale'), + LogStat('log_slot_yield_race', 'slot join calls atomic updates raced'), + LogStat('log_slot_yield_sleep', 'slot join calls slept'), LogStat('log_sync', 'log sync operations'), LogStat('log_sync_dir', 'log sync_dir operations'), LogStat('log_sync_dir_duration', 'log sync_dir time duration (usecs)', 'no_clear,no_scale'), @@ -424,6 +440,7 @@ connection_stats = [ TxnStat('txn_snapshots_created', 'number of named snapshots created'), TxnStat('txn_snapshots_dropped', 'number of named snapshots dropped'), TxnStat('txn_sync', 'transaction sync calls'), + TxnStat('txn_update_conflict', 'update conflicts'), ########################################## # Yield statistics @@ -546,10 +563,12 @@ dsrc_stats = [ CursorStat('cursor_insert', 'insert calls'), CursorStat('cursor_insert_bulk', 'bulk-loaded cursor-insert calls'), CursorStat('cursor_insert_bytes', 'cursor-insert key and value bytes inserted', 'size'), + CursorStat('cursor_modify', 'modify calls'), CursorStat('cursor_next', 'next calls'), CursorStat('cursor_prev', 'prev calls'), CursorStat('cursor_remove', 'remove calls'), CursorStat('cursor_remove_bytes', 'cursor-remove key bytes removed', 'size'), + CursorStat('cursor_reserve', 'reserve calls'), CursorStat('cursor_reset', 'reset calls'), CursorStat('cursor_restart', 'restarted searches'), CursorStat('cursor_search', 'search calls'), diff --git a/examples/c/ex_access.c b/examples/c/ex_access.c index d7f3cc557ad..6f24139182d 100644 --- a/examples/c/ex_access.c +++ b/examples/c/ex_access.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c index 82620673fe1..5e1fa4bbcc5 100644 --- a/examples/c/ex_all.c +++ b/examples/c/ex_all.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -299,6 +299,49 @@ cursor_ops(WT_SESSION *session) } { + /*! [Reserve a record] */ + const char *key = "some key"; + ret = session->open_cursor( + session, "table:mytable", NULL, NULL, &cursor); + cursor->set_key(cursor, key); + ret = cursor->reserve(cursor); + /*! [Reserve a record] */ + } + + { + /*! [Modify an existing record] */ + WT_MODIFY entries[3]; + const char *key = "some key"; + ret = session->open_cursor( + session, "table:mytable", NULL, NULL, &cursor); + + /* Position the cursor. */ + cursor->set_key(cursor, key); + ret = cursor->search(cursor); + + /* Replace 20 bytes starting at byte offset 5. */ + entries[0].data.data = "some data"; + entries[0].data.size = strlen(entries[0].data.data); + entries[0].offset = 5; + entries[0].size = 20; + + /* Insert data at byte offset 40. */ + entries[1].data.data = "and more data"; + entries[1].data.size = strlen(entries[1].data.data); + entries[1].offset = 40; + entries[1].size = 0; + + /* Replace 2 bytes starting at byte offset 10. */ + entries[2].data.data = "and more data"; + entries[2].data.size = strlen(entries[2].data.data); + entries[2].offset = 10; + entries[2].size = 2; + + ret = cursor->modify(cursor, entries, 3); + /*! [Modify an existing record] */ + } + + { /*! [Update an existing record or insert a new record] */ const char *key = "some key", *value = "some value"; ret = session->open_cursor( diff --git a/examples/c/ex_async.c b/examples/c/ex_async.c index 5cfafca0418..83cddc2824d 100644 --- a/examples/c/ex_async.c +++ b/examples/c/ex_async.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/c/ex_backup.c b/examples/c/ex_backup.c index 83cc9b22ecc..ff7d979f286 100644 --- a/examples/c/ex_backup.c +++ b/examples/c/ex_backup.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/c/ex_call_center.c b/examples/c/ex_call_center.c index cd53a1cdaf9..4483e8b1603 100644 --- a/examples/c/ex_call_center.c +++ b/examples/c/ex_call_center.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/c/ex_config_parse.c b/examples/c/ex_config_parse.c index 40508b38204..c9720325129 100644 --- a/examples/c/ex_config_parse.c +++ b/examples/c/ex_config_parse.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/c/ex_cursor.c b/examples/c/ex_cursor.c index b8ed6ab169d..0982aa43073 100644 --- a/examples/c/ex_cursor.c +++ b/examples/c/ex_cursor.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/c/ex_data_source.c b/examples/c/ex_data_source.c index 387248f6ae2..d40008e0a0e 100644 --- a/examples/c/ex_data_source.c +++ b/examples/c/ex_data_source.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/c/ex_encrypt.c b/examples/c/ex_encrypt.c index 1520bd286cd..1710d5af16f 100644 --- a/examples/c/ex_encrypt.c +++ b/examples/c/ex_encrypt.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/c/ex_event_handler.c b/examples/c/ex_event_handler.c index 03809cae7c8..acd9d9beecc 100644 --- a/examples/c/ex_event_handler.c +++ b/examples/c/ex_event_handler.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -70,6 +70,10 @@ handle_wiredtiger_error(WT_EVENT_HANDLER *handler, "app_id %s, thread context %p, error %d, message %s\n", custom_handler->app_id, (void *)session, error, message); + /* Exit if the database has a fatal error. */ + if (error == WT_PANIC) + exit (1); + return (0); } diff --git a/examples/c/ex_extending.c b/examples/c/ex_extending.c index f276cdd3e1e..7364fa4bc9e 100644 --- a/examples/c/ex_extending.c +++ b/examples/c/ex_extending.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/c/ex_extractor.c b/examples/c/ex_extractor.c index f9d7af4af0f..3aaaf90ac90 100644 --- a/examples/c/ex_extractor.c +++ b/examples/c/ex_extractor.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/c/ex_file_system.c b/examples/c/ex_file_system.c index e807ac54d3b..e454d228c39 100644 --- a/examples/c/ex_file_system.c +++ b/examples/c/ex_file_system.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -583,13 +583,13 @@ demo_fs_size(WT_FILE_SYSTEM *file_system, static int demo_fs_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *session) { - DEMO_FILE_HANDLE *demo_fh; + DEMO_FILE_HANDLE *demo_fh, *demo_fh_tmp; DEMO_FILE_SYSTEM *demo_fs; int ret = 0, tret; demo_fs = (DEMO_FILE_SYSTEM *)file_system; - while ((demo_fh = TAILQ_FIRST(&demo_fs->fileq)) != NULL) + TAILQ_FOREACH_SAFE(demo_fh, &demo_fs->fileq, q, demo_fh_tmp) if ((tret = demo_handle_remove(session, demo_fh)) != 0 && ret == 0) ret = tret; diff --git a/examples/c/ex_hello.c b/examples/c/ex_hello.c index 99534ee8868..616049aaddb 100644 --- a/examples/c/ex_hello.c +++ b/examples/c/ex_hello.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/c/ex_log.c b/examples/c/ex_log.c index 0d8fbf97233..d4de195ddee 100644 --- a/examples/c/ex_log.c +++ b/examples/c/ex_log.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/c/ex_pack.c b/examples/c/ex_pack.c index 86725123f55..37b864e62a4 100644 --- a/examples/c/ex_pack.c +++ b/examples/c/ex_pack.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/c/ex_process.c b/examples/c/ex_process.c index 217730c4288..4bab6a1cd70 100644 --- a/examples/c/ex_process.c +++ b/examples/c/ex_process.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/c/ex_schema.c b/examples/c/ex_schema.c index a59d9480780..9249ecc1e1a 100644 --- a/examples/c/ex_schema.c +++ b/examples/c/ex_schema.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/c/ex_stat.c b/examples/c/ex_stat.c index cf9e8fb97d1..7097b53a060 100644 --- a/examples/c/ex_stat.c +++ b/examples/c/ex_stat.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/c/ex_sync.c b/examples/c/ex_sync.c index b2d74b52f7f..c333ac42e1e 100644 --- a/examples/c/ex_sync.c +++ b/examples/c/ex_sync.c @@ -1,5 +1,5 @@ -/* - * Public Domain 2014-2016 MongoDB, Inc. +/*- + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/c/ex_thread.c b/examples/c/ex_thread.c index fa82bd5f113..ad2ff7f68a0 100644 --- a/examples/c/ex_thread.c +++ b/examples/c/ex_thread.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -34,22 +34,14 @@ #include <stdlib.h> #include <string.h> -#ifndef _WIN32 -#include <pthread.h> -#else -#include "windows_shim.h" -#endif - -#include <wiredtiger.h> +#include "wt_internal.h" static const char *home; -void *scan_thread(void *arg); - #define NUM_THREADS 10 /*! [thread scan] */ -void * +static WT_THREAD_RET scan_thread(void *conn_arg) { WT_CONNECTION *conn; @@ -74,7 +66,7 @@ scan_thread(void *conn_arg) fprintf(stderr, "WT_CURSOR.next: %s\n", session->strerror(session, ret)); - return (NULL); + return (WT_THREAD_RET_VALUE); } /*! [thread scan] */ @@ -85,7 +77,7 @@ main(void) WT_CONNECTION *conn; WT_SESSION *session; WT_CURSOR *cursor; - pthread_t threads[NUM_THREADS]; + wt_thread_t threads[NUM_THREADS]; int i, ret; /* @@ -114,10 +106,10 @@ main(void) ret = session->close(session, NULL); for (i = 0; i < NUM_THREADS; i++) - ret = pthread_create(&threads[i], NULL, scan_thread, conn); + ret = __wt_thread_create(NULL, &threads[i], scan_thread, conn); for (i = 0; i < NUM_THREADS; i++) - ret = pthread_join(threads[i], NULL); + ret = __wt_thread_join(NULL, threads[i]); ret = conn->close(conn, NULL); diff --git a/examples/java/com/wiredtiger/examples/ex_access.java b/examples/java/com/wiredtiger/examples/ex_access.java index 104f86d5545..ed96ebce7d7 100644 --- a/examples/java/com/wiredtiger/examples/ex_access.java +++ b/examples/java/com/wiredtiger/examples/ex_access.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/java/com/wiredtiger/examples/ex_all.java b/examples/java/com/wiredtiger/examples/ex_all.java index cf8491aa4f8..ff7d371fabd 100644 --- a/examples/java/com/wiredtiger/examples/ex_all.java +++ b/examples/java/com/wiredtiger/examples/ex_all.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/java/com/wiredtiger/examples/ex_async.java b/examples/java/com/wiredtiger/examples/ex_async.java index 2e890095b2d..92054464747 100644 --- a/examples/java/com/wiredtiger/examples/ex_async.java +++ b/examples/java/com/wiredtiger/examples/ex_async.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/java/com/wiredtiger/examples/ex_call_center.java b/examples/java/com/wiredtiger/examples/ex_call_center.java index a3f0f56ded8..921c7f9f57c 100644 --- a/examples/java/com/wiredtiger/examples/ex_call_center.java +++ b/examples/java/com/wiredtiger/examples/ex_call_center.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/java/com/wiredtiger/examples/ex_cursor.java b/examples/java/com/wiredtiger/examples/ex_cursor.java index a0a6e48aa46..4a57f3c35da 100644 --- a/examples/java/com/wiredtiger/examples/ex_cursor.java +++ b/examples/java/com/wiredtiger/examples/ex_cursor.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -156,6 +156,41 @@ public class ex_cursor { } /*! [cursor remove] */ + /*! [cursor modify] */ + public static int + cursor_modify(Cursor cursor) + throws WiredTigerException + { + byte orig[] = new byte[4]; + for (int i = 0; i < 4; i++) + orig[i] = (byte)i; + cursor.putKeyString("key"); + cursor.putValueByteArray(orig); + cursor.insert(); // 0x0 0x1 0x2 0x3 + + byte b10[] = new byte[4]; + for (int i = 0; i < 4; i++) + b10[i] = (byte)(0x10 + i); + byte b20[] = new byte[4]; + for (int i = 0; i < 4; i++) + b20[i] = (byte)(0x20 + i); + + Modify modlist[] = new Modify[2]; + // The following Modify replaces one byte at position one by: + // (0x10 0x11 0x12 0x13), leaving: + // 0x0 0x10 0x11 0x12 0x13 0x2 0x3 + modlist[0] = new Modify(b10, 1, 1); + + // The following Modify replaces one byte at position three by: + // (0x20 0x21 0x22 0x23), leaving: + // 0x0 0x10 0x11 0x20 0x21 0x22 0x23 0x13 0x2 0x3 + modlist[1] = new Modify(b20, 3, 1); + + cursor.putKeyString("key"); + return (cursor.modify(modlist)); + } + /*! [cursor modify] */ + public static int cursorExample() throws WiredTigerException @@ -219,6 +254,12 @@ public class ex_cursor { ret = cursor_remove(cursor); ret = cursor.close(); + /* Create a table with a raw value to illustrate certain operations. */ + ret = session.create("table:raw", "key_format=S,value_format=u"); + cursor = session.open_cursor("table:raw", null, null); + ret = cursor_modify(cursor); + ret = cursor.close(); + /* Note: closing the connection implicitly closes open session(s). */ if ((ret = conn.close(null)) != 0) System.err.println("Error connecting to " + home + ": " + diff --git a/examples/java/com/wiredtiger/examples/ex_log.java b/examples/java/com/wiredtiger/examples/ex_log.java index 233ad1361d8..5a76c43b13c 100644 --- a/examples/java/com/wiredtiger/examples/ex_log.java +++ b/examples/java/com/wiredtiger/examples/ex_log.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/java/com/wiredtiger/examples/ex_schema.java b/examples/java/com/wiredtiger/examples/ex_schema.java index 76bff66a688..b7aa64f0c68 100644 --- a/examples/java/com/wiredtiger/examples/ex_schema.java +++ b/examples/java/com/wiredtiger/examples/ex_schema.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/java/com/wiredtiger/examples/ex_stat.java b/examples/java/com/wiredtiger/examples/ex_stat.java index f8877a4620e..799f0396756 100644 --- a/examples/java/com/wiredtiger/examples/ex_stat.java +++ b/examples/java/com/wiredtiger/examples/ex_stat.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/java/com/wiredtiger/examples/ex_thread.java b/examples/java/com/wiredtiger/examples/ex_thread.java index 402daebbd61..2476b3a4d41 100644 --- a/examples/java/com/wiredtiger/examples/ex_thread.java +++ b/examples/java/com/wiredtiger/examples/ex_thread.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/examples/python/ex_access.py b/examples/python/ex_access.py index aa99c1f6547..58ba64607e2 100755 --- a/examples/python/ex_access.py +++ b/examples/python/ex_access.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/examples/python/ex_stat.py b/examples/python/ex_stat.py index 1772badd076..cd99c4f388b 100755 --- a/examples/python/ex_stat.py +++ b/examples/python/ex_stat.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/ext/collators/reverse/reverse_collator.c b/ext/collators/reverse/reverse_collator.c index 7e205f98193..3a589613427 100644 --- a/ext/collators/reverse/reverse_collator.c +++ b/ext/collators/reverse/reverse_collator.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/ext/collators/revint/revint_collator.c b/ext/collators/revint/revint_collator.c index cfad3989adb..9952e922077 100644 --- a/ext/collators/revint/revint_collator.c +++ b/ext/collators/revint/revint_collator.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/ext/compressors/lz4/lz4_compress.c b/ext/compressors/lz4/lz4_compress.c index 885701e564b..279f0be6c36 100644 --- a/ext/compressors/lz4/lz4_compress.c +++ b/ext/compressors/lz4/lz4_compress.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -54,8 +54,8 @@ typedef struct { /* * LZ4 decompression requires the exact compressed byte count returned by the - * LZ4_compress and LZ4_compress_destSize functions. WiredTiger doesn't track - * that value, store it in the destination buffer. + * LZ4_compress_default and LZ4_compress_destSize functions. WiredTiger doesn't + * track that value, store it in the destination buffer. * * Additionally, LZ4_compress_destSize may compress into the middle of a record, * and after decompression we return the length to the last record successfully @@ -137,11 +137,10 @@ lz4_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, (void)compressor; /* Unused parameters */ (void)session; - (void)dst_len; /* Compress, starting after the prefix bytes. */ - lz4_len = LZ4_compress( - (const char *)src, (char *)dst + sizeof(LZ4_PREFIX), (int)src_len); + lz4_len = LZ4_compress_default((const char *)src, + (char *)dst + sizeof(LZ4_PREFIX), (int)src_len, (int)dst_len); /* * If compression succeeded and the compressed length is smaller than @@ -214,7 +213,7 @@ lz4_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, */ if (dst_len < prefix.uncompressed_len) { if ((dst_tmp = wt_api->scr_alloc( - wt_api, session, (size_t)prefix.uncompressed_len)) == NULL) + wt_api, session, (size_t)prefix.uncompressed_len)) == NULL) return (ENOMEM); decoded = LZ4_decompress_safe( diff --git a/ext/compressors/nop/nop_compress.c b/ext/compressors/nop/nop_compress.c index e54013ae8b0..7cdb67c6bf2 100644 --- a/ext/compressors/nop/nop_compress.c +++ b/ext/compressors/nop/nop_compress.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/ext/compressors/snappy/snappy_compress.c b/ext/compressors/snappy/snappy_compress.c index 32f1ddcb9a0..a86de5c3803 100644 --- a/ext/compressors/snappy/snappy_compress.c +++ b/ext/compressors/snappy/snappy_compress.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/ext/compressors/zlib/zlib_compress.c b/ext/compressors/zlib/zlib_compress.c index 09a793646e7..3263b84bfaa 100644 --- a/ext/compressors/zlib/zlib_compress.c +++ b/ext/compressors/zlib/zlib_compress.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/ext/compressors/zstd/zstd_compress.c b/ext/compressors/zstd/zstd_compress.c index ea8ec97602f..d2ebaf20c4e 100644 --- a/ext/compressors/zstd/zstd_compress.c +++ b/ext/compressors/zstd/zstd_compress.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/ext/datasources/helium/helium.c b/ext/datasources/helium/helium.c index c584141b00d..5af954ba1de 100644 --- a/ext/datasources/helium/helium.c +++ b/ext/datasources/helium/helium.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/ext/encryptors/nop/nop_encrypt.c b/ext/encryptors/nop/nop_encrypt.c index af65f397549..3bc0f0f1c71 100644 --- a/ext/encryptors/nop/nop_encrypt.c +++ b/ext/encryptors/nop/nop_encrypt.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/ext/encryptors/rotn/rotn_encrypt.c b/ext/encryptors/rotn/rotn_encrypt.c index 0b905a0540d..5ffc8fcc1a3 100644 --- a/ext/encryptors/rotn/rotn_encrypt.c +++ b/ext/encryptors/rotn/rotn_encrypt.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/ext/extractors/csv/csv_extractor.c b/ext/extractors/csv/csv_extractor.c index e47ce6e2255..9866e1d5b34 100644 --- a/ext/extractors/csv/csv_extractor.c +++ b/ext/extractors/csv/csv_extractor.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/ext/test/fail_fs/fail_fs.c b/ext/test/fail_fs/fail_fs.c index d0d8a14c8c2..fd01ec66c68 100644 --- a/ext/test/fail_fs/fail_fs.c +++ b/ext/test/fail_fs/fail_fs.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -740,12 +740,12 @@ fail_fs_size(WT_FILE_SYSTEM *file_system, static int fail_fs_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *session) { - FAIL_FILE_HANDLE *fail_fh; + FAIL_FILE_HANDLE *fail_fh, *fail_fh_tmp; FAIL_FILE_SYSTEM *fail_fs; fail_fs = (FAIL_FILE_SYSTEM *)file_system; - while ((fail_fh = TAILQ_FIRST(&fail_fs->fileq)) != NULL) + TAILQ_FOREACH_SAFE(fail_fh, &fail_fs->fileq, q, fail_fh_tmp) fail_file_handle_remove(session, fail_fh); fail_fs_destroy_lock(&fail_fs->lock); diff --git a/ext/test/kvs_bdb/kvs_bdb.c b/ext/test/kvs_bdb/kvs_bdb.c index 0791b077750..8f857285b2b 100644 --- a/ext/test/kvs_bdb/kvs_bdb.c +++ b/ext/test/kvs_bdb/kvs_bdb.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/lang/java/Makefile.am b/lang/java/Makefile.am index 2ff822a5d08..71515c430fd 100644 --- a/lang/java/Makefile.am +++ b/lang/java/Makefile.am @@ -18,6 +18,7 @@ JAVA_SRC = \ $(JAVADESTFULL)/AsyncOpType.java \ $(JAVADESTFULL)/Connection.java \ $(JAVADESTFULL)/Cursor.java \ + $(JAVADESTFULL)/Modify.java \ $(JAVADESTFULL)/SearchStatus.java \ $(JAVADESTFULL)/PackFormatInputStream.java \ $(JAVADESTFULL)/PackInputStream.java \ @@ -31,6 +32,7 @@ JAVA_SRC = \ $(JAVADESTFULL)/wiredtiger.java \ $(JAVADESTFULL)/wiredtigerConstants.java \ $(JAVADESTFULL)/wiredtigerJNI.java \ + $(JAVADESTFULL)/WT_MODIFY_LIST.java \ $(JAVAEXAMPLES)/ex_access.java \ $(JAVAEXAMPLES)/ex_all.java \ $(JAVAEXAMPLES)/ex_async.java \ diff --git a/lang/java/java_doc.i b/lang/java/java_doc.i index 3606bed1d69..f9e017ee43a 100644 --- a/lang/java/java_doc.i +++ b/lang/java/java_doc.i @@ -12,8 +12,10 @@ COPYDOC(__wt_cursor, WT_CURSOR, reset) COPYDOC(__wt_cursor, WT_CURSOR, search) COPYDOC(__wt_cursor, WT_CURSOR, search_near) COPYDOC(__wt_cursor, WT_CURSOR, insert) +COPYDOC(__wt_cursor, WT_CURSOR, modify) COPYDOC(__wt_cursor, WT_CURSOR, update) COPYDOC(__wt_cursor, WT_CURSOR, remove) +COPYDOC(__wt_cursor, WT_CURSOR, reserve) COPYDOC(__wt_cursor, WT_CURSOR, close) COPYDOC(__wt_cursor, WT_CURSOR, reconfigure) COPYDOC(__wt_async_op, WT_ASYNC_OP, get_key) diff --git a/lang/java/src/com/wiredtiger/db/AsyncCallback.java b/lang/java/src/com/wiredtiger/db/AsyncCallback.java index ff428fae4fd..b272d611255 100644 --- a/lang/java/src/com/wiredtiger/db/AsyncCallback.java +++ b/lang/java/src/com/wiredtiger/db/AsyncCallback.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/lang/java/src/com/wiredtiger/db/PackFormatInputStream.java b/lang/java/src/com/wiredtiger/db/PackFormatInputStream.java index 4f05e153607..5cf52a067b8 100644 --- a/lang/java/src/com/wiredtiger/db/PackFormatInputStream.java +++ b/lang/java/src/com/wiredtiger/db/PackFormatInputStream.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/lang/java/src/com/wiredtiger/db/PackInputStream.java b/lang/java/src/com/wiredtiger/db/PackInputStream.java index 732bf450acd..013f9601edb 100644 --- a/lang/java/src/com/wiredtiger/db/PackInputStream.java +++ b/lang/java/src/com/wiredtiger/db/PackInputStream.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/lang/java/src/com/wiredtiger/db/PackOutputStream.java b/lang/java/src/com/wiredtiger/db/PackOutputStream.java index b6804a2992f..9af63db83c9 100644 --- a/lang/java/src/com/wiredtiger/db/PackOutputStream.java +++ b/lang/java/src/com/wiredtiger/db/PackOutputStream.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/lang/java/src/com/wiredtiger/db/PackUtil.java b/lang/java/src/com/wiredtiger/db/PackUtil.java index d47119eaf30..43b627cbf15 100644 --- a/lang/java/src/com/wiredtiger/db/PackUtil.java +++ b/lang/java/src/com/wiredtiger/db/PackUtil.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/lang/java/src/com/wiredtiger/db/WiredTigerException.java b/lang/java/src/com/wiredtiger/db/WiredTigerException.java index 13481efd9e4..233e1598c5d 100644 --- a/lang/java/src/com/wiredtiger/db/WiredTigerException.java +++ b/lang/java/src/com/wiredtiger/db/WiredTigerException.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/lang/java/src/com/wiredtiger/db/WiredTigerPackingException.java b/lang/java/src/com/wiredtiger/db/WiredTigerPackingException.java index 7dd1cfc24be..73d279f9e85 100644 --- a/lang/java/src/com/wiredtiger/db/WiredTigerPackingException.java +++ b/lang/java/src/com/wiredtiger/db/WiredTigerPackingException.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/lang/java/src/com/wiredtiger/db/WiredTigerPanicException.java b/lang/java/src/com/wiredtiger/db/WiredTigerPanicException.java index 8c0e08a77fb..4d82ad3d5df 100644 --- a/lang/java/src/com/wiredtiger/db/WiredTigerPanicException.java +++ b/lang/java/src/com/wiredtiger/db/WiredTigerPanicException.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/lang/java/src/com/wiredtiger/db/WiredTigerRollbackException.java b/lang/java/src/com/wiredtiger/db/WiredTigerRollbackException.java index 47d079c139d..84c7e0803a3 100644 --- a/lang/java/src/com/wiredtiger/db/WiredTigerRollbackException.java +++ b/lang/java/src/com/wiredtiger/db/WiredTigerRollbackException.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/lang/java/wiredtiger.i b/lang/java/wiredtiger.i index 275b708090c..4c22a0af43b 100644 --- a/lang/java/wiredtiger.i +++ b/lang/java/wiredtiger.i @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -47,6 +47,7 @@ %} %{ +#include "wiredtiger.h" #include "src/include/wt_internal.h" /* @@ -108,6 +109,23 @@ static void throwWiredTigerException(JNIEnv *jenv, int err) { (*jenv)->ThrowNew(jenv, excep, wiredtiger_strerror(err)); } +struct __wt_java_modify_impl; +struct __wt_java_modify_list; +typedef struct __wt_java_modify_impl WT_MODIFY_IMPL; +typedef struct __wt_java_modify_list WT_MODIFY_LIST; +static void modify_impl_release(WT_MODIFY_IMPL *impl); +static void modify_list_release(WT_MODIFY_LIST *impl); + +/* + * An extension to the WT_MODIFY struct, so we can associate some Java-specific + * information with it. + */ +typedef struct __wt_java_modify_impl { + WT_MODIFY modify; + JNIEnv *jnienv; + jobject ref; +} WT_MODIFY_IMPL; + %} /* No finalizers */ @@ -159,6 +177,32 @@ static void throwWiredTigerException(JNIEnv *jenv, int err) { } %} +/* + * In some cases, for an internal interface, we need something like a WT_ITEM, + * but we need to hold onto the memory past the method call, and release it + * later. A WT_ITEM_HOLD serves the purpose, it retains the java object + * for the byte array that we make into a global reference. + */ +%typemap(jni) WT_ITEM_HOLD, WT_ITEM_HOLD * "jbyteArray" +%typemap(jtype) WT_ITEM_HOLD, WT_ITEM_HOLD * "byte[]" +%typemap(jstype) WT_ITEM_HOLD, WT_ITEM_HOLD * "byte[]" + +%typemap(javain) WT_ITEM_HOLD, WT_ITEM_HOLD * "$javainput" +%typemap(javaout) WT_ITEM_HOLD, WT_ITEM_HOLD * { + return ($jnicall); +} +%typemap(in) WT_ITEM_HOLD * (WT_ITEM_HOLD item) %{ + $1 = &item; + $1->data = (*jenv)->GetByteArrayElements(jenv, $input, 0); + $1->size = (size_t)(*jenv)->GetArrayLength(jenv, $input); + $1->jnienv = jenv; + $1->ref = (*jenv)->NewGlobalRef(jenv, $input); +%} + +%typemap(argout) WT_ITEM_HOLD * %{ + /* Explicitly don't release the byte array elements here. */ +%} + /* Don't require empty config strings. */ %typemap(default) const char *config %{ $1 = NULL; %} @@ -309,6 +353,10 @@ WT_CLASS(struct __wt_async_op, WT_ASYNC_OP, op) %rename (prev_wrap) __wt_cursor::prev; %javamethodmodifiers __wt_cursor::key_format "protected"; %javamethodmodifiers __wt_cursor::value_format "protected"; +%ignore __wt_modify::data; +%ignore __wt_modify::position; +%ignore __wt_modify::size; +%ignore __wt_cursor::modify; %ignore __wt_cursor::compare(WT_CURSOR *, WT_CURSOR *, int *); %rename (compare_wrap) __wt_cursor::compare; @@ -1224,6 +1272,47 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; JCALL1(DeleteLocalRef, jcb->jnienv, jcursor); return (0); } + + int modify_wrap(WT_MODIFY_LIST *list, WT_ITEM *k) { + int ret; + + $self->set_key($self, k); + ret = $self->modify(self, list->mod_array, list->count); + modify_list_release(list); + return (ret); + } + + /* + * Called internally after a new call. The artificial constructor for + * WT_MODIFY_LIST has no opportunity to throw an exception on a memory + * allocation failure, so the the null check must be made within a + * method on WT_CURSOR. + */ + bool _new_check_modify_list(WT_MODIFY_LIST *list) { + JAVA_CALLBACK *jcb; + if (list == NULL) { + jcb = (JAVA_CALLBACK *)$self->lang_private; + throwWiredTigerException(jcb->jnienv, ENOMEM); + return (false); + } + return (true); + } + + /* + * Called internally after a new call. The artificial constructor for + * WT_MODIFY has no opportunity to throw an exception on a memory + * allocation failure, so the the null check must be made within a + * method on WT_CURSOR. + */ + bool _new_check_modify(WT_MODIFY *mod) { + JAVA_CALLBACK *jcb; + if (mod == NULL) { + jcb = (JAVA_CALLBACK *)$self->lang_private; + throwWiredTigerException(jcb->jnienv, ENOMEM); + return (false); + } + return (true); + } } /* Cache key/value formats in Cursor */ @@ -1820,6 +1909,149 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; return new PackInputStream(valueFormat, get_value_wrap(), _java_raw()); } + + /** + * Modify an existing record. + * + * The cursor must already be positioned, and the key's value will be + * updated. + * + * \param mods an array of modifications. + * \return 0 on success, errno on error. + */ + public int modify(Modify mods[]) + throws WiredTigerException { + byte[] key = keyPacker.getValue(); + keyPacker.reset(); + + WT_MODIFY_LIST l = new WT_MODIFY_LIST(mods.length); + if (!_new_check_modify_list(l)) + return (0); // exception is already thrown + int pos = 0; + + for (Modify m : mods) { + if (!_new_check_modify(m)) + return (0); // exception is already thrown + l.set(pos, m); + pos++; + } + return modify_wrap(l, key); + } +%} + +/* + * Support for WT_CURSOR.modify. + */ + +%inline %{ +typedef struct __wt_java_item_hold { +#ifndef SWIG + void *data; + size_t size; + JNIEnv *jnienv; + jobject ref; +#endif +} WT_ITEM_HOLD; + +/* + * An internal Java class encapsulates a list of Modify objects (stored as a + * WT_MODIFY array in C). + */ +typedef struct __wt_java_modify_list { +#ifndef SWIG + WT_MODIFY *mod_array; + jobject *ref_array; + JNIEnv *jnienv; + int count; +#endif +} WT_MODIFY_LIST; +%} +%extend __wt_java_modify_list { + __wt_java_modify_list(int count) { + WT_MODIFY_LIST *self; + if (__wt_calloc_def(NULL, 1, &self) != 0) + return (NULL); + if (__wt_calloc_def(NULL, (size_t)count, + &self->mod_array) != 0) { + __wt_free(NULL, self); + return (NULL); + } + if (__wt_calloc_def(NULL, (size_t)count, + &self->ref_array) != 0) { + __wt_free(NULL, self->mod_array); + __wt_free(NULL, self); + return (NULL); + } + self->count = count; + return (self); + } + ~__wt_java_modify_list() { + modify_list_release(self); + __wt_free(NULL, self); + } + void set(int i, WT_MODIFY *m) { + WT_MODIFY_IMPL *impl = (WT_MODIFY_IMPL *)m; + self->mod_array[i] = *m; + self->ref_array[i] = impl->ref; + impl->ref = (jobject)0; + self->jnienv = impl->jnienv; + } +}; + +%extend __wt_modify { + __wt_modify() { + WT_MODIFY_IMPL *self; + if (__wt_calloc_def(NULL, 1, &self) != 0) + return (NULL); + self->modify.data.data = NULL; + self->modify.data.size = 0; + self->modify.offset = 0; + self->modify.size = 0; + return (&self->modify); + } + __wt_modify(WT_ITEM_HOLD *itemdata, + size_t offset, size_t size) { + WT_MODIFY_IMPL *self; + if (__wt_calloc_def(NULL, 1, &self) != 0) + return (NULL); + self->modify.data.data = itemdata->data; + self->modify.data.size = itemdata->size; + self->modify.offset = offset; + self->modify.size = size; + self->ref = itemdata->ref; + self->jnienv = itemdata->jnienv; + return (&self->modify); + } + ~__wt_modify() { + modify_impl_release((WT_MODIFY_IMPL *)self); + __wt_free(NULL, self); + } +}; + +%{ +static void modify_list_release(WT_MODIFY_LIST *list) { + for (int i = 0; i < list->count; i++) + if (list->ref_array[i] != (jobject)0) { + (*list->jnienv)->ReleaseByteArrayElements( + list->jnienv, list->ref_array[i], + (jbyte *)list->mod_array[i].data.data, 0); + (*list->jnienv)->DeleteGlobalRef( + list->jnienv, list->ref_array[i]); + } + __wt_free(NULL, list->ref_array); + __wt_free(NULL, list->mod_array); + list->count = 0; +} + +static void modify_impl_release(WT_MODIFY_IMPL *impl) { + if (impl->ref != (jobject)0) { + (*impl->jnienv)->ReleaseByteArrayElements( + impl->jnienv, impl->ref, + (jbyte *)impl->modify.data.data, 0); + (*impl->jnienv)->DeleteGlobalRef(impl->jnienv, impl->ref); + impl->ref = (jobject)0; + } +} %} /* Put a WiredTigerException on all wrapped methods. We'd like this @@ -1902,6 +2134,7 @@ REQUIRE_WRAP(WT_ASYNC_OP::get_id, __wt_async_op::get_id,getId) %rename(AsyncOp) __wt_async_op; %rename(Cursor) __wt_cursor; +%rename(Modify) __wt_modify; %rename(Session) __wt_session; %rename(Connection) __wt_connection; diff --git a/lang/python/setup.py b/lang/python/setup.py index 9063a891fb9..c88b268fcff 100644 --- a/lang/python/setup.py +++ b/lang/python/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/lang/python/setup_pip.py b/lang/python/setup_pip.py index 636eecab80a..2ddca407e6b 100644 --- a/lang/python/setup_pip.py +++ b/lang/python/setup_pip.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/lang/python/wiredtiger.i b/lang/python/wiredtiger.i index 7bc84066d64..61c7fc62c43 100644 --- a/lang/python/wiredtiger.i +++ b/lang/python/wiredtiger.i @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -151,6 +151,74 @@ from packing import pack, unpack } } +%typemap(in) WT_MODIFY * (int len, WT_MODIFY *modarray, int i) { + len = PyList_Size($input); + /* + * We allocate an extra cleared WT_MODIFY struct, the first + * entry will be used solely to transmit the array length to + * the call site. + */ + if (__wt_calloc_def(NULL, (size_t)len + 1, &modarray) != 0) + SWIG_exception_fail(SWIG_MemoryError, "WT calloc failed"); + modarray[0].size = (size_t)len; + for (i = 1; i <= len; i++) { + PyObject *dataobj, *modobj, *offsetobj, *sizeobj; + char *datadata; + long offset, size; + Py_ssize_t datasize; + + if ((modobj = PySequence_GetItem($input, i - 1)) == NULL) + SWIG_exception_fail(SWIG_IndexError, + "Modify sequence failed"); + + WT_GETATTR(dataobj, modobj, "data"); + if (PyString_AsStringAndSize(dataobj, &datadata, + &datasize) < 0) { + Py_DECREF(dataobj); + Py_DECREF(modobj); + SWIG_exception_fail(SWIG_AttributeError, + "Modify.data bad value"); + } + modarray[i].data.data = malloc(datasize); + memcpy(modarray[i].data.data, datadata, datasize); + modarray[i].data.size = datasize; + Py_DECREF(dataobj); + + WT_GETATTR(offsetobj, modobj, "offset"); + if ((offset = PyInt_AsLong(offsetobj)) < 0) { + Py_DECREF(offsetobj); + Py_DECREF(modobj); + SWIG_exception_fail(SWIG_RuntimeError, + "Modify.offset bad value"); + } + modarray[i].offset = offset; + Py_DECREF(offsetobj); + + WT_GETATTR(sizeobj, modobj, "size"); + if ((size = PyInt_AsLong(sizeobj)) < 0) { + Py_DECREF(sizeobj); + Py_DECREF(modobj); + SWIG_exception_fail(SWIG_RuntimeError, + "Modify.size bad value"); + } + modarray[i].size = size; + Py_DECREF(sizeobj); + Py_DECREF(modobj); + } + $1 = modarray; +} + +%typemap(freearg) WT_MODIFY * { + /* The WT_MODIFY arg is in position 2. Is there a better way? */ + WT_MODIFY *modarray = modarray2; + size_t i, len; + + len = modarray[0].size; + for (i = 1; i <= len; i++) + __wt_free(NULL, modarray[i].data.data); + __wt_free(NULL, modarray); +} + /* 64 bit typemaps. */ %typemap(in) uint64_t { $1 = PyLong_AsUnsignedLongLong($input); @@ -244,6 +312,13 @@ static PyObject *wtError; static int sessionFreeHandler(WT_SESSION *session_arg); static int cursorFreeHandler(WT_CURSOR *cursor_arg); + +#define WT_GETATTR(var, parent, name) \ + do if ((var = PyObject_GetAttrString(parent, name)) == NULL) { \ + Py_DECREF(parent); \ + SWIG_exception_fail(SWIG_AttributeError, \ + "Modify." #name " get failed"); \ + } while(0) %} %init %{ @@ -373,8 +448,8 @@ retry: } %enddef -/* Any API that returns an enum type uses this. */ -%define ENUM_OK(m) +/* An API that returns a value that shouldn't be checked uses this. */ +%define ANY_OK(m) %exception m { $action } @@ -408,12 +483,14 @@ retry: %enddef EBUSY_OK(__wt_connection::async_new_op) -ENUM_OK(__wt_async_op::get_type) +ANY_OK(__wt_async_op::get_type) NOTFOUND_OK(__wt_cursor::next) NOTFOUND_OK(__wt_cursor::prev) NOTFOUND_OK(__wt_cursor::remove) NOTFOUND_OK(__wt_cursor::search) NOTFOUND_OK(__wt_cursor::update) +ANY_OK(__wt_modify::__wt_modify) +ANY_OK(__wt_modify::~__wt_modify) COMPARE_OK(__wt_cursor::_compare) COMPARE_OK(__wt_cursor::_equals) @@ -448,6 +525,11 @@ COMPARE_NOTFOUND_OK(__wt_cursor::_search_near) %ignore __wt_cursor::get_value; %ignore __wt_cursor::set_key; %ignore __wt_cursor::set_value; +%ignore __wt_cursor::modify(WT_CURSOR *, WT_MODIFY *, int); +%rename (modify) __wt_cursor::_modify; +%ignore __wt_modify::data; +%ignore __wt_modify::offset; +%ignore __wt_modify::size; /* Next, override methods that return integers via arguments. */ %ignore __wt_cursor::compare(WT_CURSOR *, WT_CURSOR *, int *); @@ -772,6 +854,15 @@ typedef int int_void; return (cursorFreeHandler($self)); } + /* + * modify: the size of the array was put into the first element by the + * typemap. + */ + int _modify(WT_MODIFY *list) { + int count = (int)list[0].size; + return (self->modify(self, &list[1], count)); + } + %pythoncode %{ def get_key(self): '''get_key(self) -> object @@ -870,6 +961,21 @@ typedef int int_void; %} }; +/* + * Support for WT_CURSOR.modify. The WT_MODIFY object is known to + * SWIG, but its attributes are regular Python attributes. + * We extract the attributes at the call site to WT_CURSOR.modify + * so we don't have to deal with managing Python objects references. + */ +%extend __wt_modify { +%pythoncode %{ + def __init__(self, data = '', offset = 0, size = 0): + self.data = data + self.offset = offset + self.size = size +%} +}; + %extend __wt_session { int _log_printf(const char *msg) { return self->log_printf(self, "%s", msg); @@ -951,6 +1057,7 @@ OVERRIDE_METHOD(__wt_session, WT_SESSION, log_printf, (self, msg)) %rename(AsyncOp) __wt_async_op; %rename(Cursor) __wt_cursor; +%rename(Modify) __wt_modify; %rename(Session) __wt_session; %rename(Connection) __wt_connection; @@ -974,7 +1081,7 @@ writeToPythonStream(const char *streamname, const char *message) written = NULL; arglist = arglist2 = NULL; msglen = strlen(message); - msg = malloc(msglen + 2); + WT_RET(__wt_malloc(NULL, msglen + 2, &msg)); strcpy(msg, message); strcpy(&msg[msglen], "\n"); @@ -1010,8 +1117,7 @@ err: Py_XDECREF(arglist2); /* Release python Global Interpreter Lock */ SWIG_PYTHON_THREAD_END_BLOCK; - if (msg) - free(msg); + __wt_free(NULL, msg); return (ret); } @@ -1232,4 +1338,3 @@ _rename_with_prefix('WT_STAT_CONN_', stat.conn) _rename_with_prefix('WT_STAT_DSRC_', stat.dsrc) del _rename_with_prefix %} - diff --git a/lang/python/wiredtiger/fpacking.py b/lang/python/wiredtiger/fpacking.py index cc009a29764..8ae4c1cf99f 100644 --- a/lang/python/wiredtiger/fpacking.py +++ b/lang/python/wiredtiger/fpacking.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/lang/python/wiredtiger/intpacking.py b/lang/python/wiredtiger/intpacking.py index 023c25ab5b3..ed1f00ceb37 100644 --- a/lang/python/wiredtiger/intpacking.py +++ b/lang/python/wiredtiger/intpacking.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/lang/python/wiredtiger/packing.py b/lang/python/wiredtiger/packing.py index 5d21b539888..fb674538b76 100644 --- a/lang/python/wiredtiger/packing.py +++ b/lang/python/wiredtiger/packing.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. @@ -94,6 +94,9 @@ def unpack(fmt, s): elif f == 'S': size = s.find('\0') elif f == 'u' and offset == len(fmt) - 1: + # A WT_ITEM with a NULL data field will be appear as None. + if s == None: + s = '' size = len(s) else: # Note: 'U' is used internally, and may be exposed to us. @@ -169,7 +172,7 @@ def pack(fmt, *values): result += val[:l] if f == 'S' and not havesize: result += '\0' - elif size > l: + elif size > l and havesize: result += '\0' * (size - l) elif f in 't': # bit type, size is number of bits diff --git a/lang/python/wiredtiger/pip_init.py b/lang/python/wiredtiger/pip_init.py index d59c8218976..71c35fabd57 100644 --- a/lang/python/wiredtiger/pip_init.py +++ b/lang/python/wiredtiger/pip_init.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/src/async/async_api.c b/src/async/async_api.c index b9cc995f5a5..0f3e376fbfd 100644 --- a/src/async/async_api.c +++ b/src/async/async_api.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -395,13 +395,12 @@ __wt_async_reconfig(WT_SESSION_IMPL *session, const char *cfg[]) * Join any worker we're stopping. * After the thread is stopped, close its session. */ - WT_ASSERT(session, async->worker_tids[i] != 0); + WT_ASSERT(session, async->worker_tids[i].created); WT_ASSERT(session, async->worker_sessions[i] != NULL); F_CLR(async->worker_sessions[i], WT_SESSION_SERVER_ASYNC); WT_TRET(__wt_thread_join( session, async->worker_tids[i])); - async->worker_tids[i] = 0; wt_session = &async->worker_sessions[i]->iface; WT_TRET(wt_session->close(wt_session, NULL)); async->worker_sessions[i] = NULL; @@ -420,7 +419,7 @@ int __wt_async_destroy(WT_SESSION_IMPL *session) { WT_ASYNC *async; - WT_ASYNC_FORMAT *af, *afnext; + WT_ASYNC_FORMAT *af; WT_ASYNC_OP *op; WT_CONNECTION_IMPL *conn; WT_DECL_RET; @@ -435,12 +434,8 @@ __wt_async_destroy(WT_SESSION_IMPL *session) F_CLR(conn, WT_CONN_SERVER_ASYNC); for (i = 0; i < conn->async_workers; i++) - if (async->worker_tids[i] != 0) { - WT_TRET(__wt_thread_join( - session, async->worker_tids[i])); - async->worker_tids[i] = 0; - } - WT_TRET(__wt_cond_destroy(session, &async->flush_cond)); + WT_TRET(__wt_thread_join(session, async->worker_tids[i])); + __wt_cond_destroy(session, &async->flush_cond); /* Close the server threads' sessions. */ for (i = 0; i < conn->async_workers; i++) @@ -459,15 +454,13 @@ __wt_async_destroy(WT_SESSION_IMPL *session) } /* Free format resources */ - af = TAILQ_FIRST(&async->formatqh); - while (af != NULL) { - afnext = TAILQ_NEXT(af, q); + while ((af = TAILQ_FIRST(&async->formatqh)) != NULL) { + TAILQ_REMOVE(&async->formatqh, af, q); __wt_free(session, af->uri); __wt_free(session, af->config); __wt_free(session, af->key_format); __wt_free(session, af->value_format); __wt_free(session, af); - af = afnext; } __wt_free(session, async->async_queue); __wt_free(session, async->async_ops); @@ -499,7 +492,7 @@ __wt_async_flush(WT_SESSION_IMPL *session) */ workers = 0; for (i = 0; i < conn->async_workers; ++i) - if (async->worker_tids[i] != 0) + if (async->worker_tids[i].created) ++workers; if (workers == 0) return (0); diff --git a/src/async/async_op.c b/src/async/async_op.c index 6908802dbff..d4ca754b95f 100644 --- a/src/async/async_op.c +++ b/src/async/async_op.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/async/async_worker.c b/src/async/async_worker.c index 11f59ed14f1..57ebe5d8bb1 100644 --- a/src/async/async_worker.c +++ b/src/async/async_worker.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -57,7 +57,6 @@ retry: return (0); if (!F_ISSET(conn, WT_CONN_SERVER_ASYNC)) return (0); - WT_RET(WT_SESSION_CHECK_PANIC(session)); WT_ORDERED_READ(last_consume, async->alloc_tail); } if (async->flush_state == WT_ASYNC_FLUSHING) @@ -282,7 +281,7 @@ WT_THREAD_RET __wt_async_worker(void *arg) { WT_ASYNC *async; - WT_ASYNC_CURSOR *ac, *acnext; + WT_ASYNC_CURSOR *ac; WT_ASYNC_OP_IMPL *op; WT_ASYNC_WORKER_STATE worker; WT_CONNECTION_IMPL *conn; @@ -301,11 +300,10 @@ __wt_async_worker(void *arg) WT_ERR(__async_op_dequeue(conn, session, &op)); if (op != NULL && op != &async->flush_op) { /* - * If an operation fails, we want the worker thread to - * keep running, unless there is a panic. + * Operation failure doesn't cause the worker thread to + * exit. */ (void)__async_worker_op(session, op, &worker); - WT_ERR(WT_SESSION_CHECK_PANIC(session)); } else if (async->flush_state == WT_ASYNC_FLUSHING) { /* * Worker flushing going on. Last worker to the party @@ -342,12 +340,10 @@ err: WT_PANIC_MSG(session, ret, "async worker error"); * Worker thread cleanup, close our cached cursors and free all the * WT_ASYNC_CURSOR structures. */ - ac = TAILQ_FIRST(&worker.cursorqh); - while (ac != NULL) { - acnext = TAILQ_NEXT(ac, q); + while ((ac = TAILQ_FIRST(&worker.cursorqh)) != NULL) { + TAILQ_REMOVE(&worker.cursorqh, ac, q); WT_TRET(ac->c->close(ac->c)); __wt_free(session, ac); - ac = acnext; } return (WT_THREAD_RET_VALUE); } diff --git a/src/block/block_addr.c b/src/block/block_addr.c index a67efca62a3..6a016776175 100644 --- a/src/block/block_addr.c +++ b/src/block/block_addr.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/block/block_ckpt.c b/src/block/block_ckpt.c index 05e4dcc098e..c20a294c07b 100644 --- a/src/block/block_ckpt.c +++ b/src/block/block_ckpt.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/block/block_compact.c b/src/block/block_compact.c index eb6647dd03c..e7b9beafb01 100644 --- a/src/block/block_compact.c +++ b/src/block/block_compact.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -242,8 +242,10 @@ __block_dump_avail(WT_SESSION_IMPL *session, WT_BLOCK *block, bool start) memset(percentile, 0, sizeof(percentile)); WT_EXT_FOREACH(ext, el->off) for (i = 0; i < ext->size / 512; ++i) { - ++decile[((ext->off + i * 512) * 10) / size]; - ++percentile[((ext->off + i * 512) * 100) / size]; + ++decile[ + ((ext->off + (wt_off_t)i * 512) * 10) / size]; + ++percentile[ + ((ext->off + (wt_off_t)i * 512) * 100) / size]; } #ifdef __VERBOSE_OUTPUT_PERCENTILE diff --git a/src/block/block_ext.c b/src/block/block_ext.c index da7a06d873d..6ef861b59c9 100644 --- a/src/block/block_ext.c +++ b/src/block/block_ext.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -1272,7 +1272,7 @@ __wt_block_extlist_write(WT_SESSION_IMPL *session, * entries: the initial WT_BLOCK_EXTLIST_MAGIC/0 pair and the list- * terminating WT_BLOCK_INVALID_OFFSET/0 pair. */ - size = (entries + 2) * 2 * WT_INTPACK64_MAXSIZE; + size = ((size_t)entries + 2) * 2 * WT_INTPACK64_MAXSIZE; WT_RET(__wt_block_write_size(session, block, &size)); WT_RET(__wt_scr_alloc(session, size, &tmp)); dsk = tmp->mem; diff --git a/src/block/block_map.c b/src/block/block_map.c index b7afa61cc55..847f2393043 100644 --- a/src/block/block_map.c +++ b/src/block/block_map.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/block/block_mgr.c b/src/block/block_mgr.c index 653ae3dbb6b..d09d7e7925c 100644 --- a/src/block/block_mgr.c +++ b/src/block/block_mgr.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/block/block_open.c b/src/block/block_open.c index 07ceb4c8159..d35a934b0f3 100644 --- a/src/block/block_open.c +++ b/src/block/block_open.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/block/block_read.c b/src/block/block_read.c index 8d4aec7df75..86b0cad13db 100644 --- a/src/block/block_read.c +++ b/src/block/block_read.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/block/block_session.c b/src/block/block_session.c index 6223751effa..e951897e25d 100644 --- a/src/block/block_session.c +++ b/src/block/block_session.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/block/block_slvg.c b/src/block/block_slvg.c index b06a5062f50..888d93772a2 100644 --- a/src/block/block_slvg.c +++ b/src/block/block_slvg.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/block/block_vrfy.c b/src/block/block_vrfy.c index 154765ed079..1058f16bde6 100644 --- a/src/block/block_vrfy.c +++ b/src/block/block_vrfy.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/block/block_write.c b/src/block/block_write.c index ea7859d6a38..7d689fc9bcf 100644 --- a/src/block/block_write.c +++ b/src/block/block_write.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/bloom/bloom.c b/src/bloom/bloom.c index b8d75678835..bfbfa34078f 100644 --- a/src/bloom/bloom.c +++ b/src/bloom/bloom.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -133,8 +133,12 @@ __bloom_open_cursor(WT_BLOOM *bloom, WT_CURSOR *owner) c = NULL; WT_RET(__wt_open_cursor(session, bloom->uri, owner, cfg, &c)); - /* Bump the cache priority for Bloom filters. */ - __wt_evict_priority_set(session, WT_EVICT_INT_SKEW); + /* + * Bump the cache priority for Bloom filters: this makes eviction favor + * pages from other trees over Bloom filters. + */ +#define WT_EVICT_BLOOM_SKEW 1000 + __wt_evict_priority_set(session, WT_EVICT_BLOOM_SKEW); bloom->c = c; return (0); diff --git a/src/btree/bt_compact.c b/src/btree/bt_compact.c index 2edcac76d0b..c6a412aa84e 100644 --- a/src/btree/bt_compact.c +++ b/src/btree/bt_compact.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -60,7 +60,7 @@ __compact_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) */ if (mod->rec_result == WT_PM_REC_REPLACE || mod->rec_result == WT_PM_REC_MULTIBLOCK) - __wt_writelock(session, &page->page_lock); + WT_PAGE_LOCK(session, page); if (mod->rec_result == WT_PM_REC_REPLACE) ret = bm->compact_page_skip(bm, session, @@ -80,7 +80,7 @@ __compact_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) if (mod->rec_result == WT_PM_REC_REPLACE || mod->rec_result == WT_PM_REC_MULTIBLOCK) - __wt_writeunlock(session, &page->page_lock); + WT_PAGE_UNLOCK(session, page); return (ret); } @@ -228,12 +228,8 @@ __wt_compact_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) bm, session, addr, addr_size, skipp); } - /* - * Reset the WT_REF state and push the change. The full-barrier isn't - * necessary, but it's better to keep pages in circulation than not. - */ + /* Reset the WT_REF state. */ ref->state = WT_REF_DISK; - WT_FULL_BARRIER(); return (ret); } diff --git a/src/btree/bt_curnext.c b/src/btree/bt_curnext.c index 21e575ffca9..7b92a58991d 100644 --- a/src/btree/bt_curnext.c +++ b/src/btree/bt_curnext.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -142,7 +142,7 @@ new_page: if (cbt->ins == NULL) __cursor_set_recno(cbt, WT_INSERT_RECNO(cbt->ins)); if ((upd = __wt_txn_read(session, cbt->ins->upd)) == NULL) continue; - if (WT_UPDATE_DELETED_ISSET(upd)) { + if (upd->type == WT_UPDATE_DELETED) { if (__wt_txn_visible_all(session, upd->txnid)) ++cbt->page_deleted_count; continue; @@ -205,7 +205,7 @@ new_page: /* Find the matching WT_COL slot. */ upd = cbt->ins == NULL ? NULL : __wt_txn_read(session, cbt->ins->upd); if (upd != NULL) { - if (WT_UPDATE_DELETED_ISSET(upd)) { + if (upd->type == WT_UPDATE_DELETED) { if (__wt_txn_visible_all(session, upd->txnid)) ++cbt->page_deleted_count; continue; @@ -325,7 +325,7 @@ __cursor_row_next(WT_CURSOR_BTREE *cbt, bool newpage) new_insert: if ((ins = cbt->ins) != NULL) { if ((upd = __wt_txn_read(session, ins->upd)) == NULL) continue; - if (WT_UPDATE_DELETED_ISSET(upd)) { + if (upd->type == WT_UPDATE_DELETED) { if (__wt_txn_visible_all(session, upd->txnid)) ++cbt->page_deleted_count; continue; @@ -358,7 +358,7 @@ new_insert: if ((ins = cbt->ins) != NULL) { cbt->slot = cbt->row_iteration_slot / 2 - 1; rip = &page->pg_row[cbt->slot]; upd = __wt_txn_read(session, WT_ROW_UPDATE(page, rip)); - if (upd != NULL && WT_UPDATE_DELETED_ISSET(upd)) { + if (upd != NULL && upd->type == WT_UPDATE_DELETED) { if (__wt_txn_visible_all(session, upd->txnid)) ++cbt->page_deleted_count; continue; diff --git a/src/btree/bt_curprev.c b/src/btree/bt_curprev.c index bf4bdad6529..55b5095fe91 100644 --- a/src/btree/bt_curprev.c +++ b/src/btree/bt_curprev.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -288,7 +288,7 @@ new_page: if (cbt->ins == NULL) __cursor_set_recno(cbt, WT_INSERT_RECNO(cbt->ins)); if ((upd = __wt_txn_read(session, cbt->ins->upd)) == NULL) continue; - if (WT_UPDATE_DELETED_ISSET(upd)) { + if (upd->type == WT_UPDATE_DELETED) { if (__wt_txn_visible_all(session, upd->txnid)) ++cbt->page_deleted_count; continue; @@ -352,7 +352,7 @@ new_page: if (cbt->recno < cbt->ref->ref_recno) upd = cbt->ins == NULL ? NULL : __wt_txn_read(session, cbt->ins->upd); if (upd != NULL) { - if (WT_UPDATE_DELETED_ISSET(upd)) { + if (upd->type == WT_UPDATE_DELETED) { if (__wt_txn_visible_all(session, upd->txnid)) ++cbt->page_deleted_count; continue; @@ -482,7 +482,7 @@ __cursor_row_prev(WT_CURSOR_BTREE *cbt, bool newpage) new_insert: if ((ins = cbt->ins) != NULL) { if ((upd = __wt_txn_read(session, ins->upd)) == NULL) continue; - if (WT_UPDATE_DELETED_ISSET(upd)) { + if (upd->type == WT_UPDATE_DELETED) { if (__wt_txn_visible_all(session, upd->txnid)) ++cbt->page_deleted_count; continue; @@ -517,7 +517,7 @@ new_insert: if ((ins = cbt->ins) != NULL) { cbt->slot = cbt->row_iteration_slot / 2 - 1; rip = &page->pg_row[cbt->slot]; upd = __wt_txn_read(session, WT_ROW_UPDATE(page, rip)); - if (upd != NULL && WT_UPDATE_DELETED_ISSET(upd)) { + if (upd != NULL && upd->type == WT_UPDATE_DELETED) { if (__wt_txn_visible_all(session, upd->txnid)) ++cbt->page_deleted_count; continue; diff --git a/src/btree/bt_cursor.c b/src/btree/bt_cursor.c index 944e276fc01..52435eeefed 100644 --- a/src/btree/bt_cursor.c +++ b/src/btree/bt_cursor.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -64,29 +64,6 @@ __cursor_page_pinned(WT_CURSOR_BTREE *cbt) } /* - * __cursor_copy_int_key -- - * If we're pointing into the tree, save the key into local memory. - */ -static inline int -__cursor_copy_int_key(WT_CURSOR *cursor) -{ - /* - * We're about to discard the cursor's position and the cursor layer - * might retry the operation. We discard pinned pages on error, which - * will invalidate pinned keys. Clear WT_CURSTD_KEY_INT in all cases, - * the underlying page is gone whether we can allocate memory or not. - */ - if (F_ISSET(cursor, WT_CURSTD_KEY_INT)) { - F_CLR(cursor, WT_CURSTD_KEY_INT); - if (!WT_DATA_IN_ITEM(&cursor->key)) - WT_RET(__wt_buf_set((WT_SESSION_IMPL *)cursor->session, - &cursor->key, cursor->key.data, cursor->key.size)); - F_SET(cursor, WT_CURSTD_KEY_EXT); - } - return (0); -} - -/* * __cursor_size_chk -- * Return if an inserted item is too large. */ @@ -247,7 +224,7 @@ __wt_cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp) */ if (cbt->ins != NULL && (upd = __wt_txn_read(session, cbt->ins->upd)) != NULL) { - if (WT_UPDATE_DELETED_ISSET(upd)) + if (upd->type == WT_UPDATE_DELETED) return (false); if (updp != NULL) *updp = upd; @@ -320,7 +297,7 @@ __wt_cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp) page->modify->mod_row_update != NULL && (upd = __wt_txn_read(session, page->modify->mod_row_update[cbt->slot])) != NULL) { - if (WT_UPDATE_DELETED_ISSET(upd)) + if (upd->type == WT_UPDATE_DELETED) return (false); if (updp != NULL) *updp = upd; @@ -366,10 +343,10 @@ __cursor_row_search( */ static inline int __cursor_col_modify( - WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, bool is_remove) + WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, u_int modify_type) { - return (__wt_col_modify(session, - cbt, cbt->iface.recno, &cbt->iface.value, NULL, is_remove)); + return (__wt_col_modify(session, cbt, + cbt->iface.recno, &cbt->iface.value, NULL, modify_type, false)); } /* @@ -378,10 +355,10 @@ __cursor_col_modify( */ static inline int __cursor_row_modify( - WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, bool is_remove) + WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, u_int modify_type) { - return (__wt_row_modify(session, - cbt, &cbt->iface.key, &cbt->iface.value, NULL, is_remove)); + return (__wt_row_modify(session, cbt, + &cbt->iface.key, &cbt->iface.value, NULL, modify_type, false)); } /* @@ -431,10 +408,14 @@ __wt_btcur_search(WT_CURSOR_BTREE *cbt) __cursor_state_save(cursor, &state); /* - * The pinned page goes away if we do a search, make sure there's a - * local copy of any key, then re-save the cursor state. + * The pinned page goes away if we search the tree, get a local copy of + * any pinned key and discard any pinned value, then re-save the cursor + * state. Done before searching pinned pages (unlike other cursor + * functions), because we don't anticipate applications searching for a + * key they currently have pinned.) */ - WT_ERR(__cursor_copy_int_key(cursor)); + WT_ERR(__cursor_localkey(cursor)); + __cursor_novalue(cursor); __cursor_state_save(cursor, &state); /* @@ -516,10 +497,14 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) __cursor_state_save(cursor, &state); /* - * The pinned page goes away if we do a search, make sure there's a - * local copy of any key, then re-save the cursor state. + * The pinned page goes away if we search the tree, get a local copy of + * any pinned key and discard any pinned value, then re-save the cursor + * state. Done before searching pinned pages (unlike other cursor + * functions), because we don't anticipate applications searching for a + * key they currently have pinned.) */ - WT_ERR(__cursor_copy_int_key(cursor)); + WT_ERR(__cursor_localkey(cursor)); + __cursor_novalue(cursor); __cursor_state_save(cursor, &state); /* @@ -640,8 +625,6 @@ __wt_btcur_insert(WT_CURSOR_BTREE *cbt) WT_STAT_DATA_INCRV(session, cursor_insert_bytes, cursor->key.size + cursor->value.size); - __cursor_state_save(cursor, &state); - if (btree->type == BTREE_ROW) WT_RET(__cursor_size_chk(session, &cursor->key)); WT_RET(__cursor_size_chk(session, &cursor->value)); @@ -658,6 +641,9 @@ __wt_btcur_insert(WT_CURSOR_BTREE *cbt) append_key = F_ISSET(cursor, WT_CURSTD_APPEND) && btree->type != BTREE_ROW; + /* Save the cursor state. */ + __cursor_state_save(cursor, &state); + /* * If inserting with overwrite configured, and positioned to an on-page * key, the update doesn't require another search. The cursor won't be @@ -676,28 +662,30 @@ __wt_btcur_insert(WT_CURSOR_BTREE *cbt) */ cbt->compare = 0; ret = btree->type == BTREE_ROW ? - __cursor_row_modify(session, cbt, false) : - __cursor_col_modify(session, cbt, false); + __cursor_row_modify(session, cbt, WT_UPDATE_STANDARD) : + __cursor_col_modify(session, cbt, WT_UPDATE_STANDARD); if (ret == 0) goto done; /* - * The pinned page goes away if we fail for any reason, make - * sure there's a local copy of any key. (Restart could still + * The pinned page goes away if we fail for any reason, get a + * local copy of any pinned key or value. (Restart could still * use the pinned page, but that's an unlikely path.) Re-save * the cursor state: we may retry but eventually fail. */ - WT_TRET(__cursor_copy_int_key(cursor)); + WT_TRET(__cursor_localkey(cursor)); + WT_TRET(__cursor_localvalue(cursor)); __cursor_state_save(cursor, &state); goto err; } /* - * The pinned page goes away if we do a search, make sure there's a - * local copy of any key. Re-save the cursor state: we may retry but + * The pinned page goes away if we do a search, get a local copy of any + * pinned key or value. Re-save the cursor state: we may retry but * eventually fail. */ - WT_ERR(__cursor_copy_int_key(cursor)); + WT_ERR(__cursor_localkey(cursor)); + WT_ERR(__cursor_localvalue(cursor)); __cursor_state_save(cursor, &state); retry: WT_ERR(__cursor_func_init(cbt, true)); @@ -712,7 +700,7 @@ retry: WT_ERR(__cursor_func_init(cbt, true)); cbt->compare == 0 && __wt_cursor_valid(cbt, NULL)) WT_ERR(WT_DUPLICATE_KEY); - ret = __cursor_row_modify(session, cbt, false); + ret = __cursor_row_modify(session, cbt, WT_UPDATE_STANDARD); } else { /* * Optionally insert a new record (ignoring the application's @@ -735,7 +723,7 @@ retry: WT_ERR(__cursor_func_init(cbt, true)); (cbt->compare != 0 && __cursor_fix_implicit(btree, cbt)))) WT_ERR(WT_DUPLICATE_KEY); - WT_ERR(__cursor_col_modify(session, cbt, false)); + WT_ERR(__cursor_col_modify(session, cbt, WT_UPDATE_STANDARD)); if (append_key) cbt->iface.recno = cbt->recno; @@ -812,12 +800,13 @@ __wt_btcur_insert_check(WT_CURSOR_BTREE *cbt) session = (WT_SESSION_IMPL *)cursor->session; /* - * The pinned page goes away if we do a search, make sure there's a - * local copy of any key. Unlike most of the btree cursor routines, - * we don't have to save/restore the cursor key state, none of the - * work done here changes the key state. + * The pinned page goes away if we do a search, get a local copy of any + * pinned key and discard any pinned value. Unlike most of the btree + * cursor routines, we don't have to save/restore the cursor key state, + * none of the work done here changes the cursor state. */ - WT_ERR(__cursor_copy_int_key(cursor)); + WT_ERR(__cursor_localkey(cursor)); + __cursor_novalue(cursor); retry: WT_ERR(__cursor_func_init(cbt, true)); @@ -865,14 +854,15 @@ __wt_btcur_remove(WT_CURSOR_BTREE *cbt) WT_STAT_DATA_INCR(session, cursor_remove); WT_STAT_DATA_INCRV(session, cursor_remove_bytes, cursor->key.size); - __cursor_state_save(cursor, &state); - /* * WT_CURSOR.remove has a unique semantic, the cursor stays positioned * if it starts positioned, otherwise clear the cursor on completion. */ positioned = F_ISSET(cursor, WT_CURSTD_KEY_INT); + /* Save the cursor state. */ + __cursor_state_save(cursor, &state); + /* * If remove positioned to an on-page key, the remove doesn't require * another search. We don't care about the "overwrite" configuration @@ -891,28 +881,33 @@ __wt_btcur_remove(WT_CURSOR_BTREE *cbt) */ cbt->compare = 0; ret = btree->type == BTREE_ROW ? - __cursor_row_modify(session, cbt, true) : - __cursor_col_modify(session, cbt, true); + __cursor_row_modify(session, cbt, WT_UPDATE_DELETED) : + __cursor_col_modify(session, cbt, WT_UPDATE_DELETED); if (ret == 0) goto done; /* - * The pinned page goes away if we fail for any reason, make - * sure there's a local copy of any key. (Restart could still - * use the pinned page, but that's an unlikely path.) Re-save - * the cursor state: we may retry but eventually fail. + * The pinned page goes away if we fail for any reason, get a + * local copy of any pinned key and discard any value (remove + * discards any previous value on success or failure). (Restart + * could still use the pinned page, but that's an unlikely + * path.) Re-save the cursor state: we may retry but eventually + * fail. */ - WT_TRET(__cursor_copy_int_key(cursor)); + WT_TRET(__cursor_localkey(cursor)); + F_CLR(cursor, WT_CURSTD_VALUE_SET); __cursor_state_save(cursor, &state); goto err; } /* - * The pinned page goes away if we do a search, make sure there's a - * local copy of any key. Re-save the cursor state: we may retry but - * eventually fail. + * The pinned page goes away if we do a search, get a local copy of any + * pinned key and discard any value (remove discards any previous + * value on success or failure). Re-save the cursor state: we may retry + * but eventually fail. */ - WT_ERR(__cursor_copy_int_key(cursor)); + WT_ERR(__cursor_localkey(cursor)); + F_CLR(cursor, WT_CURSTD_VALUE_SET); __cursor_state_save(cursor, &state); retry: WT_ERR(__cursor_func_init(cbt, true)); @@ -926,7 +921,7 @@ retry: WT_ERR(__cursor_func_init(cbt, true)); if (cbt->compare != 0 || !__wt_cursor_valid(cbt, NULL)) WT_ERR(WT_NOTFOUND); - ret = __cursor_row_modify(session, cbt, true); + ret = __cursor_row_modify(session, cbt, WT_UPDATE_DELETED); } else { WT_ERR(__cursor_col_search(session, cbt, NULL)); @@ -953,7 +948,8 @@ retry: WT_ERR(__cursor_func_init(cbt, true)); */ cbt->recno = cursor->recno; } else - ret = __cursor_col_modify(session, cbt, true); + ret = __cursor_col_modify( + session, cbt, WT_UPDATE_DELETED); } err: if (ret == WT_RESTART) { @@ -987,11 +983,11 @@ done: /* } /* - * __wt_btcur_update -- + * __btcur_update -- * Update a record in the tree. */ -int -__wt_btcur_update(WT_CURSOR_BTREE *cbt) +static int +__btcur_update(WT_CURSOR_BTREE *cbt, u_int modify_type) { WT_BTREE *btree; WT_CURFILE_STATE state; @@ -1003,19 +999,12 @@ __wt_btcur_update(WT_CURSOR_BTREE *cbt) cursor = &cbt->iface; session = (WT_SESSION_IMPL *)cursor->session; - WT_STAT_CONN_INCR(session, cursor_update); - WT_STAT_DATA_INCR(session, cursor_update); - WT_STAT_DATA_INCRV(session, cursor_update_bytes, cursor->value.size); - - __cursor_state_save(cursor, &state); - - if (btree->type == BTREE_ROW) - WT_RET(__cursor_size_chk(session, &cursor->key)); - WT_RET(__cursor_size_chk(session, &cursor->value)); - /* It's no longer possible to bulk-load into the tree. */ __cursor_disable_bulk(session, btree); + /* Save the cursor state. */ + __cursor_state_save(cursor, &state); + /* * If update positioned to an on-page key, the update doesn't require * another search. We don't care about the "overwrite" configuration @@ -1033,28 +1022,30 @@ __wt_btcur_update(WT_CURSOR_BTREE *cbt) */ cbt->compare = 0; ret = btree->type == BTREE_ROW ? - __cursor_row_modify(session, cbt, false) : - __cursor_col_modify(session, cbt, false); + __cursor_row_modify(session, cbt, modify_type) : + __cursor_col_modify(session, cbt, modify_type); if (ret == 0) goto done; /* - * The pinned page goes away if we fail for any reason, make - * sure there's a local copy of any key. (Restart could still + * The pinned page goes away if we fail for any reason, get a + * a local copy of any pinned key or value. (Restart could still * use the pinned page, but that's an unlikely path.) Re-save * the cursor state: we may retry but eventually fail. */ - WT_TRET(__cursor_copy_int_key(cursor)); + WT_TRET(__cursor_localkey(cursor)); + WT_TRET(__cursor_localvalue(cursor)); __cursor_state_save(cursor, &state); goto err; } /* - * The pinned page goes away if we do a search, make sure there's a - * local copy of any key. Re-save the cursor state: we may retry but + * The pinned page goes away if we do a search, get a local copy of any + * pinned key or value. Re-save the cursor state: we may retry but * eventually fail. */ - WT_ERR(__cursor_copy_int_key(cursor)); + WT_ERR(__cursor_localkey(cursor)); + WT_ERR(__cursor_localvalue(cursor)); __cursor_state_save(cursor, &state); retry: WT_ERR(__cursor_func_init(cbt, true)); @@ -1070,7 +1061,7 @@ retry: WT_ERR(__cursor_func_init(cbt, true)); if (cbt->compare != 0 || !__wt_cursor_valid(cbt, NULL)) WT_ERR(WT_NOTFOUND); } - ret = __cursor_row_modify(session, cbt, false); + ret = __cursor_row_modify(session, cbt, modify_type); } else { WT_ERR(__cursor_col_search(session, cbt, NULL)); @@ -1089,7 +1080,7 @@ retry: WT_ERR(__cursor_func_init(cbt, true)); !__cursor_fix_implicit(btree, cbt)) WT_ERR(WT_NOTFOUND); } - ret = __cursor_col_modify(session, cbt, false); + ret = __cursor_col_modify(session, cbt, modify_type); } err: if (ret == WT_RESTART) { @@ -1106,8 +1097,14 @@ err: if (ret == WT_RESTART) { * To make this work, we add a field to the btree cursor to pass back a * pointer to the modify function's allocated update structure. */ -done: if (ret == 0) - WT_TRET(__wt_kv_return(session, cbt, cbt->modify_update)); +done: if (ret == 0) { + if (modify_type == WT_UPDATE_RESERVED) { + F_CLR(cursor, WT_CURSTD_VALUE_SET); + WT_TRET(__wt_key_return(session, cbt)); + } else + WT_TRET( + __wt_kv_return(session, cbt, cbt->modify_update)); + } if (ret != 0) { WT_TRET(__cursor_reset(cbt)); @@ -1118,6 +1115,59 @@ done: if (ret == 0) } /* + * __wt_btcur_reserve -- + * Reserve a record in the tree. + */ +int +__wt_btcur_reserve(WT_CURSOR_BTREE *cbt) +{ + WT_CURSOR *cursor; + WT_DECL_RET; + WT_SESSION_IMPL *session; + bool overwrite; + + cursor = &cbt->iface; + session = (WT_SESSION_IMPL *)cursor->session; + + WT_STAT_CONN_INCR(session, cursor_reserve); + WT_STAT_DATA_INCR(session, cursor_reserve); + + /* WT_CURSOR.reserve is update-without-overwrite and a special value. */ + overwrite = F_ISSET(cursor, WT_CURSTD_OVERWRITE); + F_CLR(cursor, WT_CURSTD_OVERWRITE); + ret = __btcur_update(cbt, WT_UPDATE_RESERVED); + if (overwrite) + F_SET(cursor, WT_CURSTD_OVERWRITE); + return (ret); +} + +/* + * __wt_btcur_update -- + * Update a record in the tree. + */ +int +__wt_btcur_update(WT_CURSOR_BTREE *cbt) +{ + WT_BTREE *btree; + WT_CURSOR *cursor; + WT_SESSION_IMPL *session; + + btree = cbt->btree; + cursor = &cbt->iface; + session = (WT_SESSION_IMPL *)cursor->session; + + WT_STAT_CONN_INCR(session, cursor_update); + WT_STAT_DATA_INCR(session, cursor_update); + WT_STAT_DATA_INCRV(session, cursor_update_bytes, cursor->value.size); + + if (btree->type == BTREE_ROW) + WT_RET(__cursor_size_chk(session, &cursor->key)); + WT_RET(__cursor_size_chk(session, &cursor->value)); + + return (__btcur_update(cbt, WT_UPDATE_STANDARD)); +} + +/* * __wt_btcur_compare -- * Return a comparison between two cursors. */ @@ -1237,7 +1287,7 @@ __wt_btcur_equals(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *equalp) static int __cursor_truncate(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop, - int (*rmfunc)(WT_SESSION_IMPL *, WT_CURSOR_BTREE *, bool)) + int (*rmfunc)(WT_SESSION_IMPL *, WT_CURSOR_BTREE *, u_int)) { WT_DECL_RET; @@ -1265,7 +1315,7 @@ retry: WT_RET(__wt_btcur_search(start)); F_MASK((WT_CURSOR *)start, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT); for (;;) { - if ((ret = rmfunc(session, start, 1)) != 0) + if ((ret = rmfunc(session, start, WT_UPDATE_DELETED)) != 0) break; if (stop != NULL && __cursor_equals(start, stop)) @@ -1292,7 +1342,7 @@ retry: WT_RET(__wt_btcur_search(start)); static int __cursor_truncate_fix(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop, - int (*rmfunc)(WT_SESSION_IMPL *, WT_CURSOR_BTREE *, bool)) + int (*rmfunc)(WT_SESSION_IMPL *, WT_CURSOR_BTREE *, u_int)) { WT_DECL_RET; const uint8_t *value; @@ -1323,7 +1373,7 @@ retry: WT_RET(__wt_btcur_search(start)); for (;;) { value = (const uint8_t *)start->iface.value.data; if (*value != 0 && - (ret = rmfunc(session, start, 1)) != 0) + (ret = rmfunc(session, start, WT_UPDATE_DELETED)) != 0) break; if (stop != NULL && __cursor_equals(start, stop)) diff --git a/src/btree/bt_debug.c b/src/btree/bt_debug.c index d3f02e29b90..394ac6c7b84 100644 --- a/src/btree/bt_debug.c +++ b/src/btree/bt_debug.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -689,8 +689,6 @@ __debug_page_metadata(WT_DBG *ds, WT_REF *ref) WT_RET(ds->f(ds, ", entries %" PRIu32, entries)); WT_RET(ds->f(ds, ", %s", __wt_page_is_modified(page) ? "dirty" : "clean")); - WT_RET(ds->f(ds, ", %s", __wt_rwlock_islocked( - session, &page->page_lock) ? "locked" : "unlocked")); if (F_ISSET_ATOMIC(page, WT_PAGE_BUILD_KEYS)) WT_RET(ds->f(ds, ", keys-built")); @@ -985,8 +983,10 @@ static int __debug_update(WT_DBG *ds, WT_UPDATE *upd, bool hexbyte) { for (; upd != NULL; upd = upd->next) - if (WT_UPDATE_DELETED_ISSET(upd)) + if (upd->type == WT_UPDATE_DELETED) WT_RET(ds->f(ds, "\tvalue {deleted}\n")); + else if (upd->type == WT_UPDATE_RESERVED) + WT_RET(ds->f(ds, "\tvalue {reserved}\n")); else if (hexbyte) { WT_RET(ds->f(ds, "\t{")); WT_RET(__debug_hex_byte(ds, diff --git a/src/btree/bt_delete.c b/src/btree/bt_delete.c index b55ad291c5e..4a88b672d47 100644 --- a/src/btree/bt_delete.c +++ b/src/btree/bt_delete.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -333,7 +333,7 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) */ for (i = 0, size = 0; i < page->entries; ++i) { WT_ERR(__wt_calloc_one(session, &upd)); - WT_UPDATE_DELETED_SET(upd); + upd->type = WT_UPDATE_DELETED; if (page_del == NULL) upd->txnid = WT_TXN_NONE; /* Globally visible */ diff --git a/src/btree/bt_discard.c b/src/btree/bt_discard.c index bab7b8145d6..bfa8eb25aac 100644 --- a/src/btree/bt_discard.c +++ b/src/btree/bt_discard.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -98,7 +98,6 @@ __page_out_int(WT_SESSION_IMPL *session, WT_PAGE **pagep, bool rewrite) */ WT_ASSERT(session, !__wt_page_is_modified(page)); WT_ASSERT(session, !F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU)); - WT_ASSERT(session, !__wt_rwlock_islocked(session, &page->page_lock)); /* * If a root page split, there may be one or more pages linked from the @@ -254,6 +253,7 @@ __free_page_modify(WT_SESSION_IMPL *session, WT_PAGE *page) __wt_ovfl_discard_free(session, page); __wt_free(session, page->modify->ovfl_track); + __wt_spin_destroy(session, &page->modify->page_lock); __wt_free(session, page->modify); } diff --git a/src/btree/bt_handle.c b/src/btree/bt_handle.c index d76720b19ae..06fbd6b74c7 100644 --- a/src/btree/bt_handle.c +++ b/src/btree/bt_handle.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -418,15 +418,13 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) WT_RET(__wt_compressor_config(session, &cval, &btree->compressor)); /* - * We do not use __wt_config_gets_none here because "none" - * and the empty string have different meanings. The - * empty string means inherit the system encryption setting - * and "none" means this table is in the clear even if the - * database is encrypted. If this is the metadata handle - * always inherit from the connection. + * We do not use __wt_config_gets_none here because "none" and the empty + * string have different meanings. The empty string means inherit the + * system encryption setting and "none" means this table is in the clear + * even if the database is encrypted. */ WT_RET(__wt_config_gets(session, cfg, "encryption.name", &cval)); - if (WT_IS_METADATA(btree->dhandle) || cval.len == 0) + if (cval.len == 0) btree->kencryptor = conn->kencryptor; else if (WT_STRING_MATCH("none", cval.str, cval.len)) btree->kencryptor = NULL; @@ -444,12 +442,14 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) } /* Initialize locks. */ - __wt_rwlock_init(session, &btree->ovfl_lock); + WT_RET(__wt_rwlock_init(session, &btree->ovfl_lock)); WT_RET(__wt_spin_init(session, &btree->flush_lock, "btree flush")); - btree->checkpointing = WT_CKPT_OFF; /* Not checkpointing */ btree->modified = false; /* Clean */ - btree->write_gen = ckpt->write_gen; /* Write generation */ + + btree->checkpointing = WT_CKPT_OFF; /* Not checkpointing */ + btree->write_gen = ckpt->write_gen; /* Write generation */ + btree->checkpoint_gen = __wt_gen(session, WT_GEN_CHECKPOINT); return (0); } diff --git a/src/btree/bt_huffman.c b/src/btree/bt_huffman.c index 918791d9c6e..c5cc9ccf0b0 100644 --- a/src/btree/bt_huffman.c +++ b/src/btree/bt_huffman.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/btree/bt_io.c b/src/btree/bt_io.c index b5e4d52394a..262532a4eab 100644 --- a/src/btree/bt_io.c +++ b/src/btree/bt_io.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/btree/bt_misc.c b/src/btree/bt_misc.c index 3bec65c2567..04b607082d1 100644 --- a/src/btree/bt_misc.c +++ b/src/btree/bt_misc.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/btree/bt_ovfl.c b/src/btree/bt_ovfl.c index ae0da62af57..3d09f655c65 100644 --- a/src/btree/bt_ovfl.c +++ b/src/btree/bt_ovfl.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/btree/bt_page.c b/src/btree/bt_page.c index f20f6398e37..ca5f05fe3dc 100644 --- a/src/btree/bt_page.c +++ b/src/btree/bt_page.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/btree/bt_random.c b/src/btree/bt_random.c index c5948ec4ab5..1bdf0fd1c8b 100644 --- a/src/btree/bt_random.c +++ b/src/btree/bt_random.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -395,8 +395,7 @@ __wt_btcur_next_random(WT_CURSOR_BTREE *cbt) */ for (skip = cbt->next_random_leaf_skip; cbt->ref == NULL || skip > 0;) { n = skip; - WT_ERR(__wt_tree_walk_skip(session, &cbt->ref, &skip, - WT_READ_NO_GEN | WT_READ_SKIP_INTL | WT_READ_WONT_NEED)); + WT_ERR(__wt_tree_walk_skip(session, &cbt->ref, &skip)); if (n == skip) { if (skip == 0) break; diff --git a/src/btree/bt_read.c b/src/btree/bt_read.c index 64874547b9c..3f85e58f088 100644 --- a/src/btree/bt_read.c +++ b/src/btree/bt_read.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -8,6 +8,8 @@ #include "wt_internal.h" +static void __btree_verbose_lookaside_read(WT_SESSION_IMPL *); + /* * __wt_las_remove_block -- * Remove all records matching a key prefix from the lookaside store. @@ -19,8 +21,7 @@ __wt_las_remove_block(WT_SESSION_IMPL *session, WT_DECL_ITEM(las_addr); WT_DECL_ITEM(las_key); WT_DECL_RET; - uint64_t las_counter, las_txnid; - int64_t remove_cnt; + uint64_t las_counter, las_txnid, remove_cnt; uint32_t las_id; int exact; @@ -74,7 +75,7 @@ err: __wt_scr_free(session, &las_addr); if (remove_cnt > S2C(session)->las_record_cnt) S2C(session)->las_record_cnt = 0; else if (remove_cnt > 0) - (void)__wt_atomic_subi64( + (void)__wt_atomic_sub64( &S2C(session)->las_record_cnt, remove_cnt); return (ret); @@ -90,7 +91,8 @@ __col_instantiate(WT_SESSION_IMPL *session, { /* Search the page and add updates. */ WT_RET(__wt_col_search(session, recno, ref, cbt)); - WT_RET(__wt_col_modify(session, cbt, recno, NULL, upd, false)); + WT_RET(__wt_col_modify( + session, cbt, recno, NULL, upd, WT_UPDATE_STANDARD, false)); return (0); } @@ -104,7 +106,8 @@ __row_instantiate(WT_SESSION_IMPL *session, { /* Search the page and add updates. */ WT_RET(__wt_row_search(session, key, ref, cbt, true)); - WT_RET(__wt_row_modify(session, cbt, key, NULL, upd, false)); + WT_RET(__wt_row_modify( + session, cbt, key, NULL, upd, WT_UPDATE_STANDARD, false)); return (0); } @@ -127,7 +130,8 @@ __las_page_instantiate(WT_SESSION_IMPL *session, WT_UPDATE *first_upd, *last_upd, *upd; size_t incr, total_incr; uint64_t current_recno, las_counter, las_txnid, recno, upd_txnid; - uint32_t las_id, upd_size, session_flags; + uint32_t las_id, session_flags; + uint8_t upd_type; int exact; const uint8_t *p; @@ -188,10 +192,10 @@ __las_page_instantiate(WT_SESSION_IMPL *session, /* Allocate the WT_UPDATE structure. */ WT_ERR(cursor->get_value( - cursor, &upd_txnid, &upd_size, las_value)); - WT_ERR(__wt_update_alloc(session, - (upd_size == WT_UPDATE_DELETED_VALUE) ? NULL : las_value, - &upd, &incr)); + cursor, &upd_txnid, &upd_type, las_value)); + WT_ERR(__wt_update_alloc(session, las_value, &upd, &incr, + upd_type == WT_UPDATE_DELETED ? + WT_UPDATE_DELETED : WT_UPDATE_STANDARD)); total_incr += incr; upd->txnid = upd_txnid; @@ -448,6 +452,7 @@ __page_read(WT_SESSION_IMPL *session, WT_REF *ref) */ dsk = tmp.data; if (F_ISSET(dsk, WT_PAGE_LAS_UPDATE) && __wt_las_is_written(session)) { + __btree_verbose_lookaside_read(session); WT_STAT_CONN_INCR(session, cache_read_lookaside); WT_STAT_DATA_INCR(session, cache_read_lookaside); @@ -586,15 +591,10 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags * if the page qualifies for forced eviction and update * the page's generation number. If eviction isn't being * done on this file, we're done. - * In-memory split of large pages is allowed while - * no_eviction is set on btree, whereas reconciliation - * is not allowed. */ if (LF_ISSET(WT_READ_NO_EVICT) || F_ISSET(session, WT_SESSION_NO_EVICTION) || - btree->lsm_primary || - (btree->evict_disabled > 0 && - !F_ISSET(btree, WT_BTREE_ALLOW_SPLITS))) + btree->evict_disabled > 0 || btree->lsm_primary) goto skip_evict; /* @@ -682,3 +682,43 @@ skip_evict: __wt_sleep(0, sleep_cnt); } } + +/* + * __btree_verbose_lookaside_read -- + * Create a verbose message to display at most once per checkpoint when + * performing a lookaside table read. + */ +static void +__btree_verbose_lookaside_read(WT_SESSION_IMPL *session) +{ +#ifdef HAVE_VERBOSE + WT_CONNECTION_IMPL *conn; + uint64_t ckpt_gen_current, ckpt_gen_last; + + if (!WT_VERBOSE_ISSET(session, WT_VERB_LOOKASIDE)) return; + + conn = S2C(session); + ckpt_gen_current = __wt_gen(session, WT_GEN_CHECKPOINT); + ckpt_gen_last = conn->las_verb_gen_read; + + /* + * This message is throttled to one per checkpoint. To do this we + * track the generation of the last checkpoint for which the message + * was printed and check against the current checkpoint generation. + */ + if (ckpt_gen_current > ckpt_gen_last) { + /* + * Attempt to atomically replace the last checkpoint generation + * for which this message was printed. If the atomic swap fails + * we have raced and the winning thread will print the message. + */ + if (__wt_atomic_casv64(&conn->las_verb_gen_read, + ckpt_gen_last, ckpt_gen_current)) { + __wt_verbose(session, WT_VERB_LOOKASIDE, + "Read from lookaside file triggered."); + } + } +#else + WT_UNUSED(session); +#endif +} diff --git a/src/btree/bt_rebalance.c b/src/btree/bt_rebalance.c index 68848c7c8f5..47c7888af35 100644 --- a/src/btree/bt_rebalance.c +++ b/src/btree/bt_rebalance.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/btree/bt_ret.c b/src/btree/bt_ret.c index f17fa1b85d1..7212de72d6e 100644 --- a/src/btree/bt_ret.c +++ b/src/btree/bt_ret.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -147,9 +147,13 @@ __wt_key_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) cursor = &cbt->iface; /* - * We may already have an internal key, in which case the cursor may - * not be set up to get another copy (for example, when we rely on a - * search-function result). + * We may already have an internal key and the cursor may not be set up + * to get another copy, so we have to leave it alone. Consider a cursor + * search followed by an update: the update doesn't repeat the search, + * it simply updates the currently referenced key's value. We will end + * up here with the correct internal key, but we can't "return" the key + * again even if we wanted to do the additional work, the cursor isn't + * set up for that because we didn't just complete a search. */ F_CLR(cursor, WT_CURSTD_KEY_EXT); if (!F_ISSET(cursor, WT_CURSTD_KEY_INT)) { diff --git a/src/btree/bt_slvg.c b/src/btree/bt_slvg.c index 165f932afb2..eb39301abc7 100644 --- a/src/btree/bt_slvg.c +++ b/src/btree/bt_slvg.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/btree/bt_split.c b/src/btree/bt_split.c index 49043c8bab4..c1b7b6c4001 100644 --- a/src/btree/bt_split.c +++ b/src/btree/bt_split.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -31,143 +31,6 @@ typedef enum { } WT_SPLIT_ERROR_PHASE; /* - * __split_oldest_gen -- - * Calculate the oldest active split generation. - */ -static uint64_t -__split_oldest_gen(WT_SESSION_IMPL *session) -{ - WT_CONNECTION_IMPL *conn; - WT_SESSION_IMPL *s; - uint64_t gen, oldest; - u_int i, session_cnt; - - conn = S2C(session); - WT_ORDERED_READ(session_cnt, conn->session_cnt); - for (i = 0, s = conn->sessions, oldest = conn->split_gen + 1; - i < session_cnt; - i++, s++) - if (((gen = s->split_gen) != 0) && gen < oldest) - oldest = gen; - - return (oldest); -} - -/* - * __wt_split_obsolete -- - * Check if it is safe to free / evict based on split generation. - */ -bool -__wt_split_obsolete(WT_SESSION_IMPL *session, uint64_t split_gen) -{ - return (split_gen < __split_oldest_gen(session)); -} - -/* - * __split_stash_add -- - * Add a new entry into the session's split stash list. - */ -static int -__split_stash_add( - WT_SESSION_IMPL *session, uint64_t split_gen, void *p, size_t len) -{ - WT_CONNECTION_IMPL *conn; - WT_SPLIT_STASH *stash; - - WT_ASSERT(session, p != NULL); - - conn = S2C(session); - - /* Grow the list as necessary. */ - WT_RET(__wt_realloc_def(session, &session->split_stash_alloc, - session->split_stash_cnt + 1, &session->split_stash)); - - stash = session->split_stash + session->split_stash_cnt++; - stash->split_gen = split_gen; - stash->p = p; - stash->len = len; - - (void)__wt_atomic_add64(&conn->split_stashed_bytes, len); - (void)__wt_atomic_add64(&conn->split_stashed_objects, 1); - - /* See if we can free any previous entries. */ - if (session->split_stash_cnt > 1) - __wt_split_stash_discard(session); - - return (0); -} - -/* - * __wt_split_stash_discard -- - * Discard any memory from a session's split stash that we can. - */ -void -__wt_split_stash_discard(WT_SESSION_IMPL *session) -{ - WT_CONNECTION_IMPL *conn; - WT_SPLIT_STASH *stash; - uint64_t oldest; - size_t i; - - conn = S2C(session); - - /* Get the oldest split generation. */ - oldest = __split_oldest_gen(session); - - for (i = 0, stash = session->split_stash; - i < session->split_stash_cnt; - ++i, ++stash) { - if (stash->p == NULL) - continue; - if (stash->split_gen >= oldest) - break; - /* - * It's a bad thing if another thread is in this memory after - * we free it, make sure nothing good happens to that thread. - */ - (void)__wt_atomic_sub64(&conn->split_stashed_bytes, stash->len); - (void)__wt_atomic_sub64(&conn->split_stashed_objects, 1); - __wt_overwrite_and_free_len(session, stash->p, stash->len); - } - - /* - * If there are enough free slots at the beginning of the list, shuffle - * everything down. - */ - if (i > 100 || i == session->split_stash_cnt) - if ((session->split_stash_cnt -= i) > 0) - memmove(session->split_stash, stash, - session->split_stash_cnt * sizeof(*stash)); -} - -/* - * __wt_split_stash_discard_all -- - * Discard all memory from a session's split stash. - */ -void -__wt_split_stash_discard_all( - WT_SESSION_IMPL *session_safe, WT_SESSION_IMPL *session) -{ - WT_SPLIT_STASH *stash; - size_t i; - - /* - * This function is called during WT_CONNECTION.close to discard any - * memory that remains. For that reason, we take two WT_SESSION_IMPL - * arguments: session_safe is still linked to the WT_CONNECTION and - * can be safely used for calls to other WiredTiger functions, while - * session is the WT_SESSION_IMPL we're cleaning up. - */ - for (i = 0, stash = session->split_stash; - i < session->split_stash_cnt; - ++i, ++stash) - __wt_free(session_safe, stash->p); - - __wt_free(session_safe, session->split_stash); - session->split_stash_cnt = session->split_stash_alloc = 0; -} - -/* * __split_safe_free -- * Free a buffer if we can be sure no thread is accessing it, or schedule * it to be freed otherwise. @@ -177,13 +40,14 @@ __split_safe_free(WT_SESSION_IMPL *session, uint64_t split_gen, bool exclusive, void *p, size_t s) { /* We should only call safe free if we aren't pinning the memory. */ - WT_ASSERT(session, session->split_gen != split_gen); + WT_ASSERT(session, + __wt_session_gen(session, WT_GEN_SPLIT) != split_gen); /* * We have swapped something in a page: if we don't have exclusive * access, check whether there are other threads in the same tree. */ - if (!exclusive && __split_oldest_gen(session) > split_gen) + if (!exclusive && __wt_gen_oldest(session, WT_GEN_SPLIT) > split_gen) exclusive = true; if (exclusive) { @@ -191,7 +55,7 @@ __split_safe_free(WT_SESSION_IMPL *session, return (0); } - return (__split_stash_add(session, split_gen, p, s)); + return (__wt_stash_add(session, WT_GEN_SPLIT, split_gen, p, s)); } #ifdef HAVE_DIAGNOSTIC @@ -645,7 +509,8 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) * generation to block splits in newly created pages, so get one. */ WT_ENTER_PAGE_INDEX(session); - __split_ref_prepare(session, alloc_index, session->split_gen, false); + __split_ref_prepare(session, alloc_index, + __wt_session_gen(session, WT_GEN_SPLIT), false); /* * Confirm the root page's index hasn't moved, then update it, which @@ -662,7 +527,7 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) * after the new index is swapped into place in order to know that no * readers are looking at the old index. */ - split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); + split_gen = __wt_gen_next(session, WT_GEN_SPLIT); root->pg_intl_split_gen = split_gen; #ifdef HAVE_DIAGNOSTIC @@ -848,7 +713,7 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, * the new index is swapped into place in order to know that no readers * are looking at the old index. */ - split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); + split_gen = __wt_gen_next(session, WT_GEN_SPLIT); parent->pg_intl_split_gen = split_gen; /* @@ -1173,7 +1038,8 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) * generation to block splits in newly created pages, so get one. */ WT_ENTER_PAGE_INDEX(session); - __split_ref_prepare(session, alloc_index, session->split_gen, true); + __split_ref_prepare(session, alloc_index, + __wt_session_gen(session, WT_GEN_SPLIT), true); /* Split into the parent. */ if ((ret = __split_parent(session, page_ref, alloc_index->index, @@ -1194,7 +1060,7 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) * after the new index is swapped into place in order to know that no * readers are looking at the old index. */ - split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); + split_gen = __wt_gen_next(session, WT_GEN_SPLIT); page->pg_intl_split_gen = split_gen; #ifdef HAVE_DIAGNOSTIC @@ -1256,12 +1122,12 @@ err: switch (complete) { } /* - * __split_internal_lock -- + * __split_internal_lock_worker -- * Lock an internal page. */ static int -__split_internal_lock(WT_SESSION_IMPL *session, WT_REF *ref, bool trylock, - WT_PAGE **parentp, bool *hazardp) +__split_internal_lock_worker(WT_SESSION_IMPL *session, + WT_REF *ref, bool trylock, WT_PAGE **parentp, bool *hazardp) { WT_DECL_RET; WT_PAGE *parent; @@ -1300,13 +1166,19 @@ __split_internal_lock(WT_SESSION_IMPL *session, WT_REF *ref, bool trylock, for (;;) { parent = ref->home; + /* + * The page will be marked dirty, and we can only lock a page + * with a modify structure. + */ + WT_RET(__wt_page_modify_init(session, parent)); + if (trylock) - WT_RET(__wt_try_writelock(session, &parent->page_lock)); + WT_RET(WT_PAGE_TRYLOCK(session, parent)); else - __wt_writelock(session, &parent->page_lock); + WT_PAGE_LOCK(session, parent); if (parent == ref->home) break; - __wt_writeunlock(session, &parent->page_lock); + WT_PAGE_UNLOCK(session, parent); } /* @@ -1329,7 +1201,33 @@ __split_internal_lock(WT_SESSION_IMPL *session, WT_REF *ref, bool trylock, *parentp = parent; return (0); -err: __wt_writeunlock(session, &parent->page_lock); +err: WT_PAGE_UNLOCK(session, parent); + return (ret); +} + +/* + * __split_internal_lock -- + * Lock an internal page. + */ +static int +__split_internal_lock(WT_SESSION_IMPL *session, + WT_REF *ref, bool trylock, WT_PAGE **parentp, bool *hazardp) +{ + WT_DECL_RET; + + /* + * There's no lock on our parent page and we're about to acquire one, + * which implies using the WT_REF.home field to reference our parent + * page. As a child of the parent page, we prevent its eviction, but + * that's a weak guarantee. If the parent page splits, and our WT_REF + * were to move with the split, the WT_REF.home field might change + * underneath us and we could race, and end up attempting to access + * an evicted page. Set the session page-index generation so if the + * parent splits, it still can't be evicted. + */ + WT_WITH_PAGE_INDEX(session, + ret = __split_internal_lock_worker( + session, ref, trylock, parentp, hazardp)); return (ret); } @@ -1345,7 +1243,7 @@ __split_internal_unlock(WT_SESSION_IMPL *session, WT_PAGE *parent, bool hazard) if (hazard) ret = __wt_hazard_clear(session, parent->pg_intl_parent_ref); - __wt_writeunlock(session, &parent->page_lock); + WT_PAGE_UNLOCK(session, parent); return (ret); } @@ -1558,8 +1456,8 @@ __split_multi_inmem( WT_ERR(__wt_col_search(session, recno, ref, &cbt)); /* Apply the modification. */ - WT_ERR(__wt_col_modify( - session, &cbt, recno, NULL, upd, false)); + WT_ERR(__wt_col_modify(session, + &cbt, recno, NULL, upd, WT_UPDATE_STANDARD, true)); break; case WT_PAGE_ROW_LEAF: /* Build a key. */ @@ -1580,8 +1478,8 @@ __split_multi_inmem( WT_ERR(__wt_row_search(session, key, ref, &cbt, true)); /* Apply the modification. */ - WT_ERR(__wt_row_modify( - session, &cbt, key, NULL, upd, false)); + WT_ERR(__wt_row_modify(session, &cbt, + key, NULL, upd, WT_UPDATE_STANDARD, true)); break; WT_ILLEGAL_VALUE_ERR(session); } diff --git a/src/btree/bt_stat.c b/src/btree/bt_stat.c index 0da0e0807bd..e3b9bbced48 100644 --- a/src/btree/bt_stat.c +++ b/src/btree/bt_stat.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -178,7 +178,9 @@ __stat_page_col_var( */ WT_SKIP_FOREACH(ins, WT_COL_UPDATE(page, cip)) { upd = ins->upd; - if (WT_UPDATE_DELETED_ISSET(upd)) { + if (upd->type == WT_UPDATE_RESERVED) + continue; + if (upd->type == WT_UPDATE_DELETED) { if (!orig_deleted) { ++deleted_cnt; --entry_cnt; @@ -192,11 +194,14 @@ __stat_page_col_var( } /* Walk any append list. */ - WT_SKIP_FOREACH(ins, WT_COL_APPEND(page)) - if (WT_UPDATE_DELETED_ISSET(ins->upd)) + WT_SKIP_FOREACH(ins, WT_COL_APPEND(page)) { + if (ins->upd->type == WT_UPDATE_RESERVED) + continue; + if (ins->upd->type == WT_UPDATE_DELETED) ++deleted_cnt; else ++entry_cnt; + } WT_STAT_INCRV(session, stats, btree_column_deleted, deleted_cnt); WT_STAT_INCRV(session, stats, btree_column_rle, rle_cnt); @@ -263,7 +268,8 @@ __stat_page_row_leaf( * key on the page. */ WT_SKIP_FOREACH(ins, WT_ROW_INSERT_SMALLEST(page)) - if (!WT_UPDATE_DELETED_ISSET(ins->upd)) + if (ins->upd->type != WT_UPDATE_DELETED && + ins->upd->type != WT_UPDATE_RESERVED) ++entry_cnt; /* @@ -272,16 +278,19 @@ __stat_page_row_leaf( */ WT_ROW_FOREACH(page, rip, i) { upd = WT_ROW_UPDATE(page, rip); - if (upd == NULL || !WT_UPDATE_DELETED_ISSET(upd)) + if (upd == NULL || + (upd->type != WT_UPDATE_DELETED && + upd->type != WT_UPDATE_RESERVED)) ++entry_cnt; if (upd == NULL && (cell = __wt_row_leaf_value_cell(page, rip, NULL)) != NULL && __wt_cell_type(cell) == WT_CELL_VALUE_OVFL) - ++ovfl_cnt; + ++ovfl_cnt; /* Walk K/V pairs inserted after the on-page K/V pair. */ WT_SKIP_FOREACH(ins, WT_ROW_INSERT(page, rip)) - if (!WT_UPDATE_DELETED_ISSET(ins->upd)) + if (ins->upd->type != WT_UPDATE_DELETED && + ins->upd->type != WT_UPDATE_RESERVED) ++entry_cnt; } diff --git a/src/btree/bt_sync.c b/src/btree/bt_sync.c index ead6ccc4ac0..5b0bf53dc6c 100644 --- a/src/btree/bt_sync.c +++ b/src/btree/bt_sync.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -179,22 +179,9 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) * Set the checkpointing flag to block such actions and wait for * any problematic eviction or page splits to complete. */ - WT_PUBLISH(btree->checkpointing, WT_CKPT_PREPARE); - - /* - * Sync for checkpoint allows splits to happen while the queue - * is being drained, but not reconciliation. We need to do this, - * since draining the queue can take long enough for hot pages - * to grow significantly larger than the configured maximum - * size. - */ - F_SET(btree, WT_BTREE_ALLOW_SPLITS); - ret = __wt_evict_file_exclusive_on(session); - F_CLR(btree, WT_BTREE_ALLOW_SPLITS); - WT_ERR(ret); - __wt_evict_file_exclusive_off(session); - - WT_PUBLISH(btree->checkpointing, WT_CKPT_RUNNING); + btree->checkpointing = WT_CKPT_PREPARE; + (void)__wt_gen_next_drain(session, WT_GEN_EVICT); + btree->checkpointing = WT_CKPT_RUNNING; /* Write all dirty in-cache pages. */ flags |= WT_READ_NO_EVICT; @@ -268,9 +255,8 @@ err: /* On error, clear any left-over tree walk. */ saved_pinned_id == WT_TXN_NONE) __wt_txn_release_snapshot(session); - /* Clear the checkpoint flag and push the change. */ - if (btree->checkpointing != WT_CKPT_OFF) - WT_PUBLISH(btree->checkpointing, WT_CKPT_OFF); + /* Clear the checkpoint flag. */ + btree->checkpointing = WT_CKPT_OFF; __wt_spin_unlock(session, &btree->flush_lock); diff --git a/src/btree/bt_upgrade.c b/src/btree/bt_upgrade.c index a9ff16ad496..a7fe3283218 100644 --- a/src/btree/bt_upgrade.c +++ b/src/btree/bt_upgrade.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/btree/bt_vrfy.c b/src/btree/bt_vrfy.c index 7475811adc5..21ba2d7a715 100644 --- a/src/btree/bt_vrfy.c +++ b/src/btree/bt_vrfy.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/btree/bt_vrfy_dsk.c b/src/btree/bt_vrfy_dsk.c index a4071c44aee..55c96bbed55 100644 --- a/src/btree/bt_vrfy_dsk.c +++ b/src/btree/bt_vrfy_dsk.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/btree/bt_walk.c b/src/btree/bt_walk.c index 86484feb7c9..225e6812aa1 100644 --- a/src/btree/bt_walk.c +++ b/src/btree/bt_walk.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -497,29 +497,21 @@ restart: /* } /* - * Optionally skip leaf pages: skip all leaf pages if - * WT_READ_SKIP_LEAF is set, when the skip-leaf-count - * variable is non-zero, skip some count of leaf pages. - * If this page is disk-based, crack the cell to figure - * out it's a leaf page without reading it. + * Optionally skip leaf pages: when the skip-leaf-count + * variable is non-zero, skip some count of leaf pages, + * then take the next leaf page we can. * - * If skipping some number of leaf pages, decrement the - * count of pages to zero, and then take the next leaf - * page we can. Be cautious around the page decrement, - * if for some reason don't take this particular page, - * we can take the next one, and, there are additional - * tests/decrements when we're about to return a leaf - * page. + * The reason to do some of this work here (rather than + * in our caller), is because we can look at the cell + * and know it's a leaf page without reading it into + * memory. If this page is disk-based, crack the cell + * to figure out it's a leaf page without reading it. */ - if (skipleafcntp != NULL || LF_ISSET(WT_READ_SKIP_LEAF)) - if (__ref_is_leaf(ref)) { - if (LF_ISSET(WT_READ_SKIP_LEAF)) - break; - if (*skipleafcntp > 0) { - --*skipleafcntp; - break; - } - } + if (skipleafcntp != NULL && + *skipleafcntp > 0 && __ref_is_leaf(ref)) { + --*skipleafcntp; + break; + } ret = __wt_page_swap(session, couple, ref, WT_READ_NOTFOUND_OK | WT_READ_RESTART_OK | flags); @@ -626,34 +618,18 @@ descend: empty_internal = true; session, ref, &pindex); slot = pindex->entries - 1; } - } else { - /* - * At the lowest tree level (considering a leaf - * page), turn off the initial-descent state. - * Descent race tests are different when moving - * through the tree vs. the initial descent. - */ - initial_descent = false; - - /* - * Optionally skip leaf pages, the second half. - * We didn't have an on-page cell to figure out - * if it was a leaf page, we had to acquire the - * hazard pointer and look at the page. - */ - if (skipleafcntp != NULL || - LF_ISSET(WT_READ_SKIP_LEAF)) { - if (LF_ISSET(WT_READ_SKIP_LEAF)) - break; - if (*skipleafcntp > 0) { - --*skipleafcntp; - break; - } - } - - *refp = ref; - goto done; + continue; } + + /* + * The tree-walk restart code knows we return any leaf + * page we acquire (never hazard-pointer coupling on + * after acquiring a leaf page), and asserts no restart + * happens while holding a leaf page. This page must be + * returned to our caller. + */ + *refp = ref; + goto done; } } @@ -690,8 +666,29 @@ __wt_tree_walk_count(WT_SESSION_IMPL *session, * of leaf pages before returning. */ int -__wt_tree_walk_skip(WT_SESSION_IMPL *session, - WT_REF **refp, uint64_t *skipleafcntp, uint32_t flags) +__wt_tree_walk_skip( + WT_SESSION_IMPL *session, WT_REF **refp, uint64_t *skipleafcntp) { - return (__tree_walk_internal(session, refp, NULL, skipleafcntp, flags)); + /* + * Optionally skip leaf pages, the second half. The tree-walk function + * didn't have an on-page cell it could use to figure out if the page + * was a leaf page or not, it had to acquire the hazard pointer and look + * at the page. The tree-walk code never acquires a hazard pointer on a + * leaf page without returning it, and it's not trivial to change that. + * So, the tree-walk code returns all leaf pages here and we deal with + * decrementing the count. + */ + do { + WT_RET(__tree_walk_internal(session, refp, NULL, skipleafcntp, + WT_READ_NO_GEN | WT_READ_SKIP_INTL | WT_READ_WONT_NEED)); + + /* + * The walk skipped internal pages, any page returned must be a + * leaf page. + */ + if (*skipleafcntp > 0) + --*skipleafcntp; + } while (*skipleafcntp > 0); + + return (0); } diff --git a/src/btree/col_modify.c b/src/btree/col_modify.c index 9ccb9728189..2a64ec03952 100644 --- a/src/btree/col_modify.c +++ b/src/btree/col_modify.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -17,13 +17,14 @@ static int __col_insert_alloc( */ int __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, - uint64_t recno, WT_ITEM *value, WT_UPDATE *upd_arg, bool is_remove) + uint64_t recno, const WT_ITEM *value, + WT_UPDATE *upd_arg, u_int modify_type, bool exclusive) { + static const WT_ITEM col_fix_remove = { "", 1, NULL, 0, 0 }; WT_BTREE *btree; WT_DECL_RET; WT_INSERT *ins; WT_INSERT_HEAD *ins_head, **ins_headp; - WT_ITEM _value; WT_PAGE *page; WT_PAGE_MODIFY *mod; WT_UPDATE *old_upd, *upd; @@ -37,14 +38,17 @@ __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, upd = upd_arg; append = logged = false; - /* This code expects a remove to have a NULL value. */ - if (is_remove) { - if (btree->type == BTREE_COL_FIX) { - value = &_value; - value->data = ""; - value->size = 1; - } else - value = NULL; + if (modify_type == WT_UPDATE_DELETED || + modify_type == WT_UPDATE_RESERVED) { + /* + * Fixed-size column-store doesn't have on-page deleted values, + * it's a nul byte. + */ + if (modify_type == WT_UPDATE_DELETED && + btree->type == BTREE_COL_FIX) { + modify_type = WT_UPDATE_STANDARD; + value = &col_fix_remove; + } } else { /* * There's some chance the application specified a record past @@ -83,11 +87,11 @@ __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ASSERT(session, upd_arg == NULL); /* Make sure the update can proceed. */ - WT_ERR(__wt_txn_update_check( - session, old_upd = cbt->ins->upd)); + WT_ERR(__wt_txn_update_check(session, old_upd = cbt->ins->upd)); /* Allocate a WT_UPDATE structure and transaction ID. */ - WT_ERR(__wt_update_alloc(session, value, &upd, &upd_size)); + WT_ERR(__wt_update_alloc(session, + value, &upd, &upd_size, modify_type)); WT_ERR(__wt_txn_modify(session, upd)); logged = true; @@ -103,7 +107,7 @@ __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, /* Serialize the update. */ WT_ERR(__wt_update_serial( - session, page, &cbt->ins->upd, &upd, upd_size)); + session, page, &cbt->ins->upd, &upd, upd_size, false)); } else { /* Allocate the append/update list reference as necessary. */ if (append) { @@ -147,8 +151,8 @@ __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, mod->mod_col_split_recno > recno)); if (upd_arg == NULL) { - WT_ERR( - __wt_update_alloc(session, value, &upd, &upd_size)); + WT_ERR(__wt_update_alloc(session, + value, &upd, &upd_size, modify_type)); WT_ERR(__wt_txn_modify(session, upd)); logged = true; @@ -185,15 +189,15 @@ __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, if (append) WT_ERR(__wt_col_append_serial( session, page, cbt->ins_head, cbt->ins_stack, - &ins, ins_size, &cbt->recno, skipdepth)); + &ins, ins_size, &cbt->recno, skipdepth, exclusive)); else WT_ERR(__wt_insert_serial( session, page, cbt->ins_head, cbt->ins_stack, - &ins, ins_size, skipdepth)); + &ins, ins_size, skipdepth, exclusive)); } /* If the update was successful, add it to the in-memory log. */ - if (logged) + if (logged && modify_type != WT_UPDATE_RESERVED) WT_ERR(__wt_txn_log_op(session, cbt)); if (0) { diff --git a/src/btree/col_srch.c b/src/btree/col_srch.c index c72d66f8796..78ee367dc69 100644 --- a/src/btree/col_srch.c +++ b/src/btree/col_srch.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/btree/row_key.c b/src/btree/row_key.c index 032fdf7d897..a016568898f 100644 --- a/src/btree/row_key.c +++ b/src/btree/row_key.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -471,6 +471,8 @@ __wt_row_ikey_alloc(WT_SESSION_IMPL *session, { WT_IKEY *ikey; + WT_ASSERT(session, key != NULL); /* quiet clang scan-build */ + /* * Allocate memory for the WT_IKEY structure and the key, then copy * the key into place. diff --git a/src/btree/row_modify.c b/src/btree/row_modify.c index b1a81ca3d9f..cab07341a1c 100644 --- a/src/btree/row_modify.c +++ b/src/btree/row_modify.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -15,18 +15,13 @@ int __wt_page_modify_alloc(WT_SESSION_IMPL *session, WT_PAGE *page) { - WT_CONNECTION_IMPL *conn; + WT_DECL_RET; WT_PAGE_MODIFY *modify; - conn = S2C(session); - WT_RET(__wt_calloc_one(session, &modify)); - /* - * Select a spinlock for the page; let the barrier immediately below - * keep things from racing too badly. - */ - modify->page_lock = ++conn->page_lock_cnt % WT_PAGE_LOCKS; + /* Initialize the spinlock for the page. */ + WT_ERR(__wt_spin_init(session, &modify->page_lock, "btree page")); /* * Multiple threads of control may be searching and deciding to modify @@ -37,8 +32,8 @@ __wt_page_modify_alloc(WT_SESSION_IMPL *session, WT_PAGE *page) if (__wt_atomic_cas_ptr(&page->modify, NULL, modify)) __wt_cache_page_inmem_incr(session, page, sizeof(*modify)); else - __wt_free(session, modify); - return (0); +err: __wt_free(session, modify); + return (ret); } /* @@ -47,7 +42,8 @@ __wt_page_modify_alloc(WT_SESSION_IMPL *session, WT_PAGE *page) */ int __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, - WT_ITEM *key, WT_ITEM *value, WT_UPDATE *upd_arg, bool is_remove) + const WT_ITEM *key, const WT_ITEM *value, + WT_UPDATE *upd_arg, u_int modify_type, bool exclusive) { WT_DECL_RET; WT_INSERT *ins; @@ -65,10 +61,6 @@ __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, upd = upd_arg; logged = false; - /* This code expects a remove to have a NULL value. */ - if (is_remove) - value = NULL; - /* If we don't yet have a modify structure, we'll need one. */ WT_RET(__wt_page_modify_init(session, page)); mod = page->modify; @@ -99,8 +91,8 @@ __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, session, old_upd = *upd_entry)); /* Allocate a WT_UPDATE structure and transaction ID. */ - WT_ERR( - __wt_update_alloc(session, value, &upd, &upd_size)); + WT_ERR(__wt_update_alloc(session, + value, &upd, &upd_size, modify_type)); WT_ERR(__wt_txn_modify(session, upd)); logged = true; @@ -132,7 +124,7 @@ __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, /* Serialize the update. */ WT_ERR(__wt_update_serial( - session, page, upd_entry, &upd, upd_size)); + session, page, upd_entry, &upd, upd_size, exclusive)); } else { /* * Allocate the insert array as necessary. @@ -170,8 +162,8 @@ __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, cbt->ins = ins; if (upd_arg == NULL) { - WT_ERR( - __wt_update_alloc(session, value, &upd, &upd_size)); + WT_ERR(__wt_update_alloc(session, + value, &upd, &upd_size, modify_type)); WT_ERR(__wt_txn_modify(session, upd)); logged = true; @@ -207,10 +199,10 @@ __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, /* Insert the WT_INSERT structure. */ WT_ERR(__wt_insert_serial( session, page, cbt->ins_head, cbt->ins_stack, - &ins, ins_size, skipdepth)); + &ins, ins_size, skipdepth, exclusive)); } - if (logged) + if (logged && modify_type != WT_UPDATE_RESERVED) WT_ERR(__wt_txn_log_op(session, cbt)); if (0) { @@ -235,7 +227,7 @@ err: /* */ int __wt_row_insert_alloc(WT_SESSION_IMPL *session, - WT_ITEM *key, u_int skipdepth, WT_INSERT **insp, size_t *ins_sizep) + const WT_ITEM *key, u_int skipdepth, WT_INSERT **insp, size_t *ins_sizep) { WT_INSERT *ins; size_t ins_size; @@ -263,11 +255,10 @@ __wt_row_insert_alloc(WT_SESSION_IMPL *session, * Allocate a WT_UPDATE structure and associated value and fill it in. */ int -__wt_update_alloc( - WT_SESSION_IMPL *session, WT_ITEM *value, WT_UPDATE **updp, size_t *sizep) +__wt_update_alloc(WT_SESSION_IMPL *session, const WT_ITEM *value, + WT_UPDATE **updp, size_t *sizep, u_int modify_type) { WT_UPDATE *upd; - size_t size; *updp = NULL; @@ -275,15 +266,18 @@ __wt_update_alloc( * Allocate the WT_UPDATE structure and room for the value, then copy * the value into place. */ - size = value == NULL ? 0 : value->size; - WT_RET(__wt_calloc(session, 1, sizeof(WT_UPDATE) + size, &upd)); - if (value == NULL) - WT_UPDATE_DELETED_SET(upd); + if (modify_type == WT_UPDATE_DELETED || + modify_type == WT_UPDATE_RESERVED) + WT_RET(__wt_calloc(session, 1, sizeof(WT_UPDATE), &upd)); else { - upd->size = WT_STORE_SIZE(size); - if (size != 0) - memcpy(WT_UPDATE_DATA(upd), value->data, size); + WT_RET(__wt_calloc( + session, 1, sizeof(WT_UPDATE) + value->size, &upd)); + if (value->size != 0) { + upd->size = WT_STORE_SIZE(value->size); + memcpy(WT_UPDATE_DATA(upd), value->data, value->size); + } } + upd->type = (uint8_t)modify_type; *updp = upd; *sizep = WT_UPDATE_MEMSIZE(upd); diff --git a/src/btree/row_srch.c b/src/btree/row_srch.c index 9c3d467340e..76bebde7de7 100644 --- a/src/btree/row_srch.c +++ b/src/btree/row_srch.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/cache/cache_las.c b/src/cache/cache_las.c index 41da4225f3a..a2233514223 100644 --- a/src/cache/cache_las.c +++ b/src/cache/cache_las.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -140,8 +140,9 @@ __wt_las_set_written(WT_SESSION_IMPL *session) conn->las_written = true; /* - * Push the flag: unnecessary, but from now page reads must deal - * with lookaside table records, and we only do the write once. + * Future page reads must deal with lookaside table records. + * No write could be cached until a future read might matter, + * the barrier is more documentation than requirement. */ WT_FULL_BARRIER(); } @@ -291,8 +292,7 @@ __wt_las_sweep(WT_SESSION_IMPL *session) WT_DECL_ITEM(las_key); WT_DECL_RET; WT_ITEM *key; - uint64_t cnt, las_counter, las_txnid; - int64_t remove_cnt; + uint64_t cnt, las_counter, las_txnid, remove_cnt; uint32_t las_id, session_flags; int notused; @@ -341,7 +341,7 @@ __wt_las_sweep(WT_SESSION_IMPL *session) * blocks in the cache in order to get rid of them, and slowly review * lookaside blocks that have already been evicted. */ - cnt = (uint64_t)WT_MAX(100, conn->las_record_cnt / 30); + cnt = WT_MAX(100, conn->las_record_cnt / 30); /* Discard pages we read as soon as we're done with them. */ F_SET(session, WT_SESSION_NO_CACHE); @@ -389,14 +389,13 @@ err: __wt_buf_free(session, key); WT_TRET(__wt_las_cursor_close(session, &cursor, session_flags)); /* - * If there were races to remove records, we can over-count. All - * arithmetic is signed, so underflow isn't fatal, but check anyway so - * we don't skew low over time. + * If there were races to remove records, we can over-count. Underflow + * isn't fatal, but check anyway so we don't skew low over time. */ - if (remove_cnt > S2C(session)->las_record_cnt) - S2C(session)->las_record_cnt = 0; + if (remove_cnt > conn->las_record_cnt) + conn->las_record_cnt = 0; else if (remove_cnt > 0) - (void)__wt_atomic_subi64(&conn->las_record_cnt, remove_cnt); + (void)__wt_atomic_sub64(&conn->las_record_cnt, remove_cnt); F_CLR(session, WT_SESSION_NO_CACHE); diff --git a/src/checksum/arm64/crc32-arm64.c b/src/checksum/arm64/crc32-arm64.c index 38b4f623044..4316ee3d14e 100644 --- a/src/checksum/arm64/crc32-arm64.c +++ b/src/checksum/arm64/crc32-arm64.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -28,7 +28,7 @@ #include "wt_internal.h" -#if defined(HAVE_CRC32_HARDWARE) +#if defined(__linux__) && defined(HAVE_CRC32_HARDWARE) #include <asm/hwcap.h> #include <sys/auxv.h> @@ -82,7 +82,7 @@ __wt_checksum_hw(const void *chunk, size_t len) return (~crc); } -#endif /* HAVE_CRC32_HARDWARE */ +#endif /* * __wt_checksum_init -- @@ -91,7 +91,7 @@ __wt_checksum_hw(const void *chunk, size_t len) void __wt_checksum_init(void) { -#if defined(HAVE_CRC32_HARDWARE) +#if defined(__linux__) && defined(HAVE_CRC32_HARDWARE) unsigned long caps = getauxval(AT_HWCAP); if (caps & HWCAP_CRC32) @@ -99,7 +99,7 @@ __wt_checksum_init(void) else __wt_process.checksum = __wt_checksum_sw; -#else /* !HAVE_CRC32_HARDWARE */ +#else __wt_process.checksum = __wt_checksum_sw; -#endif /* HAVE_CRC32_HARDWARE */ +#endif } diff --git a/src/checksum/software/checksum.c b/src/checksum/software/checksum.c index 65ed74bbe06..a880d38894c 100644 --- a/src/checksum/software/checksum.c +++ b/src/checksum/software/checksum.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/src/checksum/x86/crc32-x86.c b/src/checksum/x86/crc32-x86.c index 82814ecc34d..1c2c08fa1c1 100644 --- a/src/checksum/x86/crc32-x86.c +++ b/src/checksum/x86/crc32-x86.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/src/checksum/zseries/crc32-s390x.c b/src/checksum/zseries/crc32-s390x.c index 28b46594220..ae024391ff7 100644 --- a/src/checksum/zseries/crc32-s390x.c +++ b/src/checksum/zseries/crc32-s390x.c @@ -11,8 +11,7 @@ #include <sys/types.h> #include <endian.h> -#if defined(HAVE_CRC32_HARDWARE) - +#if defined(__linux__) && defined(HAVE_CRC32_HARDWARE) #include <sys/auxv.h> /* RHEL 7 has kernel support, but does not define this constant in the lib c headers. */ @@ -100,7 +99,7 @@ __wt_checksum_hw(const void *chunk, size_t len) void __wt_checksum_init(void) { -#if defined(HAVE_CRC32_HARDWARE) +#if defined(__linux__) && defined(HAVE_CRC32_HARDWARE) unsigned long caps = getauxval(AT_HWCAP); if (caps & HWCAP_S390_VX) @@ -108,7 +107,7 @@ __wt_checksum_init(void) else __wt_process.checksum = __wt_checksum_sw; -#else /* !HAVE_CRC32_HARDWARE */ +#else __wt_process.checksum = __wt_checksum_sw; #endif } diff --git a/src/config/config.c b/src/config/config.c index a47dfe76aec..33eb988fc5a 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/config/config_api.c b/src/config/config_api.c index c1299baaafe..31efb278d2a 100644 --- a/src/config/config_api.c +++ b/src/config/config_api.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -158,11 +158,11 @@ wiredtiger_config_validate(WT_SESSION *wt_session, } /* - * __wt_conn_foc_add -- + * __conn_foc_add -- * Add a new entry into the connection's free-on-close list. */ -void -__wt_conn_foc_add(WT_SESSION_IMPL *session, const void *p) +static void +__conn_foc_add(WT_SESSION_IMPL *session, const void *p) { WT_CONNECTION_IMPL *conn; @@ -327,12 +327,12 @@ __wt_configure_method(WT_SESSION_IMPL *session, * order to avoid freeing chunks of memory twice. Again, this isn't a * commonly used API and it shouldn't ever happen, just leak it. */ - __wt_conn_foc_add(session, entry->base); - __wt_conn_foc_add(session, entry); - __wt_conn_foc_add(session, checks); - __wt_conn_foc_add(session, newcheck->type); - __wt_conn_foc_add(session, newcheck->checks); - __wt_conn_foc_add(session, newcheck_name); + __conn_foc_add(session, entry->base); + __conn_foc_add(session, entry); + __conn_foc_add(session, checks); + __conn_foc_add(session, newcheck->type); + __conn_foc_add(session, newcheck->checks); + __conn_foc_add(session, newcheck_name); /* * Instead of using locks to protect configuration information, assume diff --git a/src/config/config_check.c b/src/config/config_check.c index 2f372651cb9..8038ae89413 100644 --- a/src/config/config_check.c +++ b/src/config/config_check.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/config/config_collapse.c b/src/config/config_collapse.c index 5abe7556a03..155b700f2dd 100644 --- a/src/config/config_collapse.c +++ b/src/config/config_collapse.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/config/config_def.c b/src/config/config_def.c index f152fbacad4..a7397d21c6a 100644 --- a/src/config/config_def.c +++ b/src/config/config_def.c @@ -148,11 +148,12 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," - "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"transaction\"," - "\"verify\",\"version\",\"write\"]", + "\"handleops\",\"log\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"transaction\",\"verify\",\"version\"," + "\"write\"]", NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -751,11 +752,12 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," - "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"transaction\"," - "\"verify\",\"version\",\"write\"]", + "\"handleops\",\"log\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"transaction\",\"verify\",\"version\"," + "\"write\"]", NULL, 0 }, { "write_through", "list", NULL, "choices=[\"data\",\"log\"]", @@ -838,11 +840,12 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," - "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"transaction\"," - "\"verify\",\"version\",\"write\"]", + "\"handleops\",\"log\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"transaction\",\"verify\",\"version\"," + "\"write\"]", NULL, 0 }, { "version", "string", NULL, NULL, NULL, 0 }, { "write_through", "list", @@ -920,11 +923,12 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," - "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"transaction\"," - "\"verify\",\"version\",\"write\"]", + "\"handleops\",\"log\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"transaction\",\"verify\",\"version\"," + "\"write\"]", NULL, 0 }, { "version", "string", NULL, NULL, NULL, 0 }, { "write_through", "list", @@ -1002,11 +1006,12 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," - "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," - "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," - "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," - "\"split\",\"temporary\",\"thread_group\",\"transaction\"," - "\"verify\",\"version\",\"write\"]", + "\"handleops\",\"log\",\"lookaside_activity\",\"lsm\"," + "\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\",\"read\"," + "\"rebalance\",\"reconcile\",\"recovery\",\"recovery_progress\"," + "\"salvage\",\"shared_cache\",\"split\",\"temporary\"," + "\"thread_group\",\"transaction\",\"verify\",\"version\"," + "\"write\"]", NULL, 0 }, { "write_through", "list", NULL, "choices=[\"data\",\"log\"]", diff --git a/src/config/config_ext.c b/src/config/config_ext.c index 88f1390843a..d9e3771c707 100644 --- a/src/config/config_ext.c +++ b/src/config/config_ext.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/config/config_upgrade.c b/src/config/config_upgrade.c index e9ba38c6693..5f2770b1691 100644 --- a/src/config/config_upgrade.c +++ b/src/config/config_upgrade.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/conn/api_strerror.c b/src/conn/api_strerror.c index edb11957556..63f982deb07 100644 --- a/src/conn/api_strerror.c +++ b/src/conn/api_strerror.c @@ -18,8 +18,6 @@ const char * __wt_wiredtiger_error(int error) { - const char *p; - /* * Check for WiredTiger specific errors. */ @@ -42,14 +40,20 @@ __wt_wiredtiger_error(int error) return ("WT_CACHE_FULL: operation would overflow cache"); } + /* Windows strerror doesn't support ENOTSUP. */ + if (error == ENOTSUP) + return ("Operation not supported"); + /* - * POSIX errors are non-negative integers; check for 0 explicitly incase - * the underlying strerror doesn't handle 0, some historically didn't. + * Check for 0 in case the underlying strerror doesn't handle it, some + * historically didn't. */ if (error == 0) return ("Successful return: 0"); - if (error > 0 && (p = strerror(error)) != NULL) - return (p); + + /* POSIX errors are non-negative integers. */ + if (error > 0) + return (strerror(error)); return (NULL); } diff --git a/src/conn/api_version.c b/src/conn/api_version.c index a36cdb8d8eb..c4f3d978c1e 100644 --- a/src/conn/api_version.c +++ b/src/conn/api_version.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 68d45678965..70e96aa8473 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -175,13 +175,13 @@ __wt_conn_remove_collator(WT_SESSION_IMPL *session) conn = S2C(session); while ((ncoll = TAILQ_FIRST(&conn->collqh)) != NULL) { + /* Remove from the connection's list, free memory. */ + TAILQ_REMOVE(&conn->collqh, ncoll, q); /* Call any termination method. */ if (ncoll->collator->terminate != NULL) WT_TRET(ncoll->collator->terminate( ncoll->collator, (WT_SESSION *)session)); - /* Remove from the connection's list, free memory. */ - TAILQ_REMOVE(&conn->collqh, ncoll, q); __wt_free(session, ncoll->name); __wt_free(session, ncoll); } @@ -281,13 +281,13 @@ __wt_conn_remove_compressor(WT_SESSION_IMPL *session) conn = S2C(session); while ((ncomp = TAILQ_FIRST(&conn->compqh)) != NULL) { + /* Remove from the connection's list, free memory. */ + TAILQ_REMOVE(&conn->compqh, ncomp, q); /* Call any termination method. */ if (ncomp->compressor->terminate != NULL) WT_TRET(ncomp->compressor->terminate( ncomp->compressor, (WT_SESSION *)session)); - /* Remove from the connection's list, free memory. */ - TAILQ_REMOVE(&conn->compqh, ncomp, q); __wt_free(session, ncomp->name); __wt_free(session, ncomp); } @@ -346,13 +346,13 @@ __wt_conn_remove_data_source(WT_SESSION_IMPL *session) conn = S2C(session); while ((ndsrc = TAILQ_FIRST(&conn->dsrcqh)) != NULL) { + /* Remove from the connection's list, free memory. */ + TAILQ_REMOVE(&conn->dsrcqh, ndsrc, q); /* Call any termination method. */ if (ndsrc->dsrc->terminate != NULL) WT_TRET(ndsrc->dsrc->terminate( ndsrc->dsrc, (WT_SESSION *)session)); - /* Remove from the connection's list, free memory. */ - TAILQ_REMOVE(&conn->dsrcqh, ndsrc, q); __wt_free(session, ndsrc->prefix); __wt_free(session, ndsrc); } @@ -536,14 +536,16 @@ __wt_conn_remove_encryptor(WT_SESSION_IMPL *session) conn = S2C(session); while ((nenc = TAILQ_FIRST(&conn->encryptqh)) != NULL) { + /* Remove from the connection's list, free memory. */ + TAILQ_REMOVE(&conn->encryptqh, nenc, q); while ((kenc = TAILQ_FIRST(&nenc->keyedqh)) != NULL) { + /* Remove from the connection's list, free memory. */ + TAILQ_REMOVE(&nenc->keyedqh, kenc, q); /* Call any termination method. */ if (kenc->owned && kenc->encryptor->terminate != NULL) WT_TRET(kenc->encryptor->terminate( kenc->encryptor, (WT_SESSION *)session)); - /* Remove from the connection's list, free memory. */ - TAILQ_REMOVE(&nenc->keyedqh, kenc, q); __wt_free(session, kenc->keyid); __wt_free(session, kenc); } @@ -553,8 +555,6 @@ __wt_conn_remove_encryptor(WT_SESSION_IMPL *session) WT_TRET(nenc->encryptor->terminate( nenc->encryptor, (WT_SESSION *)session)); - /* Remove from the connection's list, free memory. */ - TAILQ_REMOVE(&conn->encryptqh, nenc, q); __wt_free(session, nenc->name); __wt_free(session, nenc); } @@ -680,13 +680,13 @@ __wt_conn_remove_extractor(WT_SESSION_IMPL *session) conn = S2C(session); while ((nextractor = TAILQ_FIRST(&conn->extractorqh)) != NULL) { + /* Remove from the connection's list, free memory. */ + TAILQ_REMOVE(&conn->extractorqh, nextractor, q); /* Call any termination method. */ if (nextractor->extractor->terminate != NULL) WT_TRET(nextractor->extractor->terminate( nextractor->extractor, (WT_SESSION *)session)); - /* Remove from the connection's list, free memory. */ - TAILQ_REMOVE(&conn->extractorqh, nextractor, q); __wt_free(session, nextractor->name); __wt_free(session, nextractor); } @@ -1803,6 +1803,7 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) { "fileops", WT_VERB_FILEOPS }, { "handleops", WT_VERB_HANDLEOPS }, { "log", WT_VERB_LOG }, + { "lookaside_activity", WT_VERB_LOOKASIDE }, { "lsm", WT_VERB_LSM }, { "lsm_manager", WT_VERB_LSM_MANAGER }, { "metadata", WT_VERB_METADATA }, diff --git a/src/conn/conn_cache.c b/src/conn/conn_cache.c index 28dd06332e0..5515eb026ca 100644 --- a/src/conn/conn_cache.c +++ b/src/conn/conn_cache.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -312,7 +312,7 @@ __wt_cache_destroy(WT_SESSION_IMPL *session) cache->bytes_dirty_intl + cache->bytes_dirty_leaf, cache->pages_dirty_intl + cache->pages_dirty_leaf); - WT_TRET(__wt_cond_destroy(session, &cache->evict_cond)); + __wt_cond_destroy(session, &cache->evict_cond); __wt_spin_destroy(session, &cache->evict_pass_lock); __wt_spin_destroy(session, &cache->evict_queue_lock); __wt_spin_destroy(session, &cache->evict_walk_lock); diff --git a/src/conn/conn_cache_pool.c b/src/conn/conn_cache_pool.c index ed078991581..adc2e2bffc3 100644 --- a/src/conn/conn_cache_pool.c +++ b/src/conn/conn_cache_pool.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -225,7 +225,7 @@ err: __wt_spin_unlock(session, &__wt_process.spinlock); __wt_free(session, pool_name); if (ret != 0 && created) { __wt_free(session, cp->name); - WT_TRET(__wt_cond_destroy(session, &cp->cache_pool_cond)); + __wt_cond_destroy(session, &cp->cache_pool_cond); __wt_free(session, cp); } return (ret); @@ -277,7 +277,7 @@ __wt_conn_cache_pool_open(WT_SESSION_IMPL *session) * the active connection shuts down. */ F_SET(cp, WT_CACHE_POOL_ACTIVE); - F_SET(cache, WT_CACHE_POOL_RUN); + FLD_SET(cache->pool_flags, WT_CACHE_POOL_RUN); WT_RET(__wt_thread_create(session, &cache->cp_tid, __wt_cache_pool_server, cache->cp_session)); @@ -340,7 +340,7 @@ __wt_conn_cache_pool_destroy(WT_SESSION_IMPL *session) __wt_spin_unlock(session, &cp->cache_pool_lock); cp_locked = false; - F_CLR(cache, WT_CACHE_POOL_RUN); + FLD_CLR(cache->pool_flags, WT_CACHE_POOL_RUN); __wt_cond_signal(session, cp->cache_pool_cond); WT_TRET(__wt_thread_join(session, cache->cp_tid)); @@ -391,7 +391,7 @@ __wt_conn_cache_pool_destroy(WT_SESSION_IMPL *session) __wt_free(session, cp->name); __wt_spin_destroy(session, &cp->cache_pool_lock); - WT_TRET(__wt_cond_destroy(session, &cp->cache_pool_cond)); + __wt_cond_destroy(session, &cp->cache_pool_cond); __wt_free(session, cp); } @@ -399,7 +399,7 @@ __wt_conn_cache_pool_destroy(WT_SESSION_IMPL *session) __wt_spin_unlock(session, &cp->cache_pool_lock); /* Notify other participants if we were managing */ - if (F_ISSET(cache, WT_CACHE_POOL_MANAGER)) { + if (FLD_ISSET(cache->pool_flags, WT_CACHE_POOL_MANAGER)) { cp->pool_managed = 0; __wt_verbose(session, WT_VERB_SHARED_CACHE, "Shutting down shared cache manager connection"); @@ -449,7 +449,8 @@ __cache_pool_balance(WT_SESSION_IMPL *session, bool forward) for (i = 0; i < 2 * WT_CACHE_POOL_BUMP_THRESHOLD && F_ISSET(cp, WT_CACHE_POOL_ACTIVE) && - F_ISSET(S2C(session)->cache, WT_CACHE_POOL_RUN); i++) { + FLD_ISSET(S2C(session)->cache->pool_flags, WT_CACHE_POOL_RUN); + i++) { __cache_pool_adjust( session, highest, bump_threshold, forward, &adjusted); /* @@ -760,7 +761,7 @@ __wt_cache_pool_server(void *arg) forward = true; while (F_ISSET(cp, WT_CACHE_POOL_ACTIVE) && - F_ISSET(cache, WT_CACHE_POOL_RUN)) { + FLD_ISSET(cache->pool_flags, WT_CACHE_POOL_RUN)) { if (cp->currently_used <= cp->size) __wt_cond_wait( session, cp->cache_pool_cond, WT_MILLION, NULL); @@ -770,12 +771,12 @@ __wt_cache_pool_server(void *arg) * lock on shutdown. */ if (!F_ISSET(cp, WT_CACHE_POOL_ACTIVE) && - F_ISSET(cache, WT_CACHE_POOL_RUN)) + FLD_ISSET(cache->pool_flags, WT_CACHE_POOL_RUN)) break; /* Try to become the managing thread */ if (__wt_atomic_cas8(&cp->pool_managed, 0, 1)) { - F_SET(cache, WT_CACHE_POOL_MANAGER); + FLD_SET(cache->pool_flags, WT_CACHE_POOL_MANAGER); __wt_verbose(session, WT_VERB_SHARED_CACHE, "Cache pool switched manager thread"); } @@ -784,7 +785,7 @@ __wt_cache_pool_server(void *arg) * Continue even if there was an error. Details of errors are * reported in the balance function. */ - if (F_ISSET(cache, WT_CACHE_POOL_MANAGER)) { + if (FLD_ISSET(cache->pool_flags, WT_CACHE_POOL_MANAGER)) { __cache_pool_balance(session, forward); forward = !forward; } diff --git a/src/conn/conn_ckpt.c b/src/conn/conn_ckpt.c index 7797ed4421c..a47524af2d7 100644 --- a/src/conn/conn_ckpt.c +++ b/src/conn/conn_ckpt.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -231,7 +231,7 @@ __wt_checkpoint_server_destroy(WT_SESSION_IMPL *session) WT_TRET(__wt_thread_join(session, conn->ckpt_tid)); conn->ckpt_tid_set = false; } - WT_TRET(__wt_cond_destroy(session, &conn->ckpt_cond)); + __wt_cond_destroy(session, &conn->ckpt_cond); /* Close the server thread's session. */ if (conn->ckpt_session != NULL) { diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c index 657cdebf7ee..97fdc7557ee 100644 --- a/src/conn/conn_dhandle.c +++ b/src/conn/conn_dhandle.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -52,7 +52,7 @@ __wt_conn_dhandle_alloc( WT_RET(__wt_calloc_one(session, &dhandle)); - __wt_rwlock_init(session, &dhandle->rwlock); + WT_ERR(__wt_rwlock_init(session, &dhandle->rwlock)); dhandle->name_hash = __wt_hash_city64(uri, strlen(uri)); WT_ERR(__wt_strdup(session, uri, &dhandle->name)); WT_ERR(__wt_strdup(session, checkpoint, &dhandle->checkpoint)); @@ -199,8 +199,13 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) /* Reset the tree's eviction priority (if any). */ __wt_evict_priority_clear(session); } - if (!marked_dead || final) - WT_ERR(__wt_checkpoint_close(session, final)); + if (!marked_dead || final) { + if ((ret = __wt_checkpoint_close( + session, final)) == EBUSY) + WT_ERR(ret); + else + WT_TRET(ret); + } } WT_TRET(__wt_btree_close(session)); @@ -364,8 +369,8 @@ __wt_conn_btree_open( F_SET(dhandle, WT_DHANDLE_OPEN); /* - * Checkpoint handles are read only, so eviction calculations - * based on the number of btrees are better to ignore them. + * Checkpoint handles are read-only, so eviction calculations based on + * the number of btrees are better to ignore them. */ if (dhandle->checkpoint == NULL) ++S2C(session)->open_btree_count; @@ -476,6 +481,49 @@ err: WT_DHANDLE_RELEASE(dhandle); } /* + * __conn_dhandle_close_one -- + * Lock and, if necessary, close a data handle. + */ +static int +__conn_dhandle_close_one(WT_SESSION_IMPL *session, + const char *uri, const char *checkpoint, bool force) +{ + WT_DECL_RET; + + /* + * Lock the handle exclusively. If this is part of schema-changing + * operation (indicated by metadata tracking being enabled), hold the + * lock for the duration of the operation. + */ + WT_RET(__wt_session_get_btree(session, uri, checkpoint, + NULL, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_LOCK_ONLY)); + if (WT_META_TRACKING(session)) + WT_RET(__wt_meta_track_handle_lock(session, false)); + + /* + * We have an exclusive lock, which means there are no cursors open at + * this point. Close the handle, if necessary. + */ + if (F_ISSET(session->dhandle, WT_DHANDLE_OPEN)) { + __wt_meta_track_sub_on(session); + ret = __wt_conn_btree_sync_and_close(session, false, force); + + /* + * If the close succeeded, drop any locks it acquired. If + * there was a failure, this function will fail and the whole + * transaction will be rolled back. + */ + if (ret == 0) + ret = __wt_meta_track_sub_off(session); + } + + if (!WT_META_TRACKING(session)) + WT_TRET(__wt_session_release_btree(session)); + + return (ret); +} + +/* * __wt_conn_dhandle_close_all -- * Close all data handles handles with matching name (including all * checkpoint handles). @@ -495,48 +543,22 @@ __wt_conn_dhandle_close_all( F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE)); WT_ASSERT(session, session->dhandle == NULL); + /* + * Lock the live handle first. This ordering is important: we rely on + * locking the live handle to fail fast if the tree is busy (e.g., with + * cursors open or in a checkpoint). + */ + WT_ERR(__conn_dhandle_close_one(session, uri, NULL, force)); + bucket = __wt_hash_city64(uri, strlen(uri)) % WT_HASH_ARRAY_SIZE; TAILQ_FOREACH(dhandle, &conn->dhhash[bucket], hashq) { if (strcmp(dhandle->name, uri) != 0 || + dhandle->checkpoint == NULL || F_ISSET(dhandle, WT_DHANDLE_DEAD)) continue; - session->dhandle = dhandle; - - /* - * Lock the handle exclusively. If this is part of - * schema-changing operation (indicated by metadata tracking - * being enabled), hold the lock for the duration of the - * operation. - */ - WT_ERR(__wt_session_get_btree(session, - dhandle->name, dhandle->checkpoint, - NULL, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_LOCK_ONLY)); - if (WT_META_TRACKING(session)) - WT_ERR(__wt_meta_track_handle_lock(session, false)); - - /* - * We have an exclusive lock, which means there are no cursors - * open at this point. Close the handle, if necessary. - */ - if (F_ISSET(dhandle, WT_DHANDLE_OPEN)) { - __wt_meta_track_sub_on(session); - ret = __wt_conn_btree_sync_and_close( - session, false, force); - - /* - * If the close succeeded, drop any locks it acquired. - * If there was a failure, this function will fail and - * the whole transaction will be rolled back. - */ - if (ret == 0) - ret = __wt_meta_track_sub_off(session); - } - - if (!WT_META_TRACKING(session)) - WT_TRET(__wt_session_release_btree(session)); - - WT_ERR(ret); + WT_ERR(__conn_dhandle_close_one( + session, dhandle->name, dhandle->checkpoint, force)); } err: session->dhandle = NULL; @@ -634,7 +656,7 @@ int __wt_conn_dhandle_discard(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; - WT_DATA_HANDLE *dhandle; + WT_DATA_HANDLE *dhandle, *dhandle_tmp; WT_DECL_RET; conn = S2C(session); @@ -680,10 +702,11 @@ restart: WT_TRET(session->meta_cursor->close(session->meta_cursor)); /* Close the metadata file handle. */ - while ((dhandle = TAILQ_FIRST(&conn->dhqh)) != NULL) + WT_TAILQ_SAFE_REMOVE_BEGIN(dhandle, &conn->dhqh, q, dhandle_tmp) { WT_WITH_DHANDLE(session, dhandle, WT_TRET(__wt_conn_dhandle_discard_single( session, true, F_ISSET(conn, WT_CONN_IN_MEMORY)))); + } WT_TAILQ_SAFE_REMOVE_END return (ret); } diff --git a/src/conn/conn_handle.c b/src/conn/conn_handle.c index 287e9ca7b99..2f3f9488b58 100644 --- a/src/conn/conn_handle.c +++ b/src/conn/conn_handle.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -62,14 +62,9 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_RET(__wt_spin_init(session, &conn->turtle_lock, "turtle file")); /* Read-write locks */ - __wt_rwlock_init(session, &conn->dhandle_lock); - __wt_rwlock_init(session, &conn->hot_backup_lock); - __wt_rwlock_init(session, &conn->table_lock); - - WT_RET(__wt_calloc_def(session, WT_PAGE_LOCKS, &conn->page_lock)); - for (i = 0; i < WT_PAGE_LOCKS; ++i) - WT_RET( - __wt_spin_init(session, &conn->page_lock[i], "btree page")); + WT_RWLOCK_INIT_TRACKED(session, &conn->dhandle_lock, dhandle); + WT_RET(__wt_rwlock_init(session, &conn->hot_backup_lock)); + WT_RWLOCK_INIT_TRACKED(session, &conn->table_lock, table); /* Setup the spin locks for the LSM manager queues. */ WT_RET(__wt_spin_init(session, @@ -81,15 +76,8 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_RET(__wt_cond_alloc( session, "LSM worker cond", &conn->lsm_manager.work_cond)); - /* - * Generation numbers. - * - * Start split generations at one. Threads publish this generation - * number before examining tree structures, and zero when they leave. - * We need to distinguish between threads that are in a tree before the - * first split has happened, and threads that are not in a tree. - */ - conn->split_gen = 1; + /* Initialize the generation manager. */ + __wt_gen_init(session); /* * Block manager. @@ -113,7 +101,6 @@ void __wt_connection_destroy(WT_CONNECTION_IMPL *conn) { WT_SESSION_IMPL *session; - u_int i; /* Check there's something to destroy. */ if (conn == NULL) @@ -144,9 +131,6 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) __wt_spin_destroy(session, &conn->schema_lock); __wt_rwlock_destroy(session, &conn->table_lock); __wt_spin_destroy(session, &conn->turtle_lock); - for (i = 0; i < WT_PAGE_LOCKS; ++i) - __wt_spin_destroy(session, &conn->page_lock[i]); - __wt_free(session, conn->page_lock); /* Free allocated memory. */ __wt_free(session, conn->cfg); diff --git a/src/conn/conn_log.c b/src/conn/conn_log.c index b8b5bd2a908..37acbe4a1a4 100644 --- a/src/conn/conn_log.c +++ b/src/conn/conn_log.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -391,13 +391,11 @@ __log_file_server(void *arg) WT_ERR(__wt_log_extract_lognum(session, close_fh->name, &filenum)); /* - * We update the close file handle before updating the - * close LSN when changing files. It is possible we - * could see mismatched settings. If we do, yield - * until it is set. This should rarely happen. + * The closing file handle should have a correct close + * LSN. */ - while (log->log_close_lsn.l.file < filenum) - __wt_yield(); + WT_ASSERT(session, + log->log_close_lsn.l.file == filenum); if (__wt_log_cmp( &log->write_lsn, &log->log_close_lsn) >= 0) { @@ -522,7 +520,7 @@ __log_file_server(void *arg) } if (0) { -err: __wt_err(session, ret, "log close server error"); +err: WT_PANIC_MSG(session, ret, "log close server error"); } if (locked) __wt_spin_unlock(session, &log->log_sync_lock); @@ -740,7 +738,8 @@ __log_wrlsn_server(void *arg) WT_ERR(__wt_log_force_write(session, 1, NULL)); __wt_log_wrlsn(session, NULL); if (0) { -err: __wt_err(session, ret, "log wrlsn server error"); +err: WT_PANIC_MSG(session, ret, "log wrlsn server error"); + } return (WT_THREAD_RET_VALUE); } @@ -844,7 +843,7 @@ __log_server(void *arg) } if (0) { -err: __wt_err(session, ret, "log server error"); +err: WT_PANIC_MSG(session, ret, "log server error"); } return (WT_THREAD_RET_VALUE); } @@ -880,7 +879,7 @@ __wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_spin_init(session, &log->log_sync_lock, "log sync")); WT_RET(__wt_spin_init(session, &log->log_writelsn_lock, "log write LSN")); - __wt_rwlock_init(session, &log->log_archive_lock); + WT_RET(__wt_rwlock_init(session, &log->log_archive_lock)); if (FLD_ISSET(conn->direct_io, WT_DIRECT_IO_LOG)) log->allocsize = (uint32_t) WT_MAX(conn->buffer_alignment, WT_LOG_ALIGN); @@ -902,7 +901,7 @@ __wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_cond_alloc(session, "log sync", &log->log_sync_cond)); WT_RET(__wt_cond_alloc(session, "log write", &log->log_write_cond)); WT_RET(__wt_log_open(session)); - WT_RET(__wt_log_slot_init(session)); + WT_RET(__wt_log_slot_init(session, true)); return (0); } @@ -1043,12 +1042,12 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) } /* Destroy the condition variables now that all threads are stopped */ - WT_TRET(__wt_cond_destroy(session, &conn->log_cond)); - WT_TRET(__wt_cond_destroy(session, &conn->log_file_cond)); - WT_TRET(__wt_cond_destroy(session, &conn->log_wrlsn_cond)); + __wt_cond_destroy(session, &conn->log_cond); + __wt_cond_destroy(session, &conn->log_file_cond); + __wt_cond_destroy(session, &conn->log_wrlsn_cond); - WT_TRET(__wt_cond_destroy(session, &conn->log->log_sync_cond)); - WT_TRET(__wt_cond_destroy(session, &conn->log->log_write_cond)); + __wt_cond_destroy(session, &conn->log->log_sync_cond); + __wt_cond_destroy(session, &conn->log->log_write_cond); __wt_rwlock_destroy(session, &conn->log->log_archive_lock); __wt_spin_destroy(session, &conn->log->log_lock); __wt_spin_destroy(session, &conn->log->log_slot_lock); diff --git a/src/conn/conn_open.c b/src/conn/conn_open.c index eb3c79422a0..ab7253c2828 100644 --- a/src/conn/conn_open.c +++ b/src/conn/conn_open.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -192,7 +192,7 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) for (i = 0; i < conn->session_size; ++s, ++i) { __wt_free(session, s->dhhash); __wt_free(session, s->tablehash); - __wt_split_stash_discard_all(session, s); + __wt_stash_discard_all(session, s); __wt_free(session, s->hazard); } diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c index d89392b66c6..f38d81a7f7a 100644 --- a/src/conn/conn_stat.c +++ b/src/conn/conn_stat.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -83,9 +83,9 @@ __wt_conn_stat_init(WT_SESSION_IMPL *session) stats, session_cursor_open, conn->open_cursor_count); WT_STAT_SET(session, stats, dh_conn_handle_count, conn->dhandle_count); WT_STAT_SET(session, - stats, rec_split_stashed_objects, conn->split_stashed_objects); + stats, rec_split_stashed_objects, conn->stashed_objects); WT_STAT_SET(session, - stats, rec_split_stashed_bytes, conn->split_stashed_bytes); + stats, rec_split_stashed_bytes, conn->stashed_bytes); } /* @@ -648,7 +648,7 @@ __wt_statlog_destroy(WT_SESSION_IMPL *session, bool is_close) WT_TRET(__wt_thread_join(session, conn->stat_tid)); conn->stat_tid_set = false; } - WT_TRET(__wt_cond_destroy(session, &conn->stat_cond)); + __wt_cond_destroy(session, &conn->stat_cond); /* Log a set of statistics on shutdown if configured. */ if (is_close) diff --git a/src/conn/conn_sweep.c b/src/conn/conn_sweep.c index 22d90b08438..df60a3c784d 100644 --- a/src/conn/conn_sweep.c +++ b/src/conn/conn_sweep.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -219,15 +219,12 @@ static int __sweep_remove_handles(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; - WT_DATA_HANDLE *dhandle, *dhandle_next; + WT_DATA_HANDLE *dhandle, *dhandle_tmp; WT_DECL_RET; conn = S2C(session); - for (dhandle = TAILQ_FIRST(&conn->dhqh); - dhandle != NULL; - dhandle = dhandle_next) { - dhandle_next = TAILQ_NEXT(dhandle, q); + TAILQ_FOREACH_SAFE(dhandle, &conn->dhqh, q, dhandle_tmp) { if (WT_IS_METADATA(dhandle)) continue; if (!WT_DHANDLE_CAN_DISCARD(dhandle)) @@ -432,7 +429,7 @@ __wt_sweep_destroy(WT_SESSION_IMPL *session) WT_TRET(__wt_thread_join(session, conn->sweep_tid)); conn->sweep_tid_set = 0; } - WT_TRET(__wt_cond_destroy(session, &conn->sweep_cond)); + __wt_cond_destroy(session, &conn->sweep_cond); if (conn->sweep_session != NULL) { wt_session = &conn->sweep_session->iface; diff --git a/src/cursor/cur_backup.c b/src/cursor/cur_backup.c index 61ced8d11e7..60750b88900 100644 --- a/src/cursor/cur_backup.c +++ b/src/cursor/cur_backup.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -119,8 +119,10 @@ __wt_curbackup_open(WT_SESSION_IMPL *session, __wt_cursor_notsup, /* search */ __wt_cursor_search_near_notsup, /* search-near */ __wt_cursor_notsup, /* insert */ + __wt_cursor_modify_notsup, /* modify */ __wt_cursor_notsup, /* update */ __wt_cursor_notsup, /* remove */ + __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ __curbackup_close); /* close */ WT_CURSOR *cursor; diff --git a/src/cursor/cur_bulk.c b/src/cursor/cur_bulk.c index 68611e30ff1..56bcbb741f7 100644 --- a/src/cursor/cur_bulk.c +++ b/src/cursor/cur_bulk.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -58,11 +58,11 @@ __curbulk_insert_fix(WT_CURSOR *cursor) if (F_ISSET(cursor, WT_CURSTD_APPEND)) recno = cbulk->recno + 1; else { - WT_CURSOR_CHECKKEY(cursor); + WT_ERR(__cursor_checkkey(cursor)); if ((recno = cursor->recno) <= cbulk->recno) WT_ERR(__bulk_col_keycmp_err(cbulk)); } - WT_CURSOR_CHECKVALUE(cursor); + WT_ERR(__cursor_checkvalue(cursor)); /* * Insert any skipped records as deleted records, update the current @@ -101,7 +101,7 @@ __curbulk_insert_fix_bitmap(WT_CURSOR *cursor) CURSOR_API_CALL(cursor, session, insert, btree); WT_STAT_DATA_INCR(session, cursor_insert_bulk); - WT_CURSOR_CHECKVALUE(cursor); + WT_ERR(__cursor_checkvalue(cursor)); /* Insert the current record. */ ret = __wt_bulk_insert_fix_bitmap(session, cbulk); @@ -140,11 +140,11 @@ __curbulk_insert_var(WT_CURSOR *cursor) if (F_ISSET(cursor, WT_CURSTD_APPEND)) recno = cbulk->recno + 1; else { - WT_CURSOR_CHECKKEY(cursor); + WT_ERR(__cursor_checkkey(cursor)); if ((recno = cursor->recno) <= cbulk->recno) WT_ERR(__bulk_col_keycmp_err(cbulk)); } - WT_CURSOR_CHECKVALUE(cursor); + WT_ERR(__cursor_checkvalue(cursor)); if (!cbulk->first_insert) { /* @@ -241,8 +241,8 @@ __curbulk_insert_row(WT_CURSOR *cursor) CURSOR_API_CALL(cursor, session, insert, btree); WT_STAT_DATA_INCR(session, cursor_insert_bulk); - WT_CURSOR_CHECKKEY(cursor); - WT_CURSOR_CHECKVALUE(cursor); + WT_ERR(__cursor_checkkey(cursor)); + WT_ERR(__cursor_checkvalue(cursor)); /* * If this isn't the first key inserted, compare it against the last key @@ -288,8 +288,8 @@ __curbulk_insert_row_skip_check(WT_CURSOR *cursor) CURSOR_API_CALL(cursor, session, insert, btree); WT_STAT_DATA_INCR(session, cursor_insert_bulk); - WT_CURSOR_CHECKKEY(cursor); - WT_CURSOR_CHECKVALUE(cursor); + WT_ERR(__cursor_checkkey(cursor)); + WT_ERR(__cursor_checkvalue(cursor)); ret = __wt_bulk_insert_row(session, cbulk); diff --git a/src/cursor/cur_config.c b/src/cursor/cur_config.c index 4001188e21c..6c198315e33 100644 --- a/src/cursor/cur_config.c +++ b/src/cursor/cur_config.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -39,8 +39,10 @@ __wt_curconfig_open(WT_SESSION_IMPL *session, __wt_cursor_notsup, /* search */ __wt_cursor_search_near_notsup, /* search-near */ __wt_cursor_notsup, /* insert */ + __wt_cursor_modify_notsup, /* modify */ __wt_cursor_notsup, /* update */ __wt_cursor_notsup, /* remove */ + __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ __curconfig_close); WT_CURSOR_CONFIG *cconfig; diff --git a/src/cursor/cur_ds.c b/src/cursor/cur_ds.c index 131d1ffa930..10de133be75 100644 --- a/src/cursor/cur_ds.c +++ b/src/cursor/cur_ds.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -42,7 +42,7 @@ __curds_key_set(WT_CURSOR *cursor) source = ((WT_CURSOR_DATA_SOURCE *)cursor)->source; - WT_CURSOR_NEEDKEY(cursor); + WT_ERR(__cursor_needkey(cursor)); source->recno = cursor->recno; source->key.data = cursor->key.data; @@ -63,7 +63,7 @@ __curds_value_set(WT_CURSOR *cursor) source = ((WT_CURSOR_DATA_SOURCE *)cursor)->source; - WT_CURSOR_NEEDVALUE(cursor); + WT_ERR(__cursor_needvalue(cursor)); source->value.data = cursor->value.data; source->value.size = cursor->value.size; @@ -142,8 +142,8 @@ __curds_compare(WT_CURSOR *a, WT_CURSOR *b, int *cmpp) WT_ERR_MSG(session, EINVAL, "Cursors must reference the same object"); - WT_CURSOR_NEEDKEY(a); - WT_CURSOR_NEEDKEY(b); + WT_ERR(__cursor_needkey(a)); + WT_ERR(__cursor_needkey(b)); if (WT_CURSOR_RECNO(a)) { if (a->recno < b->recno) @@ -317,7 +317,7 @@ __curds_insert(WT_CURSOR *cursor) source = ((WT_CURSOR_DATA_SOURCE *)cursor)->source; - CURSOR_UPDATE_API_CALL(cursor, session, insert, NULL); + CURSOR_UPDATE_API_CALL(cursor, session, insert); __curds_txn_enter(session); @@ -350,7 +350,7 @@ __curds_update(WT_CURSOR *cursor) source = ((WT_CURSOR_DATA_SOURCE *)cursor)->source; - CURSOR_UPDATE_API_CALL(cursor, session, update, NULL); + CURSOR_UPDATE_API_CALL(cursor, session, update); WT_STAT_CONN_INCR(session, cursor_update); WT_STAT_DATA_INCR(session, cursor_update); @@ -458,8 +458,10 @@ __wt_curds_open( __curds_search, /* search */ __curds_search_near, /* search-near */ __curds_insert, /* insert */ + __wt_cursor_modify_notsup, /* modify */ __curds_update, /* update */ __curds_remove, /* remove */ + __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ __curds_close); /* close */ WT_CONFIG_ITEM cval, metadata; diff --git a/src/cursor/cur_dump.c b/src/cursor/cur_dump.c index d7f18bb61ac..3e90d321db6 100644 --- a/src/cursor/cur_dump.c +++ b/src/cursor/cur_dump.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -369,8 +369,10 @@ __wt_curdump_create(WT_CURSOR *child, WT_CURSOR *owner, WT_CURSOR **cursorp) __curdump_search, /* search */ __curdump_search_near, /* search-near */ __curdump_insert, /* insert */ + __wt_cursor_modify_notsup, /* modify */ __curdump_update, /* update */ __curdump_remove, /* remove */ + __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ __curdump_close); /* close */ WT_CURSOR *cursor; diff --git a/src/cursor/cur_file.c b/src/cursor/cur_file.c index 205afb607c3..3b6328a2d93 100644 --- a/src/cursor/cur_file.c +++ b/src/cursor/cur_file.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -31,8 +31,8 @@ __curfile_compare(WT_CURSOR *a, WT_CURSOR *b, int *cmpp) WT_ERR_MSG(session, EINVAL, "Cursors must reference the same object"); - WT_CURSOR_CHECKKEY(a); - WT_CURSOR_CHECKKEY(b); + WT_ERR(__cursor_checkkey(a)); + WT_ERR(__cursor_checkkey(b)); ret = __wt_btcur_compare( (WT_CURSOR_BTREE *)a, (WT_CURSOR_BTREE *)b, cmpp); @@ -63,8 +63,8 @@ __curfile_equals(WT_CURSOR *a, WT_CURSOR *b, int *equalp) WT_ERR_MSG(session, EINVAL, "Cursors must reference the same object"); - WT_CURSOR_CHECKKEY(a); - WT_CURSOR_CHECKKEY(b); + WT_ERR(__cursor_checkkey(a)); + WT_ERR(__cursor_checkkey(b)); ret = __wt_btcur_equals( (WT_CURSOR_BTREE *)a, (WT_CURSOR_BTREE *)b, equalp); @@ -182,9 +182,7 @@ __curfile_search(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL(cursor, session, search, cbt->btree); - - WT_CURSOR_CHECKKEY(cursor); - WT_CURSOR_NOVALUE(cursor); + WT_ERR(__cursor_checkkey(cursor)); WT_ERR(__wt_btcur_search(cbt)); @@ -209,9 +207,7 @@ __curfile_search_near(WT_CURSOR *cursor, int *exact) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL(cursor, session, search_near, cbt->btree); - - WT_CURSOR_CHECKKEY(cursor); - WT_CURSOR_NOVALUE(cursor); + WT_ERR(__cursor_checkkey(cursor)); WT_ERR(__wt_btcur_search_near(cbt, exact)); @@ -235,11 +231,11 @@ __curfile_insert(WT_CURSOR *cursor) WT_SESSION_IMPL *session; cbt = (WT_CURSOR_BTREE *)cursor; - CURSOR_UPDATE_API_CALL(cursor, session, insert, cbt->btree); + CURSOR_UPDATE_API_CALL_BTREE(cursor, session, insert, cbt->btree); if (!F_ISSET(cursor, WT_CURSTD_APPEND)) - WT_CURSOR_CHECKKEY(cursor); - WT_CURSOR_CHECKVALUE(cursor); + WT_ERR(__cursor_checkkey(cursor)); + WT_ERR(__cursor_checkvalue(cursor)); WT_ERR(__wt_btcur_insert(cbt)); @@ -269,10 +265,8 @@ __wt_curfile_insert_check(WT_CURSOR *cursor) WT_SESSION_IMPL *session; cbt = (WT_CURSOR_BTREE *)cursor; - CURSOR_UPDATE_API_CALL(cursor, session, update, cbt->btree); - - WT_CURSOR_CHECKKEY(cursor); - WT_CURSOR_NOVALUE(cursor); + CURSOR_UPDATE_API_CALL_BTREE(cursor, session, update, cbt->btree); + WT_ERR(__cursor_checkkey(cursor)); ret = __wt_btcur_insert_check(cbt); @@ -292,10 +286,9 @@ __curfile_update(WT_CURSOR *cursor) WT_SESSION_IMPL *session; cbt = (WT_CURSOR_BTREE *)cursor; - CURSOR_UPDATE_API_CALL(cursor, session, update, cbt->btree); - - WT_CURSOR_CHECKKEY(cursor); - WT_CURSOR_CHECKVALUE(cursor); + CURSOR_UPDATE_API_CALL_BTREE(cursor, session, update, cbt->btree); + WT_ERR(__cursor_checkkey(cursor)); + WT_ERR(__cursor_checkvalue(cursor)); WT_ERR(__wt_btcur_update(cbt)); @@ -321,9 +314,7 @@ __curfile_remove(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_REMOVE_API_CALL(cursor, session, cbt->btree); - - WT_CURSOR_CHECKKEY(cursor); - WT_CURSOR_NOVALUE(cursor); + WT_ERR(__cursor_checkkey(cursor)); WT_ERR(__wt_btcur_remove(cbt)); @@ -343,6 +334,47 @@ err: CURSOR_UPDATE_API_END(session, ret); } /* + * __curfile_reserve -- + * WT_CURSOR->reserve method for the btree cursor type. + */ +static int +__curfile_reserve(WT_CURSOR *cursor) +{ + WT_CURSOR_BTREE *cbt; + WT_DECL_RET; + WT_SESSION_IMPL *session; + + cbt = (WT_CURSOR_BTREE *)cursor; + CURSOR_UPDATE_API_CALL_BTREE(cursor, session, reserve, cbt->btree); + WT_ERR(__cursor_checkkey(cursor)); + + WT_ERR(__wt_txn_context_check(session, true)); + + WT_ERR(__wt_btcur_reserve(cbt)); + + /* + * Reserve maintains a position and key, which doesn't match the library + * API, where reserve maintains a value. Fix the API by searching after + * each successful reserve operation. + */ + WT_ASSERT(session, + F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT); + WT_ASSERT(session, F_MASK(cursor, WT_CURSTD_VALUE_SET) == 0); + +err: CURSOR_UPDATE_API_END(session, ret); + + /* + * The application might do a WT_CURSOR.get_value call when we return, + * so we need a value and the underlying functions didn't set one up. + * For various reasons, those functions may not have done a search and + * any previous value in the cursor might race with WT_CURSOR.reserve + * (and in cases like LSM, the reserve never encountered the original + * key). For simplicity, repeat the search here. + */ + return (ret == 0 ? cursor->search(cursor) : ret); +} + +/* * __curfile_close -- * WT_CURSOR->close method for the btree cursor type. */ @@ -403,8 +435,10 @@ __curfile_create(WT_SESSION_IMPL *session, __curfile_search, /* search */ __curfile_search_near, /* search-near */ __curfile_insert, /* insert */ + __wt_cursor_modify_notsup, /* modify */ __curfile_update, /* update */ __curfile_remove, /* remove */ + __curfile_reserve, /* reserve */ __wt_cursor_reconfigure, /* reconfigure */ __curfile_close); /* close */ WT_BTREE *btree; @@ -486,7 +520,14 @@ __curfile_create(WT_SESSION_IMPL *session, WT_STAT_DATA_INCR(session, cursor_create); if (0) { -err: WT_TRET(__curfile_close(cursor)); +err: /* + * Our caller expects to release the data handle if we fail. + * Disconnect it from the cursor before closing. + */ + if (session->dhandle != NULL) + __wt_cursor_dhandle_decr_use(session); + cbt->btree = NULL; + WT_TRET(__curfile_close(cursor)); *cursorp = NULL; } diff --git a/src/cursor/cur_index.c b/src/cursor/cur_index.c index 6fc01c0421f..e8fcb1b2702 100644 --- a/src/cursor/cur_index.c +++ b/src/cursor/cur_index.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -66,8 +66,8 @@ __curindex_compare(WT_CURSOR *a, WT_CURSOR *b, int *cmpp) WT_ERR_MSG(session, EINVAL, "Cursors must reference the same object"); - WT_CURSOR_CHECKKEY(a); - WT_CURSOR_CHECKKEY(b); + WT_ERR(__cursor_checkkey(a)); + WT_ERR(__cursor_checkkey(b)); ret = __wt_compare( session, cindex->index->collator, &a->key, &b->key, cmpp); @@ -449,8 +449,10 @@ __wt_curindex_open(WT_SESSION_IMPL *session, __curindex_search, /* search */ __curindex_search_near, /* search-near */ __wt_cursor_notsup, /* insert */ + __wt_cursor_modify_notsup, /* modify */ __wt_cursor_notsup, /* update */ __wt_cursor_notsup, /* remove */ + __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ __curindex_close); /* close */ WT_CURSOR_INDEX *cindex; diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index 80afaf798dc..e4ccb90139e 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -34,6 +34,7 @@ static int __curjoin_split_key(WT_SESSION_IMPL *, WT_CURSOR_JOIN *, WT_ITEM *, */ int __wt_curjoin_joined(WT_CURSOR *cursor) + WT_GCC_FUNC_ATTRIBUTE((cold)) { WT_SESSION_IMPL *session; @@ -590,8 +591,10 @@ __curjoin_entry_member(WT_SESSION_IMPL *session, WT_CURSOR_JOIN_ENTRY *entry, __wt_cursor_notsup, /* search */ __wt_cursor_search_near_notsup, /* search-near */ __curjoin_extract_insert, /* insert */ + __wt_cursor_modify_notsup, /* modify */ __wt_cursor_notsup, /* update */ __wt_cursor_notsup, /* remove */ + __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ __wt_cursor_notsup); /* close */ WT_DECL_RET; @@ -1291,8 +1294,10 @@ __wt_curjoin_open(WT_SESSION_IMPL *session, __wt_cursor_notsup, /* search */ __wt_cursor_search_near_notsup, /* search-near */ __wt_cursor_notsup, /* insert */ + __wt_cursor_modify_notsup, /* modify */ __wt_cursor_notsup, /* update */ __wt_cursor_notsup, /* remove */ + __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ __curjoin_close); /* close */ WT_CURSOR *cursor; diff --git a/src/cursor/cur_json.c b/src/cursor/cur_json.c index e8ddb767863..99b4fc1ce4f 100644 --- a/src/cursor/cur_json.c +++ b/src/cursor/cur_json.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/cursor/cur_log.c b/src/cursor/cur_log.c index e5b56aa406f..38e9d4a1784 100644 --- a/src/cursor/cur_log.c +++ b/src/cursor/cur_log.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -342,8 +342,10 @@ __wt_curlog_open(WT_SESSION_IMPL *session, __curlog_search, /* search */ __wt_cursor_search_near_notsup, /* search-near */ __wt_cursor_notsup, /* insert */ + __wt_cursor_modify_notsup, /* modify */ __wt_cursor_notsup, /* update */ __wt_cursor_notsup, /* remove */ + __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ __curlog_close); /* close */ WT_CURSOR *cursor; diff --git a/src/cursor/cur_metadata.c b/src/cursor/cur_metadata.c index fbfc73956e2..d9aeed1fccd 100644 --- a/src/cursor/cur_metadata.c +++ b/src/cursor/cur_metadata.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -13,7 +13,7 @@ * backing metadata table cursor. */ #define WT_MD_CURSOR_NEEDKEY(cursor) do { \ - WT_CURSOR_NEEDKEY(cursor); \ + WT_ERR(__cursor_needkey(cursor)); \ WT_ERR(__wt_buf_set(session, \ &((WT_CURSOR_METADATA *)(cursor))->file_cursor->key, \ (cursor)->key.data, (cursor)->key.size)); \ @@ -22,7 +22,7 @@ } while (0) #define WT_MD_CURSOR_NEEDVALUE(cursor) do { \ - WT_CURSOR_NEEDVALUE(cursor); \ + WT_ERR(__cursor_needvalue(cursor)); \ WT_ERR(__wt_buf_set(session, \ &((WT_CURSOR_METADATA *)(cursor))->file_cursor->value, \ (cursor)->value.data, (cursor)->value.size)); \ @@ -550,8 +550,10 @@ __wt_curmetadata_open(WT_SESSION_IMPL *session, __curmetadata_search, /* search */ __curmetadata_search_near, /* search-near */ __curmetadata_insert, /* insert */ + __wt_cursor_modify_notsup, /* modify */ __curmetadata_update, /* update */ __curmetadata_remove, /* remove */ + __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ __curmetadata_close); /* close */ WT_CURSOR *cursor; diff --git a/src/cursor/cur_stat.c b/src/cursor/cur_stat.c index 0bff642370d..a1ec1d75918 100644 --- a/src/cursor/cur_stat.c +++ b/src/cursor/cur_stat.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -54,7 +54,7 @@ __curstat_get_key(WT_CURSOR *cursor, ...) va_start(ap, cursor); CURSOR_API_CALL(cursor, session, get_key, NULL); - WT_CURSOR_NEEDKEY(cursor); + WT_ERR(__cursor_needkey(cursor)); if (F_ISSET(cursor, WT_CURSTD_RAW)) { WT_ERR(__wt_struct_size( @@ -93,7 +93,7 @@ __curstat_get_value(WT_CURSOR *cursor, ...) va_start(ap, cursor); CURSOR_API_CALL(cursor, session, get_value, NULL); - WT_CURSOR_NEEDVALUE(cursor); + WT_ERR(__cursor_needvalue(cursor)); WT_ERR(cst->stats_desc(cst, WT_STAT_KEY_OFFSET(cst), &desc)); if (F_ISSET(cursor, WT_CURSTD_RAW)) { @@ -287,7 +287,7 @@ __curstat_search(WT_CURSOR *cursor) cst = (WT_CURSOR_STAT *)cursor; CURSOR_API_CALL(cursor, session, search, NULL); - WT_CURSOR_NEEDKEY(cursor); + WT_ERR(__cursor_needkey(cursor)); F_CLR(cursor, WT_CURSTD_VALUE_SET | WT_CURSTD_VALUE_SET); /* Initialize on demand. */ @@ -478,7 +478,7 @@ __curstat_join_desc(WT_CURSOR_STAT *cst, int slot, const char **resultp) strlen(static_desc) + 1; WT_RET(__wt_realloc(session, NULL, len, &cst->desc_buf)); WT_RET(__wt_snprintf( - cst->desc_buf, len, "join: %s%s", sgrp->desc_prefix, static_desc)); + cst->desc_buf, len, "join: %s%s", sgrp->desc_prefix, static_desc)); *resultp = cst->desc_buf; return (0); } @@ -576,8 +576,10 @@ __wt_curstat_open(WT_SESSION_IMPL *session, __curstat_search, /* search */ __wt_cursor_search_near_notsup, /* search-near */ __wt_cursor_notsup, /* insert */ + __wt_cursor_modify_notsup, /* modify */ __wt_cursor_notsup, /* update */ __wt_cursor_notsup, /* remove */ + __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ __curstat_close); /* close */ WT_CONFIG_ITEM cval, sval; diff --git a/src/cursor/cur_std.c b/src/cursor/cur_std.c index 99a9e373354..91995ab0e0a 100644 --- a/src/cursor/cur_std.c +++ b/src/cursor/cur_std.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -90,6 +90,19 @@ __wt_cursor_equals_notsup(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp) } /* + * __wt_cursor_modify_notsup -- + * Unsupported cursor modify. + */ +int +__wt_cursor_modify_notsup(WT_CURSOR *cursor, WT_MODIFY *entries, int nentries) +{ + WT_UNUSED(entries); + WT_UNUSED(nentries); + + return (__wt_cursor_notsup(cursor)); +} + +/* * __wt_cursor_search_near_notsup -- * Unsupported cursor search-near. */ @@ -136,6 +149,7 @@ __wt_cursor_set_notsup(WT_CURSOR *cursor) cursor->insert = __wt_cursor_notsup; cursor->update = __wt_cursor_notsup; cursor->remove = __wt_cursor_notsup; + cursor->reserve = __wt_cursor_notsup; } /* @@ -275,7 +289,7 @@ __wt_cursor_get_keyv(WT_CURSOR *cursor, uint32_t flags, va_list ap) const char *fmt; CURSOR_API_CALL(cursor, session, get_key, NULL); - if (!F_ISSET(cursor, WT_CURSTD_KEY_EXT | WT_CURSTD_KEY_INT)) + if (!F_ISSET(cursor, WT_CURSTD_KEY_SET)) WT_ERR(__wt_cursor_kv_not_set(cursor, true)); if (WT_CURSOR_RECNO(cursor)) { @@ -581,6 +595,100 @@ err: API_END(session, ret); } /* + * __cursor_modify -- + * WT_CURSOR->modify default implementation. + */ +static int +__cursor_modify(WT_CURSOR *cursor, WT_MODIFY *entries, int nentries) +{ + WT_DECL_RET; + WT_SESSION_IMPL *session; + WT_DECL_ITEM(ta); + WT_DECL_ITEM(tb); + WT_DECL_ITEM(tmp); + size_t len, size; + int i; + + CURSOR_UPDATE_API_CALL(cursor, session, modify); + WT_ERR(__cursor_checkkey(cursor)); + + /* Check for a rational modify vector count. */ + if (nentries <= 0) + WT_ERR_MSG( + session, EINVAL, "Illegal modify vector of %d", nentries); + + WT_STAT_CONN_INCR(session, cursor_modify); + WT_STAT_DATA_INCR(session, cursor_modify); + + /* Acquire position and value. */ + WT_ERR(cursor->search(cursor)); + + /* + * Process the entries to figure out how large a buffer we need. This is + * a bit pessimistic because we're ignoring replacement bytes, but it's + * a simpler calculation. + */ + for (size = cursor->value.size, i = 0; i < nentries; ++i) { + if (entries[i].offset >= size) + size = entries[i].offset; + size += entries[i].data.size; + } + + /* Allocate a pair of buffers. */ + WT_ERR(__wt_scr_alloc(session, size, &ta)); + WT_ERR(__wt_scr_alloc(session, size, &tb)); + + /* Apply the change vector to the value. */ + WT_ERR(__wt_buf_set( + session, ta, cursor->value.data, cursor->value.size)); + for (i = 0; i < nentries; ++i) { + /* Take leading bytes from the original, plus any gap bytes. */ + if (entries[i].offset >= ta->size) { + memcpy(tb->mem, ta->mem, ta->size); + if (entries[i].offset > ta->size) + memset((uint8_t *)tb->mem + ta->size, + '\0', entries[i].offset - ta->size); + } else + if (entries[i].offset > 0) + memcpy(tb->mem, ta->mem, entries[i].offset); + tb->size = entries[i].offset; + + /* Take replacement bytes. */ + if (entries[i].data.size > 0) { + memcpy((uint8_t *)tb->mem + tb->size, + entries[i].data.data, entries[i].data.size); + tb->size += entries[i].data.size; + } + + /* Take trailing bytes from the original. */ + len = entries[i].offset + entries[i].size; + if (ta->size > len) { + memcpy((uint8_t *)tb->mem + tb->size, + (uint8_t *)ta->mem + len, ta->size - len); + tb->size += ta->size - len; + } + WT_ASSERT(session, tb->size <= size); + + tmp = ta; + ta = tb; + tb = tmp; + } + + /* Set the cursor's value. */ + ta->data = ta->mem; + cursor->set_value(cursor, ta); + + /* We know both key and value are set, "overwrite" doesn't matter. */ + ret = cursor->update(cursor); + +err: __wt_scr_free(session, &ta); + __wt_scr_free(session, &tb); + + CURSOR_UPDATE_API_END(session, ret); + return (ret); +} + +/* * __wt_cursor_reconfigure -- * Set runtime-configurable settings. */ @@ -705,15 +813,17 @@ __wt_cursor_init(WT_CURSOR *cursor, WT_RET(__wt_config_gets_def(session, cfg, "checkpoint", 0, &cval)); if (cval.len != 0) { cursor->insert = __wt_cursor_notsup; - cursor->update = __wt_cursor_notsup; cursor->remove = __wt_cursor_notsup; + cursor->reserve = __wt_cursor_notsup; + cursor->update = __wt_cursor_notsup; } else { WT_RET( __wt_config_gets_def(session, cfg, "readonly", 0, &cval)); if (cval.val != 0 || F_ISSET(S2C(session), WT_CONN_READONLY)) { cursor->insert = __wt_cursor_notsup; - cursor->update = __wt_cursor_notsup; cursor->remove = __wt_cursor_notsup; + cursor->reserve = __wt_cursor_notsup; + cursor->update = __wt_cursor_notsup; } } @@ -754,6 +864,14 @@ __wt_cursor_init(WT_CURSOR *cursor, F_SET(cursor, WT_CURSTD_RAW); /* + * WT_CURSOR.modify supported on 'u' value formats, but may have been + * already initialized. + */ + if (WT_STREQ(cursor->value_format, "u") && + cursor->modify == __wt_cursor_modify_notsup) + cursor->modify = __cursor_modify; + + /* * Cursors that are internal to some other cursor (such as file cursors * inside a table cursor) should be closed after the containing cursor. * Arrange for that to happen by putting internal cursors after their diff --git a/src/cursor/cur_table.c b/src/cursor/cur_table.c index 3b72bb0730f..000fcae99f2 100644 --- a/src/cursor/cur_table.c +++ b/src/cursor/cur_table.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -91,8 +91,10 @@ __wt_apply_single_idx(WT_SESSION_IMPL *session, WT_INDEX *idx, __wt_cursor_notsup, /* search */ __wt_cursor_search_near_notsup, /* search-near */ __curextract_insert, /* insert */ + __wt_cursor_modify_notsup, /* modify */ __wt_cursor_notsup, /* update */ __wt_cursor_notsup, /* remove */ + __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ __wt_cursor_notsup); /* close */ WT_CURSOR_EXTRACTOR extract_cursor; @@ -110,8 +112,7 @@ __wt_apply_single_idx(WT_SESSION_IMPL *session, WT_INDEX *idx, WT_RET(__wt_cursor_get_raw_key(&ctable->iface, &key)); WT_RET(__wt_cursor_get_raw_value(&ctable->iface, &value)); ret = idx->extractor->extract(idx->extractor, - &session->iface, &key, &value, - &extract_cursor.iface); + &session->iface, &key, &value, &extract_cursor.iface); __wt_buf_free(session, &extract_cursor.iface.key); WT_RET(ret); @@ -190,12 +191,13 @@ __wt_curtable_get_value(WT_CURSOR *cursor, ...) WT_SESSION_IMPL *session; va_list ap; - va_start(ap, cursor); JOINABLE_CURSOR_API_CALL(cursor, session, get_value, NULL); - WT_ERR(__wt_curtable_get_valuev(cursor, ap)); -err: va_end(ap); - API_END_RET(session, ret); + va_start(ap, cursor); + ret = __wt_curtable_get_valuev(cursor, ap); + va_end(ap); + +err: API_END_RET(session, ret); } /* @@ -323,8 +325,8 @@ __curtable_compare(WT_CURSOR *a, WT_CURSOR *b, int *cmpp) if (strcmp(a->internal_uri, b->internal_uri) != 0) WT_ERR_MSG(session, EINVAL, "comparison method cursors must reference the same object"); - WT_CURSOR_CHECKKEY(WT_CURSOR_PRIMARY(a)); - WT_CURSOR_CHECKKEY(WT_CURSOR_PRIMARY(b)); + WT_ERR(__cursor_checkkey(WT_CURSOR_PRIMARY(a))); + WT_ERR(__cursor_checkkey(WT_CURSOR_PRIMARY(b))); ret = WT_CURSOR_PRIMARY(a)->compare( WT_CURSOR_PRIMARY(a), WT_CURSOR_PRIMARY(b), cmpp); @@ -483,7 +485,7 @@ __curtable_insert(WT_CURSOR *cursor) u_int i; ctable = (WT_CURSOR_TABLE *)cursor; - JOINABLE_CURSOR_UPDATE_API_CALL(cursor, session, insert, NULL); + JOINABLE_CURSOR_UPDATE_API_CALL(cursor, session, insert); WT_ERR(__curtable_open_indices(ctable)); /* @@ -562,7 +564,7 @@ __curtable_update(WT_CURSOR *cursor) WT_SESSION_IMPL *session; ctable = (WT_CURSOR_TABLE *)cursor; - JOINABLE_CURSOR_UPDATE_API_CALL(cursor, session, update, NULL); + JOINABLE_CURSOR_UPDATE_API_CALL(cursor, session, update); WT_ERR(__curtable_open_indices(ctable)); /* @@ -661,6 +663,47 @@ err: CURSOR_UPDATE_API_END(session, ret); } /* + * __curtable_reserve -- + * WT_CURSOR->reserve method for the table cursor type. + */ +static int +__curtable_reserve(WT_CURSOR *cursor) +{ + WT_CURSOR_TABLE *ctable; + WT_DECL_RET; + WT_SESSION_IMPL *session; + + ctable = (WT_CURSOR_TABLE *)cursor; + JOINABLE_CURSOR_UPDATE_API_CALL(cursor, session, update); + + /* + * We don't have to open the indices here, but it makes the code similar + * to other cursor functions, and it's odd for a reserve call to succeed + * but the subsequent update fail opening indices. + * + * Check for a transaction before index open, opening the indices will + * start a transaction if one isn't running. + */ + WT_ERR(__wt_txn_context_check(session, true)); + WT_ERR(__curtable_open_indices(ctable)); + + /* Reserve in column groups, ignore indices. */ + APPLY_CG(ctable, reserve); + +err: CURSOR_UPDATE_API_END(session, ret); + + /* + * The application might do a WT_CURSOR.get_value call when we return, + * so we need a value and the underlying functions didn't set one up. + * For various reasons, those functions may not have done a search and + * any previous value in the cursor might race with WT_CURSOR.reserve + * (and in cases like LSM, the reserve never encountered the original + * key). For simplicity, repeat the search here. + */ + return (ret == 0 ? cursor->search(cursor) : ret); +} + +/* * __wt_table_range_truncate -- * Truncate of a cursor range, table implementation. */ @@ -907,8 +950,10 @@ __wt_curtable_open(WT_SESSION_IMPL *session, __curtable_search, /* search */ __curtable_search_near, /* search-near */ __curtable_insert, /* insert */ + __wt_cursor_modify_notsup, /* modify */ __curtable_update, /* update */ __curtable_remove, /* remove */ + __curtable_reserve, /* reserve */ __wt_cursor_reconfigure, /* reconfigure */ __curtable_close); /* close */ WT_CONFIG_ITEM cval; @@ -943,6 +988,12 @@ __wt_curtable_open(WT_SESSION_IMPL *session, table->cgroups[0]->source, NULL, cfg, cursorp); __wt_schema_release_table(session, table); + if (ret == 0) { + /* Fix up the public URI to match what was passed in. */ + cursor = *cursorp; + __wt_free(session, cursor->uri); + WT_TRET(__wt_strdup(session, uri, &cursor->uri)); + } return (ret); } diff --git a/src/docs/Doxyfile b/src/docs/Doxyfile index 3d8c46962f1..e7382e2bc5e 100644 --- a/src/docs/Doxyfile +++ b/src/docs/Doxyfile @@ -1582,6 +1582,7 @@ PREDEFINED = DOXYGEN \ __wt_file_system:=WT_FILE_SYSTEM \ __wt_item:=WT_ITEM \ __wt_lsn:=WT_LSN \ + __wt_modify:=WT_MODIFY \ __wt_session:=WT_SESSION \ __wt_txn_notify:=WT_TXN_NOTIFY \ WT_HANDLE_CLOSED(x):=x \ diff --git a/src/docs/backup.dox b/src/docs/backup.dox index 45edc85d6a5..91b15da9275 100644 --- a/src/docs/backup.dox +++ b/src/docs/backup.dox @@ -59,6 +59,11 @@ During the period the backup cursor is open, database checkpoints can be created, but no checkpoints can be deleted. This may result in significant file growth. +Additionally, if a crash occurs during the period the backup cursor is open and +logging is disabled, then the system will be restored to the most recent +checkpoint prior to the opening of the backup cursor, even if later database +checkpoints were created. + The following is a programmatic example of creating a backup: @snippet ex_all.c backup diff --git a/src/docs/build-javadoc.sh b/src/docs/build-javadoc.sh index be886937070..69cb1186467 100755 --- a/src/docs/build-javadoc.sh +++ b/src/docs/build-javadoc.sh @@ -8,5 +8,5 @@ CLASSPATH=$THRIFT_HOME/libthrift.jar:$SLF4J_JAR javadoc -public -d $DOCS/java \ -stylesheetfile $DOCS/style/javadoc.css \ -use -link http://java.sun.com/j2se/1.5.0/docs/api/ \ -header '<b>WiredTiger API</b><br><font size="-1"> version '$WT_VERSION'</font>' \ - -windowtitle 'WiredTiger Java API' -bottom '<font size=1>Copyright (c) 2008-2016 MongoDB, Inc. All rights reserved.</font>' \ + -windowtitle 'WiredTiger Java API' -bottom '<font size=1>Copyright (c) 2008-2017 MongoDB, Inc. All rights reserved.</font>' \ com.wiredtiger com.wiredtiger.util diff --git a/src/docs/error-handling.dox b/src/docs/error-handling.dox index 62be498fc15..eb9ca6bb82a 100644 --- a/src/docs/error-handling.dox +++ b/src/docs/error-handling.dox @@ -17,13 +17,18 @@ are thrown as \c WiredTigerException, which may be caught by the application. The \c WiredTigerRollbackException is a specific type of \c WiredTigerException, -it is thrown when there is a conflict between concurrent operations. +thrown when there is a conflict between concurrent operations. An application that catches this exception should call rollback() on the relevant transaction, and retry as necessary. The \c WiredTigerPanicException is a specific type of \c WiredTigerException, -it is thrown when there is an underlying problem that requires the -application to exit and restart. +thrown when there is a fatal error requiring database restart. Applications +will normally handle \c WiredTigerPanicException as a special case. A +correctly-written WiredTiger application will likely catch +\c WiredTigerPanicException and immediately exit or otherwise handle fatal +errors. Note that no further WiredTiger calls are required after +\c WiredTigerPanicException is caught (and further calls will themselves +immediately fail). The following is a complete list of possible WiredTiger-specific return values, all constants defined in the com.wiredtiger.db.wiredtiger class: @@ -47,7 +52,7 @@ This error is returned when an error is not covered by a specific error return. This error indicates an operation did not find a value to return. This includes cursor search and other operations where no record matched the cursor's search key such as WT_CURSOR::update or WT_CURSOR::remove. @par <code>WT_PANIC</code> -This error indicates an underlying problem that requires the application exit and restart. The application can exit immediately when \c WT_PANIC is returned from a WiredTiger interface, no further WiredTiger calls are required. +This error indicates an underlying problem that requires a database restart. The application may exit immediately, no further WiredTiger calls are required (and further calls will themselves immediately fail). @par <code>WT_RUN_RECOVERY</code> This error is generated when wiredtiger_open is configured to return an error if recovery is required to use the database. @@ -73,7 +78,7 @@ Note that ::wiredtiger_strerror is not thread-safe. @m_if{c} @section error_handling_event Error handling using the WT_EVENT_HANDLER -More complex error handling can be configured by passing an implementation +Specific error handling can be configured by passing an implementation of WT_EVENT_HANDLER to ::wiredtiger_open or WT_CONNECTION::open_session. For example, both informational and error messages might be passed to an @@ -81,6 +86,17 @@ application-specific logging function that added a timestamp and logged the message to a file, and error messages might additionally be output to the \c stderr file stream. +Additionally, applications will normally handle \c WT_PANIC as a special +case. WiredTiger will always call the error handler callback with +\c WT_PANIC in the case of a fatal error requiring database restart, +however, WiredTiger cannot guarantee applications will see an application +thread return \c WT_PANIC from a WiredTiger API call. For this reason, a +correctly-written WiredTiger application will likely specify at least an +error handler which will immediately exit or otherwise handle fatal errors. +Note that no further WiredTiger calls are required after an error handler +is called with \c WT_PANIC (and further calls will themselves immediately +fail). + @snippet ex_event_handler.c Function event_handler @snippet ex_event_handler.c Configure event_handler diff --git a/src/docs/programming.dox b/src/docs/programming.dox index aa76bef4614..205e7544c6c 100644 --- a/src/docs/programming.dox +++ b/src/docs/programming.dox @@ -65,19 +65,20 @@ each of which is ordered by one or more columns. - @subpage_single wtperf - @subpage_single wtstats <p> -- @subpage_single tune_memory_allocator -- @subpage_single tune_page_size_and_comp -- @subpage_single tune_cache +- @subpage_single tune_build_options - @subpage_single tune_bulk_load +- @subpage_single tune_cache +- @subpage_single tune_checksum +- @subpage_single tune_close - @subpage_single tune_cursor_persist -- @subpage_single tune_read_only - @subpage_single tune_durability -- @subpage_single tune_checksum - @subpage_single tune_file_alloc +- @subpage_single tune_memory_allocator +- @subpage_single tune_mutex +- @subpage_single tune_page_size_and_comp +- @subpage_single tune_read_only - @subpage_single tune_system_buffer_cache - @subpage_single tune_transparent_huge_pages -- @subpage_single tune_close -- @subpage_single tune_mutex - @subpage_single tune_zone_reclaim */ diff --git a/src/docs/spell.ok b/src/docs/spell.ok index bc2e16b1122..5d629f4c49f 100644 --- a/src/docs/spell.ok +++ b/src/docs/spell.ok @@ -237,6 +237,7 @@ fput freelist fsync ftruncate +fvisibility gcc gdbm ge diff --git a/src/docs/style/footer.html b/src/docs/style/footer.html index e5a7b30eef5..12d25422f89 100644 --- a/src/docs/style/footer.html +++ b/src/docs/style/footer.html @@ -3,13 +3,13 @@ <div id="nav-path" class="navpath"><!-- id is needed for treeview function! --> <ul> $navpath - <li class="footer">Copyright (c) 2008-2016 MongoDB, Inc. All rights reserved. Contact <a href="mailto:info@wiredtiger.com">info@wiredtiger.com</a> for more information.</li> + <li class="footer">Copyright (c) 2008-2017 MongoDB, Inc. All rights reserved. Contact <a href="mailto:info@wiredtiger.com">info@wiredtiger.com</a> for more information.</li> </ul> </div> <!--END GENERATE_TREEVIEW--> <!--BEGIN !GENERATE_TREEVIEW--> <hr class="footer"/><address class="footer"><small> -Copyright (c) 2008-2016 MongoDB, Inc. All rights reserved. Contact <a href="mailto:info@wiredtiger.com">info@wiredtiger.com</a> for more information. +Copyright (c) 2008-2017 MongoDB, Inc. All rights reserved. Contact <a href="mailto:info@wiredtiger.com">info@wiredtiger.com</a> for more information. </small></address> <!--END !GENERATE_TREEVIEW--> </body> diff --git a/src/docs/tools/doxfilter.py b/src/docs/tools/doxfilter.py index f1c3308c689..301142269c3 100755 --- a/src/docs/tools/doxfilter.py +++ b/src/docs/tools/doxfilter.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/src/docs/tools/fixlinks.py b/src/docs/tools/fixlinks.py index 7163246e3bd..1887665d5be 100755 --- a/src/docs/tools/fixlinks.py +++ b/src/docs/tools/fixlinks.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/src/docs/top/main.dox b/src/docs/top/main.dox index 6b28bd0062f..1bfb623c0a0 100644 --- a/src/docs/top/main.dox +++ b/src/docs/top/main.dox @@ -6,12 +6,12 @@ WiredTiger is an high performance, scalable, production quality, NoSQL, @section releases Releases <table> -@row{<b>WiredTiger 2.9.2</b> (current), +@row{<b>WiredTiger 2.9.3</b> (current), + <a href="releases/wiredtiger-2.9.3.tar.bz2"><b>[Release package]</b></a>, + <a href="2.9.3/index.html"><b>[Documentation]</b></a>} +@row{<b>WiredTiger 2.9.2</b> (previous), <a href="releases/wiredtiger-2.9.2.tar.bz2"><b>[Release package]</b></a>, <a href="2.9.2/index.html"><b>[Documentation]</b></a>} -@row{<b>WiredTiger 2.8.0</b> (previous), - <a href="releases/wiredtiger-2.8.0.tar.bz2"><b>[Release package]</b></a>, - <a href="2.8.0/index.html"><b>[Documentation]</b></a>} @row{<b>Development branch</b>, <a href="https://github.com/wiredtiger/wiredtiger"><b>[Source code]</b></a>, <a href="develop/index.html"><b>[Documentation]</b></a>} diff --git a/src/docs/tune-build-options.dox b/src/docs/tune-build-options.dox new file mode 100644 index 00000000000..79cd60b1105 --- /dev/null +++ b/src/docs/tune-build-options.dox @@ -0,0 +1,9 @@ +/*! @page tune_build_options gcc/clang build options + +WiredTiger can be built using the gcc/clang \c -fvisibility=hidden flag, +which may significantly reduce the size and load time of the WiredTiger +library when built as a dynamic shared object, and allow the optimizer +to produce better code (for example, by eliminating most lookups in the +procedure linkage table). + + */ diff --git a/src/docs/upgrading.dox b/src/docs/upgrading.dox index e5fce3d0d5d..8640991e7cd 100644 --- a/src/docs/upgrading.dox +++ b/src/docs/upgrading.dox @@ -1,5 +1,18 @@ /*! @page upgrading Upgrading WiredTiger applications +@section version_293 Upgrading to Version 2.9.3 +<dl> + +<dt>Logging subsystem statistics</dt> +<dd> +Two logging subsystem statistics have been removed as they were a duplicate of +other statistics. The \c log_slot_joins and \c log_slot_transitions statistics +are no longer present. They were duplicates of \c log_writes and +\c log_slot_closes respectively. Several new logging related statistics have +been added. +</dd> + +</dl><hr> @section version_292 Upgrading to Version 2.9.2 <dl> @@ -16,6 +29,15 @@ have switched that lock from a spin lock to a read-write lock, and consequently changed the statistics tracking lock related wait time. </dd> +<dt>Logging subsystem statistics</dt> +<dd> +Two logging subsystem statistics have been removed as they were a duplicate of +other statistics. The \c log_slot_joins and \c log_slot_transitions statistics +are no longer present. They were duplicates of \c log_writes and +\c log_slot_closes respectively. Several new logging related statistics have +been added. +</dd> + <dt>Forced and named checkpoint error conditions changed</dt> <dd> There are new cases where checkpoints created with an explicit name or the diff --git a/src/evict/evict_file.c b/src/evict/evict_file.c index 3d8f4a61ca7..56638934305 100644 --- a/src/evict/evict_file.c +++ b/src/evict/evict_file.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index 26bbf9f679b..46291eb63de 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -15,7 +15,7 @@ static int __evict_lru_walk(WT_SESSION_IMPL *); static int __evict_page(WT_SESSION_IMPL *, bool); static int __evict_pass(WT_SESSION_IMPL *); static int __evict_server(WT_SESSION_IMPL *, bool *); -static int __evict_tune_workers(WT_SESSION_IMPL *session); +static void __evict_tune_workers(WT_SESSION_IMPL *session); static int __evict_walk(WT_SESSION_IMPL *, WT_EVICT_QUEUE *); static int __evict_walk_file( WT_SESSION_IMPL *, WT_EVICT_QUEUE *, u_int, u_int *); @@ -31,28 +31,17 @@ static int __evict_walk_file( static int __evict_lock_handle_list(WT_SESSION_IMPL *session) { - struct timespec enter, leave; WT_CACHE *cache; WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_RWLOCK *dh_lock; u_int spins; - bool dh_stats; conn = S2C(session); cache = conn->cache; dh_lock = &conn->dhandle_lock; /* - * Setup tracking of handle lock acquisition wait time if statistics - * are enabled. - */ - dh_stats = WT_STAT_ENABLED(session); - - if (dh_stats) - __wt_epoch(session, &enter); - - /* * Use a custom lock acquisition back off loop so the eviction server * notices any interrupt quickly. */ @@ -64,17 +53,7 @@ __evict_lock_handle_list(WT_SESSION_IMPL *session) else __wt_sleep(0, WT_THOUSAND); } - /* - * Only record statistics on success. - */ - WT_RET(ret); - if (dh_stats) { - __wt_epoch(session, &leave); - WT_STAT_CONN_INCRV( - session, lock_handle_list_wait_eviction, - (int64_t)WT_TIMEDIFF_US(leave, enter)); - } - return (0); + return (ret); } /* @@ -95,12 +74,8 @@ __evict_entry_priority(WT_SESSION_IMPL *session, WT_REF *ref) if (page->read_gen == WT_READGEN_OLDEST) return (WT_READGEN_OLDEST); - /* - * Any leaf page from a dead tree is a great choice (not internal pages, - * they may have children and are not yet evictable). - */ - if (!WT_PAGE_IS_INTERNAL(page) && - F_ISSET(btree->dhandle, WT_DHANDLE_DEAD)) + /* Any page from a dead tree is a great choice. */ + if (F_ISSET(btree->dhandle, WT_DHANDLE_DEAD)) return (WT_READGEN_OLDEST); /* Any empty page (leaf or internal), is a good choice. */ @@ -123,8 +98,10 @@ __evict_entry_priority(WT_SESSION_IMPL *session, WT_REF *ref) read_gen = page->read_gen; read_gen += btree->evict_priority; + +#define WT_EVICT_INTL_SKEW 1000 if (WT_PAGE_IS_INTERNAL(page)) - read_gen += WT_EVICT_INT_SKEW; + read_gen += WT_EVICT_INTL_SKEW; return (read_gen); } @@ -271,8 +248,19 @@ __wt_evict_server_wake(WT_SESSION_IMPL *session) } /* + * __wt_evict_thread_chk -- + * Check to decide if the eviction thread should continue running. + */ +bool +__wt_evict_thread_chk(WT_SESSION_IMPL *session) +{ + return (F_ISSET(S2C(session), WT_CONN_EVICTION_RUN)); +} + +/* * __wt_evict_thread_run -- - * Starting point for an eviction thread. + * Entry function for an eviction thread. This is called repeatedly + * from the thread group code so it does not need to loop itself. */ int __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) @@ -285,73 +273,83 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) conn = S2C(session); cache = conn->cache; -#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) /* - * Ensure the cache stuck timer is initialized when starting eviction. + * The thread group code calls us repeatedly. So each call is one pass + * through eviction. */ - if (thread->id == 0) - __wt_epoch(session, &cache->stuck_ts); -#endif - - while (F_ISSET(conn, WT_CONN_EVICTION_RUN) && - F_ISSET(thread, WT_THREAD_RUN)) { - if (conn->evict_server_running && - __wt_spin_trylock(session, &cache->evict_pass_lock) == 0) { - /* - * Cannot use WT_WITH_PASS_LOCK because this is a try - * lock. Fix when that is supported. We set the flag - * on both sessions because we may call clear_walk when - * we are walking with the walk session, locked. - */ - F_SET(session, WT_SESSION_LOCKED_PASS); - F_SET(cache->walk_session, WT_SESSION_LOCKED_PASS); - ret = __evict_server(session, &did_work); - F_CLR(cache->walk_session, WT_SESSION_LOCKED_PASS); - F_CLR(session, WT_SESSION_LOCKED_PASS); - was_intr = cache->pass_intr != 0; - __wt_spin_unlock(session, &cache->evict_pass_lock); - WT_ERR(ret); - - /* - * If the eviction server was interrupted, wait until - * requests have been processed: the system may - * otherwise be busy so don't go to sleep. - */ - if (was_intr) { - while (cache->pass_intr != 0 && - F_ISSET(conn, WT_CONN_EVICTION_RUN) && - F_ISSET(thread, WT_THREAD_RUN)) - __wt_yield(); - continue; - } + if (conn->evict_server_running && + __wt_spin_trylock(session, &cache->evict_pass_lock) == 0) { + /* + * Cannot use WT_WITH_PASS_LOCK because this is a try lock. + * Fix when that is supported. We set the flag on both sessions + * because we may call clear_walk when we are walking with + * the walk session, locked. + */ + F_SET(session, WT_SESSION_LOCKED_PASS); + F_SET(cache->walk_session, WT_SESSION_LOCKED_PASS); + ret = __evict_server(session, &did_work); + F_CLR(cache->walk_session, WT_SESSION_LOCKED_PASS); + F_CLR(session, WT_SESSION_LOCKED_PASS); + was_intr = cache->pass_intr != 0; + __wt_spin_unlock(session, &cache->evict_pass_lock); + WT_ERR(ret); + /* + * If the eviction server was interrupted, wait until requests + * have been processed: the system may otherwise be busy so + * don't go to sleep. + */ + if (was_intr) + while (cache->pass_intr != 0 && + F_ISSET(conn, WT_CONN_EVICTION_RUN) && + F_ISSET(thread, WT_THREAD_RUN)) + __wt_yield(); + else { __wt_verbose(session, WT_VERB_EVICTSERVER, "sleeping"); /* Don't rely on signals: check periodically. */ - __wt_cond_auto_wait( - session, cache->evict_cond, did_work, NULL); + __wt_cond_auto_wait(session, + cache->evict_cond, did_work, NULL); __wt_verbose(session, WT_VERB_EVICTSERVER, "waking"); - } else - WT_ERR(__evict_lru_pages(session, false)); + } + } else + WT_ERR(__evict_lru_pages(session, false)); + + if (0) { +err: WT_PANIC_MSG(session, ret, "cache eviction thread error"); } + return (ret); +} + +/* + * __wt_evict_thread_stop -- + * Shutdown function for an eviction thread. + */ +int +__wt_evict_thread_stop(WT_SESSION_IMPL *session, WT_THREAD *thread) +{ + WT_CACHE *cache; + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + + if (thread->id != 0) + return (0); + conn = S2C(session); + cache = conn->cache; /* * The only time the first eviction thread is stopped is on shutdown: * in case any trees are still open, clear all walks now so that they * can be closed. */ - if (thread->id == 0) { - WT_WITH_PASS_LOCK(session, - ret = __evict_clear_all_walks(session)); - WT_ERR(ret); - /* - * The only two cases when the eviction server is expected to - * stop are when recovery is finished or when the connection is - * closing. - */ - WT_ASSERT(session, - F_ISSET(conn, WT_CONN_CLOSING | WT_CONN_RECOVERING)); - } + WT_WITH_PASS_LOCK(session, ret = __evict_clear_all_walks(session)); + WT_ERR(ret); + /* + * The only two cases when the eviction server is expected to + * stop are when recovery is finished or when the connection is + * closing. + */ + WT_ASSERT(session, F_ISSET(conn, WT_CONN_CLOSING | WT_CONN_RECOVERING)); __wt_verbose( session, WT_VERB_EVICTSERVER, "cache eviction thread exiting"); @@ -472,7 +470,15 @@ __wt_evict_create(WT_SESSION_IMPL *session) */ WT_RET(__wt_thread_group_create(session, &conn->evict_threads, "eviction-server", conn->evict_threads_min, conn->evict_threads_max, - WT_THREAD_CAN_WAIT | WT_THREAD_PANIC_FAIL, __wt_evict_thread_run)); + WT_THREAD_CAN_WAIT | WT_THREAD_PANIC_FAIL, __wt_evict_thread_chk, + __wt_evict_thread_run, __wt_evict_thread_stop)); + +#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) + /* + * Ensure the cache stuck timer is initialized when starting eviction. + */ + __wt_epoch(session, &conn->cache->stuck_ts); +#endif /* * Allow queues to be populated now that the eviction threads @@ -535,7 +541,7 @@ __evict_update_work(WT_SESSION_IMPL *session) cache = conn->cache; /* Clear previous state. */ - F_CLR(cache, WT_CACHE_EVICT_MASK); + cache->flags = 0; if (!F_ISSET(conn, WT_CONN_EVICTION_RUN)) return (false); @@ -592,8 +598,7 @@ __evict_update_work(WT_SESSION_IMPL *session) F_CLR(cache, WT_CACHE_EVICT_CLEAN | WT_CACHE_EVICT_CLEAN_HARD); } - WT_STAT_CONN_SET(session, cache_eviction_state, - F_MASK(cache, WT_CACHE_EVICT_MASK)); + WT_STAT_CONN_SET(session, cache_eviction_state, cache->flags); return (F_ISSET(cache, WT_CACHE_EVICT_ALL | WT_CACHE_EVICT_URGENT)); } @@ -628,7 +633,7 @@ __evict_pass(WT_SESSION_IMPL *session) prev = now; if (conn->evict_threads.threads[0]->session == session) - WT_RET(__evict_tune_workers(session)); + __evict_tune_workers(session); /* * Increment the shared read generation. Do this occasionally * even if eviction is not currently required, so that pages @@ -880,10 +885,8 @@ void __wt_evict_file_exclusive_off(WT_SESSION_IMPL *session) { WT_BTREE *btree; - WT_CACHE *cache; btree = S2BT(session); - cache = S2C(session)->cache; /* * We have seen subtle bugs with multiple threads racing to turn @@ -891,12 +894,26 @@ __wt_evict_file_exclusive_off(WT_SESSION_IMPL *session) */ WT_DIAGNOSTIC_YIELD; - /* Hold the walk lock to turn on eviction. */ - __wt_spin_lock(session, &cache->evict_walk_lock); - WT_ASSERT(session, - btree->evict_ref == NULL && btree->evict_disabled > 0); - --btree->evict_disabled; - __wt_spin_unlock(session, &cache->evict_walk_lock); + /* + * Atomically decrement the evict-disabled count, without acquiring the + * eviction walk-lock. We can't acquire that lock here because there's + * a potential deadlock. When acquiring exclusive eviction access, we + * acquire the eviction walk-lock and then the cache's pass-intr lock. + * The current eviction implementation can hold the pass-intr lock and + * call into this function (see WT-3303 for the details), which might + * deadlock with another thread trying to get exclusive eviction access. + */ +#if defined(HAVE_DIAGNOSTIC) + { + int32_t v; + + WT_ASSERT(session, btree->evict_ref == NULL); + v = __wt_atomic_subi32(&btree->evict_disabled, 1); + WT_ASSERT(session, v >= 0); + } +#else + (void)__wt_atomic_subi32(&btree->evict_disabled, 1); +#endif } #define EVICT_TUNE_BATCH 1 /* Max workers to add each period */ @@ -927,13 +944,12 @@ __wt_evict_file_exclusive_off(WT_SESSION_IMPL *session) * curve. In that case, we will set the number of workers to the best observed * so far and settle into a stable state. */ -static int +static void __evict_tune_workers(WT_SESSION_IMPL *session) { struct timespec current_time; WT_CACHE *cache; WT_CONNECTION_IMPL *conn; - WT_DECL_RET; uint64_t delta_msec, delta_pages; uint64_t pgs_evicted_cur, pgs_evicted_persec_cur, time_diff; int32_t cur_threads, i, target_threads, thread_surplus; @@ -942,7 +958,7 @@ __evict_tune_workers(WT_SESSION_IMPL *session) cache = conn->cache; WT_ASSERT(session, conn->evict_threads.threads[0]->session == session); - pgs_evicted_cur = pgs_evicted_persec_cur = 0; + pgs_evicted_cur = 0; __wt_epoch(session, ¤t_time); time_diff = WT_TIMEDIFF_SEC(current_time, conn->evict_tune_last_time); @@ -953,7 +969,7 @@ __evict_tune_workers(WT_SESSION_IMPL *session) */ if (conn->evict_tune_stable) { if (time_diff < EVICT_FORCE_RETUNE) - return (0); + return; /* * Stable state was reached a long time ago. Let's re-tune. @@ -972,8 +988,8 @@ __evict_tune_workers(WT_SESSION_IMPL *session) (int32_t)conn->evict_threads_min; for (i = 0; i < thread_surplus; i++) { - WT_ERR(__wt_thread_group_stop_one( - session, &conn->evict_threads, false)); + __wt_thread_group_stop_one( + session, &conn->evict_threads); WT_STAT_CONN_INCR(session, cache_eviction_worker_removed); } @@ -985,7 +1001,7 @@ __evict_tune_workers(WT_SESSION_IMPL *session) * anything unless enough time has passed since the last * time we have taken any action in this function. */ - return (0); + return; /* * Measure the number of evicted pages so far. Eviction rate correlates @@ -1001,7 +1017,7 @@ __evict_tune_workers(WT_SESSION_IMPL *session) * Otherwise, we just record the number of evicted pages and return. */ if (conn->evict_tune_pgs_last == 0) - goto err; + goto done; delta_msec = WT_TIMEDIFF_MS(current_time, conn->evict_tune_last_time); delta_pages = pgs_evicted_cur - conn->evict_tune_pgs_last; @@ -1052,15 +1068,10 @@ __evict_tune_workers(WT_SESSION_IMPL *session) (int32_t)conn->evict_tune_workers_best; for (i = 0; i < thread_surplus; i++) { - /* - * If we get an error, it should be because we - * were unable to acquire the thread group lock. - * Break out of trying. - */ - WT_ERR(__wt_thread_group_stop_one( - session, &conn->evict_threads, false)); + __wt_thread_group_stop_one( + session, &conn->evict_threads); WT_STAT_CONN_INCR(session, - cache_eviction_worker_removed); + cache_eviction_worker_removed); } WT_STAT_CONN_SET(session, cache_eviction_stable_state_workers, @@ -1068,7 +1079,7 @@ __evict_tune_workers(WT_SESSION_IMPL *session) conn->evict_tune_stable = true; WT_STAT_CONN_SET(session, cache_eviction_active_workers, conn->evict_threads.current_threads); - goto err; + goto done; } } @@ -1091,13 +1102,8 @@ __evict_tune_workers(WT_SESSION_IMPL *session) * Start the new threads. */ for (i = cur_threads; i < target_threads; ++i) { - /* - * If we get an error, it should be because we were - * unable to acquire the thread group lock. Break out - * of trying. - */ - WT_ERR(__wt_thread_group_start_one(session, - &conn->evict_threads, false)); + __wt_thread_group_start_one(session, + &conn->evict_threads, false); WT_STAT_CONN_INCR(session, cache_eviction_worker_created); __wt_verbose(session, WT_VERB_EVICTSERVER, @@ -1109,15 +1115,8 @@ __evict_tune_workers(WT_SESSION_IMPL *session) WT_STAT_CONN_SET(session, cache_eviction_active_workers, conn->evict_threads.current_threads); -err: conn->evict_tune_last_time = current_time; +done: conn->evict_tune_last_time = current_time; conn->evict_tune_pgs_last = pgs_evicted_cur; - /* - * If we got an EBUSY trying to acquire the lock just return. - * We can try to tune the workers next time. - */ - if (ret == EBUSY) - ret = 0; - return (ret); } /* @@ -1136,13 +1135,13 @@ __evict_lru_pages(WT_SESSION_IMPL *session, bool is_server) * Reconcile and discard some pages: EBUSY is returned if a page fails * eviction because it's unavailable, continue in that case. */ - while (F_ISSET(S2C(session), WT_CONN_EVICTION_RUN) && ret == 0) + while (F_ISSET(conn, WT_CONN_EVICTION_RUN) && ret == 0) if ((ret = __evict_page(session, is_server)) == EBUSY) ret = 0; /* If a worker thread found the queue empty, pause. */ if (ret == WT_NOTFOUND && !is_server && - F_ISSET(S2C(session), WT_CONN_EVICTION_RUN)) + F_ISSET(conn, WT_CONN_EVICTION_RUN)) __wt_cond_wait( session, conn->evict_threads.wait_cond, 10000, NULL); @@ -1327,7 +1326,7 @@ __evict_walk(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue) bool dhandle_locked, incr; conn = S2C(session); - cache = S2C(session)->cache; + cache = conn->cache; btree = NULL; dhandle = NULL; dhandle_locked = incr = false; @@ -1470,7 +1469,8 @@ retry: while (slot < max_entries) { ret = __evict_walk_file( session, queue, max_entries, &slot)); - WT_ASSERT(session, session->split_gen == 0); + WT_ASSERT(session, __wt_session_gen( + session, WT_GEN_SPLIT) == 0); } __wt_spin_unlock(session, &cache->evict_walk_lock); WT_ERR(ret); @@ -1568,7 +1568,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_EVICT_ENTRY *end, *evict, *start; - WT_PAGE *page; + WT_PAGE *last_parent, *page; WT_PAGE_MODIFY *mod; WT_REF *ref; WT_TXN_GLOBAL *txn_global; @@ -1576,14 +1576,15 @@ __evict_walk_file(WT_SESSION_IMPL *session, uint64_t pages_seen, pages_queued, refs_walked; uint32_t remaining_slots, total_slots, walk_flags; uint32_t target_pages_clean, target_pages_dirty, target_pages; - int internal_pages, restarts; + int restarts; bool give_up, modified, urgent_queued; conn = S2C(session); btree = S2BT(session); cache = conn->cache; txn_global = &conn->txn_global; - internal_pages = restarts = 0; + last_parent = NULL; + restarts = 0; give_up = urgent_queued = false; /* @@ -1684,7 +1685,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, * whether to give up. When we are only looking for dirty pages, * search the tree for longer. */ - min_pages = 10 * target_pages; + min_pages = 10 * (uint64_t)target_pages; if (F_ISSET(cache, WT_CACHE_EVICT_DIRTY) && !F_ISSET(cache, WT_CACHE_EVICT_CLEAN)) min_pages *= 10; @@ -1738,6 +1739,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, */ for (evict = start, pages_queued = pages_seen = refs_walked = 0; evict < end && (ret == 0 || ret == WT_NOTFOUND); + last_parent = ref == NULL ? NULL : ref->home, ret = __wt_tree_walk_count( session, &ref, &refs_walked, walk_flags)) { /* @@ -1818,10 +1820,23 @@ __evict_walk_file(WT_SESSION_IMPL *session, if (modified && !F_ISSET(cache, WT_CACHE_EVICT_DIRTY)) continue; - /* Limit internal pages to 50% of the total. */ - if (WT_PAGE_IS_INTERNAL(page) && - internal_pages > (int)(evict - start) / 2) - continue; + /* + * Don't attempt eviction of internal pages with children in + * cache (indicated by seeing an internal page that is the + * parent of the last page we saw). + * + * Also skip internal page unless we get aggressive or the tree + * is idle (indicated by the tree being skipped for walks). + * The goal here is that if trees become completely idle, we + * eventually push them out of cache completely. + */ + if (WT_PAGE_IS_INTERNAL(page)) { + if (page == last_parent) + continue; + if (btree->evict_walk_period == 0 && + !__wt_cache_aggressive(session)) + continue; + } /* If eviction gets aggressive, anything else is fair game. */ if (__wt_cache_aggressive(session)) @@ -1850,9 +1865,6 @@ fast: /* If the page can't be evicted, give up. */ ++evict; ++pages_queued; - if (WT_PAGE_IS_INTERNAL(page)) - ++internal_pages; - __wt_verbose(session, WT_VERB_EVICTSERVER, "select: %p, size %" WT_SIZET_FMT, (void *)page, page->memory_footprint); diff --git a/src/evict/evict_page.c b/src/evict/evict_page.c index 85689efd0b1..d50326afb1e 100644 --- a/src/evict/evict_page.c +++ b/src/evict/evict_page.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -55,10 +55,12 @@ __wt_page_release_evict(WT_SESSION_IMPL *session, WT_REF *ref) WT_BTREE *btree; WT_DECL_RET; WT_PAGE *page; + struct timespec start, stop; bool locked, too_big; btree = S2BT(session); page = ref->page; + __wt_epoch(session, &start); /* * Take some care with order of operations: if we release the hazard @@ -75,19 +77,34 @@ __wt_page_release_evict(WT_SESSION_IMPL *session, WT_REF *ref) (void)__wt_atomic_addv32(&btree->evict_busy, 1); too_big = page->memory_footprint >= btree->splitmempage; - if ((ret = __wt_evict(session, ref, false)) == 0) { - if (too_big) + + /* + * Track how long the call to evict took. If eviction is successful then + * we have one of two pairs of stats to increment. + */ + ret = __wt_evict(session, ref, false); + __wt_epoch(session, &stop); + if (ret == 0) { + if (too_big) { WT_STAT_CONN_INCR(session, cache_eviction_force); - else + WT_STAT_CONN_INCRV(session, cache_eviction_force_time, + WT_TIMEDIFF_US(stop, start)); + } else { /* * If the page isn't too big, we are evicting it because * it had a chain of deleted entries that make traversal * expensive. */ - WT_STAT_CONN_INCR( - session, cache_eviction_force_delete); - } else + WT_STAT_CONN_INCR(session, cache_eviction_force_delete); + WT_STAT_CONN_INCRV(session, + cache_eviction_force_delete_time, + WT_TIMEDIFF_US(stop, start)); + } + } else { WT_STAT_CONN_INCR(session, cache_eviction_force_fail); + WT_STAT_CONN_INCRV(session, cache_eviction_force_fail_time, + WT_TIMEDIFF_US(stop, start)); + } (void)__wt_atomic_subv32(&btree->evict_busy, 1); @@ -113,6 +130,9 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool closing) /* Checkpoints should never do eviction. */ WT_ASSERT(session, !WT_SESSION_IS_CHECKPOINT(session)); + /* Enter the eviction generation. */ + __wt_session_gen_enter(session, WT_GEN_EVICT); + page = ref->page; tree_dead = F_ISSET(session->dhandle, WT_DHANDLE_DEAD); @@ -133,7 +153,7 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool closing) * we want: there is nothing more to do. */ if (LF_ISSET(WT_EVICT_INMEM_SPLIT)) - return (0); + goto done; /* Count evictions of internal pages during normal operation. */ if (!closing && WT_PAGE_IS_INTERNAL(page)) { @@ -156,7 +176,7 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool closing) /* Update the reference and discard the page. */ if (__wt_ref_is_root(ref)) __wt_ref_out(session, ref); - else if ((clean_page && !LF_ISSET(WT_EVICT_IN_MEMORY)) || tree_dead) + else if ((clean_page && !F_ISSET(conn, WT_CONN_IN_MEMORY)) || tree_dead) /* * Pages that belong to dead trees never write back to disk * and can't support page splits. @@ -182,6 +202,9 @@ err: if (!closing) WT_STAT_DATA_INCR(session, cache_eviction_fail); } +done: /* Leave the eviction generation. */ + __wt_session_gen_leave(session, WT_GEN_EVICT); + return (ret); } @@ -202,8 +225,8 @@ __evict_delete_ref(WT_SESSION_IMPL *session, WT_REF *ref, bool closing) return (0); /* - * Avoid doing reverse splits when closing the file, it is - * wasted work and some structure may already have been freed. + * Avoid doing reverse splits when closing the file, it is wasted work + * and some structures may have already been freed. */ if (!closing) { parent = ref->home; @@ -393,11 +416,13 @@ __evict_review( WT_SESSION_IMPL *session, WT_REF *ref, uint32_t *flagsp, bool closing) { WT_CACHE *cache; + WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_PAGE *page; uint32_t flags; - bool lookaside_retry, modified; + bool lookaside_retry, *lookaside_retryp, modified; + conn = S2C(session); flags = WT_EVICTING; *flagsp = flags; @@ -453,7 +478,7 @@ __evict_review( * Clean pages can't be evicted when running in memory only. This * should be uncommon - we don't add clean pages to the queue. */ - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY) && !modified && !closing) + if (F_ISSET(conn, WT_CONN_IN_MEMORY) && !modified && !closing) return (EBUSY); /* Check if the page can be evicted. */ @@ -479,10 +504,6 @@ __evict_review( */ if (LF_ISSET(WT_EVICT_INMEM_SPLIT)) return (__wt_split_insert(session, ref)); - - /* If splits are the only permitted operation, we're done. */ - if (F_ISSET(S2BT(session), WT_BTREE_ALLOW_SPLITS)) - return (EBUSY); } /* If the page is clean, we're done and we can evict. */ @@ -519,11 +540,14 @@ __evict_review( * Additionally, if we aren't trying to free space in the cache, scrub * the page and keep it in memory. */ - cache = S2C(session)->cache; + cache = conn->cache; + lookaside_retry = false; + lookaside_retryp = NULL; + if (closing) LF_SET(WT_VISIBILITY_ERR); else if (!WT_PAGE_IS_INTERNAL(page)) { - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) + if (F_ISSET(conn, WT_CONN_IN_MEMORY)) LF_SET(WT_EVICT_IN_MEMORY | WT_EVICT_SCRUB | WT_EVICT_UPDATE_RESTORE); else { @@ -531,21 +555,26 @@ __evict_review( if (F_ISSET(cache, WT_CACHE_EVICT_SCRUB)) LF_SET(WT_EVICT_SCRUB); + + /* + * Check if reconciliation suggests trying the + * lookaside table. + */ + lookaside_retryp = &lookaside_retry; } } /* Reconcile the page. */ - ret = __wt_reconcile(session, ref, NULL, flags, &lookaside_retry); + ret = __wt_reconcile(session, ref, NULL, flags, lookaside_retryp); /* - * If reconciliation fails, eviction is stuck and reconciliation reports - * it might succeed if we use the lookaside table (the page didn't have - * uncommitted updates, it was not-yet-globally visible updates causing - * the problem), configure reconciliation to write those updates to the - * lookaside table, allowing the eviction of pages we'd otherwise have - * to retain in cache to support older readers. + * If reconciliation fails, eviction is stuck and reconciliation + * reports it might succeed if we use the lookaside table, then + * configure reconciliation to write those updates to the lookaside + * table, allowing the eviction of pages we'd otherwise have to retain + * in cache to support older readers. */ - if (ret == EBUSY && __wt_cache_stuck(session) && lookaside_retry) { + if (ret == EBUSY && lookaside_retry && __wt_cache_stuck(session)) { LF_CLR(WT_EVICT_SCRUB | WT_EVICT_UPDATE_RESTORE); LF_SET(WT_EVICT_LOOKASIDE); ret = __wt_reconcile(session, ref, NULL, flags, NULL); diff --git a/src/evict/evict_stat.c b/src/evict/evict_stat.c index 7c2d5722a63..276e737ebbb 100644 --- a/src/evict/evict_stat.c +++ b/src/evict/evict_stat.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/api.h b/src/include/api.h index a3636eb8040..372ba063cd3 100644 --- a/src/include/api.h +++ b/src/include/api.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -12,21 +12,19 @@ const char *__oldname = (s)->name; \ (s)->dhandle = (dh); \ (s)->name = (s)->lastop = #h "." #n; \ - -#define API_CALL_NOCONF(s, h, n, dh) do { \ - API_SESSION_INIT(s, h, n, dh); \ WT_ERR(WT_SESSION_CHECK_PANIC(s)); \ __wt_verbose((s), WT_VERB_API, "CALL: " #h ":" #n) +#define API_CALL_NOCONF(s, h, n, dh) do { \ + API_SESSION_INIT(s, h, n, dh) + #define API_CALL(s, h, n, dh, config, cfg) do { \ const char *(cfg)[] = \ { WT_CONFIG_BASE(s, h##_##n), config, NULL }; \ API_SESSION_INIT(s, h, n, dh); \ - WT_ERR(WT_SESSION_CHECK_PANIC(s)); \ if ((config) != NULL) \ WT_ERR(__wt_config_check((s), \ - WT_CONFIG_REF(session, h##_##n), (config), 0)); \ - __wt_verbose((s), WT_VERB_API, "CALL: " #h ":" #n) + WT_CONFIG_REF(session, h##_##n), (config), 0)) #define API_END(s, ret) \ if ((s) != NULL) { \ @@ -49,9 +47,9 @@ F_SET(&(s)->txn, WT_TXN_AUTOCOMMIT) /* An API call wrapped in a transaction if necessary. */ -#define TXN_API_CALL_NOCONF(s, h, n, bt) do { \ +#define TXN_API_CALL_NOCONF(s, h, n, dh) do { \ bool __autotxn = false; \ - API_CALL_NOCONF(s, h, n, bt); \ + API_CALL_NOCONF(s, h, n, dh); \ __autotxn = !F_ISSET(&(s)->txn, WT_TXN_AUTOCOMMIT | WT_TXN_RUNNING);\ if (__autotxn) \ F_SET(&(s)->txn, WT_TXN_AUTOCOMMIT) @@ -135,17 +133,21 @@ CURSOR_REMOVE_API_CALL(cur, s, bt); \ JOINABLE_CURSOR_CALL_CHECK(cur) -#define CURSOR_UPDATE_API_CALL(cur, s, n, bt) \ +#define CURSOR_UPDATE_API_CALL_BTREE(cur, s, n, bt) \ (s) = (WT_SESSION_IMPL *)(cur)->session; \ - TXN_API_CALL_NOCONF(s, WT_CURSOR, n, \ - ((bt) == NULL) ? NULL : ((WT_BTREE *)(bt))->dhandle); \ + TXN_API_CALL_NOCONF( \ + s, WT_CURSOR, n, ((WT_BTREE *)(bt))->dhandle); \ if (F_ISSET(S2C(s), WT_CONN_IN_MEMORY) && \ !F_ISSET((WT_BTREE *)(bt), WT_BTREE_IGNORE_CACHE) && \ __wt_cache_full(s)) \ WT_ERR(WT_CACHE_FULL); -#define JOINABLE_CURSOR_UPDATE_API_CALL(cur, s, n, bt) \ - CURSOR_UPDATE_API_CALL(cur, s, n, bt); \ +#define CURSOR_UPDATE_API_CALL(cur, s, n) \ + (s) = (WT_SESSION_IMPL *)(cur)->session; \ + TXN_API_CALL_NOCONF(s, WT_CURSOR, n, NULL); + +#define JOINABLE_CURSOR_UPDATE_API_CALL(cur, s, n) \ + CURSOR_UPDATE_API_CALL(cur, s, n); \ JOINABLE_CURSOR_CALL_CHECK(cur) #define CURSOR_UPDATE_API_END(s, ret) \ diff --git a/src/include/async.h b/src/include/async.h index 7a415a4a17a..53a7d982ba5 100644 --- a/src/include/async.h +++ b/src/include/async.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/bitstring.i b/src/include/bitstring.i index 118dc0bba01..d3dc3bebd0f 100644 --- a/src/include/bitstring.i +++ b/src/include/bitstring.i @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/src/include/block.h b/src/include/block.h index 0cf76e34367..b93ed948eee 100644 --- a/src/include/block.h +++ b/src/include/block.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/bloom.h b/src/include/bloom.h index ddc2d64a118..a0efc0bf1fa 100644 --- a/src/include/bloom.h +++ b/src/include/bloom.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/btmem.h b/src/include/btmem.h index b1d5df4e9d2..32839192a96 100644 --- a/src/include/btmem.h +++ b/src/include/btmem.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -208,7 +208,7 @@ struct __wt_ovfl_txnc { */ #define WT_LAS_FORMAT \ "key_format=" WT_UNCHECKED_STRING(IuQQu) \ - ",value_format=" WT_UNCHECKED_STRING(QIu) + ",value_format=" WT_UNCHECKED_STRING(QBu) /* * WT_PAGE_MODIFY -- @@ -414,18 +414,20 @@ struct __wt_page_modify { size_t discard_allocated; } *ovfl_track; +#define WT_PAGE_LOCK(s, p) \ + __wt_spin_lock((s), &(p)->modify->page_lock) +#define WT_PAGE_TRYLOCK(s, p) \ + __wt_spin_trylock((s), &(p)->modify->page_lock) +#define WT_PAGE_UNLOCK(s, p) \ + __wt_spin_unlock((s), &(p)->modify->page_lock) + WT_SPINLOCK page_lock; /* Page's spinlock */ + /* * The write generation is incremented when a page is modified, a page * is clean if the write generation is 0. */ uint32_t write_gen; -#define WT_PAGE_LOCK(s, p) \ - __wt_spin_lock((s), &S2C(s)->page_lock[(p)->modify->page_lock]) -#define WT_PAGE_UNLOCK(s, p) \ - __wt_spin_unlock((s), &S2C(s)->page_lock[(p)->modify->page_lock]) - uint8_t page_lock; /* Page's spinlock */ - #define WT_PM_REC_EMPTY 1 /* Reconciliation: no replacement */ #define WT_PM_REC_MULTIBLOCK 2 /* Reconciliation: multiple blocks */ #define WT_PM_REC_REPLACE 3 /* Reconciliation: single block */ @@ -507,7 +509,8 @@ struct __wt_page { #define WT_INTL_INDEX_GET_SAFE(page) \ ((page)->u.intl.__index) #define WT_INTL_INDEX_GET(session, page, pindex) do { \ - WT_ASSERT(session, (session)->split_gen != 0); \ + WT_ASSERT(session, \ + __wt_session_gen(session, WT_GEN_SPLIT) != 0); \ (pindex) = WT_INTL_INDEX_GET_SAFE(page); \ } while (0) #define WT_INTL_INDEX_SET(page, v) do { \ @@ -603,13 +606,6 @@ struct __wt_page { uint8_t unused[2]; /* Unused padding */ /* - * Used to protect and co-ordinate splits for internal pages and - * reconciliation for all pages. Only used to co-ordinate among the - * uncommon cases that require exclusive access to a page. - */ - WT_RWLOCK page_lock; - - /* * The page's read generation acts as an LRU value for each page in the * tree; it is used by the eviction server thread to select pages to be * discarded from the in-memory tree. @@ -635,8 +631,6 @@ struct __wt_page { #define WT_READGEN_STEP 100 uint64_t read_gen; - uint64_t evict_pass_gen; /* Eviction pass generation */ - size_t memory_footprint; /* Memory attached to the page */ /* Page's on-disk representation: NULL for pages created in memory. */ @@ -644,6 +638,10 @@ struct __wt_page { /* If/when the page is modified, we need lots more information. */ WT_PAGE_MODIFY *modify; + + /* This is the 64 byte boundary, try to keep hot fields above here. */ + + uint64_t evict_pass_gen; /* Eviction pass generation */ }; /* @@ -808,11 +806,11 @@ struct __wt_row { /* On-page key, on-page cell, or off-page WT_IKEY */ * Walk the entries of an in-memory row-store leaf page. */ #define WT_ROW_FOREACH(page, rip, i) \ - for ((i) = (page)->entries, \ + for ((i) = (page)->entries, \ (rip) = (page)->pg_row; (i) > 0; ++(rip), --(i)) #define WT_ROW_FOREACH_REVERSE(page, rip, i) \ - for ((i) = (page)->entries, \ - (rip) = (page)->pg_row + ((page)->entries - 1); \ + for ((i) = (page)->entries, \ + (rip) = (page)->pg_row + ((page)->entries - 1); \ (i) > 0; --(rip), --(i)) /* @@ -860,7 +858,7 @@ struct __wt_col { * Walk the entries of variable-length column-store leaf page. */ #define WT_COL_FOREACH(page, cip, i) \ - for ((i) = (page)->entries, \ + for ((i) = (page)->entries, \ (cip) = (page)->pg_var; (i) > 0; ++(cip), --(i)) /* @@ -907,19 +905,16 @@ struct __wt_ikey { * list. */ WT_PACKED_STRUCT_BEGIN(__wt_update) - uint64_t txnid; /* update transaction */ + uint64_t txnid; /* transaction */ WT_UPDATE *next; /* forward-linked list */ - /* - * We use the maximum size as an is-deleted flag, which means we can't - * store 4GB objects; I'd rather do that than increase the size of this - * structure for a flag bit. - */ -#define WT_UPDATE_DELETED_VALUE UINT32_MAX -#define WT_UPDATE_DELETED_SET(upd) ((upd)->size = WT_UPDATE_DELETED_VALUE) -#define WT_UPDATE_DELETED_ISSET(upd) ((upd)->size == WT_UPDATE_DELETED_VALUE) - uint32_t size; /* update length */ + uint32_t size; /* data length */ + +#define WT_UPDATE_STANDARD 0 +#define WT_UPDATE_DELETED 1 +#define WT_UPDATE_RESERVED 2 + uint8_t type; /* type (one byte to conserve memory) */ /* The untyped value immediately follows the WT_UPDATE structure. */ #define WT_UPDATE_DATA(upd) \ @@ -931,9 +926,13 @@ WT_PACKED_STRUCT_BEGIN(__wt_update) * cache overhead calculation. */ #define WT_UPDATE_MEMSIZE(upd) \ - WT_ALIGN(sizeof(WT_UPDATE) + \ - (WT_UPDATE_DELETED_ISSET(upd) ? 0 : (upd)->size), 32) + WT_ALIGN(sizeof(WT_UPDATE) + (upd)->size, 32) WT_PACKED_STRUCT_END +/* + * WT_UPDATE_SIZE is the expected structure size -- we verify the build to + * ensure the compiler hasn't inserted padding. + */ +#define WT_UPDATE_SIZE 21 /* * WT_INSERT -- @@ -1097,22 +1096,16 @@ struct __wt_insert_head { * already have a split generation, leave it alone. If our caller is examining * an index, we don't want the oldest split generation to move forward and * potentially free it. - * - * Check that we haven't raced with a split_gen update after publishing: we - * rely on the published value not being missed when scanning for the oldest - * active split_gen. */ #define WT_ENTER_PAGE_INDEX(session) do { \ - uint64_t __prev_split_gen = (session)->split_gen; \ + uint64_t __prev_split_gen = \ + __wt_session_gen(session, WT_GEN_SPLIT); \ if (__prev_split_gen == 0) \ - do { \ - WT_PUBLISH((session)->split_gen, \ - S2C(session)->split_gen); \ - } while ((session)->split_gen != S2C(session)->split_gen) + __wt_session_gen_enter(session, WT_GEN_SPLIT); #define WT_LEAVE_PAGE_INDEX(session) \ if (__prev_split_gen == 0) \ - (session)->split_gen = 0; \ + __wt_session_gen_leave(session, WT_GEN_SPLIT); \ } while (0) #define WT_WITH_PAGE_INDEX(session, e) \ diff --git a/src/include/btree.h b/src/include/btree.h index 28fe1b94b23..95af9e154f8 100644 --- a/src/include/btree.h +++ b/src/include/btree.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -134,9 +134,12 @@ struct __wt_btree { WT_BM *bm; /* Block manager reference */ u_int block_header; /* WT_PAGE_HEADER_BYTE_SIZE */ - uint64_t checkpoint_gen; /* Checkpoint generation */ - uint64_t rec_max_txn; /* Maximum txn seen (clean trees) */ uint64_t write_gen; /* Write generation */ + uint64_t rec_max_txn; /* Maximum txn seen (clean trees) */ + uint64_t checkpoint_gen; /* Checkpoint generation */ + volatile enum { + WT_CKPT_OFF, WT_CKPT_PREPARE, WT_CKPT_RUNNING + } checkpointing; /* Checkpoint in progress */ uint64_t bytes_inmem; /* Cache bytes in memory. */ uint64_t bytes_dirty_intl; /* Bytes in dirty internal pages. */ @@ -147,13 +150,10 @@ struct __wt_btree { u_int evict_walk_period; /* Skip this many LRU walks */ u_int evict_walk_saved; /* Saved walk skips for checkpoints */ u_int evict_walk_skips; /* Number of walks skipped */ - int evict_disabled; /* Eviction disabled count */ + int32_t evict_disabled; /* Eviction disabled count */ volatile uint32_t evict_busy; /* Count of threads in eviction */ int evict_start_type; /* Start position for eviction walk (see WT_EVICT_WALK_START). */ - enum { - WT_CKPT_OFF, WT_CKPT_PREPARE, WT_CKPT_RUNNING - } checkpointing; /* Checkpoint in progress */ /* * We flush pages from the tree (in order to make checkpoint faster), @@ -163,19 +163,18 @@ struct __wt_btree { WT_SPINLOCK flush_lock; /* Lock to flush the tree's pages */ /* Flags values up to 0xff are reserved for WT_DHANDLE_* */ -#define WT_BTREE_ALLOW_SPLITS 0x000100 /* Allow splits, even with no evict */ -#define WT_BTREE_BULK 0x000200 /* Bulk-load handle */ -#define WT_BTREE_CLOSED 0x000400 /* Handle closed */ -#define WT_BTREE_IGNORE_CACHE 0x000800 /* Cache-resident object */ -#define WT_BTREE_IN_MEMORY 0x001000 /* Cache-resident object */ -#define WT_BTREE_LOOKASIDE 0x002000 /* Look-aside table */ -#define WT_BTREE_NO_CHECKPOINT 0x004000 /* Disable checkpoints */ -#define WT_BTREE_NO_LOGGING 0x008000 /* Disable logging */ -#define WT_BTREE_REBALANCE 0x020000 /* Handle is for rebalance */ -#define WT_BTREE_SALVAGE 0x040000 /* Handle is for salvage */ -#define WT_BTREE_SKIP_CKPT 0x080000 /* Handle skipped checkpoint */ -#define WT_BTREE_UPGRADE 0x100000 /* Handle is for upgrade */ -#define WT_BTREE_VERIFY 0x200000 /* Handle is for verify */ +#define WT_BTREE_BULK 0x000100 /* Bulk-load handle */ +#define WT_BTREE_CLOSED 0x000200 /* Handle closed */ +#define WT_BTREE_IGNORE_CACHE 0x000400 /* Cache-resident object */ +#define WT_BTREE_IN_MEMORY 0x000800 /* Cache-resident object */ +#define WT_BTREE_LOOKASIDE 0x001000 /* Look-aside table */ +#define WT_BTREE_NO_CHECKPOINT 0x002000 /* Disable checkpoints */ +#define WT_BTREE_NO_LOGGING 0x004000 /* Disable logging */ +#define WT_BTREE_REBALANCE 0x008000 /* Handle is for rebalance */ +#define WT_BTREE_SALVAGE 0x010000 /* Handle is for salvage */ +#define WT_BTREE_SKIP_CKPT 0x020000 /* Handle skipped checkpoint */ +#define WT_BTREE_UPGRADE 0x040000 /* Handle is for upgrade */ +#define WT_BTREE_VERIFY 0x080000 /* Handle is for verify */ uint32_t flags; }; diff --git a/src/include/btree.i b/src/include/btree.i index 1d6fcd6272c..d4db65b2033 100644 --- a/src/include/btree.i +++ b/src/include/btree.i @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -424,7 +424,7 @@ __wt_cache_page_evict(WT_SESSION_IMPL *session, WT_PAGE *page, bool rewrite) modify = page->modify; /* Update the bytes in-memory to reflect the eviction. */ - __wt_cache_decr_check_uint64(session, &S2BT(session)->bytes_inmem, + __wt_cache_decr_check_uint64(session, &btree->bytes_inmem, page->memory_footprint, "WT_BTREE.bytes_inmem"); __wt_cache_decr_check_uint64(session, &cache->bytes_inmem, page->memory_footprint, "WT_CACHE.bytes_inmem"); @@ -1023,33 +1023,6 @@ __wt_row_leaf_key(WT_SESSION_IMPL *session, } /* - * __wt_cursor_row_leaf_key -- - * Set a buffer to reference a cursor-referenced row-store leaf page key. - */ -static inline int -__wt_cursor_row_leaf_key(WT_CURSOR_BTREE *cbt, WT_ITEM *key) -{ - WT_PAGE *page; - WT_ROW *rip; - WT_SESSION_IMPL *session; - - /* - * If the cursor references a WT_INSERT item, take the key from there, - * else take the key from the original page. - */ - if (cbt->ins == NULL) { - session = (WT_SESSION_IMPL *)cbt->iface.session; - page = cbt->ref->page; - rip = &page->pg_row[cbt->slot]; - WT_RET(__wt_row_leaf_key(session, page, rip, key, false)); - } else { - key->data = WT_INSERT_KEY(cbt->ins); - key->size = WT_INSERT_KEY_SIZE(cbt->ins); - } - return (0); -} - -/* * __wt_row_leaf_value_cell -- * Return a pointer to the value cell for a row-store leaf page key, or * NULL if there isn't one. @@ -1313,6 +1286,16 @@ __wt_page_can_evict( return (true); /* + * We can't split or evict multiblock row-store pages where the parent's + * key for the page is an overflow item, because the split into the + * parent frees the backing blocks for any no-longer-used overflow keys, + * which will corrupt the checkpoint's block management. + */ + if (btree->checkpointing != WT_CKPT_OFF && + F_ISSET_ATOMIC(ref->home, WT_PAGE_OVERFLOW_KEYS)) + return (false); + + /* * Check for in-memory splits before other eviction tests. If the page * should split in-memory, return success immediately and skip more * detailed eviction tests. We don't need further tests since the page @@ -1339,16 +1322,6 @@ __wt_page_can_evict( } /* - * We can't evict clean, multiblock row-store pages where the parent's - * key for the page is an overflow item, because the split into the - * parent frees the backing blocks for any no-longer-used overflow keys, - * which will corrupt the checkpoint's block management. - */ - if (btree->checkpointing != WT_CKPT_OFF && - F_ISSET_ATOMIC(ref->home, WT_PAGE_OVERFLOW_KEYS)) - return (false); - - /* * If a split created new internal pages, those newly created internal * pages cannot be evicted until all threads are known to have exited * the original parent page's index, because evicting an internal page @@ -1360,8 +1333,8 @@ __wt_page_can_evict( * that case, no readers can be looking at an old index. */ if (!F_ISSET(session->dhandle, WT_DHANDLE_EXCLUSIVE) && - WT_PAGE_IS_INTERNAL(page) && !__wt_split_obsolete( - session, page->pg_intl_split_gen)) + WT_PAGE_IS_INTERNAL(page) && + page->pg_intl_split_gen >= __wt_gen_oldest(session, WT_GEN_SPLIT)) return (false); /* diff --git a/src/include/btree_cmp.i b/src/include/btree_cmp.i index 23a462e4e50..c1354a7ea4b 100644 --- a/src/include/btree_cmp.i +++ b/src/include/btree_cmp.i @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/buf.i b/src/include/buf.i index d192e292dcf..17f67afefce 100644 --- a/src/include/buf.i +++ b/src/include/buf.i @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/cache.h b/src/include/cache.h index 04920c3585a..a3fc17b9740 100644 --- a/src/include/cache.h +++ b/src/include/cache.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -10,14 +10,10 @@ * Tuning constants: I hesitate to call this tuning, but we want to review some * number of pages from each file's in-memory tree for each page we evict. */ -#define WT_EVICT_INT_SKEW (1<<20) /* Prefer leaf pages over internal - pages by this many increments of the - read generation. */ +#define WT_EVICT_MAX_TREES 1000 /* Maximum walk points */ #define WT_EVICT_WALK_BASE 300 /* Pages tracked across file visits */ #define WT_EVICT_WALK_INCR 100 /* Pages added each walk */ -#define WT_EVICT_MAX_TREES 1000 /* Maximum walk points */ - /* Ways to position when starting an eviction walk. */ typedef enum { WT_EVICT_WALK_NEXT, @@ -183,6 +179,10 @@ struct __wt_cache { /* * Flags. */ +#define WT_CACHE_POOL_MANAGER 0x001 /* The active cache pool manager */ +#define WT_CACHE_POOL_RUN 0x002 /* Cache pool thread running */ + uint32_t pool_flags; /* Cache pool flags */ + #define WT_CACHE_EVICT_CLEAN 0x001 /* Evict clean pages */ #define WT_CACHE_EVICT_CLEAN_HARD 0x002 /* Clean % blocking app threads */ #define WT_CACHE_EVICT_DIRTY 0x004 /* Evict dirty pages */ @@ -190,9 +190,6 @@ struct __wt_cache { #define WT_CACHE_EVICT_SCRUB 0x010 /* Scrub dirty pages */ #define WT_CACHE_EVICT_URGENT 0x020 /* Pages are in the urgent queue */ #define WT_CACHE_EVICT_ALL (WT_CACHE_EVICT_CLEAN | WT_CACHE_EVICT_DIRTY) -#define WT_CACHE_EVICT_MASK 0x0FF -#define WT_CACHE_POOL_MANAGER 0x100 /* The active cache pool manager */ -#define WT_CACHE_POOL_RUN 0x200 /* Cache pool thread running */ uint32_t flags; }; diff --git a/src/include/cache.i b/src/include/cache.i index 90dd1bcdda8..1e058a3ec1b 100644 --- a/src/include/cache.i +++ b/src/include/cache.i @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/cell.i b/src/include/cell.i index 71c2515daf0..0dbf29d21c3 100644 --- a/src/include/cell.i +++ b/src/include/cell.i @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/column.i b/src/include/column.i index 07b627315e6..c95d338f980 100644 --- a/src/include/column.i +++ b/src/include/column.i @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/compact.h b/src/include/compact.h index 96797f6b275..d74090c286c 100644 --- a/src/include/compact.h +++ b/src/include/compact.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/config.h b/src/include/config.h index f2746fc76d9..1f21693511b 100644 --- a/src/include/config.h +++ b/src/include/config.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/connection.h b/src/include/connection.h index 6c23492e926..56d801cd361 100644 --- a/src/include/connection.h +++ b/src/include/connection.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -175,21 +175,6 @@ struct __wt_connection_impl { WT_SPINLOCK turtle_lock; /* Turtle file spinlock */ WT_RWLOCK dhandle_lock; /* Data handle list lock */ - /* - * We distribute the btree page locks across a set of spin locks. Don't - * use too many: they are only held for very short operations, each one - * is 64 bytes, so 256 will fill the L1 cache on most CPUs. - * - * Use a prime number of buckets rather than assuming a good hash - * (Reference Sedgewick, Algorithms in C, "Hash Functions"). - * - * Note: this can't be an array, we impose cache-line alignment and gcc - * doesn't support that for arrays smaller than the alignment. - */ -#define WT_PAGE_LOCKS 17 - WT_SPINLOCK *page_lock; /* Btree page spinlocks */ - u_int page_lock_cnt; /* Next spinlock to use */ - /* Connection queue */ TAILQ_ENTRY(__wt_connection_impl) q; /* Cache pool queue */ @@ -210,10 +195,6 @@ struct __wt_connection_impl { WT_FH *lock_fh; /* Lock file handle */ - volatile uint64_t split_gen; /* Generation number for splits */ - uint64_t split_stashed_bytes; /* Atomic: split statistics */ - uint64_t split_stashed_objects; - /* * The connection keeps a cache of data handles. The set of handles * can grow quite large so we maintain both a simple list and a hash @@ -329,9 +310,10 @@ struct __wt_connection_impl { #define WT_CONN_LOG_ARCHIVE 0x01 /* Archive is enabled */ #define WT_CONN_LOG_ENABLED 0x02 /* Logging is enabled */ #define WT_CONN_LOG_EXISTED 0x04 /* Log files found */ -#define WT_CONN_LOG_RECOVER_DONE 0x08 /* Recovery completed */ -#define WT_CONN_LOG_RECOVER_ERR 0x10 /* Error if recovery required */ -#define WT_CONN_LOG_ZERO_FILL 0x20 /* Manually zero files */ +#define WT_CONN_LOG_RECOVER_DIRTY 0x08 /* Recovering unclean */ +#define WT_CONN_LOG_RECOVER_DONE 0x10 /* Recovery completed */ +#define WT_CONN_LOG_RECOVER_ERR 0x20 /* Error if recovery required */ +#define WT_CONN_LOG_ZERO_FILL 0x40 /* Manually zero files */ uint32_t log_flags; /* Global logging configuration */ WT_CONDVAR *log_cond; /* Log server wait mutex */ WT_SESSION_IMPL *log_session; /* Log server session */ @@ -378,7 +360,15 @@ struct __wt_connection_impl { bool las_written; /* Lookaside table has been written */ WT_ITEM las_sweep_key; /* Sweep server's saved key */ - int64_t las_record_cnt;/* Count of lookaside records */ + uint64_t las_record_cnt;/* Count of lookaside records */ + + /* + * The "lookaside_activity" verbose messages are throttled to once per + * checkpoint. To accomplish this we track the checkpoint generation + * for the most recent read and write verbose messages. + */ + volatile uint64_t las_verb_gen_read; + volatile uint64_t las_verb_gen_write; /* Locked: collator list */ TAILQ_HEAD(__wt_coll_qh, __wt_named_collator) collqh; @@ -401,7 +391,10 @@ struct __wt_connection_impl { /* If non-zero, all buffers used for I/O will be aligned to this. */ size_t buffer_alignment; - uint32_t schema_gen; /* Schema generation number */ + uint64_t stashed_bytes; /* Atomic: stashed memory statistics */ + uint64_t stashed_objects; + /* Generations manager */ + volatile uint64_t generations[WT_GENERATIONS]; wt_off_t data_extend_len; /* file_extend data length */ wt_off_t log_extend_len; /* file_extend log length */ diff --git a/src/include/ctype.i b/src/include/ctype.i index b4a1ad9f318..3855ae653a5 100644 --- a/src/include/ctype.i +++ b/src/include/ctype.i @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/cursor.h b/src/include/cursor.h index f32b4250d30..8d2f2c80c2a 100644 --- a/src/include/cursor.h +++ b/src/include/cursor.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -22,8 +22,10 @@ search, \ search_near, \ insert, \ + modify, \ update, \ remove, \ + reserve, \ reconfigure, \ close) \ static const WT_CURSOR n = { \ @@ -43,8 +45,10 @@ search, \ search_near, \ insert, \ + modify, \ update, \ remove, \ + reserve, \ close, \ reconfigure, \ { NULL, NULL }, /* TAILQ_ENTRY q */ \ @@ -497,57 +501,5 @@ struct __wt_cursor_table { #define WT_CURSOR_RECNO(cursor) WT_STREQ((cursor)->key_format, "r") -/* - * WT_CURSOR_NEEDKEY, WT_CURSOR_NEEDVALUE -- - * Check if we have a key/value set. There's an additional semantic - * implemented here: if we're pointing into the tree, and about to perform - * a cursor operation, get a local copy of whatever we're referencing in - * the tree, there's an obvious race with the cursor moving and the key or - * value reference, and it's better to solve it here than in the underlying - * data-source layers. - * - * WT_CURSOR_CHECKKEY -- - * Check if a key is set without making a copy. - * - * WT_CURSOR_NOVALUE -- - * Release any cached value before an operation that could update the - * transaction context and free data a value is pointing to. - */ -#define WT_CURSOR_CHECKKEY(cursor) do { \ - if (!F_ISSET(cursor, WT_CURSTD_KEY_SET)) \ - WT_ERR(__wt_cursor_kv_not_set(cursor, true)); \ -} while (0) -#define WT_CURSOR_CHECKVALUE(cursor) do { \ - if (!F_ISSET(cursor, WT_CURSTD_VALUE_SET)) \ - WT_ERR(__wt_cursor_kv_not_set(cursor, false)); \ -} while (0) -#define WT_CURSOR_NEEDKEY(cursor) do { \ - if (F_ISSET(cursor, WT_CURSTD_KEY_INT)) { \ - if (!WT_DATA_IN_ITEM(&(cursor)->key)) \ - WT_ERR(__wt_buf_set( \ - (WT_SESSION_IMPL *)(cursor)->session, \ - &(cursor)->key, \ - (cursor)->key.data, (cursor)->key.size)); \ - F_CLR(cursor, WT_CURSTD_KEY_INT); \ - F_SET(cursor, WT_CURSTD_KEY_EXT); \ - } \ - WT_CURSOR_CHECKKEY(cursor); \ -} while (0) -#define WT_CURSOR_NEEDVALUE(cursor) do { \ - if (F_ISSET(cursor, WT_CURSTD_VALUE_INT)) { \ - if (!WT_DATA_IN_ITEM(&(cursor)->value)) \ - WT_ERR(__wt_buf_set( \ - (WT_SESSION_IMPL *)(cursor)->session, \ - &(cursor)->value, \ - (cursor)->value.data, (cursor)->value.size));\ - F_CLR(cursor, WT_CURSTD_VALUE_INT); \ - F_SET(cursor, WT_CURSTD_VALUE_EXT); \ - } \ - WT_CURSOR_CHECKVALUE(cursor); \ -} while (0) -#define WT_CURSOR_NOVALUE(cursor) do { \ - F_CLR(cursor, WT_CURSTD_VALUE_INT); \ -} while (0) - #define WT_CURSOR_RAW_OK \ (WT_CURSTD_DUMP_HEX | WT_CURSTD_DUMP_PRINT | WT_CURSTD_RAW) diff --git a/src/include/cursor.i b/src/include/cursor.i index 12044e0e228..75fd935fc91 100644 --- a/src/include/cursor.i +++ b/src/include/cursor.i @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -18,6 +18,102 @@ __cursor_set_recno(WT_CURSOR_BTREE *cbt, uint64_t v) } /* + * __cursor_novalue -- + * Release any cached value before an operation that could update the + * transaction context and free data a value is pointing to. + */ +static inline void +__cursor_novalue(WT_CURSOR *cursor) +{ + F_CLR(cursor, WT_CURSTD_VALUE_INT); +} + +/* + * __cursor_checkkey -- + * Check if a key is set without making a copy. + */ +static inline int +__cursor_checkkey(WT_CURSOR *cursor) +{ + return (F_ISSET(cursor, WT_CURSTD_KEY_SET) ? + 0 : __wt_cursor_kv_not_set(cursor, true)); +} + +/* + * __cursor_checkvalue -- + * Check if a value is set without making a copy. + */ +static inline int +__cursor_checkvalue(WT_CURSOR *cursor) +{ + return (F_ISSET(cursor, WT_CURSTD_VALUE_SET) ? + 0 : __wt_cursor_kv_not_set(cursor, false)); +} + +/* + * __cursor_localkey -- + * If the key points into the tree, get a local copy. + */ +static inline int +__cursor_localkey(WT_CURSOR *cursor) +{ + if (F_ISSET(cursor, WT_CURSTD_KEY_INT)) { + if (!WT_DATA_IN_ITEM(&cursor->key)) + WT_RET(__wt_buf_set((WT_SESSION_IMPL *)cursor->session, + &cursor->key, cursor->key.data, cursor->key.size)); + F_CLR(cursor, WT_CURSTD_KEY_INT); + F_SET(cursor, WT_CURSTD_KEY_EXT); + } + return (0); +} + +/* + * __cursor_localvalue -- + * If the value points into the tree, get a local copy. + */ +static inline int +__cursor_localvalue(WT_CURSOR *cursor) +{ + if (F_ISSET(cursor, WT_CURSTD_VALUE_INT)) { + if (!WT_DATA_IN_ITEM(&cursor->value)) + WT_RET(__wt_buf_set((WT_SESSION_IMPL *)cursor->session, + &cursor->value, + cursor->value.data, cursor->value.size)); + F_CLR(cursor, WT_CURSTD_VALUE_INT); + F_SET(cursor, WT_CURSTD_VALUE_EXT); + } + return (0); +} + +/* + * __cursor_needkey -- + * + * Check if we have a key set. There's an additional semantic here: if we're + * pointing into the tree, get a local copy of whatever we're referencing in + * the tree, there's an obvious race with the cursor moving and the reference. + */ +static inline int +__cursor_needkey(WT_CURSOR *cursor) +{ + WT_RET(__cursor_localkey(cursor)); + return (__cursor_checkkey(cursor)); +} + +/* + * __cursor_needvalue -- + * + * Check if we have a value set. There's an additional semantic here: if we're + * pointing into the tree, get a local copy of whatever we're referencing in + * the tree, there's an obvious race with the cursor moving and the reference. + */ +static inline int +__cursor_needvalue(WT_CURSOR *cursor) +{ + WT_RET(__cursor_localkey(cursor)); + return (__cursor_checkvalue(cursor)); +} + +/* * __cursor_pos_clear -- * Reset the cursor's location. */ @@ -129,27 +225,24 @@ static inline int __wt_curindex_get_valuev(WT_CURSOR *cursor, va_list ap) { WT_CURSOR_INDEX *cindex; - WT_DECL_RET; WT_ITEM *item; WT_SESSION_IMPL *session; cindex = (WT_CURSOR_INDEX *)cursor; session = (WT_SESSION_IMPL *)cursor->session; - WT_CURSOR_NEEDVALUE(cursor); + WT_RET(__cursor_checkvalue(cursor)); if (F_ISSET(cursor, WT_CURSOR_RAW_OK)) { - ret = __wt_schema_project_merge(session, + WT_RET(__wt_schema_project_merge(session, cindex->cg_cursors, cindex->value_plan, - cursor->value_format, &cursor->value); - if (ret == 0) { - item = va_arg(ap, WT_ITEM *); - item->data = cursor->value.data; - item->size = cursor->value.size; - } + cursor->value_format, &cursor->value)); + item = va_arg(ap, WT_ITEM *); + item->data = cursor->value.data; + item->size = cursor->value.size; } else - ret = __wt_schema_project_out(session, - cindex->cg_cursors, cindex->value_plan, ap); -err: return (ret); + WT_RET(__wt_schema_project_out(session, + cindex->cg_cursors, cindex->value_plan, ap)); + return (0); } /* @@ -161,28 +254,25 @@ __wt_curtable_get_valuev(WT_CURSOR *cursor, va_list ap) { WT_CURSOR *primary; WT_CURSOR_TABLE *ctable; - WT_DECL_RET; WT_ITEM *item; WT_SESSION_IMPL *session; ctable = (WT_CURSOR_TABLE *)cursor; session = (WT_SESSION_IMPL *)cursor->session; primary = *ctable->cg_cursors; - WT_CURSOR_NEEDVALUE(primary); + WT_RET(__cursor_checkvalue(primary)); if (F_ISSET(cursor, WT_CURSOR_RAW_OK)) { - ret = __wt_schema_project_merge(session, + WT_RET(__wt_schema_project_merge(session, ctable->cg_cursors, ctable->plan, - cursor->value_format, &cursor->value); - if (ret == 0) { - item = va_arg(ap, WT_ITEM *); - item->data = cursor->value.data; - item->size = cursor->value.size; - } + cursor->value_format, &cursor->value)); + item = va_arg(ap, WT_ITEM *); + item->data = cursor->value.data; + item->size = cursor->value.size; } else - ret = __wt_schema_project_out(session, - ctable->cg_cursors, ctable->plan, ap); -err: return (ret); + WT_RET(__wt_schema_project_out(session, + ctable->cg_cursors, ctable->plan, ap)); + return (0); } /* diff --git a/src/include/dhandle.h b/src/include/dhandle.h index 8861e96112b..0db59d45691 100644 --- a/src/include/dhandle.h +++ b/src/include/dhandle.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/dlh.h b/src/include/dlh.h index 9e49c2ff3cb..d02523b03d1 100644 --- a/src/include/dlh.h +++ b/src/include/dlh.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/error.h b/src/include/error.h index c338acb370f..465ab4fa859 100644 --- a/src/include/error.h +++ b/src/include/error.h @@ -1,12 +1,12 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * * See the file LICENSE for redistribution information. */ -#define WT_DEBUG_POINT ((void *)0xdeadbeef) +#define WT_DEBUG_POINT ((void *)(uintptr_t)0xdeadbeef) #define WT_DEBUG_BYTE (0xab) /* In DIAGNOSTIC mode, yield in places where we want to encourage races. */ diff --git a/src/include/extern.h b/src/include/extern.h index 55ba1bada7c..f055e4810b3 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -1,638 +1,639 @@ /* DO NOT EDIT: automatically built by dist/s_prototypes. */ -extern void __wt_async_stats_update(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_async_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_async_reconfig(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_async_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_async_flush(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_async_new_op(WT_SESSION_IMPL *session, const char *uri, const char *config, const char *cfg[], WT_ASYNC_CALLBACK *cb, WT_ASYNC_OP_IMPL **opp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_async_op_enqueue(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_async_op_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern WT_THREAD_RET __wt_async_worker(void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_addr_to_buffer(WT_BLOCK *block, uint8_t **pp, wt_off_t offset, uint32_t size, uint32_t checksum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_addr_invalid(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, bool live) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_addr_string(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_buffer_to_ckpt(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *p, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern void __wt_async_stats_update(WT_SESSION_IMPL *session); +extern int __wt_async_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_async_reconfig(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_async_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_async_flush(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_async_new_op(WT_SESSION_IMPL *session, const char *uri, const char *config, const char *cfg[], WT_ASYNC_CALLBACK *cb, WT_ASYNC_OP_IMPL **opp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_async_op_enqueue(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_async_op_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern WT_THREAD_RET __wt_async_worker(void *arg); +extern int __wt_block_addr_to_buffer(WT_BLOCK *block, uint8_t **pp, wt_off_t offset, uint32_t size, uint32_t checksum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_addr_invalid(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, bool live) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_addr_string(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_buffer_to_ckpt(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *p, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_ckpt_decode(WT_SESSION *wt_session, size_t allocsize, const uint8_t *p, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_ckpt_to_buffer(WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t **pp, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_ckpt_init( WT_SESSION_IMPL *session, WT_BLOCK_CKPT *ci, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, uint8_t *root_addr, size_t *root_addr_sizep, bool checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_checkpoint_unload( WT_SESSION_IMPL *session, WT_BLOCK *block, bool checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_block_ckpt_destroy(WT_SESSION_IMPL *session, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_checkpoint(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, WT_CKPT *ckptbase, bool data_checksum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_checkpoint_resolve(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_compact_start(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_compact_end(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_compact_page_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, bool *skipp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_misplaced(WT_SESSION_IMPL *session, WT_BLOCK *block, const char *tag, wt_off_t offset, uint32_t size, bool live) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_off_remove_overlap(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, wt_off_t off, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_alloc( WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t *offp, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_free(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_off_free( WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t offset, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_extlist_check( WT_SESSION_IMPL *session, WT_EXTLIST *al, WT_EXTLIST *bl) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_extlist_overlap( WT_SESSION_IMPL *session, WT_BLOCK *block, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_extlist_merge(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *a, WT_EXTLIST *b) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_insert_ext(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, wt_off_t off, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_extlist_read_avail(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, wt_off_t ckpt_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_extlist_read(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, wt_off_t ckpt_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_extlist_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, WT_EXTLIST *additional) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_extlist_truncate( WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_extlist_init(WT_SESSION_IMPL *session, WT_EXTLIST *el, const char *name, const char *extname, bool track_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_block_extlist_free(WT_SESSION_IMPL *session, WT_EXTLIST *el) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_map(WT_SESSION_IMPL *session, WT_BLOCK *block, void *mapped_regionp, size_t *lengthp, void *mapped_cookiep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_unmap(WT_SESSION_IMPL *session, WT_BLOCK *block, void *mapped_region, size_t length, void *mapped_cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_manager_open(WT_SESSION_IMPL *session, const char *filename, const char *cfg[], bool forced_salvage, bool readonly, uint32_t allocsize, WT_BM **bmp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_panic(WT_SESSION_IMPL *session, int error, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_manager_drop( WT_SESSION_IMPL *session, const char *filename, bool durable) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_manager_create( WT_SESSION_IMPL *session, const char *filename, uint32_t allocsize) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_block_configure_first_fit(WT_BLOCK *block, bool on) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_open(WT_SESSION_IMPL *session, const char *filename, const char *cfg[], bool forced_salvage, bool readonly, uint32_t allocsize, WT_BLOCK **blockp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_close(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_desc_write(WT_SESSION_IMPL *session, WT_FH *fh, uint32_t allocsize) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_block_stat(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_DSRC_STATS *stats) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_manager_size(WT_BM *bm, WT_SESSION_IMPL *session, wt_off_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_manager_named_size( WT_SESSION_IMPL *session, const char *name, wt_off_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_bm_preload( WT_BM *bm, WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_bm_read(WT_BM *bm, WT_SESSION_IMPL *session, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_read_off_blind( WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_off_t offset) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_off_t offset, uint32_t size, uint32_t checksum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_ext_alloc(WT_SESSION_IMPL *session, WT_EXT **extp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_block_ext_free(WT_SESSION_IMPL *session, WT_EXT *ext) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_size_alloc(WT_SESSION_IMPL *session, WT_SIZE **szp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_block_size_free(WT_SESSION_IMPL *session, WT_SIZE *sz) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_ext_prealloc(WT_SESSION_IMPL *session, u_int max) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_ext_discard(WT_SESSION_IMPL *session, u_int max) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_salvage_start(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_salvage_end(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern bool __wt_block_offset_invalid(WT_BLOCK *block, wt_off_t offset, uint32_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_salvage_next(WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t *addr, size_t *addr_sizep, bool *eofp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_salvage_valid(WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t *addr, size_t addr_size, bool valid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_verify_start(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_verify_end(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_verify_ckpt_load( WT_SESSION_IMPL *session, WT_BLOCK *block, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_verify_ckpt_unload(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_verify_addr(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_truncate(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t len) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_discard(WT_SESSION_IMPL *session, WT_BLOCK *block, size_t added_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_write_size(WT_SESSION_IMPL *session, WT_BLOCK *block, size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, bool data_checksum, bool checkpoint_io) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump, bool data_checksum, bool checkpoint_io, bool caller_locked) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_block_ckpt_to_buffer(WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t **pp, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_ckpt_init( WT_SESSION_IMPL *session, WT_BLOCK_CKPT *ci, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, uint8_t *root_addr, size_t *root_addr_sizep, bool checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_checkpoint_unload( WT_SESSION_IMPL *session, WT_BLOCK *block, bool checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_block_ckpt_destroy(WT_SESSION_IMPL *session, WT_BLOCK_CKPT *ci); +extern int __wt_block_checkpoint(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, WT_CKPT *ckptbase, bool data_checksum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_checkpoint_resolve(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_compact_start(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_compact_end(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_compact_page_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, bool *skipp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_misplaced(WT_SESSION_IMPL *session, WT_BLOCK *block, const char *tag, wt_off_t offset, uint32_t size, bool live) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_off_remove_overlap(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, wt_off_t off, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_alloc( WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t *offp, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_free(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_off_free( WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t offset, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_extlist_check( WT_SESSION_IMPL *session, WT_EXTLIST *al, WT_EXTLIST *bl) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_extlist_overlap( WT_SESSION_IMPL *session, WT_BLOCK *block, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_extlist_merge(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *a, WT_EXTLIST *b) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_insert_ext(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, wt_off_t off, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_extlist_read_avail(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, wt_off_t ckpt_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_extlist_read(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, wt_off_t ckpt_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_extlist_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, WT_EXTLIST *additional) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_extlist_truncate( WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_extlist_init(WT_SESSION_IMPL *session, WT_EXTLIST *el, const char *name, const char *extname, bool track_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_block_extlist_free(WT_SESSION_IMPL *session, WT_EXTLIST *el); +extern int __wt_block_map(WT_SESSION_IMPL *session, WT_BLOCK *block, void *mapped_regionp, size_t *lengthp, void *mapped_cookiep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_unmap(WT_SESSION_IMPL *session, WT_BLOCK *block, void *mapped_region, size_t length, void *mapped_cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_manager_open(WT_SESSION_IMPL *session, const char *filename, const char *cfg[], bool forced_salvage, bool readonly, uint32_t allocsize, WT_BM **bmp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_panic(WT_SESSION_IMPL *session, int error, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_manager_drop( WT_SESSION_IMPL *session, const char *filename, bool durable) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_manager_create( WT_SESSION_IMPL *session, const char *filename, uint32_t allocsize) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_block_configure_first_fit(WT_BLOCK *block, bool on); +extern int __wt_block_open(WT_SESSION_IMPL *session, const char *filename, const char *cfg[], bool forced_salvage, bool readonly, uint32_t allocsize, WT_BLOCK **blockp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_close(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_desc_write(WT_SESSION_IMPL *session, WT_FH *fh, uint32_t allocsize) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_block_stat(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_DSRC_STATS *stats); +extern int __wt_block_manager_size(WT_BM *bm, WT_SESSION_IMPL *session, wt_off_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_manager_named_size( WT_SESSION_IMPL *session, const char *name, wt_off_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bm_preload( WT_BM *bm, WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bm_read(WT_BM *bm, WT_SESSION_IMPL *session, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_read_off_blind( WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_off_t offset) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_off_t offset, uint32_t size, uint32_t checksum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_ext_alloc(WT_SESSION_IMPL *session, WT_EXT **extp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_block_ext_free(WT_SESSION_IMPL *session, WT_EXT *ext); +extern int __wt_block_size_alloc(WT_SESSION_IMPL *session, WT_SIZE **szp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_block_size_free(WT_SESSION_IMPL *session, WT_SIZE *sz); +extern int __wt_block_ext_prealloc(WT_SESSION_IMPL *session, u_int max) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_ext_discard(WT_SESSION_IMPL *session, u_int max) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_salvage_start(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_salvage_end(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern bool __wt_block_offset_invalid(WT_BLOCK *block, wt_off_t offset, uint32_t size); +extern int __wt_block_salvage_next(WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t *addr, size_t *addr_sizep, bool *eofp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_salvage_valid(WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t *addr, size_t addr_size, bool valid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_verify_start(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_verify_end(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_verify_ckpt_load( WT_SESSION_IMPL *session, WT_BLOCK *block, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_verify_ckpt_unload(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_verify_addr(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_truncate(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t len) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_discard(WT_SESSION_IMPL *session, WT_BLOCK *block, size_t added_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_write_size(WT_SESSION_IMPL *session, WT_BLOCK *block, size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, bool data_checksum, bool checkpoint_io) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump, bool data_checksum, bool checkpoint_io, bool caller_locked) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bloom_create( WT_SESSION_IMPL *session, const char *uri, const char *config, uint64_t count, uint32_t factor, uint32_t k, WT_BLOOM **bloomp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bloom_open(WT_SESSION_IMPL *session, const char *uri, uint32_t factor, uint32_t k, WT_CURSOR *owner, WT_BLOOM **bloomp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_bloom_insert(WT_BLOOM *bloom, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_bloom_finalize(WT_BLOOM *bloom) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_bloom_hash(WT_BLOOM *bloom, WT_ITEM *key, WT_BLOOM_HASH *bhash) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_bloom_hash_get(WT_BLOOM *bloom, WT_BLOOM_HASH *bhash) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern void __wt_bloom_hash(WT_BLOOM *bloom, WT_ITEM *key, WT_BLOOM_HASH *bhash); +extern int __wt_bloom_hash_get(WT_BLOOM *bloom, WT_BLOOM_HASH *bhash) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bloom_get(WT_BLOOM *bloom, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_bloom_inmem_get(WT_BLOOM *bloom, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_bloom_intersection(WT_BLOOM *bloom, WT_BLOOM *other) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_bloom_inmem_get(WT_BLOOM *bloom, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bloom_intersection(WT_BLOOM *bloom, WT_BLOOM *other) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bloom_close(WT_BLOOM *bloom) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bloom_drop(WT_BLOOM *bloom, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_compact(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_compact_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_key_order_check( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, bool next) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_key_order_init(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_cursor_key_order_reset(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_btcur_iterate_setup(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btcur_next(WT_CURSOR_BTREE *cbt, bool truncating) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern bool __wt_cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btcur_reset(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btcur_search(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btcur_insert(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btcur_insert_check(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btcur_remove(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btcur_update(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btcur_compare(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *cmpp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btcur_equals(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *equalp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btcur_range_truncate(WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_btcur_init(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_btcur_open(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btcur_close(WT_CURSOR_BTREE *cbt, bool lowlevel) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_debug_set_verbose(WT_SESSION_IMPL *session, const char *v) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_debug_addr_print( WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_debug_addr(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_debug_offset_blind( WT_SESSION_IMPL *session, wt_off_t offset, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_debug_offset(WT_SESSION_IMPL *session, wt_off_t offset, uint32_t size, uint32_t checksum, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_debug_disk( WT_SESSION_IMPL *session, const WT_PAGE_HEADER *dsk, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_debug_tree_shape( WT_SESSION_IMPL *session, WT_PAGE *page, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_debug_tree_all( WT_SESSION_IMPL *session, WT_BTREE *btree, WT_REF *ref, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_debug_tree( WT_SESSION_IMPL *session, WT_BTREE *btree, WT_REF *ref, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_debug_page(WT_SESSION_IMPL *session, WT_REF *ref, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_delete_page(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_delete_page_rollback(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern bool __wt_delete_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, bool visible_all) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_ref_out_int(WT_SESSION_IMPL *session, WT_REF *ref, bool rewrite) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_ref_out(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_page_out(WT_SESSION_IMPL *session, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_free_ref( WT_SESSION_IMPL *session, WT_REF *ref, int page_type, bool free_pages) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_free_ref_index(WT_SESSION_IMPL *session, WT_PAGE *page, WT_PAGE_INDEX *pindex, bool free_pages) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_free_update_list(WT_SESSION_IMPL *session, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btree_close(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btree_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_root_ref_init(WT_REF *root_ref, WT_PAGE *root, bool is_recno) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btree_tree_open( WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btree_new_leaf_page(WT_SESSION_IMPL *session, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btree_huffman_open(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_btree_huffman_close(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_bt_read(WT_SESSION_IMPL *session, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, bool checkpoint, bool checkpoint_io, bool compressed) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_compact(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_compact_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_key_order_check( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, bool next) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_key_order_init(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cursor_key_order_reset(WT_CURSOR_BTREE *cbt); +extern void __wt_btcur_iterate_setup(WT_CURSOR_BTREE *cbt); +extern int __wt_btcur_next(WT_CURSOR_BTREE *cbt, bool truncating) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern bool __wt_cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp); +extern int __wt_btcur_reset(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btcur_search(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btcur_insert(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btcur_insert_check(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btcur_remove(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btcur_reserve(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btcur_update(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btcur_compare(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *cmpp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btcur_equals(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *equalp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btcur_range_truncate(WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_btcur_init(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt); +extern void __wt_btcur_open(WT_CURSOR_BTREE *cbt); +extern int __wt_btcur_close(WT_CURSOR_BTREE *cbt, bool lowlevel) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_debug_set_verbose(WT_SESSION_IMPL *session, const char *v) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_debug_addr_print( WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_debug_addr(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_debug_offset_blind( WT_SESSION_IMPL *session, wt_off_t offset, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_debug_offset(WT_SESSION_IMPL *session, wt_off_t offset, uint32_t size, uint32_t checksum, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_debug_disk( WT_SESSION_IMPL *session, const WT_PAGE_HEADER *dsk, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_debug_tree_shape( WT_SESSION_IMPL *session, WT_PAGE *page, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_debug_tree_all( WT_SESSION_IMPL *session, WT_BTREE *btree, WT_REF *ref, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_debug_tree( WT_SESSION_IMPL *session, WT_BTREE *btree, WT_REF *ref, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_debug_page(WT_SESSION_IMPL *session, WT_REF *ref, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_delete_page(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_delete_page_rollback(WT_SESSION_IMPL *session, WT_REF *ref); +extern bool __wt_delete_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, bool visible_all); +extern int __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_ref_out_int(WT_SESSION_IMPL *session, WT_REF *ref, bool rewrite); +extern void __wt_ref_out(WT_SESSION_IMPL *session, WT_REF *ref); +extern void __wt_page_out(WT_SESSION_IMPL *session, WT_PAGE **pagep); +extern void __wt_free_ref( WT_SESSION_IMPL *session, WT_REF *ref, int page_type, bool free_pages); +extern void __wt_free_ref_index(WT_SESSION_IMPL *session, WT_PAGE *page, WT_PAGE_INDEX *pindex, bool free_pages); +extern void __wt_free_update_list(WT_SESSION_IMPL *session, WT_UPDATE *upd); +extern int __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btree_close(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btree_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_root_ref_init(WT_REF *root_ref, WT_PAGE *root, bool is_recno); +extern int __wt_btree_tree_open( WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btree_new_leaf_page(WT_SESSION_IMPL *session, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btree_huffman_open(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_btree_huffman_close(WT_SESSION_IMPL *session); +extern int __wt_bt_read(WT_SESSION_IMPL *session, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, bool checkpoint, bool checkpoint_io, bool compressed) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_page_type_string(u_int type) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern const char *__wt_cell_type_string(uint8_t type) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern const char *__wt_page_addr_string(WT_SESSION_IMPL *session, WT_REF *ref, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern const char *__wt_addr_string(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ovfl_read(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL_UNPACK *unpack, WT_ITEM *store) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ovfl_cache(WT_SESSION_IMPL *session, WT_PAGE *page, void *cookie, WT_CELL_UNPACK *vpack) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ovfl_discard(WT_SESSION_IMPL *session, WT_CELL *cell) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_page_alloc(WT_SESSION_IMPL *session, uint8_t type, uint32_t alloc_entries, bool alloc_refs, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_page_inmem(WT_SESSION_IMPL *session, WT_REF *ref, const void *image, size_t memsize, uint32_t flags, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_random_descent(WT_SESSION_IMPL *session, WT_REF **refp, bool eviction) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btcur_next_random(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_las_remove_block(WT_SESSION_IMPL *session, WT_CURSOR *cursor, uint32_t btree_id, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern const char *__wt_cell_type_string(uint8_t type); +extern const char *__wt_page_addr_string(WT_SESSION_IMPL *session, WT_REF *ref, WT_ITEM *buf); +extern const char *__wt_addr_string(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size, WT_ITEM *buf); +extern int __wt_ovfl_read(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL_UNPACK *unpack, WT_ITEM *store) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ovfl_cache(WT_SESSION_IMPL *session, WT_PAGE *page, void *cookie, WT_CELL_UNPACK *vpack) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ovfl_discard(WT_SESSION_IMPL *session, WT_CELL *cell) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_page_alloc(WT_SESSION_IMPL *session, uint8_t type, uint32_t alloc_entries, bool alloc_refs, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_page_inmem(WT_SESSION_IMPL *session, WT_REF *ref, const void *image, size_t memsize, uint32_t flags, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_random_descent(WT_SESSION_IMPL *session, WT_REF **refp, bool eviction) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btcur_next_random(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_las_remove_block(WT_SESSION_IMPL *session, WT_CURSOR *cursor, uint32_t btree_id, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags #ifdef HAVE_DIAGNOSTIC , const char *file, int line #endif - ) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_key_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_kv_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_bt_salvage(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern bool __wt_split_obsolete(WT_SESSION_IMPL *session, uint64_t split_gen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_split_stash_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_split_stash_discard_all( WT_SESSION_IMPL *session_safe, WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_multi_to_ref(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi, WT_REF **refp, size_t *incrp, bool closing) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int closing) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_split_reverse(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_split_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, WT_MULTI *multi) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_btree_stat_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cache_op(WT_SESSION_IMPL *session, WT_CACHE_OP op) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_upgrade(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_verify_dsk_image(WT_SESSION_IMPL *session, const char *tag, const WT_PAGE_HEADER *dsk, size_t size, bool empty_page_ok) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_verify_dsk(WT_SESSION_IMPL *session, const char *tag, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_tree_walk(WT_SESSION_IMPL *session, WT_REF **refp, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_tree_walk_count(WT_SESSION_IMPL *session, WT_REF **refp, uint64_t *walkcntp, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_tree_walk_skip(WT_SESSION_IMPL *session, WT_REF **refp, uint64_t *skipleafcntp, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, uint64_t recno, WT_ITEM *value, WT_UPDATE *upd_arg, bool is_remove) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_col_search(WT_SESSION_IMPL *session, uint64_t search_recno, WT_REF *leaf, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_row_leaf_keys(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_row_leaf_key_copy( WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW *rip, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_row_leaf_key_work(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW *rip_arg, WT_ITEM *keyb, bool instantiate) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_row_ikey_alloc(WT_SESSION_IMPL *session, uint32_t cell_offset, const void *key, size_t size, WT_IKEY **ikeyp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_row_ikey_incr(WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t cell_offset, const void *key, size_t size, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_row_ikey(WT_SESSION_IMPL *session, uint32_t cell_offset, const void *key, size_t size, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_page_modify_alloc(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, WT_ITEM *value, WT_UPDATE *upd_arg, bool is_remove) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_row_insert_alloc(WT_SESSION_IMPL *session, WT_ITEM *key, u_int skipdepth, WT_INSERT **insp, size_t *ins_sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_update_alloc( WT_SESSION_IMPL *session, WT_ITEM *value, WT_UPDATE **updp, size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern WT_UPDATE *__wt_update_obsolete_check( WT_SESSION_IMPL *session, WT_PAGE *page, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_update_obsolete_free( WT_SESSION_IMPL *session, WT_PAGE *page, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_search_insert(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_INSERT_HEAD *ins_head, WT_ITEM *srch_key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_row_search(WT_SESSION_IMPL *session, WT_ITEM *srch_key, WT_REF *leaf, WT_CURSOR_BTREE *cbt, bool insert) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_las_stats_update(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_las_create(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_las_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_las_set_written(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern bool __wt_las_is_written(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_las_cursor_open(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_las_cursor( WT_SESSION_IMPL *session, WT_CURSOR **cursorp, uint32_t *session_flags) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_las_cursor_close( WT_SESSION_IMPL *session, WT_CURSOR **cursorp, uint32_t session_flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_las_sweep(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); + ); +extern int __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_key_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_kv_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bt_salvage(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_multi_to_ref(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi, WT_REF **refp, size_t *incrp, bool closing) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int closing) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_split_reverse(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_split_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, WT_MULTI *multi) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btree_stat_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cache_op(WT_SESSION_IMPL *session, WT_CACHE_OP op) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_upgrade(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_verify_dsk_image(WT_SESSION_IMPL *session, const char *tag, const WT_PAGE_HEADER *dsk, size_t size, bool empty_page_ok) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_verify_dsk(WT_SESSION_IMPL *session, const char *tag, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tree_walk(WT_SESSION_IMPL *session, WT_REF **refp, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tree_walk_count(WT_SESSION_IMPL *session, WT_REF **refp, uint64_t *walkcntp, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tree_walk_skip( WT_SESSION_IMPL *session, WT_REF **refp, uint64_t *skipleafcntp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_UPDATE *upd_arg, u_int modify_type, bool exclusive) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_col_search(WT_SESSION_IMPL *session, uint64_t search_recno, WT_REF *leaf, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_row_leaf_keys(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_row_leaf_key_copy( WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW *rip, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_row_leaf_key_work(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW *rip_arg, WT_ITEM *keyb, bool instantiate) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_row_ikey_alloc(WT_SESSION_IMPL *session, uint32_t cell_offset, const void *key, size_t size, WT_IKEY **ikeyp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_row_ikey_incr(WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t cell_offset, const void *key, size_t size, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_row_ikey(WT_SESSION_IMPL *session, uint32_t cell_offset, const void *key, size_t size, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_page_modify_alloc(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, WT_UPDATE *upd_arg, u_int modify_type, bool exclusive) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_row_insert_alloc(WT_SESSION_IMPL *session, const WT_ITEM *key, u_int skipdepth, WT_INSERT **insp, size_t *ins_sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_update_alloc(WT_SESSION_IMPL *session, const WT_ITEM *value, WT_UPDATE **updp, size_t *sizep, u_int modify_type) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern WT_UPDATE *__wt_update_obsolete_check( WT_SESSION_IMPL *session, WT_PAGE *page, WT_UPDATE *upd); +extern void __wt_update_obsolete_free( WT_SESSION_IMPL *session, WT_PAGE *page, WT_UPDATE *upd); +extern int __wt_search_insert(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_INSERT_HEAD *ins_head, WT_ITEM *srch_key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_row_search(WT_SESSION_IMPL *session, WT_ITEM *srch_key, WT_REF *leaf, WT_CURSOR_BTREE *cbt, bool insert) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_las_stats_update(WT_SESSION_IMPL *session); +extern int __wt_las_create(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_las_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_las_set_written(WT_SESSION_IMPL *session); +extern bool __wt_las_is_written(WT_SESSION_IMPL *session); +extern int __wt_las_cursor_open(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_las_cursor( WT_SESSION_IMPL *session, WT_CURSOR **cursorp, uint32_t *session_flags); +extern int __wt_las_cursor_close( WT_SESSION_IMPL *session, WT_CURSOR **cursorp, uint32_t session_flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_las_sweep(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern uint32_t __wt_checksum_sw(const void *chunk, size_t len) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern void __wt_checksum_init(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_config_initn( WT_SESSION_IMPL *session, WT_CONFIG *conf, const char *str, size_t len) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_config_init(WT_SESSION_IMPL *session, WT_CONFIG *conf, const char *str) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_config_subinit( WT_SESSION_IMPL *session, WT_CONFIG *conf, WT_CONFIG_ITEM *item) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_config_next(WT_CONFIG *conf, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_config_get(WT_SESSION_IMPL *session, const char **cfg_arg, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_config_gets(WT_SESSION_IMPL *session, const char **cfg, const char *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_config_gets_none(WT_SESSION_IMPL *session, const char **cfg, const char *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_config_getone(WT_SESSION_IMPL *session, const char *config, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_config_getones(WT_SESSION_IMPL *session, const char *config, const char *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_config_getones_none(WT_SESSION_IMPL *session, const char *config, const char *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_config_gets_def(WT_SESSION_IMPL *session, const char **cfg, const char *key, int def, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_config_subgetraw(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cfg, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_config_subgets(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cfg, const char *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_conn_foc_add(WT_SESSION_IMPL *session, const void *p) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_conn_foc_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_configure_method(WT_SESSION_IMPL *session, const char *method, const char *uri, const char *config, const char *type, const char *check) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_config_check(WT_SESSION_IMPL *session, const WT_CONFIG_ENTRY *entry, const char *config, size_t config_len) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_config_collapse( WT_SESSION_IMPL *session, const char **cfg, char **config_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern void __wt_checksum_init(void); +extern void __wt_config_initn( WT_SESSION_IMPL *session, WT_CONFIG *conf, const char *str, size_t len); +extern void __wt_config_init(WT_SESSION_IMPL *session, WT_CONFIG *conf, const char *str); +extern void __wt_config_subinit( WT_SESSION_IMPL *session, WT_CONFIG *conf, WT_CONFIG_ITEM *item); +extern int __wt_config_next(WT_CONFIG *conf, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_config_get(WT_SESSION_IMPL *session, const char **cfg_arg, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_config_gets(WT_SESSION_IMPL *session, const char **cfg, const char *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_config_gets_none(WT_SESSION_IMPL *session, const char **cfg, const char *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_config_getone(WT_SESSION_IMPL *session, const char *config, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_config_getones(WT_SESSION_IMPL *session, const char *config, const char *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_config_getones_none(WT_SESSION_IMPL *session, const char *config, const char *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_config_gets_def(WT_SESSION_IMPL *session, const char **cfg, const char *key, int def, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_config_subgetraw(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cfg, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_config_subgets(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cfg, const char *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_conn_foc_discard(WT_SESSION_IMPL *session); +extern int __wt_configure_method(WT_SESSION_IMPL *session, const char *method, const char *uri, const char *config, const char *type, const char *check) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_config_check(WT_SESSION_IMPL *session, const WT_CONFIG_ENTRY *entry, const char *config, size_t config_len) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_config_collapse( WT_SESSION_IMPL *session, const char **cfg, char **config_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_merge(WT_SESSION_IMPL *session, const char **cfg, const char *cfg_strip, const char **config_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_conn_config_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_conn_config_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern const WT_CONFIG_ENTRY *__wt_conn_config_match(const char *method) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_config_get(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, WT_CONFIG_ARG *cfg_arg, const char *key, WT_CONFIG_ITEM *cval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_config_get_string(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *config, const char *key, WT_CONFIG_ITEM *cval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_config_parser_open_arg(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, WT_CONFIG_ARG *cfg_arg, WT_CONFIG_PARSER **config_parserp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_config_upgrade(WT_SESSION_IMPL *session, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern const char *__wt_wiredtiger_error(int error) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_collator_config(WT_SESSION_IMPL *session, const char *uri, WT_CONFIG_ITEM *cname, WT_CONFIG_ITEM *metadata, WT_COLLATOR **collatorp, int *ownp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_conn_remove_collator(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_compressor_config( WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval, WT_COMPRESSOR **compressorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_conn_remove_compressor(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_conn_remove_data_source(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_encryptor_config(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval, WT_CONFIG_ITEM *keyid, WT_CONFIG_ARG *cfg_arg, WT_KEYED_ENCRYPTOR **kencryptorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_conn_remove_encryptor(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_extractor_config(WT_SESSION_IMPL *session, const char *uri, const char *config, WT_EXTRACTOR **extractorp, int *ownp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_conn_remove_extractor(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cache_config(WT_SESSION_IMPL *session, bool reconfigure, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cache_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_cache_stats_update(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cache_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cache_pool_config(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_conn_cache_pool_open(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_conn_cache_pool_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern WT_THREAD_RET __wt_cache_pool_server(void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_checkpoint_server_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_checkpoint_server_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_checkpoint_signal(WT_SESSION_IMPL *session, wt_off_t logsize) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_conn_dhandle_alloc( WT_SESSION_IMPL *session, const char *uri, const char *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_conn_dhandle_find( WT_SESSION_IMPL *session, const char *uri, const char *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_conn_btree_open( WT_SESSION_IMPL *session, const char *cfg[], uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_conn_dhandle_close_all( WT_SESSION_IMPL *session, const char *uri, bool force) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_conn_dhandle_discard_single( WT_SESSION_IMPL *session, bool final, bool force) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_conn_dhandle_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_connection_destroy(WT_CONNECTION_IMPL *conn) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logmgr_reconfig(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_truncate_files( WT_SESSION_IMPL *session, WT_CURSOR *cursor, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_log_wrlsn(WT_SESSION_IMPL *session, int *yield) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logmgr_open(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logmgr_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_connection_open(WT_CONNECTION_IMPL *conn, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_connection_close(WT_CONNECTION_IMPL *conn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_connection_workers(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_conn_stat_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_statlog_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_statlog_destroy(WT_SESSION_IMPL *session, bool is_close) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_sweep_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_sweep_create(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_sweep_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_backup_file_remove(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curbulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool bitmap, bool skip_sort_check) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curconfig_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curds_open( WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_DATA_SOURCE *dsrc, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curdump_create(WT_CURSOR *child, WT_CURSOR *owner, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curfile_next_random(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curfile_insert_check(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curfile_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curindex_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curjoin_joined(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curjoin_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT_INDEX *idx, WT_CURSOR *ref_cursor, uint8_t flags, uint8_t range, uint64_t count, uint32_t bloom_bit_count, uint32_t bloom_hash_count) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_json_alloc_unpack(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, WT_CURSOR_JSON *json, bool iskey, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_json_close(WT_SESSION_IMPL *session, WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_conn_config_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_conn_config_discard(WT_SESSION_IMPL *session); +extern const WT_CONFIG_ENTRY *__wt_conn_config_match(const char *method); +extern int __wt_ext_config_get(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, WT_CONFIG_ARG *cfg_arg, const char *key, WT_CONFIG_ITEM *cval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_config_get_string(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *config, const char *key, WT_CONFIG_ITEM *cval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_config_parser_open_arg(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, WT_CONFIG_ARG *cfg_arg, WT_CONFIG_PARSER **config_parserp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_config_upgrade(WT_SESSION_IMPL *session, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern const char *__wt_wiredtiger_error(int error); +extern int __wt_collator_config(WT_SESSION_IMPL *session, const char *uri, WT_CONFIG_ITEM *cname, WT_CONFIG_ITEM *metadata, WT_COLLATOR **collatorp, int *ownp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_remove_collator(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_compressor_config( WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval, WT_COMPRESSOR **compressorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_remove_compressor(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_remove_data_source(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_encryptor_config(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval, WT_CONFIG_ITEM *keyid, WT_CONFIG_ARG *cfg_arg, WT_KEYED_ENCRYPTOR **kencryptorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_remove_encryptor(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_extractor_config(WT_SESSION_IMPL *session, const char *uri, const char *config, WT_EXTRACTOR **extractorp, int *ownp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_remove_extractor(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cache_config(WT_SESSION_IMPL *session, bool reconfigure, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cache_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cache_stats_update(WT_SESSION_IMPL *session); +extern int __wt_cache_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cache_pool_config(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_cache_pool_open(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_cache_pool_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern WT_THREAD_RET __wt_cache_pool_server(void *arg); +extern int __wt_checkpoint_server_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_checkpoint_server_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_checkpoint_signal(WT_SESSION_IMPL *session, wt_off_t logsize); +extern int __wt_conn_dhandle_alloc( WT_SESSION_IMPL *session, const char *uri, const char *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_dhandle_find( WT_SESSION_IMPL *session, const char *uri, const char *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_btree_open( WT_SESSION_IMPL *session, const char *cfg[], uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_dhandle_close_all( WT_SESSION_IMPL *session, const char *uri, bool force) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_dhandle_discard_single( WT_SESSION_IMPL *session, bool final, bool force) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_dhandle_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_connection_destroy(WT_CONNECTION_IMPL *conn); +extern int __wt_logmgr_reconfig(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_truncate_files( WT_SESSION_IMPL *session, WT_CURSOR *cursor, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_log_wrlsn(WT_SESSION_IMPL *session, int *yield); +extern int __wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logmgr_open(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logmgr_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_connection_open(WT_CONNECTION_IMPL *conn, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_connection_close(WT_CONNECTION_IMPL *conn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_connection_workers(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_conn_stat_init(WT_SESSION_IMPL *session); +extern int __wt_statlog_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_statlog_destroy(WT_SESSION_IMPL *session, bool is_close) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_sweep_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_sweep_create(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_sweep_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_backup_file_remove(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curbulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool bitmap, bool skip_sort_check) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curconfig_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curds_open( WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_DATA_SOURCE *dsrc, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curdump_create(WT_CURSOR *child, WT_CURSOR *owner, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curfile_next_random(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curfile_insert_check(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curfile_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curindex_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curjoin_joined(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curjoin_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT_INDEX *idx, WT_CURSOR *ref_cursor, uint8_t flags, uint8_t range, uint64_t count, uint32_t bloom_bit_count, uint32_t bloom_hash_count) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_json_alloc_unpack(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, WT_CURSOR_JSON *json, bool iskey, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_json_close(WT_SESSION_IMPL *session, WT_CURSOR *cursor); extern size_t __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern void __wt_json_column_init(WT_CURSOR *cursor, const char *uri, const char *keyformat, const WT_CONFIG_ITEM *idxconf, const WT_CONFIG_ITEM *colconf) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern void __wt_json_column_init(WT_CURSOR *cursor, const char *uri, const char *keyformat, const WT_CONFIG_ITEM *idxconf, const WT_CONFIG_ITEM *colconf); extern int __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, const char **tokstart, size_t *toklen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_json_tokname(int toktype) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern int __wt_json_to_item(WT_SESSION_IMPL *session, const char *jstr, const char *format, WT_CURSOR_JSON *json, bool iskey, WT_ITEM *item) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_json_to_item(WT_SESSION_IMPL *session, const char *jstr, const char *format, WT_CURSOR_JSON *json, bool iskey, WT_ITEM *item) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern ssize_t __wt_json_strlen(const char *src, size_t srclen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_json_strncpy(WT_SESSION *wt_session, char **pdst, size_t dstlen, const char *src, size_t srclen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_curlog_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curmetadata_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_curstat_dsrc_final(WT_CURSOR_STAT *cst) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curstat_init(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *curjoin, const char *cfg[], WT_CURSOR_STAT *cst) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curstat_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_noop(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_notsup(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_get_value_notsup(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_cursor_set_key_notsup(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_cursor_set_value_notsup(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_compare_notsup(WT_CURSOR *a, WT_CURSOR *b, int *cmpp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_equals_notsup(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_search_near_notsup(WT_CURSOR *cursor, int *exact) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_reconfigure_notsup(WT_CURSOR *cursor, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_cursor_set_notsup(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_kv_not_set(WT_CURSOR *cursor, bool key) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_get_key(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_cursor_set_key(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_get_raw_key(WT_CURSOR *cursor, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_cursor_set_raw_key(WT_CURSOR *cursor, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_get_raw_value(WT_CURSOR *cursor, WT_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_cursor_set_raw_value(WT_CURSOR *cursor, WT_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_get_keyv(WT_CURSOR *cursor, uint32_t flags, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_cursor_set_keyv(WT_CURSOR *cursor, uint32_t flags, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_get_value(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_get_valuev(WT_CURSOR *cursor, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_cursor_set_value(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_cursor_set_valuev(WT_CURSOR *cursor, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_close(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_equals(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_reconfigure(WT_CURSOR *cursor, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_dup_position(WT_CURSOR *to_dup, WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cursor_init(WT_CURSOR *cursor, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_apply_single_idx(WT_SESSION_IMPL *session, WT_INDEX *idx, WT_CURSOR *cur, WT_CURSOR_TABLE *ctable, int (*f)(WT_CURSOR *)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curtable_get_key(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curtable_get_value(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_curtable_set_key(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_curtable_set_value(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_table_range_truncate(WT_CURSOR_TABLE *start, WT_CURSOR_TABLE *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curtable_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_evict_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_evict_list_clear_page(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_evict_server_wake(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_evict_create(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_evict_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_evict_file_exclusive_off(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, u_int pct_full) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern bool __wt_page_evict_urgent(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_evict_priority_set(WT_SESSION_IMPL *session, uint64_t v) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_evict_priority_clear(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_verbose_dump_cache(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_page_release_evict(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool closing) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_curstat_cache_walk(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_log_ckpt(WT_SESSION_IMPL *session, WT_LSN *ckp_lsn) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_flush_lsn(WT_SESSION_IMPL *session, WT_LSN *lsn, bool start) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_log_background(WT_SESSION_IMPL *session, WT_LSN *lsn) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_needs_recovery(WT_SESSION_IMPL *session, WT_LSN *ckp_lsn, bool *recp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_log_written_reset(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_get_all_files(WT_SESSION_IMPL *session, char ***filesp, u_int *countp, uint32_t *maxid, bool active_only) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_extract_lognum( WT_SESSION_IMPL *session, const char *name, uint32_t *id) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_acquire(WT_SESSION_IMPL *session, uint64_t recsize, WT_LOGSLOT *slot) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_allocfile( WT_SESSION_IMPL *session, uint32_t lognum, const char *dest) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_remove(WT_SESSION_IMPL *session, const char *file_prefix, uint32_t lognum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_open(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_close(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, int (*func)(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, WT_LSN *next_lsnp, void *cookie, int firstrecord), void *cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_force_write(WT_SESSION_IMPL *session, bool retry, bool *did_work) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_write(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_vprintf(WT_SESSION_IMPL *session, const char *fmt, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_flush(WT_SESSION_IMPL *session, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logrec_alloc(WT_SESSION_IMPL *session, size_t size, WT_ITEM **logrecp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_logrec_free(WT_SESSION_IMPL *session, WT_ITEM **logrecp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logrec_read(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *rectypep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_read(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *optypep, uint32_t *opsizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_col_put_pack( WT_SESSION_IMPL *session, WT_ITEM *logrec, uint32_t fileid, uint64_t recno, WT_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_col_put_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *fileidp, uint64_t *recnop, WT_ITEM *valuep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_col_put_print(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_col_remove_pack( WT_SESSION_IMPL *session, WT_ITEM *logrec, uint32_t fileid, uint64_t recno) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_col_remove_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *fileidp, uint64_t *recnop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_col_remove_print(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_col_truncate_pack( WT_SESSION_IMPL *session, WT_ITEM *logrec, uint32_t fileid, uint64_t start, uint64_t stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_col_truncate_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *fileidp, uint64_t *startp, uint64_t *stopp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_col_truncate_print(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_row_put_pack( WT_SESSION_IMPL *session, WT_ITEM *logrec, uint32_t fileid, WT_ITEM *key, WT_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_row_put_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *fileidp, WT_ITEM *keyp, WT_ITEM *valuep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_row_put_print(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_row_remove_pack( WT_SESSION_IMPL *session, WT_ITEM *logrec, uint32_t fileid, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_row_remove_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *fileidp, WT_ITEM *keyp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_row_remove_print(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_row_truncate_pack( WT_SESSION_IMPL *session, WT_ITEM *logrec, uint32_t fileid, WT_ITEM *start, WT_ITEM *stop, uint32_t mode) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_row_truncate_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *fileidp, WT_ITEM *startp, WT_ITEM *stopp, uint32_t *modep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_logop_row_truncate_print(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_txn_op_printlog(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_log_slot_activate(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_slot_switch(WT_SESSION_IMPL *session, WT_MYSLOT *myslot, bool retry, bool forced, bool *did_work) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_slot_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_slot_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, uint32_t flags, WT_MYSLOT *myslot) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int64_t __wt_log_slot_release(WT_SESSION_IMPL *session, WT_MYSLOT *myslot, int64_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_log_slot_free(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_clsm_request_switch(WT_CURSOR_LSM *clsm) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_clsm_await_switch(WT_CURSOR_LSM *clsm) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_clsm_init_merge( WT_CURSOR *cursor, u_int start_chunk, uint32_t start_id, u_int nchunks) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_clsm_close(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_clsm_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_clsm_open_bulk(WT_CURSOR_LSM *clsm, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_manager_config(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_manager_reconfig(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_manager_start(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_lsm_manager_free_work_unit( WT_SESSION_IMPL *session, WT_LSM_WORK_UNIT *entry) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_manager_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_lsm_manager_clear_tree(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_manager_pop_entry( WT_SESSION_IMPL *session, uint32_t type, WT_LSM_WORK_UNIT **entryp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_manager_push_entry(WT_SESSION_IMPL *session, uint32_t type, uint32_t flags, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_merge_update_tree(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int start_chunk, u_int nchunks, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, const char *newconfig) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curstat_lsm_init( WT_SESSION_IMPL *session, const char *uri, WT_CURSOR_STAT *cst) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_tree_close_all(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_tree_bloom_name(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, uint32_t id, const char **retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_tree_chunk_name(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, uint32_t id, const char **retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_tree_set_chunk_size( WT_SESSION_IMPL *session, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_tree_setup_chunk( WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_tree_setup_bloom( WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_tree_create(WT_SESSION_IMPL *session, const char *uri, bool exclusive, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_tree_get(WT_SESSION_IMPL *session, const char *uri, bool exclusive, WT_LSM_TREE **treep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_lsm_tree_release(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_lsm_tree_throttle( WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool decrease_only) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_tree_retire_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int start_chunk, u_int nchunks) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_tree_alter( WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_tree_drop( WT_SESSION_IMPL *session, const char *name, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_tree_rename(WT_SESSION_IMPL *session, const char *olduri, const char *newuri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_tree_truncate( WT_SESSION_IMPL *session, const char *name, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_lsm_tree_readlock(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_lsm_tree_readunlock(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_lsm_tree_writelock(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_lsm_tree_writeunlock(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_tree_worker(WT_SESSION_IMPL *session, const char *uri, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[], uint32_t open_flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_get_chunk_to_flush(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool force, WT_LSM_CHUNK **chunkp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_work_switch( WT_SESSION_IMPL *session, WT_LSM_WORK_UNIT **entryp, bool *ran) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_work_bloom(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_free_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_worker_start(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_lsm_worker_stop(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_meta_apply_all(WT_SESSION_IMPL *session, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_meta_checkpoint(WT_SESSION_IMPL *session, const char *fname, const char *checkpoint, WT_CKPT *ckpt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_meta_checkpoint_last_name( WT_SESSION_IMPL *session, const char *fname, const char **namep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_meta_checkpoint_clear(WT_SESSION_IMPL *session, const char *fname) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_meta_ckptlist_get( WT_SESSION_IMPL *session, const char *fname, WT_CKPT **ckptbasep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, const char *fname, WT_CKPT *ckptbase, WT_LSN *ckptlsn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_meta_ckptlist_free(WT_SESSION_IMPL *session, WT_CKPT **ckptbasep) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_meta_checkpoint_free(WT_SESSION_IMPL *session, WT_CKPT *ckpt) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_metadata_insert(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, const char *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_metadata_remove( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_metadata_search(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, char **valuep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_metadata_update(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, const char *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_curlog_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curmetadata_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_curstat_dsrc_final(WT_CURSOR_STAT *cst); +extern int __wt_curstat_init(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *curjoin, const char *cfg[], WT_CURSOR_STAT *cst) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curstat_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_noop(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_notsup(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_get_value_notsup(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cursor_set_key_notsup(WT_CURSOR *cursor, ...); +extern void __wt_cursor_set_value_notsup(WT_CURSOR *cursor, ...); +extern int __wt_cursor_compare_notsup(WT_CURSOR *a, WT_CURSOR *b, int *cmpp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_equals_notsup(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_modify_notsup(WT_CURSOR *cursor, WT_MODIFY *entries, int nentries) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_search_near_notsup(WT_CURSOR *cursor, int *exact) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_reconfigure_notsup(WT_CURSOR *cursor, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cursor_set_notsup(WT_CURSOR *cursor); +extern int __wt_cursor_kv_not_set(WT_CURSOR *cursor, bool key) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_get_key(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cursor_set_key(WT_CURSOR *cursor, ...); +extern int __wt_cursor_get_raw_key(WT_CURSOR *cursor, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cursor_set_raw_key(WT_CURSOR *cursor, WT_ITEM *key); +extern int __wt_cursor_get_raw_value(WT_CURSOR *cursor, WT_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cursor_set_raw_value(WT_CURSOR *cursor, WT_ITEM *value); +extern int __wt_cursor_get_keyv(WT_CURSOR *cursor, uint32_t flags, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cursor_set_keyv(WT_CURSOR *cursor, uint32_t flags, va_list ap); +extern int __wt_cursor_get_value(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_get_valuev(WT_CURSOR *cursor, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cursor_set_value(WT_CURSOR *cursor, ...); +extern void __wt_cursor_set_valuev(WT_CURSOR *cursor, va_list ap); +extern int __wt_cursor_close(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_equals(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_reconfigure(WT_CURSOR *cursor, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_dup_position(WT_CURSOR *to_dup, WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_init(WT_CURSOR *cursor, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_apply_single_idx(WT_SESSION_IMPL *session, WT_INDEX *idx, WT_CURSOR *cur, WT_CURSOR_TABLE *ctable, int (*f)(WT_CURSOR *)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curtable_get_key(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curtable_get_value(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_curtable_set_key(WT_CURSOR *cursor, ...); +extern void __wt_curtable_set_value(WT_CURSOR *cursor, ...); +extern int __wt_table_range_truncate(WT_CURSOR_TABLE *start, WT_CURSOR_TABLE *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curtable_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_evict_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_evict_list_clear_page(WT_SESSION_IMPL *session, WT_REF *ref); +extern void __wt_evict_server_wake(WT_SESSION_IMPL *session); +extern bool __wt_evict_thread_chk(WT_SESSION_IMPL *session); +extern int __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_evict_thread_stop(WT_SESSION_IMPL *session, WT_THREAD *thread) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_evict_create(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_evict_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_evict_file_exclusive_off(WT_SESSION_IMPL *session); +extern int __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, u_int pct_full) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern bool __wt_page_evict_urgent(WT_SESSION_IMPL *session, WT_REF *ref); +extern void __wt_evict_priority_set(WT_SESSION_IMPL *session, uint64_t v); +extern void __wt_evict_priority_clear(WT_SESSION_IMPL *session); +extern int __wt_verbose_dump_cache(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_page_release_evict(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool closing) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_curstat_cache_walk(WT_SESSION_IMPL *session); +extern void __wt_log_ckpt(WT_SESSION_IMPL *session, WT_LSN *ckp_lsn); +extern int __wt_log_flush_lsn(WT_SESSION_IMPL *session, WT_LSN *lsn, bool start) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_log_background(WT_SESSION_IMPL *session, WT_LSN *lsn); +extern int __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_needs_recovery(WT_SESSION_IMPL *session, WT_LSN *ckp_lsn, bool *recp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_log_written_reset(WT_SESSION_IMPL *session); +extern int __wt_log_get_all_files(WT_SESSION_IMPL *session, char ***filesp, u_int *countp, uint32_t *maxid, bool active_only) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_extract_lognum( WT_SESSION_IMPL *session, const char *name, uint32_t *id) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_reset(WT_SESSION_IMPL *session, uint32_t lognum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_acquire(WT_SESSION_IMPL *session, uint64_t recsize, WT_LOGSLOT *slot) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_allocfile( WT_SESSION_IMPL *session, uint32_t lognum, const char *dest) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_remove(WT_SESSION_IMPL *session, const char *file_prefix, uint32_t lognum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_open(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_close(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, int (*func)(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, WT_LSN *next_lsnp, void *cookie, int firstrecord), void *cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_force_write(WT_SESSION_IMPL *session, bool retry, bool *did_work) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_write(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_vprintf(WT_SESSION_IMPL *session, const char *fmt, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_flush(WT_SESSION_IMPL *session, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logrec_alloc(WT_SESSION_IMPL *session, size_t size, WT_ITEM **logrecp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_logrec_free(WT_SESSION_IMPL *session, WT_ITEM **logrecp); +extern int __wt_logrec_read(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *rectypep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_read(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *optypep, uint32_t *opsizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_col_put_pack( WT_SESSION_IMPL *session, WT_ITEM *logrec, uint32_t fileid, uint64_t recno, WT_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_col_put_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *fileidp, uint64_t *recnop, WT_ITEM *valuep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_col_put_print(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_col_remove_pack( WT_SESSION_IMPL *session, WT_ITEM *logrec, uint32_t fileid, uint64_t recno) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_col_remove_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *fileidp, uint64_t *recnop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_col_remove_print(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_col_truncate_pack( WT_SESSION_IMPL *session, WT_ITEM *logrec, uint32_t fileid, uint64_t start, uint64_t stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_col_truncate_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *fileidp, uint64_t *startp, uint64_t *stopp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_col_truncate_print(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_row_put_pack( WT_SESSION_IMPL *session, WT_ITEM *logrec, uint32_t fileid, WT_ITEM *key, WT_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_row_put_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *fileidp, WT_ITEM *keyp, WT_ITEM *valuep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_row_put_print(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_row_remove_pack( WT_SESSION_IMPL *session, WT_ITEM *logrec, uint32_t fileid, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_row_remove_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *fileidp, WT_ITEM *keyp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_row_remove_print(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_row_truncate_pack( WT_SESSION_IMPL *session, WT_ITEM *logrec, uint32_t fileid, WT_ITEM *start, WT_ITEM *stop, uint32_t mode) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_row_truncate_unpack( WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t *fileidp, WT_ITEM *startp, WT_ITEM *stopp, uint32_t *modep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_logop_row_truncate_print(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_op_printlog(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_log_slot_activate(WT_SESSION_IMPL *session, WT_LOGSLOT *slot); +extern int __wt_log_slot_switch(WT_SESSION_IMPL *session, WT_MYSLOT *myslot, bool retry, bool forced, bool *did_work) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_slot_init(WT_SESSION_IMPL *session, bool alloc) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_slot_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, uint32_t flags, WT_MYSLOT *myslot); +extern int64_t __wt_log_slot_release(WT_MYSLOT *myslot, int64_t size); +extern void __wt_log_slot_free(WT_SESSION_IMPL *session, WT_LOGSLOT *slot); +extern int __wt_clsm_request_switch(WT_CURSOR_LSM *clsm) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_clsm_await_switch(WT_CURSOR_LSM *clsm) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_clsm_init_merge( WT_CURSOR *cursor, u_int start_chunk, uint32_t start_id, u_int nchunks) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_clsm_close(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_clsm_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_clsm_open_bulk(WT_CURSOR_LSM *clsm, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_manager_config(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_manager_reconfig(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_manager_start(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_lsm_manager_free_work_unit( WT_SESSION_IMPL *session, WT_LSM_WORK_UNIT *entry); +extern int __wt_lsm_manager_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_lsm_manager_clear_tree(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); +extern int __wt_lsm_manager_pop_entry( WT_SESSION_IMPL *session, uint32_t type, WT_LSM_WORK_UNIT **entryp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_manager_push_entry(WT_SESSION_IMPL *session, uint32_t type, uint32_t flags, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_merge_update_tree(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int start_chunk, u_int nchunks, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, const char *newconfig) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curstat_lsm_init( WT_SESSION_IMPL *session, const char *uri, WT_CURSOR_STAT *cst) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_tree_close_all(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_tree_bloom_name(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, uint32_t id, const char **retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_tree_chunk_name(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, uint32_t id, const char **retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_tree_set_chunk_size( WT_SESSION_IMPL *session, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_tree_setup_chunk( WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_tree_setup_bloom( WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_tree_create(WT_SESSION_IMPL *session, const char *uri, bool exclusive, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_tree_get(WT_SESSION_IMPL *session, const char *uri, bool exclusive, WT_LSM_TREE **treep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_lsm_tree_release(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); +extern void __wt_lsm_tree_throttle( WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool decrease_only); +extern int __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_tree_retire_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int start_chunk, u_int nchunks) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_tree_alter( WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_tree_drop( WT_SESSION_IMPL *session, const char *name, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_tree_rename(WT_SESSION_IMPL *session, const char *olduri, const char *newuri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_tree_truncate( WT_SESSION_IMPL *session, const char *name, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_lsm_tree_readlock(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); +extern void __wt_lsm_tree_readunlock(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); +extern void __wt_lsm_tree_writelock(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); +extern void __wt_lsm_tree_writeunlock(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); +extern int __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_tree_worker(WT_SESSION_IMPL *session, const char *uri, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[], uint32_t open_flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_get_chunk_to_flush(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool force, WT_LSM_CHUNK **chunkp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_work_switch( WT_SESSION_IMPL *session, WT_LSM_WORK_UNIT **entryp, bool *ran) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_work_bloom(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_free_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_worker_start(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_worker_stop(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_apply_all(WT_SESSION_IMPL *session, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_checkpoint(WT_SESSION_IMPL *session, const char *fname, const char *checkpoint, WT_CKPT *ckpt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_checkpoint_last_name( WT_SESSION_IMPL *session, const char *fname, const char **namep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_checkpoint_clear(WT_SESSION_IMPL *session, const char *fname) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_ckptlist_get( WT_SESSION_IMPL *session, const char *fname, WT_CKPT **ckptbasep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, const char *fname, WT_CKPT *ckptbase, WT_LSN *ckptlsn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_meta_ckptlist_free(WT_SESSION_IMPL *session, WT_CKPT **ckptbasep); +extern void __wt_meta_checkpoint_free(WT_SESSION_IMPL *session, WT_CKPT *ckpt); +extern int __wt_ext_metadata_insert(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, const char *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_metadata_remove( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_metadata_search(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, char **valuep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_metadata_update(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, const char *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_metadata_get_ckptlist( WT_SESSION *session, const char *name, WT_CKPT **ckptbasep) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_metadata_free_ckptlist(WT_SESSION *session, WT_CKPT *ckptbase) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern int __wt_metadata_cursor_open( WT_SESSION_IMPL *session, const char *config, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_metadata_cursor(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_metadata_cursor_release(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_metadata_insert( WT_SESSION_IMPL *session, const char *key, const char *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_metadata_update( WT_SESSION_IMPL *session, const char *key, const char *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_metadata_remove(WT_SESSION_IMPL *session, const char *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_metadata_search(WT_SESSION_IMPL *session, const char *key, char **valuep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_meta_track_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_meta_track_on(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_meta_track_off(WT_SESSION_IMPL *session, bool need_sync, bool unroll) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_meta_track_sub_on(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_meta_track_sub_off(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_meta_track_checkpoint(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_meta_track_insert(WT_SESSION_IMPL *session, const char *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_meta_track_update(WT_SESSION_IMPL *session, const char *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_meta_track_fileop( WT_SESSION_IMPL *session, const char *olduri, const char *newuri) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_meta_track_drop( WT_SESSION_IMPL *session, const char *filename) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_meta_track_handle_lock(WT_SESSION_IMPL *session, bool created) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_meta_track_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_meta_track_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_turtle_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_turtle_read(WT_SESSION_IMPL *session, const char *key, char **valuep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_turtle_update(WT_SESSION_IMPL *session, const char *key, const char *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_filename(WT_SESSION_IMPL *session, const char *name, char **path) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_nfilename( WT_SESSION_IMPL *session, const char *name, size_t namelen, char **path) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_remove_if_exists(WT_SESSION_IMPL *session, const char *name, bool durable) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_metadata_cursor_open( WT_SESSION_IMPL *session, const char *config, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_metadata_cursor(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_metadata_cursor_release(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_metadata_insert( WT_SESSION_IMPL *session, const char *key, const char *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_metadata_update( WT_SESSION_IMPL *session, const char *key, const char *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_metadata_remove(WT_SESSION_IMPL *session, const char *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_metadata_search(WT_SESSION_IMPL *session, const char *key, char **valuep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_meta_track_discard(WT_SESSION_IMPL *session); +extern int __wt_meta_track_on(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_track_off(WT_SESSION_IMPL *session, bool need_sync, bool unroll) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_meta_track_sub_on(WT_SESSION_IMPL *session); +extern int __wt_meta_track_sub_off(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_track_checkpoint(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_track_insert(WT_SESSION_IMPL *session, const char *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_track_update(WT_SESSION_IMPL *session, const char *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_track_fileop( WT_SESSION_IMPL *session, const char *olduri, const char *newuri) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_track_drop( WT_SESSION_IMPL *session, const char *filename) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_track_handle_lock(WT_SESSION_IMPL *session, bool created) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_track_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_meta_track_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_turtle_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_turtle_read(WT_SESSION_IMPL *session, const char *key, char **valuep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_turtle_update(WT_SESSION_IMPL *session, const char *key, const char *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_filename(WT_SESSION_IMPL *session, const char *name, char **path) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_nfilename( WT_SESSION_IMPL *session, const char *name, size_t namelen, char **path) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_remove_if_exists(WT_SESSION_IMPL *session, const char *name, bool durable) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_copy_and_sync(WT_SESSION *wt_session, const char *from, const char *to) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); extern int __wt_calloc(WT_SESSION_IMPL *session, size_t number, size_t size, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_malloc(WT_SESSION_IMPL *session, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_realloc(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_realloc_noclear(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_realloc_aligned(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_strndup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_malloc(WT_SESSION_IMPL *session, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_realloc(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_realloc_noclear(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_realloc_aligned(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_strndup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_free_int(WT_SESSION_IMPL *session, const void *p_arg) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern int __wt_errno(void) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern const char *__wt_strerror(WT_SESSION_IMPL *session, int error, char *errbuf, size_t errlen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_map_windows_error( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, uint32_t windows_error) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern bool __wt_handle_is_open(WT_SESSION_IMPL *session, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_open(WT_SESSION_IMPL *session, const char *name, WT_FS_OPEN_FILE_TYPE file_type, u_int flags, WT_FH **fhp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_close(WT_SESSION_IMPL *session, WT_FH **fhp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_close_connection_close(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_os_inmemory(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_fopen(WT_SESSION_IMPL *session, const char *name, uint32_t open_flags, uint32_t flags, WT_FSTREAM **fstrp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_os_stdio(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_errno(void) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern const char *__wt_strerror(WT_SESSION_IMPL *session, int error, char *errbuf, size_t errlen); +extern int __wt_ext_map_windows_error( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, uint32_t windows_error) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern bool __wt_handle_is_open(WT_SESSION_IMPL *session, const char *name); +extern int __wt_open(WT_SESSION_IMPL *session, const char *name, WT_FS_OPEN_FILE_TYPE file_type, u_int flags, WT_FH **fhp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_close(WT_SESSION_IMPL *session, WT_FH **fhp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_close_connection_close(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_os_inmemory(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_fopen(WT_SESSION_IMPL *session, const char *name, uint32_t open_flags, uint32_t flags, WT_FSTREAM **fstrp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_os_stdio(WT_SESSION_IMPL *session); extern int __wt_getopt( const char *progname, int nargc, char *const *nargv, const char *ostr) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern uint64_t __wt_strtouq(const char *nptr, char **endptr, int base) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern int __wt_ext_struct_pack(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, void *buffer, size_t size, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_struct_size(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, size_t *sizep, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_struct_unpack(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const void *buffer, size_t size, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_struct_check(WT_SESSION_IMPL *session, const char *fmt, size_t len, bool *fixedp, uint32_t *fixed_lenp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_struct_confchk(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *v) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_struct_size(WT_SESSION_IMPL *session, size_t *sizep, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_struct_pack(WT_SESSION_IMPL *session, void *buffer, size_t size, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_struct_unpack(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_struct_repack(WT_SESSION_IMPL *session, const char *infmt, const char *outfmt, const WT_ITEM *inbuf, WT_ITEM *outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_pack_start(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *format, void *buffer, size_t size, WT_PACK_STREAM **psp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_unpack_start(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *format, const void *buffer, size_t size, WT_PACK_STREAM **psp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_pack_close(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, size_t *usedp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_pack_item(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, WT_ITEM *item) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_pack_int(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, int64_t i) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_pack_str(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, const char *s) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_pack_uint(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, uint64_t u) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_unpack_item(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, WT_ITEM *item) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_unpack_int(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, int64_t *ip) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_unpack_str(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, const char **sp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_unpack_uint(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, uint64_t *up) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ovfl_discard_add(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL *cell) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_ovfl_discard_free(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ovfl_reuse_search(WT_SESSION_IMPL *session, WT_PAGE *page, uint8_t **addrp, size_t *addr_sizep, const void *value, size_t value_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ovfl_reuse_add(WT_SESSION_IMPL *session, WT_PAGE *page, const uint8_t *addr, size_t addr_size, const void *value, size_t value_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_ovfl_reuse_free(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ovfl_txnc_search( WT_PAGE *page, const uint8_t *addr, size_t addr_size, WT_ITEM *store) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ovfl_txnc_add(WT_SESSION_IMPL *session, WT_PAGE *page, const uint8_t *addr, size_t addr_size, const void *value, size_t value_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_ovfl_txnc_free(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ovfl_track_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ovfl_track_wrapup_err(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage, uint32_t flags, bool *lookaside_retryp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern uint32_t __wt_split_page_size(WT_BTREE *btree, uint32_t maxpagesize) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_bulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_bulk_wrapup(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_bulk_insert_fix( WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool deleted) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_bulk_insert_fix_bitmap(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_bulk_insert_var( WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool deleted) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_alter(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_direct_io_size_check(WT_SESSION_IMPL *session, const char **cfg, const char *config_name, uint32_t *allocsizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_colgroup_source(WT_SESSION_IMPL *session, WT_TABLE *table, const char *cgname, const char *config, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_index_source(WT_SESSION_IMPL *session, WT_TABLE *table, const char *idxname, const char *config, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_create( WT_SESSION_IMPL *session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_get_table(WT_SESSION_IMPL *session, const char *name, size_t namelen, bool ok_incomplete, WT_TABLE **tablep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_schema_release_table(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_schema_destroy_colgroup(WT_SESSION_IMPL *session, WT_COLGROUP **colgroupp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_destroy_index(WT_SESSION_IMPL *session, WT_INDEX **idxp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_destroy_table(WT_SESSION_IMPL *session, WT_TABLE **tablep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_remove_table(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_close_tables(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_colgroup_name(WT_SESSION_IMPL *session, WT_TABLE *table, const char *cgname, size_t len, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_open_colgroups(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_open_index(WT_SESSION_IMPL *session, WT_TABLE *table, const char *idxname, size_t len, WT_INDEX **indexp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_open_indices(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_get_colgroup(WT_SESSION_IMPL *session, const char *uri, bool quiet, WT_TABLE **tablep, WT_COLGROUP **colgroupp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_get_index(WT_SESSION_IMPL *session, const char *uri, bool quiet, WT_TABLE **tablep, WT_INDEX **indexp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_open_table(WT_SESSION_IMPL *session, const char *name, size_t namelen, bool ok_incomplete, WT_TABLE **tablep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_colcheck(WT_SESSION_IMPL *session, const char *key_format, const char *value_format, WT_CONFIG_ITEM *colconf, u_int *kcolsp, u_int *vcolsp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_table_check(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_struct_plan(WT_SESSION_IMPL *session, WT_TABLE *table, const char *columns, size_t len, bool value_only, WT_ITEM *plan) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_struct_reformat(WT_SESSION_IMPL *session, WT_TABLE *table, const char *columns, size_t len, const char *extra_cols, bool value_only, WT_ITEM *format) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_struct_truncate(WT_SESSION_IMPL *session, const char *input_fmt, u_int ncols, WT_ITEM *format) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_project_in(WT_SESSION_IMPL *session, WT_CURSOR **cp, const char *proj_arg, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_project_out(WT_SESSION_IMPL *session, WT_CURSOR **cp, const char *proj_arg, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_project_slice(WT_SESSION_IMPL *session, WT_CURSOR **cp, const char *proj_arg, bool key_only, const char *vformat, WT_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_project_merge(WT_SESSION_IMPL *session, WT_CURSOR **cp, const char *proj_arg, const char *vformat, WT_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_rename(WT_SESSION_IMPL *session, const char *uri, const char *newuri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curstat_colgroup_init(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR_STAT *cst) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curstat_index_init(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR_STAT *cst) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_curstat_table_init(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR_STAT *cst) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_truncate( WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_range_truncate(WT_CURSOR *start, WT_CURSOR *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_range_truncate( WT_SESSION_IMPL *session, WT_CURSOR *start, WT_CURSOR *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_backup_check(WT_SESSION_IMPL *session, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern WT_DATA_SOURCE *__wt_schema_get_source(WT_SESSION_IMPL *session, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_str_name_check(WT_SESSION_IMPL *session, const char *str) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_name_check(WT_SESSION_IMPL *session, const char *str, size_t len) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_schema_worker(WT_SESSION_IMPL *session, const char *uri, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[], uint32_t open_flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_session_notsup(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_session_reset_cursors(WT_SESSION_IMPL *session, bool free_buffers) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_session_copy_values(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_session_release_resources(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_open_cursor(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_session_create( WT_SESSION_IMPL *session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_session_range_truncate(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *start, WT_CURSOR *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern const char *__wt_session_strerror(WT_SESSION *wt_session, int error) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const char *config, bool open_metadata, WT_SESSION_IMPL **sessionp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_open_internal_session(WT_CONNECTION_IMPL *conn, const char *name, bool open_metadata, uint32_t session_flags, WT_SESSION_IMPL **sessionp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_session_compact_check_timeout(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_session_compact( WT_SESSION *wt_session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_session_compact_readonly( WT_SESSION *wt_session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_session_lock_dhandle( WT_SESSION_IMPL *session, uint32_t flags, bool *is_deadp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_session_release_btree(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_session_get_btree_ckpt(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_session_close_cache(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_session_get_btree(WT_SESSION_IMPL *session, const char *uri, const char *checkpoint, const char *cfg[], uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_session_lock_checkpoint(WT_SESSION_IMPL *session, const char *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_salvage(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cond_auto_alloc(WT_SESSION_IMPL *session, const char *name, uint64_t min, uint64_t max, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_cond_auto_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_cond_auto_wait(WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool (*run_func)(WT_SESSION_IMPL *)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_decrypt(WT_SESSION_IMPL *session, WT_ENCRYPTOR *encryptor, size_t skip, WT_ITEM *in, WT_ITEM *out) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_encrypt(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t skip, WT_ITEM *in, WT_ITEM *out) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_encrypt_size(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t incoming_size, size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_event_handler_set(WT_SESSION_IMPL *session, WT_EVENT_HANDLER *handler) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, const char *file_name, int line_number, const char *fmt, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_ext_struct_pack(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, void *buffer, size_t size, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_struct_size(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, size_t *sizep, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_struct_unpack(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const void *buffer, size_t size, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_struct_check(WT_SESSION_IMPL *session, const char *fmt, size_t len, bool *fixedp, uint32_t *fixed_lenp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_struct_confchk(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *v) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_struct_size(WT_SESSION_IMPL *session, size_t *sizep, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_struct_pack(WT_SESSION_IMPL *session, void *buffer, size_t size, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_struct_unpack(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_struct_repack(WT_SESSION_IMPL *session, const char *infmt, const char *outfmt, const WT_ITEM *inbuf, WT_ITEM *outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_pack_start(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *format, void *buffer, size_t size, WT_PACK_STREAM **psp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_unpack_start(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *format, const void *buffer, size_t size, WT_PACK_STREAM **psp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_pack_close(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, size_t *usedp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_pack_item(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, WT_ITEM *item) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_pack_int(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, int64_t i) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_pack_str(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, const char *s) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_pack_uint(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, uint64_t u) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_unpack_item(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, WT_ITEM *item) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_unpack_int(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, int64_t *ip) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_unpack_str(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, const char **sp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_unpack_uint(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, uint64_t *up) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ovfl_discard_add(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL *cell) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_ovfl_discard_free(WT_SESSION_IMPL *session, WT_PAGE *page); +extern int __wt_ovfl_reuse_search(WT_SESSION_IMPL *session, WT_PAGE *page, uint8_t **addrp, size_t *addr_sizep, const void *value, size_t value_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ovfl_reuse_add(WT_SESSION_IMPL *session, WT_PAGE *page, const uint8_t *addr, size_t addr_size, const void *value, size_t value_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_ovfl_reuse_free(WT_SESSION_IMPL *session, WT_PAGE *page); +extern int __wt_ovfl_txnc_search( WT_PAGE *page, const uint8_t *addr, size_t addr_size, WT_ITEM *store) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ovfl_txnc_add(WT_SESSION_IMPL *session, WT_PAGE *page, const uint8_t *addr, size_t addr_size, const void *value, size_t value_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_ovfl_txnc_free(WT_SESSION_IMPL *session, WT_PAGE *page); +extern int __wt_ovfl_track_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ovfl_track_wrapup_err(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage, uint32_t flags, bool *lookaside_retryp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern uint32_t __wt_split_page_size(WT_BTREE *btree, uint32_t maxpagesize); +extern int __wt_bulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bulk_wrapup(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bulk_insert_fix( WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool deleted) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bulk_insert_fix_bitmap(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bulk_insert_var( WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool deleted) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_alter(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_direct_io_size_check(WT_SESSION_IMPL *session, const char **cfg, const char *config_name, uint32_t *allocsizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_colgroup_source(WT_SESSION_IMPL *session, WT_TABLE *table, const char *cgname, const char *config, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_index_source(WT_SESSION_IMPL *session, WT_TABLE *table, const char *idxname, const char *config, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_create( WT_SESSION_IMPL *session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_get_table(WT_SESSION_IMPL *session, const char *name, size_t namelen, bool ok_incomplete, WT_TABLE **tablep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_schema_release_table(WT_SESSION_IMPL *session, WT_TABLE *table); +extern void __wt_schema_destroy_colgroup(WT_SESSION_IMPL *session, WT_COLGROUP **colgroupp); +extern int __wt_schema_destroy_index(WT_SESSION_IMPL *session, WT_INDEX **idxp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_destroy_table(WT_SESSION_IMPL *session, WT_TABLE **tablep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_remove_table(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_close_tables(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_colgroup_name(WT_SESSION_IMPL *session, WT_TABLE *table, const char *cgname, size_t len, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_open_colgroups(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_open_index(WT_SESSION_IMPL *session, WT_TABLE *table, const char *idxname, size_t len, WT_INDEX **indexp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_open_indices(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_get_colgroup(WT_SESSION_IMPL *session, const char *uri, bool quiet, WT_TABLE **tablep, WT_COLGROUP **colgroupp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_get_index(WT_SESSION_IMPL *session, const char *uri, bool quiet, WT_TABLE **tablep, WT_INDEX **indexp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_open_table(WT_SESSION_IMPL *session, const char *name, size_t namelen, bool ok_incomplete, WT_TABLE **tablep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_colcheck(WT_SESSION_IMPL *session, const char *key_format, const char *value_format, WT_CONFIG_ITEM *colconf, u_int *kcolsp, u_int *vcolsp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_table_check(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_struct_plan(WT_SESSION_IMPL *session, WT_TABLE *table, const char *columns, size_t len, bool value_only, WT_ITEM *plan) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_struct_reformat(WT_SESSION_IMPL *session, WT_TABLE *table, const char *columns, size_t len, const char *extra_cols, bool value_only, WT_ITEM *format) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_struct_truncate(WT_SESSION_IMPL *session, const char *input_fmt, u_int ncols, WT_ITEM *format) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_project_in(WT_SESSION_IMPL *session, WT_CURSOR **cp, const char *proj_arg, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_project_out(WT_SESSION_IMPL *session, WT_CURSOR **cp, const char *proj_arg, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_project_slice(WT_SESSION_IMPL *session, WT_CURSOR **cp, const char *proj_arg, bool key_only, const char *vformat, WT_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_project_merge(WT_SESSION_IMPL *session, WT_CURSOR **cp, const char *proj_arg, const char *vformat, WT_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_rename(WT_SESSION_IMPL *session, const char *uri, const char *newuri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curstat_colgroup_init(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR_STAT *cst) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curstat_index_init(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR_STAT *cst) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curstat_table_init(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR_STAT *cst) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_truncate( WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_range_truncate(WT_CURSOR *start, WT_CURSOR *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_range_truncate( WT_SESSION_IMPL *session, WT_CURSOR *start, WT_CURSOR *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_backup_check(WT_SESSION_IMPL *session, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern WT_DATA_SOURCE *__wt_schema_get_source(WT_SESSION_IMPL *session, const char *name); +extern int __wt_str_name_check(WT_SESSION_IMPL *session, const char *str) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_name_check(WT_SESSION_IMPL *session, const char *str, size_t len) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_worker(WT_SESSION_IMPL *session, const char *uri, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[], uint32_t open_flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_notsup(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_reset_cursors(WT_SESSION_IMPL *session, bool free_buffers) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_copy_values(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_release_resources(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_open_cursor(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_create( WT_SESSION_IMPL *session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_range_truncate(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *start, WT_CURSOR *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern const char *__wt_session_strerror(WT_SESSION *wt_session, int error); +extern int __wt_open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const char *config, bool open_metadata, WT_SESSION_IMPL **sessionp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_open_internal_session(WT_CONNECTION_IMPL *conn, const char *name, bool open_metadata, uint32_t session_flags, WT_SESSION_IMPL **sessionp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_compact_check_timeout(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_compact( WT_SESSION *wt_session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_compact_readonly( WT_SESSION *wt_session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_lock_dhandle( WT_SESSION_IMPL *session, uint32_t flags, bool *is_deadp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_release_btree(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_get_btree_ckpt(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_session_close_cache(WT_SESSION_IMPL *session); +extern int __wt_session_get_btree(WT_SESSION_IMPL *session, const char *uri, const char *checkpoint, const char *cfg[], uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_lock_checkpoint(WT_SESSION_IMPL *session, const char *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_salvage(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cond_auto_alloc(WT_SESSION_IMPL *session, const char *name, uint64_t min, uint64_t max, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cond_auto_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled); +extern void __wt_cond_auto_wait(WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool (*run_func)(WT_SESSION_IMPL *)); +extern int __wt_decrypt(WT_SESSION_IMPL *session, WT_ENCRYPTOR *encryptor, size_t skip, WT_ITEM *in, WT_ITEM *out) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_encrypt(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t skip, WT_ITEM *in, WT_ITEM *out) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_encrypt_size(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t incoming_size, size_t *sizep); +extern void __wt_event_handler_set(WT_SESSION_IMPL *session, WT_EVENT_HANDLER *handler); +extern int __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, const char *file_name, int line_number, const char *fmt, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_err(WT_SESSION_IMPL *session, int error, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern void __wt_errx(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_err_printf( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_msg(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_msg_printf( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern const char *__wt_ext_strerror(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, int error) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_progress(WT_SESSION_IMPL *session, const char *s, uint64_t v) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern void __wt_errx(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))); +extern int __wt_ext_err_printf( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_msg(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_msg_printf( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern const char *__wt_ext_strerror(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, int error); +extern int __wt_progress(WT_SESSION_IMPL *session, const char *s, uint64_t v) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_assert(WT_SESSION_IMPL *session, int error, const char *file_name, int line_number, const char *fmt, ...) @@ -644,129 +645,139 @@ __wt_assert(WT_SESSION_IMPL *session, WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_panic(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_illegal_value(WT_SESSION_IMPL *session, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_object_unsupported(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_bad_object_type(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_unexpected_object_type( WT_SESSION_IMPL *session, const char *uri, const char *expect) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_library_init(void) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_breakpoint(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_attach(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern uint64_t __wt_hash_city64(const void *s, size_t len) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern uint64_t __wt_hash_fnv64(const void *string, size_t len) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_object_unsupported(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bad_object_type(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_unexpected_object_type( WT_SESSION_IMPL *session, const char *uri, const char *expect) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_gen_init(WT_SESSION_IMPL *session); +extern uint64_t __wt_gen(WT_SESSION_IMPL *session, int which); +extern uint64_t __wt_gen_next(WT_SESSION_IMPL *session, int which); +extern uint64_t __wt_gen_next_drain(WT_SESSION_IMPL *session, int which); +extern void __wt_gen_drain(WT_SESSION_IMPL *session, int which, uint64_t generation); +extern uint64_t __wt_gen_oldest(WT_SESSION_IMPL *session, int which); +extern uint64_t __wt_session_gen(WT_SESSION_IMPL *session, int which); +extern void __wt_session_gen_enter(WT_SESSION_IMPL *session, int which); +extern void __wt_session_gen_leave(WT_SESSION_IMPL *session, int which); +extern void __wt_stash_discard(WT_SESSION_IMPL *session); +extern int __wt_stash_add(WT_SESSION_IMPL *session, int which, uint64_t generation, void *p, size_t len) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_stash_discard_all(WT_SESSION_IMPL *session_safe, WT_SESSION_IMPL *session); +extern int __wt_library_init(void) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_breakpoint(void); +extern void __wt_attach(WT_SESSION_IMPL *session); +extern uint64_t __wt_hash_city64(const void *s, size_t len); +extern uint64_t __wt_hash_fnv64(const void *string, size_t len); extern int __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp #ifdef HAVE_DIAGNOSTIC , const char *file, int line #endif - ) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_hazard_clear(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_hazard_close(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern WT_HAZARD *__wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern u_int __wt_hazard_count(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_fill_hex(const uint8_t *src, size_t src_max, uint8_t *dest, size_t dest_max, size_t *lenp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_raw_to_hex( WT_SESSION_IMPL *session, const uint8_t *from, size_t size, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_raw_to_esc_hex( WT_SESSION_IMPL *session, const uint8_t *from, size_t size, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_hex2byte(const u_char *from, u_char *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_hex_to_raw(WT_SESSION_IMPL *session, const char *from, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_nhex_to_raw( WT_SESSION_IMPL *session, const char *from, size_t size, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_esc_hex_to_raw(WT_SESSION_IMPL *session, const char *from, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_huffman_open(WT_SESSION_IMPL *session, void *symbol_frequency_array, u_int symcnt, u_int numbytes, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_huffman_close(WT_SESSION_IMPL *session, void *huffman_arg) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_print_huffman_code(void *huffman_arg, uint16_t symbol) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_huffman_encode(WT_SESSION_IMPL *session, void *huffman_arg, const uint8_t *from_arg, size_t from_len, WT_ITEM *to_buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_huffman_decode(WT_SESSION_IMPL *session, void *huffman_arg, const uint8_t *from_arg, size_t from_len, WT_ITEM *to_buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_rwlock_init(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_rwlock_destroy(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_try_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_readlock_spin(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_readunlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_writeunlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern bool __wt_rwlock_islocked(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern uint32_t __wt_nlpo2_round(uint32_t v) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern uint32_t __wt_nlpo2(uint32_t v) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern uint32_t __wt_log2_int(uint32_t n) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern bool __wt_ispo2(uint32_t v) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern uint32_t __wt_rduppo2(uint32_t n, uint32_t po2) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); + ); +extern int __wt_hazard_clear(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_hazard_close(WT_SESSION_IMPL *session); +extern WT_HAZARD *__wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref); +extern u_int __wt_hazard_count(WT_SESSION_IMPL *session, WT_REF *ref); +extern void __wt_fill_hex(const uint8_t *src, size_t src_max, uint8_t *dest, size_t dest_max, size_t *lenp); +extern int __wt_raw_to_hex( WT_SESSION_IMPL *session, const uint8_t *from, size_t size, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_raw_to_esc_hex( WT_SESSION_IMPL *session, const uint8_t *from, size_t size, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_hex2byte(const u_char *from, u_char *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_hex_to_raw(WT_SESSION_IMPL *session, const char *from, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_nhex_to_raw( WT_SESSION_IMPL *session, const char *from, size_t size, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_esc_hex_to_raw(WT_SESSION_IMPL *session, const char *from, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_huffman_open(WT_SESSION_IMPL *session, void *symbol_frequency_array, u_int symcnt, u_int numbytes, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_huffman_close(WT_SESSION_IMPL *session, void *huffman_arg); +extern void __wt_print_huffman_code(void *huffman_arg, uint16_t symbol); +extern int __wt_huffman_encode(WT_SESSION_IMPL *session, void *huffman_arg, const uint8_t *from_arg, size_t from_len, WT_ITEM *to_buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_huffman_decode(WT_SESSION_IMPL *session, void *huffman_arg, const uint8_t *from_arg, size_t from_len, WT_ITEM *to_buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_rwlock_init(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_rwlock_destroy(WT_SESSION_IMPL *session, WT_RWLOCK *l); +extern int __wt_try_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l); +extern void __wt_readunlock(WT_SESSION_IMPL *session, WT_RWLOCK *l); +extern int __wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l); +extern void __wt_writeunlock(WT_SESSION_IMPL *session, WT_RWLOCK *l); +extern bool __wt_rwlock_islocked(WT_SESSION_IMPL *session, WT_RWLOCK *l); +extern uint32_t __wt_nlpo2_round(uint32_t v); +extern uint32_t __wt_nlpo2(uint32_t v); +extern uint32_t __wt_log2_int(uint32_t n); +extern bool __wt_ispo2(uint32_t v); +extern uint32_t __wt_rduppo2(uint32_t n, uint32_t po2); extern void __wt_random_init(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_random_init_seed( WT_SESSION_IMPL *session, WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern uint32_t __wt_random(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern uint64_t __wt_random64(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern int __wt_buf_grow_worker(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_buf_fmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_buf_catfmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern const char *__wt_buf_set_printable( WT_SESSION_IMPL *session, const void *p, size_t size, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern const char *__wt_buf_set_size( WT_SESSION_IMPL *session, uint64_t size, bool exact, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_buf_grow_worker(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_buf_fmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_buf_catfmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern const char *__wt_buf_set_printable( WT_SESSION_IMPL *session, const void *p, size_t size, WT_ITEM *buf); +extern const char *__wt_buf_set_size( WT_SESSION_IMPL *session, uint64_t size, bool exact, WT_ITEM *buf); extern int __wt_scr_alloc_func(WT_SESSION_IMPL *session, size_t size, WT_ITEM **scratchp #ifdef HAVE_DIAGNOSTIC , const char *file, int line #endif - ) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_scr_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void *__wt_ext_scr_alloc( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_ext_scr_free(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, void *p) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_stat_dsrc_desc(WT_CURSOR_STAT *cst, int slot, const char **p) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_stat_dsrc_init_single(WT_DSRC_STATS *stats) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_stat_dsrc_init( WT_SESSION_IMPL *session, WT_DATA_HANDLE *handle) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_stat_dsrc_discard( WT_SESSION_IMPL *session, WT_DATA_HANDLE *handle) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_stat_dsrc_clear_all(WT_DSRC_STATS **stats) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_stat_dsrc_aggregate_single( WT_DSRC_STATS *from, WT_DSRC_STATS *to) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_stat_dsrc_aggregate( WT_DSRC_STATS **from, WT_DSRC_STATS *to) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_stat_connection_desc(WT_CURSOR_STAT *cst, int slot, const char **p) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_stat_connection_init_single(WT_CONNECTION_STATS *stats) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_stat_connection_init( WT_SESSION_IMPL *session, WT_CONNECTION_IMPL *handle) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_stat_connection_discard( WT_SESSION_IMPL *session, WT_CONNECTION_IMPL *handle) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_stat_connection_clear_all(WT_CONNECTION_STATS **stats) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_stat_connection_aggregate( WT_CONNECTION_STATS **from, WT_CONNECTION_STATS *to) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_stat_join_desc(WT_CURSOR_STAT *cst, int slot, const char **p) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_stat_join_init_single(WT_JOIN_STATS *stats) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_stat_join_clear_single(WT_JOIN_STATS *stats) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_stat_join_clear_all(WT_JOIN_STATS **stats) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_stat_join_aggregate( WT_JOIN_STATS **from, WT_JOIN_STATS *to) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern WT_THREAD_RET __wt_thread_run(void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_thread_group_resize( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, uint32_t new_min, uint32_t new_max, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_thread_group_create( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, const char *name, uint32_t min, uint32_t max, uint32_t flags, int (*run_func)(WT_SESSION_IMPL *session, WT_THREAD *context)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_thread_group_destroy(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_thread_group_start_one( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, bool wait) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_thread_group_stop_one( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, bool wait) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_txn_release_snapshot(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_txn_get_snapshot(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_txn_release(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_txn_rollback(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_txn_init(WT_SESSION_IMPL *session, WT_SESSION_IMPL *session_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_txn_stats_update(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_txn_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_txn_global_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_verbose_dump_txn(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[], bool waiting) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_checkpoint_sync(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern uint64_t __wt_ext_transaction_id(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_transaction_isolation_level( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_transaction_notify( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, WT_TXN_NOTIFY *notify) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern uint64_t __wt_ext_transaction_oldest(WT_EXTENSION_API *wt_api) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_ext_transaction_visible( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, uint64_t transaction_id) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_txn_op_free(WT_SESSION_IMPL *session, WT_TXN_OP *op) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_txn_log_op(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_txn_log_commit(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_txn_checkpoint_logread(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, WT_LSN *ckpt_lsn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_txn_checkpoint_log( WT_SESSION_IMPL *session, bool full, uint32_t flags, WT_LSN *lsnp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_txn_truncate_log( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_txn_truncate_end(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); + ); +extern void __wt_scr_discard(WT_SESSION_IMPL *session); +extern void *__wt_ext_scr_alloc( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, size_t size); +extern void __wt_ext_scr_free(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, void *p); +extern int __wt_stat_dsrc_desc(WT_CURSOR_STAT *cst, int slot, const char **p) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_stat_dsrc_init_single(WT_DSRC_STATS *stats); +extern int __wt_stat_dsrc_init( WT_SESSION_IMPL *session, WT_DATA_HANDLE *handle) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_stat_dsrc_discard( WT_SESSION_IMPL *session, WT_DATA_HANDLE *handle); +extern void __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats); +extern void __wt_stat_dsrc_clear_all(WT_DSRC_STATS **stats); +extern void __wt_stat_dsrc_aggregate_single( WT_DSRC_STATS *from, WT_DSRC_STATS *to); +extern void __wt_stat_dsrc_aggregate( WT_DSRC_STATS **from, WT_DSRC_STATS *to); +extern int __wt_stat_connection_desc(WT_CURSOR_STAT *cst, int slot, const char **p) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_stat_connection_init_single(WT_CONNECTION_STATS *stats); +extern int __wt_stat_connection_init( WT_SESSION_IMPL *session, WT_CONNECTION_IMPL *handle) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_stat_connection_discard( WT_SESSION_IMPL *session, WT_CONNECTION_IMPL *handle); +extern void __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats); +extern void __wt_stat_connection_clear_all(WT_CONNECTION_STATS **stats); +extern void __wt_stat_connection_aggregate( WT_CONNECTION_STATS **from, WT_CONNECTION_STATS *to); +extern int __wt_stat_join_desc(WT_CURSOR_STAT *cst, int slot, const char **p) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_stat_join_init_single(WT_JOIN_STATS *stats); +extern void __wt_stat_join_clear_single(WT_JOIN_STATS *stats); +extern void __wt_stat_join_clear_all(WT_JOIN_STATS **stats); +extern void __wt_stat_join_aggregate( WT_JOIN_STATS **from, WT_JOIN_STATS *to); +extern int __wt_thread_group_resize( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, uint32_t new_min, uint32_t new_max, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_thread_group_create( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, const char *name, uint32_t min, uint32_t max, uint32_t flags, bool (*chk_func)(WT_SESSION_IMPL *session), int (*run_func)(WT_SESSION_IMPL *session, WT_THREAD *context), int (*stop_func)(WT_SESSION_IMPL *session, WT_THREAD *context)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_thread_group_destroy(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_thread_group_start_one( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, bool is_locked); +extern void __wt_thread_group_stop_one(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group); +extern void __wt_txn_release_snapshot(WT_SESSION_IMPL *session); +extern void __wt_txn_get_snapshot(WT_SESSION_IMPL *session); +extern int __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_txn_release(WT_SESSION_IMPL *session); +extern int __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_rollback(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_init(WT_SESSION_IMPL *session, WT_SESSION_IMPL *session_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_txn_stats_update(WT_SESSION_IMPL *session); +extern void __wt_txn_destroy(WT_SESSION_IMPL *session); +extern int __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_txn_global_destroy(WT_SESSION_IMPL *session); +extern int __wt_verbose_dump_txn(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[], bool waiting) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_checkpoint_sync(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern uint64_t __wt_ext_transaction_id(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session); +extern int __wt_ext_transaction_isolation_level( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_transaction_notify( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, WT_TXN_NOTIFY *notify) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern uint64_t __wt_ext_transaction_oldest(WT_EXTENSION_API *wt_api); +extern int __wt_ext_transaction_visible( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, uint64_t transaction_id) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_txn_op_free(WT_SESSION_IMPL *session, WT_TXN_OP *op); +extern int __wt_txn_log_op(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_log_commit(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_checkpoint_logread(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, WT_LSN *ckpt_lsn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_checkpoint_log( WT_SESSION_IMPL *session, bool full, uint32_t flags, WT_LSN *lsnp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_truncate_log( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_txn_truncate_end(WT_SESSION_IMPL *session); extern int __wt_txn_printlog(WT_SESSION *wt_session, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_txn_named_snapshot_begin(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_txn_named_snapshot_drop(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_txn_named_snapshot_get(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *nameval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_txn_named_snapshot_config(WT_SESSION_IMPL *session, const char *cfg[], bool *has_create, bool *has_drops) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_txn_named_snapshot_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_txn_recover(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_txn_named_snapshot_begin(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_named_snapshot_drop(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_named_snapshot_get(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *nameval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_named_snapshot_config(WT_SESSION_IMPL *session, const char *cfg[], bool *has_create, bool *has_drops) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_txn_named_snapshot_destroy(WT_SESSION_IMPL *session); +extern int __wt_txn_recover(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/include/extern_posix.h b/src/include/extern_posix.h index 57d94e392d1..b6b5ac51f73 100644 --- a/src/include/extern_posix.h +++ b/src/include/extern_posix.h @@ -1,32 +1,32 @@ /* DO NOT EDIT: automatically built by dist/s_prototypes. */ -extern int __wt_posix_directory_list(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *directory, const char *prefix, char ***dirlistp, uint32_t *countp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_posix_directory_list_free(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, char **dirlist, uint32_t count) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_dlopen(WT_SESSION_IMPL *session, const char *path, WT_DLH **dlhp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_dlsym(WT_SESSION_IMPL *session, WT_DLH *dlh, const char *name, bool fail, void *sym_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_dlclose(WT_SESSION_IMPL *session, WT_DLH *dlh) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_posix_file_extend( WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, wt_off_t offset) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_os_posix(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_posix_directory_list(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *directory, const char *prefix, char ***dirlistp, uint32_t *countp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_posix_directory_list_free(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, char **dirlist, uint32_t count) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_dlopen(WT_SESSION_IMPL *session, const char *path, WT_DLH **dlhp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_dlsym(WT_SESSION_IMPL *session, WT_DLH *dlh, const char *name, bool fail, void *sym_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_dlclose(WT_SESSION_IMPL *session, WT_DLH *dlh) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_posix_file_extend( WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, wt_off_t offset) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_os_posix(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_posix_map(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, void *mapped_regionp, size_t *lenp, void *mapped_cookiep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_posix_map_preload(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, const void *map, size_t length, void *mapped_cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_posix_map_discard(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, void *map, size_t length, void *mapped_cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_posix_unmap(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, void *mapped_region, size_t len, void *mapped_cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_cond_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_once(void (*init_routine)(void)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_get_vm_pagesize(void) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern bool __wt_absolute_path(const char *path) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern const char *__wt_path_separator(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern bool __wt_has_priv(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_posix_map(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, void *mapped_regionp, size_t *lenp, void *mapped_cookiep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_posix_map_preload(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, const void *map, size_t length, void *mapped_cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_posix_map_discard(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, void *map, size_t length, void *mapped_cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_posix_unmap(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, void *mapped_region, size_t len, void *mapped_cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cond_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled); +extern void __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond); +extern void __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp); +extern int __wt_once(void (*init_routine)(void)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_get_vm_pagesize(void) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern bool __wt_absolute_path(const char *path); +extern const char *__wt_path_separator(void); +extern bool __wt_has_priv(void); extern void __wt_stream_set_line_buffer(FILE *fp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_stream_set_no_buffer(FILE *fp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_sleep(uint64_t seconds, uint64_t micro_seconds) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_vsnprintf_len_incr( char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_id(char *buf, size_t buflen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_yield(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); diff --git a/src/include/extern_win.h b/src/include/extern_win.h index 43127a0c79f..d548ee0b2ec 100644 --- a/src/include/extern_win.h +++ b/src/include/extern_win.h @@ -1,35 +1,35 @@ /* DO NOT EDIT: automatically built by dist/s_prototypes. */ -extern int __wt_win_directory_list(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *directory, const char *prefix, char ***dirlistp, uint32_t *countp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_win_directory_list_free(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, char **dirlist, uint32_t count) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_dlopen(WT_SESSION_IMPL *session, const char *path, WT_DLH **dlhp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_dlsym(WT_SESSION_IMPL *session, WT_DLH *dlh, const char *name, bool fail, void *sym_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_dlclose(WT_SESSION_IMPL *session, WT_DLH *dlh) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_win_fs_size(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *name, wt_off_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_os_win(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_win_map(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, void *mapped_regionp, size_t *lenp, void *mapped_cookiep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_win_unmap(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, void *mapped_region, size_t length, void *mapped_cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_cond_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_once(void (*init_routine)(void)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_get_vm_pagesize(void) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern bool __wt_absolute_path(const char *path) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern const char *__wt_path_separator(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern bool __wt_has_priv(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_stream_set_line_buffer(FILE *fp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_stream_set_no_buffer(FILE *fp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_sleep(uint64_t seconds, uint64_t micro_seconds) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_vsnprintf_len_incr( char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_thread_id(char *buf, size_t buflen) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_to_utf16_string( WT_SESSION_IMPL *session, const char*utf8, WT_ITEM **outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_to_utf8_string( WT_SESSION_IMPL *session, const wchar_t*wide, WT_ITEM **outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern DWORD __wt_getlasterror(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern int __wt_map_windows_error(DWORD windows_error) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern const char *__wt_formatmessage(WT_SESSION_IMPL *session, DWORD windows_error) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); -extern void __wt_yield(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden"))); +extern int __wt_win_directory_list(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *directory, const char *prefix, char ***dirlistp, uint32_t *countp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_win_directory_list_free(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, char **dirlist, uint32_t count) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_dlopen(WT_SESSION_IMPL *session, const char *path, WT_DLH **dlhp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_dlsym(WT_SESSION_IMPL *session, WT_DLH *dlh, const char *name, bool fail, void *sym_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_dlclose(WT_SESSION_IMPL *session, WT_DLH *dlh) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_win_fs_size(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *name, wt_off_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_os_win(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_win_map(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, void *mapped_regionp, size_t *lenp, void *mapped_cookiep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_win_unmap(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, void *mapped_region, size_t length, void *mapped_cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cond_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled); +extern void __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond); +extern void __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp); +extern int __wt_once(void (*init_routine)(void)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_get_vm_pagesize(void) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern bool __wt_absolute_path(const char *path); +extern const char *__wt_path_separator(void); +extern bool __wt_has_priv(void); +extern void __wt_stream_set_line_buffer(FILE *fp); +extern void __wt_stream_set_no_buffer(FILE *fp); +extern void __wt_sleep(uint64_t seconds, uint64_t micro_seconds); +extern int __wt_vsnprintf_len_incr( char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_thread_id(char *buf, size_t buflen) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp); +extern int __wt_to_utf16_string( WT_SESSION_IMPL *session, const char*utf8, WT_ITEM **outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_to_utf8_string( WT_SESSION_IMPL *session, const wchar_t*wide, WT_ITEM **outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern DWORD __wt_getlasterror(void); +extern int __wt_map_windows_error(DWORD windows_error) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern const char *__wt_formatmessage(WT_SESSION_IMPL *session, DWORD windows_error); +extern void __wt_yield(void); diff --git a/src/include/flags.h b/src/include/flags.h index f26a45c68f5..919c0dd2f98 100644 --- a/src/include/flags.h +++ b/src/include/flags.h @@ -47,9 +47,8 @@ #define WT_READ_PREV 0x00000080 #define WT_READ_RESTART_OK 0x00000100 #define WT_READ_SKIP_INTL 0x00000200 -#define WT_READ_SKIP_LEAF 0x00000400 -#define WT_READ_TRUNCATE 0x00000800 -#define WT_READ_WONT_NEED 0x00001000 +#define WT_READ_TRUNCATE 0x00000400 +#define WT_READ_WONT_NEED 0x00000800 #define WT_SESSION_CAN_WAIT 0x00000001 #define WT_SESSION_INTERNAL 0x00000002 #define WT_SESSION_LOCKED_CHECKPOINT 0x00000004 @@ -96,25 +95,26 @@ #define WT_VERB_FILEOPS 0x00000080 #define WT_VERB_HANDLEOPS 0x00000100 #define WT_VERB_LOG 0x00000200 -#define WT_VERB_LSM 0x00000400 -#define WT_VERB_LSM_MANAGER 0x00000800 -#define WT_VERB_METADATA 0x00001000 -#define WT_VERB_MUTEX 0x00002000 -#define WT_VERB_OVERFLOW 0x00004000 -#define WT_VERB_READ 0x00008000 -#define WT_VERB_REBALANCE 0x00010000 -#define WT_VERB_RECONCILE 0x00020000 -#define WT_VERB_RECOVERY 0x00040000 -#define WT_VERB_RECOVERY_PROGRESS 0x00080000 -#define WT_VERB_SALVAGE 0x00100000 -#define WT_VERB_SHARED_CACHE 0x00200000 -#define WT_VERB_SPLIT 0x00400000 -#define WT_VERB_TEMPORARY 0x00800000 -#define WT_VERB_THREAD_GROUP 0x01000000 -#define WT_VERB_TRANSACTION 0x02000000 -#define WT_VERB_VERIFY 0x04000000 -#define WT_VERB_VERSION 0x08000000 -#define WT_VERB_WRITE 0x10000000 +#define WT_VERB_LOOKASIDE 0x00000400 +#define WT_VERB_LSM 0x00000800 +#define WT_VERB_LSM_MANAGER 0x00001000 +#define WT_VERB_METADATA 0x00002000 +#define WT_VERB_MUTEX 0x00004000 +#define WT_VERB_OVERFLOW 0x00008000 +#define WT_VERB_READ 0x00010000 +#define WT_VERB_REBALANCE 0x00020000 +#define WT_VERB_RECONCILE 0x00040000 +#define WT_VERB_RECOVERY 0x00080000 +#define WT_VERB_RECOVERY_PROGRESS 0x00100000 +#define WT_VERB_SALVAGE 0x00200000 +#define WT_VERB_SHARED_CACHE 0x00400000 +#define WT_VERB_SPLIT 0x00800000 +#define WT_VERB_TEMPORARY 0x01000000 +#define WT_VERB_THREAD_GROUP 0x02000000 +#define WT_VERB_TRANSACTION 0x04000000 +#define WT_VERB_VERIFY 0x08000000 +#define WT_VERB_VERSION 0x10000000 +#define WT_VERB_WRITE 0x20000000 #define WT_VISIBILITY_ERR 0x00000080 /* * flags section: END diff --git a/src/include/gcc.h b/src/include/gcc.h index 22d78fc165a..21eaaaef049 100644 --- a/src/include/gcc.h +++ b/src/include/gcc.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -9,7 +9,7 @@ #define WT_PTRDIFFT_FMT "td" /* ptrdiff_t format string */ #define WT_SIZET_FMT "zu" /* size_t format string */ -/* Add GCC-specific attributes to types and function declarations. */ +/* GCC-specific attributes. */ #define WT_PACKED_STRUCT_BEGIN(name) \ struct __attribute__ ((__packed__)) name { #define WT_PACKED_STRUCT_END \ diff --git a/src/include/hardware.h b/src/include/hardware.h index 2530659db21..3ff198be3c7 100644 --- a/src/include/hardware.h +++ b/src/include/hardware.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/intpack.i b/src/include/intpack.i index a534de9d9a8..51e43b21321 100644 --- a/src/include/intpack.i +++ b/src/include/intpack.i @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/lint.h b/src/include/lint.h index 2d0f47988b7..97b91c4c061 100644 --- a/src/include/lint.h +++ b/src/include/lint.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -9,6 +9,7 @@ #define WT_PTRDIFFT_FMT "td" /* ptrdiff_t format string */ #define WT_SIZET_FMT "zu" /* size_t format string */ +/* Lint-specific attributes. */ #define WT_PACKED_STRUCT_BEGIN(name) \ struct name { #define WT_PACKED_STRUCT_END \ diff --git a/src/include/log.h b/src/include/log.h index fb3c961417f..e7bc28cd220 100644 --- a/src/include/log.h +++ b/src/include/log.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -130,7 +130,7 @@ union __wt_lsn { #define WT_LOG_SLOT_FLAGS(state) ((state) & WT_LOG_SLOT_MASK_ON) #define WT_LOG_SLOT_JOINED(state) (((state) & WT_LOG_SLOT_MASK_OFF) >> 32) #define WT_LOG_SLOT_JOINED_BUFFERED(state) \ - (WT_LOG_SLOT_JOINED(state) & \ + (WT_LOG_SLOT_JOINED(state) & \ (WT_LOG_SLOT_UNBUFFERED - 1)) #define WT_LOG_SLOT_JOIN_REL(j, r, s) (((j) << 32) + (r) + (s)) #define WT_LOG_SLOT_RELEASED(state) ((int64_t)(int32_t)(state)) diff --git a/src/include/log.i b/src/include/log.i index 9e6c36291f7..8c7e5dc65e8 100644 --- a/src/include/log.i +++ b/src/include/log.i @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/lsm.h b/src/include/lsm.h index e3f6897ef9d..f8d0f480cbb 100644 --- a/src/include/lsm.h +++ b/src/include/lsm.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -240,11 +240,11 @@ struct __wt_lsm_tree { * area, copying them into place when a statistics cursor is created. */ #define WT_LSM_TREE_STAT_INCR(session, fld) do { \ - if (WT_STAT_ENABLED(session)) \ + if (WT_STAT_ENABLED(session)) \ ++(fld); \ } while (0) #define WT_LSM_TREE_STAT_INCRV(session, fld, v) do { \ - if (WT_STAT_ENABLED(session)) \ + if (WT_STAT_ENABLED(session)) \ (fld) += (int64_t)(v); \ } while (0) int64_t bloom_false_positive; diff --git a/src/include/meta.h b/src/include/meta.h index 68ac2e339d0..2dd77157caa 100644 --- a/src/include/meta.h +++ b/src/include/meta.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/misc.h b/src/include/misc.h index 9161a215fdc..c84368b235c 100644 --- a/src/include/misc.h +++ b/src/include/misc.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -276,3 +276,22 @@ union __wt_rand_state { uint32_t w, z; } x; }; + +/* + * WT_TAILQ_SAFE_REMOVE_BEGIN/END -- + * Macro to safely walk a TAILQ where we're expecting some underlying + * function to remove elements from the list, but we don't want to stop on + * error, nor do we want an error to turn into an infinite loop. Used during + * shutdown, when we're shutting down various lists. Unlike TAILQ_FOREACH_SAFE, + * this macro works even when the next element gets removed along with the + * current one. + */ +#define WT_TAILQ_SAFE_REMOVE_BEGIN(var, head, field, tvar) \ + for ((tvar) = NULL; ((var) = TAILQ_FIRST(head)) != NULL; \ + (tvar) = (var)) { \ + if ((tvar) == (var)) { \ + /* Leak the structure. */ \ + TAILQ_REMOVE(head, (var), field); \ + continue; \ + } +#define WT_TAILQ_SAFE_REMOVE_END } diff --git a/src/include/misc.i b/src/include/misc.i index 7040886cf82..36a1e1f18eb 100644 --- a/src/include/misc.i +++ b/src/include/misc.i @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -55,6 +55,31 @@ __wt_seconds(WT_SESSION_IMPL *session, time_t *timep) } /* + * __wt_time_check_monotonic -- + * Check and prevent time running backward. If we detect that it has, we + * set the time structure to the previous values, making time stand still + * until we see a time in the future of the highest value seen so far. + */ +static inline void +__wt_time_check_monotonic(WT_SESSION_IMPL *session, struct timespec *tsp) +{ + /* + * Detect time going backward. If so, use the last + * saved timestamp. + */ + if (session == NULL) + return; + + if (tsp->tv_sec < session->last_epoch.tv_sec || + (tsp->tv_sec == session->last_epoch.tv_sec && + tsp->tv_nsec < session->last_epoch.tv_nsec)) { + WT_STAT_CONN_INCR(session, time_travel); + *tsp = session->last_epoch; + } else + session->last_epoch = *tsp; +} + +/* * __wt_verbose -- * Verbose message. * @@ -177,3 +202,21 @@ __wt_snprintf_len_incr( va_end(ap); return (ret); } + +/* + * __wt_txn_context_check -- + * Complain if a transaction is/isn't running. + */ +static inline int +__wt_txn_context_check(WT_SESSION_IMPL *session, bool requires_txn) +{ + if (requires_txn && !F_ISSET(&session->txn, WT_TXN_RUNNING)) + WT_RET_MSG(session, EINVAL, + "%s: only permitted in a running transaction", + session->name); + if (!requires_txn && F_ISSET(&session->txn, WT_TXN_RUNNING)) + WT_RET_MSG(session, EINVAL, + "%s: not permitted in a running transaction", + session->name); + return (0); +} diff --git a/src/include/msvc.h b/src/include/msvc.h index 6c5c8b67647..f1fab2add9e 100644 --- a/src/include/msvc.h +++ b/src/include/msvc.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -16,9 +16,7 @@ #define WT_PTRDIFFT_FMT "Id" /* ptrdiff_t format string */ #define WT_SIZET_FMT "Iu" /* size_t format string */ -/* - * Add MSVC-specific attributes and pragmas to types and function declarations. - */ +/* MSVC-specific attributes. */ #define WT_PACKED_STRUCT_BEGIN(name) \ __pragma(pack(push,1)) \ struct name { diff --git a/src/include/mutex.h b/src/include/mutex.h index 910eb7af5b9..7aeb6160f43 100644 --- a/src/include/mutex.h +++ b/src/include/mutex.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -37,20 +37,48 @@ struct __wt_condvar { * Don't modify this structure without understanding the read/write locking * functions. */ -union __wt_rwlock { /* Read/write lock */ - uint64_t u; - struct { - uint32_t wr; /* Writers and readers */ - } i; - struct { - uint16_t writers; /* Now serving for writers */ - uint16_t readers; /* Now serving for readers */ - uint16_t next; /* Next available ticket number */ - uint16_t writers_active;/* Count of active writers */ - } s; +struct __wt_rwlock { /* Read/write lock */ + volatile union { + uint64_t v; /* Full 64-bit value */ + struct { + uint8_t current; /* Current ticket */ + uint8_t next; /* Next available ticket */ + uint8_t reader; /* Read queue ticket */ + uint8_t __notused; /* Padding */ + uint16_t readers_active;/* Count of active readers */ + uint16_t readers_queued;/* Count of queued readers */ + } s; + } u; + + int16_t stat_read_count_off; /* read acquisitions offset */ + int16_t stat_write_count_off; /* write acquisitions offset */ + int16_t stat_app_usecs_off; /* waiting application threads offset */ + int16_t stat_int_usecs_off; /* waiting server threads offset */ + + WT_CONDVAR *cond_readers; /* Blocking readers */ + WT_CONDVAR *cond_writers; /* Blocking writers */ }; /* + * WT_RWLOCK_INIT_TRACKED -- + * Read write lock initialization, with tracking. + * + * Implemented as a macro so we can pass in a statistics field and convert + * it into a statistics structure array offset. + */ +#define WT_RWLOCK_INIT_TRACKED(session, l, name) do { \ + WT_RET(__wt_rwlock_init(session, l)); \ + (l)->stat_read_count_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ + S2C(session)->stats, lock_##name##_read_count); \ + (l)->stat_write_count_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ + S2C(session)->stats, lock_##name##_write_count); \ + (l)->stat_app_usecs_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ + S2C(session)->stats, lock_##name##_wait_application); \ + (l)->stat_int_usecs_off = (int16_t)WT_STATS_FIELD_TO_OFFSET( \ + S2C(session)->stats, lock_##name##_wait_internal); \ +} while (0) + +/* * Spin locks: * * WiredTiger uses spinlocks for fast mutual exclusion (where operations done @@ -63,11 +91,11 @@ union __wt_rwlock { /* Read/write lock */ #define SPINLOCK_PTHREAD_MUTEX_ADAPTIVE 3 struct __wt_spinlock { - WT_CACHE_LINE_PAD_BEGIN #if SPINLOCK_TYPE == SPINLOCK_GCC + WT_CACHE_LINE_PAD_BEGIN volatile int lock; -#elif SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX ||\ - SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX_ADAPTIVE ||\ +#elif SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX || \ + SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX_ADAPTIVE || \ SPINLOCK_TYPE == SPINLOCK_MSVC wt_mutex_t lock; #else @@ -87,5 +115,8 @@ struct __wt_spinlock { int16_t stat_int_usecs_off; /* waiting server threads offset */ int8_t initialized; /* Lock initialized, for cleanup */ + +#if SPINLOCK_TYPE == SPINLOCK_GCC WT_CACHE_LINE_PAD_END +#endif }; diff --git a/src/include/mutex.i b/src/include/mutex.i index 2d483972ed2..5b14bb24730 100644 --- a/src/include/mutex.i +++ b/src/include/mutex.i @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -102,8 +102,8 @@ __wt_spin_unlock(WT_SESSION_IMPL *session, WT_SPINLOCK *t) __sync_lock_release(&t->lock); } -#elif SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX ||\ - SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX_ADAPTIVE +#elif SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX || \ + SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX_ADAPTIVE /* * __wt_spin_init -- @@ -142,8 +142,8 @@ __wt_spin_destroy(WT_SESSION_IMPL *session, WT_SPINLOCK *t) } } -#if SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX ||\ - SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX_ADAPTIVE +#if SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX || \ + SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX_ADAPTIVE /* * __wt_spin_trylock -- diff --git a/src/include/os.h b/src/include/os.h index 73d89268392..ec1860d19a6 100644 --- a/src/include/os.h +++ b/src/include/os.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/os_fhandle.i b/src/include/os_fhandle.i index 428b14556d9..e5177e64b57 100644 --- a/src/include/os_fhandle.i +++ b/src/include/os_fhandle.i @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/os_fs.i b/src/include/os_fs.i index 4cf1128280e..c81d3f5dec6 100644 --- a/src/include/os_fs.i +++ b/src/include/os_fs.i @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/os_fstream.i b/src/include/os_fstream.i index 98d0622f346..1561274b388 100644 --- a/src/include/os_fstream.i +++ b/src/include/os_fstream.i @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/os_windows.h b/src/include/os_windows.h index c1e5f788dc6..ea54d00af1f 100644 --- a/src/include/os_windows.h +++ b/src/include/os_windows.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -12,7 +12,10 @@ */ typedef CONDITION_VARIABLE wt_cond_t; typedef CRITICAL_SECTION wt_mutex_t; -typedef HANDLE wt_thread_t; +typedef struct { + bool created; + HANDLE id; +} wt_thread_t; /* * Thread callbacks need to match the return signature of _beginthreadex. @@ -39,9 +42,9 @@ struct timespec { * These are POSIX types which Windows lacks * Eventually WiredTiger will migrate away from these types */ -typedef uint32_t u_int; +typedef unsigned int u_int; typedef unsigned char u_char; -typedef uint64_t u_long; +typedef unsigned long u_long; /* * Windows does have ssize_t diff --git a/src/include/packing.i b/src/include/packing.i index 0eadb2f2027..6d302020f1e 100644 --- a/src/include/packing.i +++ b/src/include/packing.i @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/posix.h b/src/include/posix.h index 2593c7b6797..23a4d178e98 100644 --- a/src/include/posix.h +++ b/src/include/posix.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -25,7 +25,10 @@ */ typedef pthread_cond_t wt_cond_t; typedef pthread_mutex_t wt_mutex_t; -typedef pthread_t wt_thread_t; +typedef struct { + bool created; + pthread_t id; +} wt_thread_t; /* * Thread callbacks need to match the platform specific callback types diff --git a/src/include/schema.h b/src/include/schema.h index 50e141d9921..8b8ee5616d1 100644 --- a/src/include/schema.h +++ b/src/include/schema.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -68,8 +68,8 @@ struct __wt_table { bool cg_complete, idx_complete, is_simple; u_int ncolgroups, nindices, nkey_columns; - uint32_t refcnt; /* Number of open cursors */ - uint32_t schema_gen; /* Cached schema generation number */ + uint32_t refcnt; /* Number of open cursors */ + uint64_t schema_gen; /* Cached schema generation number */ }; /* @@ -323,7 +323,7 @@ struct __wt_table { F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST_READ); \ } \ if (__handle_write_locked) { \ - __wt_writelock(session, &__conn->dhandle_lock); \ + __wt_writelock(session, &__conn->dhandle_lock); \ F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE); \ } \ } while (0) diff --git a/src/include/serial.i b/src/include/serial.i index 982f196b0b8..bd0e498f621 100644 --- a/src/include/serial.i +++ b/src/include/serial.i @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -154,7 +154,7 @@ __col_append_serial_func(WT_SESSION_IMPL *session, WT_INSERT_HEAD *ins_head, static inline int __wt_col_append_serial(WT_SESSION_IMPL *session, WT_PAGE *page, WT_INSERT_HEAD *ins_head, WT_INSERT ***ins_stack, WT_INSERT **new_insp, - size_t new_ins_size, uint64_t *recnop, u_int skipdepth) + size_t new_ins_size, uint64_t *recnop, u_int skipdepth, bool exclusive) { WT_INSERT *new_ins = *new_insp; WT_DECL_RET; @@ -165,11 +165,16 @@ __wt_col_append_serial(WT_SESSION_IMPL *session, WT_PAGE *page, /* Clear references to memory we now own and must free on error. */ *new_insp = NULL; - /* Acquire the page's spinlock, call the worker function. */ - WT_PAGE_LOCK(session, page); + /* + * Acquire the page's spinlock unless we already have exclusive access. + * Then call the worker function. + */ + if (!exclusive) + WT_PAGE_LOCK(session, page); ret = __col_append_serial_func( session, ins_head, ins_stack, new_ins, recnop, skipdepth); - WT_PAGE_UNLOCK(session, page); + if (!exclusive) + WT_PAGE_UNLOCK(session, page); if (ret != 0) { /* Free unused memory on error. */ @@ -198,7 +203,7 @@ __wt_col_append_serial(WT_SESSION_IMPL *session, WT_PAGE *page, static inline int __wt_insert_serial(WT_SESSION_IMPL *session, WT_PAGE *page, WT_INSERT_HEAD *ins_head, WT_INSERT ***ins_stack, WT_INSERT **new_insp, - size_t new_ins_size, u_int skipdepth) + size_t new_ins_size, u_int skipdepth, bool exclusive) { WT_INSERT *new_ins = *new_insp; WT_DECL_RET; @@ -220,10 +225,12 @@ __wt_insert_serial(WT_SESSION_IMPL *session, WT_PAGE *page, ret = __insert_simple_func( session, ins_stack, new_ins, skipdepth); else { - WT_PAGE_LOCK(session, page); + if (!exclusive) + WT_PAGE_LOCK(session, page); ret = __insert_serial_func( session, ins_head, ins_stack, new_ins, skipdepth); - WT_PAGE_UNLOCK(session, page); + if (!exclusive) + WT_PAGE_UNLOCK(session, page); } if (ret != 0) { @@ -252,7 +259,8 @@ __wt_insert_serial(WT_SESSION_IMPL *session, WT_PAGE *page, */ static inline int __wt_update_serial(WT_SESSION_IMPL *session, WT_PAGE *page, - WT_UPDATE **srch_upd, WT_UPDATE **updp, size_t upd_size) + WT_UPDATE **srch_upd, WT_UPDATE **updp, size_t upd_size, + bool exclusive) { WT_DECL_RET; WT_UPDATE *obsolete, *upd = *updp; @@ -295,7 +303,7 @@ __wt_update_serial(WT_SESSION_IMPL *session, WT_PAGE *page, /* * If there are no subsequent WT_UPDATE structures we are done here. */ - if (upd->next == NULL) + if (upd->next == NULL || exclusive) return (0); /* @@ -316,11 +324,11 @@ __wt_update_serial(WT_SESSION_IMPL *session, WT_PAGE *page, } /* If we can't lock it, don't scan, that's okay. */ - if (__wt_try_writelock(session, &page->page_lock) != 0) + if (WT_PAGE_TRYLOCK(session, page) != 0) return (0); obsolete = __wt_update_obsolete_check(session, page, upd->next); - __wt_writeunlock(session, &page->page_lock); + WT_PAGE_UNLOCK(session, page); if (obsolete != NULL) __wt_update_obsolete_free(session, page, obsolete); diff --git a/src/include/session.h b/src/include/session.h index 674e92671b1..dfd84675721 100644 --- a/src/include/session.h +++ b/src/include/session.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -66,6 +66,7 @@ struct __wt_session_impl { /* Session handle reference list */ TAILQ_HEAD(__dhandles, __wt_data_handle_cache) dhandles; time_t last_sweep; /* Last sweep for dead handles */ + struct timespec last_epoch; /* Last epoch time returned */ /* Cursors closed with the session */ TAILQ_HEAD(__cursors, __wt_cursor) cursors; @@ -97,6 +98,10 @@ struct __wt_session_impl { */ TAILQ_HEAD(__tables, __wt_table) tables; + /* Current rwlock for callback. */ + WT_RWLOCK *current_rwlock; + uint8_t current_rwticket; + WT_ITEM **scratch; /* Temporary memory for any function */ u_int scratch_alloc; /* Currently allocated */ size_t scratch_cached; /* Scratch bytes cached */ @@ -167,25 +172,32 @@ struct __wt_session_impl { /* Hashed table reference list array */ TAILQ_HEAD(__tables_hash, __wt_table) *tablehash; + /* Generations manager */ +#define WT_GEN_CHECKPOINT 0 /* Checkpoint generation */ +#define WT_GEN_EVICT 1 /* Eviction generation */ +#define WT_GEN_HAZARD 2 /* Hazard pointer */ +#define WT_GEN_SCHEMA 3 /* Schema version */ +#define WT_GEN_SPLIT 4 /* Page splits */ +#define WT_GENERATIONS 5 /* Total generation manager entries */ + volatile uint64_t generations[WT_GENERATIONS]; + /* - * Split stash memory persists past session close because it's accessed - * by threads of control other than the thread owning the session. - * - * Splits can "free" memory that may still be in use, and we use a - * split generation number to track it, that is, the session stores a - * reference to the memory and allocates a split generation; when no - * session is reading from that split generation, the memory can be - * freed for real. + * Session memory persists past session close because it's accessed by + * threads of control other than the thread owning the session. For + * example, btree splits and hazard pointers can "free" memory that's + * still in use. In order to eventually free it, it's stashed here with + * with its generation number; when no thread is reading in generation, + * the memory can be freed for real. */ - struct __wt_split_stash { - uint64_t split_gen; /* Split generation */ - void *p; /* Memory, length */ - size_t len; - } *split_stash; /* Split stash array */ - size_t split_stash_cnt; /* Array entries */ - size_t split_stash_alloc; /* Allocated bytes */ - - uint64_t split_gen; /* Reading split generation */ + struct __wt_session_stash { + struct __wt_stash { + void *p; /* Memory, length */ + size_t len; + uint64_t gen; /* Generation */ + } *list; + size_t cnt; /* Array entries */ + size_t alloc; /* Allocated bytes */ + } stash[WT_GENERATIONS]; /* * Hazard pointers. diff --git a/src/include/stat.h b/src/include/stat.h index 6c274484bcb..7d7d701590a 100644 --- a/src/include/stat.h +++ b/src/include/stat.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -316,6 +316,7 @@ struct __wt_connection_stats { int64_t cache_eviction_worker_removed; int64_t cache_eviction_stable_state_workers; int64_t cache_eviction_force_fail; + int64_t cache_eviction_force_fail_time; int64_t cache_eviction_walks_active; int64_t cache_eviction_walks_started; int64_t cache_eviction_force_retune; @@ -340,7 +341,9 @@ struct __wt_connection_stats { int64_t cache_write_lookaside; int64_t cache_pages_inuse; int64_t cache_eviction_force; + int64_t cache_eviction_force_time; int64_t cache_eviction_force_delete; + int64_t cache_eviction_force_delete_time; int64_t cache_eviction_app; int64_t cache_eviction_pages_queued; int64_t cache_eviction_pages_queued_urgent; @@ -361,6 +364,7 @@ struct __wt_connection_stats { int64_t cache_eviction_clean; int64_t cond_auto_wait_reset; int64_t cond_auto_wait; + int64_t time_travel; int64_t file_open; int64_t memory_allocation; int64_t memory_free; @@ -373,9 +377,11 @@ struct __wt_connection_stats { int64_t write_io; int64_t cursor_create; int64_t cursor_insert; + int64_t cursor_modify; int64_t cursor_next; int64_t cursor_prev; int64_t cursor_remove; + int64_t cursor_reserve; int64_t cursor_reset; int64_t cursor_restart; int64_t cursor_search; @@ -393,24 +399,21 @@ struct __wt_connection_stats { int64_t lock_checkpoint_count; int64_t lock_checkpoint_wait_application; int64_t lock_checkpoint_wait_internal; - int64_t lock_handle_list_wait_eviction; + int64_t lock_dhandle_wait_application; + int64_t lock_dhandle_wait_internal; + int64_t lock_dhandle_read_count; + int64_t lock_dhandle_write_count; int64_t lock_metadata_count; int64_t lock_metadata_wait_application; int64_t lock_metadata_wait_internal; int64_t lock_schema_count; int64_t lock_schema_wait_application; int64_t lock_schema_wait_internal; - int64_t lock_table_count; int64_t lock_table_wait_application; int64_t lock_table_wait_internal; + int64_t lock_table_read_count; + int64_t lock_table_write_count; int64_t log_slot_switch_busy; - int64_t log_slot_closes; - int64_t log_slot_active_closed; - int64_t log_slot_races; - int64_t log_slot_transitions; - int64_t log_slot_joins; - int64_t log_slot_no_free_slots; - int64_t log_slot_unbuffered; int64_t log_bytes_payload; int64_t log_bytes_written; int64_t log_zero_fills; @@ -437,6 +440,19 @@ struct __wt_connection_stats { int64_t log_prealloc_files; int64_t log_prealloc_used; int64_t log_scan_records; + int64_t log_slot_close_race; + int64_t log_slot_close_unbuf; + int64_t log_slot_closes; + int64_t log_slot_races; + int64_t log_slot_yield_race; + int64_t log_slot_immediate; + int64_t log_slot_yield_close; + int64_t log_slot_yield_sleep; + int64_t log_slot_yield; + int64_t log_slot_active_closed; + int64_t log_slot_yield_duration; + int64_t log_slot_no_free_slots; + int64_t log_slot_unbuffered; int64_t log_compress_mem; int64_t log_buffer_size; int64_t log_compress_len; @@ -501,6 +517,7 @@ struct __wt_connection_stats { int64_t txn_sync; int64_t txn_commit; int64_t txn_rollback; + int64_t txn_update_conflict; }; /* @@ -602,9 +619,11 @@ struct __wt_dsrc_stats { int64_t cursor_remove_bytes; int64_t cursor_update_bytes; int64_t cursor_insert; + int64_t cursor_modify; int64_t cursor_next; int64_t cursor_prev; int64_t cursor_remove; + int64_t cursor_reserve; int64_t cursor_reset; int64_t cursor_restart; int64_t cursor_search; diff --git a/src/include/swap.h b/src/include/swap.h index 2040ca88a77..bd28296e668 100644 --- a/src/include/swap.h +++ b/src/include/swap.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/thread_group.h b/src/include/thread_group.h index 77cff00dc8d..7375f9dfd87 100644 --- a/src/include/thread_group.h +++ b/src/include/thread_group.h @@ -1,11 +1,13 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * * See the file LICENSE for redistribution information. */ +#define WT_THREAD_PAUSE 10 /* Thread pause timeout in seconds */ + /* * WT_THREAD -- * Encapsulation of a thread that belongs to a thread group. @@ -19,13 +21,24 @@ struct __wt_thread { * WT_THREAD and thread-group function flags, merged because * WT_THREAD_PANIC_FAIL appears in both groups. */ -#define WT_THREAD_CAN_WAIT 0x01 /* WT_SESSION_CAN_WAIT */ -#define WT_THREAD_PANIC_FAIL 0x02 /* panic if the thread fails */ -#define WT_THREAD_RUN 0x04 /* thread is running */ +#define WT_THREAD_ACTIVE 0x01 /* thread is active or paused */ +#define WT_THREAD_CAN_WAIT 0x02 /* WT_SESSION_CAN_WAIT */ +#define WT_THREAD_PANIC_FAIL 0x04 /* panic if the thread fails */ +#define WT_THREAD_RUN 0x08 /* thread is running */ uint32_t flags; + /* + * Condition signalled when a thread becomes active. Paused + * threads wait on this condition. + */ + WT_CONDVAR *pause_cond; + + /* The check function used by all threads. */ + bool (*chk_func)(WT_SESSION_IMPL *session); /* The runner function used by all threads. */ int (*run_func)(WT_SESSION_IMPL *session, WT_THREAD *context); + /* The stop function used by all threads. */ + int (*stop_func)(WT_SESSION_IMPL *session, WT_THREAD *context); }; /* @@ -57,6 +70,10 @@ struct __wt_thread_group { */ WT_THREAD **threads; + /* The check function used by all threads. */ + bool (*chk_func)(WT_SESSION_IMPL *session); /* The runner function used by all threads. */ int (*run_func)(WT_SESSION_IMPL *session, WT_THREAD *context); + /* The stop function used by all threads. May be NULL */ + int (*stop_func)(WT_SESSION_IMPL *session, WT_THREAD *context); }; diff --git a/src/include/txn.h b/src/include/txn.h index 7e802c188ab..c1f19ada959 100644 --- a/src/include/txn.h +++ b/src/include/txn.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -107,7 +107,6 @@ struct __wt_txn_global { */ volatile bool checkpoint_running; /* Checkpoint running */ volatile uint32_t checkpoint_id; /* Checkpoint's session ID */ - volatile uint64_t checkpoint_gen; /* Checkpoint generation */ volatile uint64_t checkpoint_pinned; /* Oldest ID for checkpoint */ volatile uint64_t checkpoint_txnid; /* Checkpoint's txn ID */ diff --git a/src/include/txn.i b/src/include/txn.i index 314c948e4d1..f4f571cb67e 100644 --- a/src/include/txn.i +++ b/src/include/txn.i @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -69,7 +69,7 @@ __wt_txn_modify(WT_SESSION_IMPL *session, WT_UPDATE *upd) if (F_ISSET(txn, WT_TXN_READONLY)) WT_RET_MSG(session, WT_ROLLBACK, - "Attempt to update in a read only transaction"); + "Attempt to update in a read-only transaction"); WT_RET(__txn_next_op(session, &op)); op->type = F_ISSET(session, WT_SESSION_LOGGING_INMEM) ? @@ -126,7 +126,7 @@ __wt_txn_oldest_id(WT_SESSION_IMPL *session) */ oldest_id = txn_global->oldest_id; include_checkpoint_txn = btree == NULL || - btree->checkpoint_gen != txn_global->checkpoint_gen; + btree->checkpoint_gen != __wt_gen(session, WT_GEN_CHECKPOINT); WT_READ_BARRIER(); checkpoint_pinned = txn_global->checkpoint_pinned; @@ -233,8 +233,11 @@ __wt_txn_visible(WT_SESSION_IMPL *session, uint64_t id) static inline WT_UPDATE * __wt_txn_read(WT_SESSION_IMPL *session, WT_UPDATE *upd) { - while (upd != NULL && !__wt_txn_visible(session, upd->txnid)) - upd = upd->next; + /* Skip reserved place-holders, they're never visible. */ + for (; upd != NULL; upd = upd->next) + if (upd->type != WT_UPDATE_RESERVED && + __wt_txn_visible(session, upd->txnid)) + break; return (upd); } @@ -421,6 +424,8 @@ __wt_txn_update_check(WT_SESSION_IMPL *session, WT_UPDATE *upd) if (txn->isolation == WT_ISO_SNAPSHOT) while (upd != NULL && !__wt_txn_visible(session, upd->txnid)) { if (upd->txnid != WT_TXN_ABORTED) { + WT_STAT_CONN_INCR( + session, txn_update_conflict); WT_STAT_DATA_INCR( session, txn_update_conflict); return (WT_ROLLBACK); @@ -449,8 +454,7 @@ __wt_txn_read_last(WT_SESSION_IMPL *session) * snapshot here: it will be restored by WT_WITH_TXN_ISOLATION. */ if ((!F_ISSET(txn, WT_TXN_RUNNING) || - txn->isolation != WT_ISO_SNAPSHOT) && - txn->forced_iso == 0) + txn->isolation != WT_ISO_SNAPSHOT) && txn->forced_iso == 0) __wt_txn_release_snapshot(session); } diff --git a/src/include/verify_build.h b/src/include/verify_build.h index 640f5e4cf5f..57189b5c2b2 100644 --- a/src/include/verify_build.h +++ b/src/include/verify_build.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -52,6 +52,7 @@ __wt_verify_build(void) /* Check specific structures weren't padded. */ WT_SIZE_CHECK(WT_BLOCK_DESC, WT_BLOCK_DESC_SIZE); WT_SIZE_CHECK(WT_REF, WT_REF_SIZE); + WT_SIZE_CHECK(WT_UPDATE, WT_UPDATE_SIZE); /* Check specific structures were padded. */ #define WT_PADDING_CHECK(s) \ @@ -59,7 +60,6 @@ __wt_verify_build(void) sizeof(s) > WT_CACHE_LINE_ALIGNMENT || \ sizeof(s) % WT_CACHE_LINE_ALIGNMENT == 0) WT_PADDING_CHECK(WT_LOGSLOT); - WT_PADDING_CHECK(WT_SPINLOCK); WT_PADDING_CHECK(WT_TXN_STATE); /* diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index ddecb2ac765..cf7117376af 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -39,10 +39,14 @@ extern "C" { #define __F(func) (*(func)) #endif -#ifdef SWIG -%{ -#include <wiredtiger.h> -%} +/* + * We support configuring WiredTiger with the gcc/clang -fvisibility=hidden + * flags, but that requires public APIs be specifically marked. + */ +#if defined(DOXYGEN) || defined(SWIG) || !defined(__GNUC__) +#define WT_ATTRIBUTE_LIBRARY_VISIBLE +#else +#define WT_ATTRIBUTE_LIBRARY_VISIBLE __attribute__((visibility("default"))) #endif /*! @@ -74,6 +78,7 @@ struct __wt_extractor; typedef struct __wt_extractor WT_EXTRACTOR; struct __wt_file_handle; typedef struct __wt_file_handle WT_FILE_HANDLE; struct __wt_file_system; typedef struct __wt_file_system WT_FILE_SYSTEM; struct __wt_item; typedef struct __wt_item WT_ITEM; +struct __wt_modify; typedef struct __wt_modify WT_MODIFY; struct __wt_session; typedef struct __wt_session WT_SESSION; #if defined(SWIGJAVA) @@ -128,6 +133,43 @@ struct __wt_item { }; /*! + * A set of modifications for a value, including a pointer to new data and a + * length, plus a target offset in the value and an optional length of data + * in the value to be replaced. + * + * WT_MODIFY structures do not need to be cleared before use. + */ +struct __wt_modify { + /*! + * New data. The size of the new data may be zero when no new data is + * provided. + */ + WT_ITEM data; + + /*! + * The zero-based byte offset in the value where the new data is placed. + * + * If the offset is past the end of the value, nul bytes are appended to + * the value up to the specified offset. + */ + size_t offset; + + /*! + * The number of bytes in the value to be replaced. + * + * If the size is zero, no bytes from the value are replaced and the new + * data is inserted. + * + * If the offset is past the end of the value, the size is ignored. + * + * If the offset plus the size overlaps the end of the previous value, + * bytes from the offset to the end of the value are replaced and any + * remaining new data is appended. + */ + size_t size; +}; + +/*! * The maximum packed size of a 64-bit integer. The ::wiredtiger_struct_pack * function will pack single long integers into at most this many bytes. */ @@ -436,6 +478,38 @@ struct __wt_cursor { int __F(insert)(WT_CURSOR *cursor); /*! + * Modify an existing record. + * + * Both the key and value must be set and the record must already exist; + * the record will be updated. + * + * Modification structures are applied in order, and later modifications + * can update earlier modifications. + * + * The modify method is only supported on raw byte arrays accessed using + * a WT_ITEM structure, that is, a format type of \c u. + * + * @snippet ex_all.c Modify an existing record + * + * On success, the cursor ends positioned at the modified record; to + * minimize cursor resources, the WT_CURSOR::reset method should be + * called as soon as the cursor no longer needs that position. + * + * The maximum length of a single column stored in a table is not fixed + * (as it partially depends on the underlying file configuration), but + * is always a small number of bytes less than 4GB. + * + * @param cursor the cursor handle + * @param entries an array of modification data structures + * @param nentries the number of modification data structures + * @errors + * In particular, if \c in_memory is configured for the database and + * the modify requires more than the configured cache size to complete, + * ::WT_CACHE_FULL is returned. + */ + int __F(modify)(WT_CURSOR *cursor, WT_MODIFY *entries, int nentries); + + /*! * Update an existing record and optionally insert a record. * * If the cursor was configured with "overwrite=true" (the default), @@ -464,7 +538,7 @@ struct __wt_cursor { * @errors * In particular, if \c overwrite=false is configured and no record with * the specified key exists, ::WT_NOTFOUND is returned. - * Also, if \c in_memory is configured for the database and the insert + * Also, if \c in_memory is configured for the database and the update * requires more than the configured cache size to complete, * ::WT_CACHE_FULL is returned. */ @@ -504,6 +578,23 @@ struct __wt_cursor { * with the specified key exists, ::WT_NOTFOUND is returned. */ int __F(remove)(WT_CURSOR *cursor); + + /*! + * Reserve an existing record so a subsequent write is less likely to + * fail due to a conflict between concurrent operations. + * + * The key must first be set and the record must already exist. + * + * @snippet ex_all.c Reserve a record + * + * On success, the cursor ends positioned at the specified record; to + * minimize cursor resources, the WT_CURSOR::reset method should be + * called as soon as the cursor no longer needs that position. + * + * @param cursor the cursor handle + * @errors + */ + int __F(reserve)(WT_CURSOR *cursor); /*! @} */ /*! @@ -1996,12 +2087,12 @@ struct __wt_connection { * list\, with values chosen from the following options: \c "api"\, \c * "block"\, \c "checkpoint"\, \c "compact"\, \c "evict"\, \c * "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c "handleops"\, \c - * "log"\, \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c - * "overflow"\, \c "read"\, \c "rebalance"\, \c "reconcile"\, \c - * "recovery"\, \c "recovery_progress"\, \c "salvage"\, \c - * "shared_cache"\, \c "split"\, \c "temporary"\, \c "thread_group"\, \c - * "transaction"\, \c "verify"\, \c "version"\, \c "write"; default - * empty.} + * "log"\, \c "lookaside_activity"\, \c "lsm"\, \c "lsm_manager"\, \c + * "metadata"\, \c "mutex"\, \c "overflow"\, \c "read"\, \c + * "rebalance"\, \c "reconcile"\, \c "recovery"\, \c + * "recovery_progress"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, + * \c "temporary"\, \c "thread_group"\, \c "transaction"\, \c "verify"\, + * \c "version"\, \c "write"; default empty.} * @configend * @errors */ @@ -2528,12 +2619,12 @@ struct __wt_connection { * list\, such as <code>"verbose=[evictserver\,read]"</code>., a list\, with * values chosen from the following options: \c "api"\, \c "block"\, \c * "checkpoint"\, \c "compact"\, \c "evict"\, \c "evict_stuck"\, \c - * "evictserver"\, \c "fileops"\, \c "handleops"\, \c "log"\, \c "lsm"\, \c - * "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, \c "read"\, \c - * "rebalance"\, \c "reconcile"\, \c "recovery"\, \c "recovery_progress"\, \c - * "salvage"\, \c "shared_cache"\, \c "split"\, \c "temporary"\, \c - * "thread_group"\, \c "transaction"\, \c "verify"\, \c "version"\, \c "write"; - * default empty.} + * "evictserver"\, \c "fileops"\, \c "handleops"\, \c "log"\, \c + * "lookaside_activity"\, \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c + * "mutex"\, \c "overflow"\, \c "read"\, \c "rebalance"\, \c "reconcile"\, \c + * "recovery"\, \c "recovery_progress"\, \c "salvage"\, \c "shared_cache"\, \c + * "split"\, \c "temporary"\, \c "thread_group"\, \c "transaction"\, \c + * "verify"\, \c "version"\, \c "write"; default empty.} * @config{write_through, Use \c FILE_FLAG_WRITE_THROUGH on Windows to write to * files. Ignored on non-Windows systems. Options are given as a list\, such * as <code>"write_through=[data]"</code>. Configuring \c write_through requires @@ -2553,7 +2644,7 @@ struct __wt_connection { */ int wiredtiger_open(const char *home, WT_EVENT_HANDLER *errhandler, const char *config, - WT_CONNECTION **connectionp); + WT_CONNECTION **connectionp) WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Return information about a WiredTiger error as a string (see @@ -2564,7 +2655,7 @@ int wiredtiger_open(const char *home, * @param error a return value from a WiredTiger, ISO C, or POSIX standard API * @returns a string representation of the error */ -const char *wiredtiger_strerror(int error); +const char *wiredtiger_strerror(int error) WT_ATTRIBUTE_LIBRARY_VISIBLE; #if !defined(SWIG) /*! @@ -2701,7 +2792,8 @@ struct __wt_event_handler { * @errors */ int wiredtiger_struct_pack(WT_SESSION *session, - void *buffer, size_t size, const char *format, ...); + void *buffer, size_t size, const char *format, ...) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Calculate the size required to pack a structure. @@ -2719,7 +2811,7 @@ int wiredtiger_struct_pack(WT_SESSION *session, * @errors */ int wiredtiger_struct_size(WT_SESSION *session, - size_t *sizep, const char *format, ...); + size_t *sizep, const char *format, ...) WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Unpack a structure from a buffer. @@ -2736,7 +2828,8 @@ int wiredtiger_struct_size(WT_SESSION *session, * @errors */ int wiredtiger_struct_unpack(WT_SESSION *session, - const void *buffer, size_t size, const char *format, ...); + const void *buffer, size_t size, const char *format, ...) + WT_ATTRIBUTE_LIBRARY_VISIBLE; #if !defined(SWIG) @@ -2763,7 +2856,8 @@ typedef struct __wt_pack_stream WT_PACK_STREAM; * @errors */ int wiredtiger_pack_start(WT_SESSION *session, - const char *format, void *buffer, size_t size, WT_PACK_STREAM **psp); + const char *format, void *buffer, size_t size, WT_PACK_STREAM **psp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Start an unpacking operation from a buffer with the given format string. @@ -2779,7 +2873,8 @@ int wiredtiger_pack_start(WT_SESSION *session, * @errors */ int wiredtiger_unpack_start(WT_SESSION *session, - const char *format, const void *buffer, size_t size, WT_PACK_STREAM **psp); + const char *format, const void *buffer, size_t size, WT_PACK_STREAM **psp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Close a packing stream. @@ -2788,7 +2883,8 @@ int wiredtiger_unpack_start(WT_SESSION *session, * @param[out] usedp the number of bytes in the buffer used by the stream * @errors */ -int wiredtiger_pack_close(WT_PACK_STREAM *ps, size_t *usedp); +int wiredtiger_pack_close(WT_PACK_STREAM *ps, size_t *usedp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Pack an item into a packing stream. @@ -2797,7 +2893,8 @@ int wiredtiger_pack_close(WT_PACK_STREAM *ps, size_t *usedp); * @param item an item to pack * @errors */ -int wiredtiger_pack_item(WT_PACK_STREAM *ps, WT_ITEM *item); +int wiredtiger_pack_item(WT_PACK_STREAM *ps, WT_ITEM *item) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Pack a signed integer into a packing stream. @@ -2806,7 +2903,8 @@ int wiredtiger_pack_item(WT_PACK_STREAM *ps, WT_ITEM *item); * @param i a signed integer to pack * @errors */ -int wiredtiger_pack_int(WT_PACK_STREAM *ps, int64_t i); +int wiredtiger_pack_int(WT_PACK_STREAM *ps, int64_t i) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Pack a string into a packing stream. @@ -2815,7 +2913,8 @@ int wiredtiger_pack_int(WT_PACK_STREAM *ps, int64_t i); * @param s a string to pack * @errors */ -int wiredtiger_pack_str(WT_PACK_STREAM *ps, const char *s); +int wiredtiger_pack_str(WT_PACK_STREAM *ps, const char *s) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Pack an unsigned integer into a packing stream. @@ -2824,7 +2923,8 @@ int wiredtiger_pack_str(WT_PACK_STREAM *ps, const char *s); * @param u an unsigned integer to pack * @errors */ -int wiredtiger_pack_uint(WT_PACK_STREAM *ps, uint64_t u); +int wiredtiger_pack_uint(WT_PACK_STREAM *ps, uint64_t u) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Unpack an item from a packing stream. @@ -2833,7 +2933,8 @@ int wiredtiger_pack_uint(WT_PACK_STREAM *ps, uint64_t u); * @param item an item to unpack * @errors */ -int wiredtiger_unpack_item(WT_PACK_STREAM *ps, WT_ITEM *item); +int wiredtiger_unpack_item(WT_PACK_STREAM *ps, WT_ITEM *item) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Unpack a signed integer from a packing stream. @@ -2842,7 +2943,8 @@ int wiredtiger_unpack_item(WT_PACK_STREAM *ps, WT_ITEM *item); * @param[out] ip the unpacked signed integer * @errors */ -int wiredtiger_unpack_int(WT_PACK_STREAM *ps, int64_t *ip); +int wiredtiger_unpack_int(WT_PACK_STREAM *ps, int64_t *ip) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Unpack a string from a packing stream. @@ -2851,7 +2953,8 @@ int wiredtiger_unpack_int(WT_PACK_STREAM *ps, int64_t *ip); * @param[out] sp the unpacked string * @errors */ -int wiredtiger_unpack_str(WT_PACK_STREAM *ps, const char **sp); +int wiredtiger_unpack_str(WT_PACK_STREAM *ps, const char **sp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Unpack an unsigned integer from a packing stream. @@ -2860,7 +2963,8 @@ int wiredtiger_unpack_str(WT_PACK_STREAM *ps, const char **sp); * @param[out] up the unpacked unsigned integer * @errors */ -int wiredtiger_unpack_uint(WT_PACK_STREAM *ps, uint64_t *up); +int wiredtiger_unpack_uint(WT_PACK_STREAM *ps, uint64_t *up) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! @} */ /*! @@ -2938,7 +3042,8 @@ struct __wt_config_item { * @snippet ex_all.c Validate a configuration string */ int wiredtiger_config_validate(WT_SESSION *session, - WT_EVENT_HANDLER *errhandler, const char *name, const char *config); + WT_EVENT_HANDLER *errhandler, const char *name, const char *config) + WT_ATTRIBUTE_LIBRARY_VISIBLE; #endif /*! @@ -2958,7 +3063,8 @@ int wiredtiger_config_validate(WT_SESSION *session, * @snippet ex_config_parse.c Create a configuration parser */ int wiredtiger_config_parser_open(WT_SESSION *session, - const char *config, size_t len, WT_CONFIG_PARSER **config_parserp); + const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * A handle that can be used to search and traverse configuration strings @@ -3047,7 +3153,8 @@ struct __wt_config_parser { * @param patchp a location where the patch version number is returned * @returns a string representation of the version */ -const char *wiredtiger_version(int *majorp, int *minorp, int *patchp); +const char *wiredtiger_version(int *majorp, int *minorp, int *patchp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /******************************************* * Error returns @@ -3100,10 +3207,9 @@ const char *wiredtiger_version(int *majorp, int *minorp, int *patchp); #define WT_NOTFOUND (-31803) /*! * WiredTiger library panic. - * This error indicates an underlying problem that requires the application exit - * and restart. The application can exit immediately when \c WT_PANIC is - * returned from a WiredTiger interface, no further WiredTiger calls are - * required. + * This error indicates an underlying problem that requires a database restart. + * The application may exit immediately, no further WiredTiger calls are + * required (and further calls will themselves immediately fail). */ #define WT_PANIC (-31804) /*! @cond internal */ @@ -4454,396 +4560,448 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_CACHE_EVICTION_WORKER_REMOVED 1056 /*! cache: eviction worker thread stable number */ #define WT_STAT_CONN_CACHE_EVICTION_STABLE_STATE_WORKERS 1057 -/*! cache: failed eviction of pages that exceeded the in-memory maximum */ +/*! + * cache: failed eviction of pages that exceeded the in-memory maximum + * count + */ #define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1058 +/*! + * cache: failed eviction of pages that exceeded the in-memory maximum + * time (usecs) + */ +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL_TIME 1059 /*! cache: files with active eviction walks */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1059 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1060 /*! cache: files with new eviction walks started */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1060 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1061 /*! cache: force re-tuning of eviction workers once in a while */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_RETUNE 1061 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_RETUNE 1062 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1062 +#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1063 /*! cache: hazard pointer check calls */ -#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1063 +#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1064 /*! cache: hazard pointer check entries walked */ -#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1064 +#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1065 /*! cache: hazard pointer maximum array length */ -#define WT_STAT_CONN_CACHE_HAZARD_MAX 1065 +#define WT_STAT_CONN_CACHE_HAZARD_MAX 1066 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1066 +#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1067 /*! cache: in-memory page splits */ -#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1067 +#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1068 /*! cache: internal pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1068 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1069 /*! cache: internal pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1069 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1070 /*! cache: leaf pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1070 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1071 /*! cache: lookaside table insert calls */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_INSERT 1071 +#define WT_STAT_CONN_CACHE_LOOKASIDE_INSERT 1072 /*! cache: lookaside table remove calls */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE 1072 +#define WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE 1073 /*! cache: maximum bytes configured */ -#define WT_STAT_CONN_CACHE_BYTES_MAX 1073 +#define WT_STAT_CONN_CACHE_BYTES_MAX 1074 /*! cache: maximum page size at eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1074 +#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1075 /*! cache: modified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1075 +#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1076 /*! cache: modified pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1076 +#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1077 /*! cache: overflow pages read into cache */ -#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1077 +#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1078 /*! cache: overflow values cached in memory */ -#define WT_STAT_CONN_CACHE_OVERFLOW_VALUE 1078 +#define WT_STAT_CONN_CACHE_OVERFLOW_VALUE 1079 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1079 +#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1080 /*! cache: page written requiring lookaside records */ -#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1080 +#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1081 /*! cache: pages currently held in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_INUSE 1081 -/*! cache: pages evicted because they exceeded the in-memory maximum */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1082 -/*! cache: pages evicted because they had chains of deleted items */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1083 +#define WT_STAT_CONN_CACHE_PAGES_INUSE 1082 +/*! cache: pages evicted because they exceeded the in-memory maximum count */ +#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1083 +/*! + * cache: pages evicted because they exceeded the in-memory maximum time + * (usecs) + */ +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_TIME 1084 +/*! cache: pages evicted because they had chains of deleted items count */ +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1085 +/*! + * cache: pages evicted because they had chains of deleted items time + * (usecs) + */ +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE_TIME 1086 /*! cache: pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP 1084 +#define WT_STAT_CONN_CACHE_EVICTION_APP 1087 /*! cache: pages queued for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1085 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1088 /*! cache: pages queued for urgent eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1086 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1089 /*! cache: pages queued for urgent eviction during walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1087 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1090 /*! cache: pages read into cache */ -#define WT_STAT_CONN_CACHE_READ 1088 +#define WT_STAT_CONN_CACHE_READ 1091 /*! cache: pages read into cache requiring lookaside entries */ -#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1089 +#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1092 /*! cache: pages requested from the cache */ -#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1090 +#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1093 /*! cache: pages seen by eviction walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1091 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1094 /*! cache: pages selected for eviction unable to be evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1092 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1095 /*! cache: pages walked for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK 1093 +#define WT_STAT_CONN_CACHE_EVICTION_WALK 1096 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1094 +#define WT_STAT_CONN_CACHE_WRITE 1097 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1095 +#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1098 /*! cache: percentage overhead */ -#define WT_STAT_CONN_CACHE_OVERHEAD 1096 +#define WT_STAT_CONN_CACHE_OVERHEAD 1099 /*! cache: tracked bytes belonging to internal pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1097 +#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1100 /*! cache: tracked bytes belonging to leaf pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_LEAF 1098 +#define WT_STAT_CONN_CACHE_BYTES_LEAF 1101 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1099 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1102 /*! cache: tracked dirty pages in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1100 +#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1103 /*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1101 +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1104 /*! connection: auto adjusting condition resets */ -#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1102 +#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1105 /*! connection: auto adjusting condition wait calls */ -#define WT_STAT_CONN_COND_AUTO_WAIT 1103 +#define WT_STAT_CONN_COND_AUTO_WAIT 1106 +/*! connection: detected system time went backwards */ +#define WT_STAT_CONN_TIME_TRAVEL 1107 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1104 +#define WT_STAT_CONN_FILE_OPEN 1108 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1105 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1109 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1106 +#define WT_STAT_CONN_MEMORY_FREE 1110 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1107 +#define WT_STAT_CONN_MEMORY_GROW 1111 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1108 +#define WT_STAT_CONN_COND_WAIT 1112 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1109 +#define WT_STAT_CONN_RWLOCK_READ 1113 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1110 +#define WT_STAT_CONN_RWLOCK_WRITE 1114 /*! connection: total fsync I/Os */ -#define WT_STAT_CONN_FSYNC_IO 1111 +#define WT_STAT_CONN_FSYNC_IO 1115 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1112 +#define WT_STAT_CONN_READ_IO 1116 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1113 +#define WT_STAT_CONN_WRITE_IO 1117 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1114 +#define WT_STAT_CONN_CURSOR_CREATE 1118 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1115 +#define WT_STAT_CONN_CURSOR_INSERT 1119 +/*! cursor: cursor modify calls */ +#define WT_STAT_CONN_CURSOR_MODIFY 1120 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1116 +#define WT_STAT_CONN_CURSOR_NEXT 1121 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1117 +#define WT_STAT_CONN_CURSOR_PREV 1122 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1118 +#define WT_STAT_CONN_CURSOR_REMOVE 1123 +/*! cursor: cursor reserve calls */ +#define WT_STAT_CONN_CURSOR_RESERVE 1124 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1119 +#define WT_STAT_CONN_CURSOR_RESET 1125 /*! cursor: cursor restarted searches */ -#define WT_STAT_CONN_CURSOR_RESTART 1120 +#define WT_STAT_CONN_CURSOR_RESTART 1126 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1121 +#define WT_STAT_CONN_CURSOR_SEARCH 1127 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1122 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1128 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1123 +#define WT_STAT_CONN_CURSOR_UPDATE 1129 /*! cursor: truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1124 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1130 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1125 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1131 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1126 +#define WT_STAT_CONN_DH_SWEEP_REF 1132 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1127 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1133 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1128 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1134 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1129 +#define WT_STAT_CONN_DH_SWEEP_TOD 1135 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1130 +#define WT_STAT_CONN_DH_SWEEPS 1136 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1131 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1137 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1132 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1138 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1133 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1139 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1134 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1140 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1135 -/*! lock: handle-list lock eviction thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_HANDLE_LIST_WAIT_EVICTION 1136 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1141 +/*! + * lock: dhandle lock application thread time waiting for the dhandle + * lock (usecs) + */ +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1142 +/*! + * lock: dhandle lock internal thread time waiting for the dhandle lock + * (usecs) + */ +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1143 +/*! lock: dhandle read lock acquisitions */ +#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1144 +/*! lock: dhandle write lock acquisitions */ +#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1145 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1137 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1146 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1138 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1147 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1139 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1148 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1140 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1149 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1141 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1150 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1142 -/*! lock: table lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_COUNT 1143 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1151 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1144 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1152 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1145 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1153 +/*! lock: table read lock acquisitions */ +#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1154 +/*! lock: table write lock acquisitions */ +#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1155 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1146 -/*! log: consolidated slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1147 -/*! log: consolidated slot join active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1148 -/*! log: consolidated slot join races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1149 -/*! log: consolidated slot join transitions */ -#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1150 -/*! log: consolidated slot joins */ -#define WT_STAT_CONN_LOG_SLOT_JOINS 1151 -/*! log: consolidated slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1152 -/*! log: consolidated slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1153 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1156 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1154 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1157 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1155 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1158 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1156 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1159 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1157 +#define WT_STAT_CONN_LOG_FLUSH 1160 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1158 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1161 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1159 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1162 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1160 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1163 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1161 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1164 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1162 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1165 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1163 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1166 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1164 +#define WT_STAT_CONN_LOG_SCANS 1167 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1165 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1168 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1166 +#define WT_STAT_CONN_LOG_WRITE_LSN 1169 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1167 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1170 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1168 +#define WT_STAT_CONN_LOG_SYNC 1171 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1169 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1172 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1170 +#define WT_STAT_CONN_LOG_SYNC_DIR 1173 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1171 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1174 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1172 +#define WT_STAT_CONN_LOG_WRITES 1175 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1173 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1176 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1174 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1177 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1175 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1178 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1176 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1179 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1177 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1180 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1178 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1181 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1179 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1182 +/*! log: slot close lost race */ +#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1183 +/*! log: slot close unbuffered waits */ +#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1184 +/*! log: slot closures */ +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1185 +/*! log: slot join atomic update races */ +#define WT_STAT_CONN_LOG_SLOT_RACES 1186 +/*! log: slot join calls atomic updates raced */ +#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1187 +/*! log: slot join calls did not yield */ +#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1188 +/*! log: slot join calls found active slot closed */ +#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1189 +/*! log: slot join calls slept */ +#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1190 +/*! log: slot join calls yielded */ +#define WT_STAT_CONN_LOG_SLOT_YIELD 1191 +/*! log: slot join found active slot closed */ +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1192 +/*! log: slot joins yield time (usecs) */ +#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1193 +/*! log: slot transitions unable to find free slot */ +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1194 +/*! log: slot unbuffered writes */ +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1195 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1180 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1196 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1181 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1197 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1182 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1198 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1183 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1199 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1184 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1200 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1185 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1201 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1186 +#define WT_STAT_CONN_REC_PAGES 1202 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1187 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1203 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1188 +#define WT_STAT_CONN_REC_PAGE_DELETE 1204 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1189 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1205 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1190 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1206 /*! session: open cursor count */ -#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1191 +#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1207 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1192 +#define WT_STAT_CONN_SESSION_OPEN 1208 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1193 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1209 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1194 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1210 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1195 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1211 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1196 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1212 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1197 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1213 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1198 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1214 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1199 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1215 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1200 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1216 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1201 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1217 /*! session: table rebalance failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1202 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1218 /*! session: table rebalance successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1203 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1219 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1204 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1220 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1205 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1221 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1206 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1222 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1207 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1223 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1208 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1224 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1209 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1225 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1210 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1226 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1211 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1227 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1212 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1228 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1213 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1229 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1214 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1230 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1215 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1231 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1216 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1232 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1217 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1233 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1218 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1234 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1219 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1235 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1220 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1236 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1221 +#define WT_STAT_CONN_PAGE_SLEEP 1237 /*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1222 +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1238 /*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1223 +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1239 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1224 +#define WT_STAT_CONN_TXN_BEGIN 1240 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1225 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1241 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1226 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1242 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1227 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1243 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1228 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1244 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1229 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1245 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1230 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1246 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1231 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1247 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1232 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1248 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1233 +#define WT_STAT_CONN_TXN_CHECKPOINT 1249 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1234 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1250 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1235 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1251 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1236 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1252 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1237 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1253 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1238 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1254 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1239 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1255 /*! * transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1240 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1256 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1241 +#define WT_STAT_CONN_TXN_SYNC 1257 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1242 +#define WT_STAT_CONN_TXN_COMMIT 1258 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1243 +#define WT_STAT_CONN_TXN_ROLLBACK 1259 +/*! transaction: update conflicts */ +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1260 /*! * @} @@ -5125,61 +5283,65 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2092 /*! cursor: insert calls */ #define WT_STAT_DSRC_CURSOR_INSERT 2093 +/*! cursor: modify calls */ +#define WT_STAT_DSRC_CURSOR_MODIFY 2094 /*! cursor: next calls */ -#define WT_STAT_DSRC_CURSOR_NEXT 2094 +#define WT_STAT_DSRC_CURSOR_NEXT 2095 /*! cursor: prev calls */ -#define WT_STAT_DSRC_CURSOR_PREV 2095 +#define WT_STAT_DSRC_CURSOR_PREV 2096 /*! cursor: remove calls */ -#define WT_STAT_DSRC_CURSOR_REMOVE 2096 +#define WT_STAT_DSRC_CURSOR_REMOVE 2097 +/*! cursor: reserve calls */ +#define WT_STAT_DSRC_CURSOR_RESERVE 2098 /*! cursor: reset calls */ -#define WT_STAT_DSRC_CURSOR_RESET 2097 +#define WT_STAT_DSRC_CURSOR_RESET 2099 /*! cursor: restarted searches */ -#define WT_STAT_DSRC_CURSOR_RESTART 2098 +#define WT_STAT_DSRC_CURSOR_RESTART 2100 /*! cursor: search calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH 2099 +#define WT_STAT_DSRC_CURSOR_SEARCH 2101 /*! cursor: search near calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2100 +#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2102 /*! cursor: truncate calls */ -#define WT_STAT_DSRC_CURSOR_TRUNCATE 2101 +#define WT_STAT_DSRC_CURSOR_TRUNCATE 2103 /*! cursor: update calls */ -#define WT_STAT_DSRC_CURSOR_UPDATE 2102 +#define WT_STAT_DSRC_CURSOR_UPDATE 2104 /*! reconciliation: dictionary matches */ -#define WT_STAT_DSRC_REC_DICTIONARY 2103 +#define WT_STAT_DSRC_REC_DICTIONARY 2105 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2104 +#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2106 /*! * reconciliation: internal page key bytes discarded using suffix * compression */ -#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2105 +#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2107 /*! reconciliation: internal page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2106 +#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2108 /*! reconciliation: internal-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2107 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2109 /*! reconciliation: leaf page key bytes discarded using prefix compression */ -#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2108 +#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2110 /*! reconciliation: leaf page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2109 +#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2111 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2110 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2112 /*! reconciliation: maximum blocks required for a page */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2111 +#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2113 /*! reconciliation: overflow values written */ -#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2112 +#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2114 /*! reconciliation: page checksum matches */ -#define WT_STAT_DSRC_REC_PAGE_MATCH 2113 +#define WT_STAT_DSRC_REC_PAGE_MATCH 2115 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_DSRC_REC_PAGES 2114 +#define WT_STAT_DSRC_REC_PAGES 2116 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_DSRC_REC_PAGES_EVICTION 2115 +#define WT_STAT_DSRC_REC_PAGES_EVICTION 2117 /*! reconciliation: pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE 2116 +#define WT_STAT_DSRC_REC_PAGE_DELETE 2118 /*! session: object compaction */ -#define WT_STAT_DSRC_SESSION_COMPACT 2117 +#define WT_STAT_DSRC_SESSION_COMPACT 2119 /*! session: open cursor count */ -#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2118 +#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2120 /*! transaction: update conflicts */ -#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2119 +#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2121 /*! * @} diff --git a/src/include/wiredtiger_ext.h b/src/include/wiredtiger_ext.h index 236d4e07e67..bc61c43e29d 100644 --- a/src/include/wiredtiger_ext.h +++ b/src/include/wiredtiger_ext.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/include/wt_internal.h b/src/include/wt_internal.h index da318ad8a86..1c9600dd27f 100644 --- a/src/include/wt_internal.h +++ b/src/include/wt_internal.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -268,6 +268,8 @@ struct __wt_ref; typedef struct __wt_ref WT_REF; struct __wt_row; typedef struct __wt_row WT_ROW; +struct __wt_rwlock; + typedef struct __wt_rwlock WT_RWLOCK; struct __wt_salvage_cookie; typedef struct __wt_salvage_cookie WT_SALVAGE_COOKIE; struct __wt_save_upd; @@ -276,12 +278,14 @@ struct __wt_scratch_track; typedef struct __wt_scratch_track WT_SCRATCH_TRACK; struct __wt_session_impl; typedef struct __wt_session_impl WT_SESSION_IMPL; +struct __wt_session_stash; + typedef struct __wt_session_stash WT_SESSION_STASH; struct __wt_size; typedef struct __wt_size WT_SIZE; struct __wt_spinlock; typedef struct __wt_spinlock WT_SPINLOCK; -struct __wt_split_stash; - typedef struct __wt_split_stash WT_SPLIT_STASH; +struct __wt_stash; + typedef struct __wt_stash WT_STASH; struct __wt_table; typedef struct __wt_table WT_TABLE; struct __wt_thread; @@ -302,8 +306,6 @@ union __wt_lsn; typedef union __wt_lsn WT_LSN; union __wt_rand_state; typedef union __wt_rand_state WT_RAND_STATE; -union __wt_rwlock; - typedef union __wt_rwlock WT_RWLOCK; /* * Forward type declarations for internal types: END * DO NOT EDIT: automatically built by dist/s_typedef. diff --git a/src/log/log.c b/src/log/log.c index 803d3e8dfab..868f5d0eaf4 100644 --- a/src/log/log.c +++ b/src/log/log.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -8,6 +8,7 @@ #include "wt_internal.h" +static int __log_newfile(WT_SESSION_IMPL *, bool, bool *); static int __log_openfile( WT_SESSION_IMPL *, WT_FH **, const char *, uint32_t, uint32_t); static int __log_write_internal( @@ -24,7 +25,7 @@ static int __log_write_internal( * __log_wait_for_earlier_slot -- * Wait for write_lsn to catch up to this slot. */ -static int +static void __log_wait_for_earlier_slot(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) { WT_CONNECTION_IMPL *conn; @@ -41,7 +42,6 @@ __log_wait_for_earlier_slot(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) * unlock in case an earlier thread is trying to switch its * slot and complete its operation. */ - WT_RET(WT_SESSION_CHECK_PANIC(session)); if (F_ISSET(session, WT_SESSION_LOCKED_SLOT)) __wt_spin_unlock(session, &log->log_slot_lock); __wt_cond_signal(session, conn->log_wrlsn_cond); @@ -52,7 +52,6 @@ __log_wait_for_earlier_slot(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) if (F_ISSET(session, WT_SESSION_LOCKED_SLOT)) __wt_spin_lock(session, &log->log_slot_lock); } - return (0); } /* @@ -72,7 +71,7 @@ __log_fs_write(WT_SESSION_IMPL *session, * be a hole at the end of the previous log file that we cannot detect. */ if (slot->slot_release_lsn.l.file < slot->slot_start_lsn.l.file) { - WT_RET(__log_wait_for_earlier_slot(session, slot)); + __log_wait_for_earlier_slot(session, slot); WT_RET(__wt_log_force_sync(session, &slot->slot_release_lsn)); } if ((ret = __wt_write(session, slot->slot_fh, offset, len, buf)) != 0) @@ -112,7 +111,6 @@ __wt_log_flush_lsn(WT_SESSION_IMPL *session, WT_LSN *lsn, bool start) conn = S2C(session); log = conn->log; - WT_RET(WT_SESSION_CHECK_PANIC(session)); WT_RET(__wt_log_force_write(session, 1, NULL)); __wt_log_wrlsn(session, NULL); if (start) @@ -177,7 +175,6 @@ __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) * log file ready to close. */ while (log->sync_lsn.l.file < min_lsn->l.file) { - WT_RET(WT_SESSION_CHECK_PANIC(session)); __wt_cond_signal(session, S2C(session)->log_file_cond); __wt_cond_wait(session, log->log_sync_cond, 10000, NULL); } @@ -442,6 +439,57 @@ __wt_log_extract_lognum( } /* + * __wt_log_reset -- + * Reset the existing log file to after the given file number. + * Called from recovery when toggling logging back on, it was off + * the previous open but it was on earlier before that toggle. + */ +int +__wt_log_reset(WT_SESSION_IMPL *session, uint32_t lognum) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + WT_LOG *log; + uint32_t old_lognum; + u_int i, logcount; + char **logfiles; + + conn = S2C(session); + log = conn->log; + + if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED) || + log->fileid > lognum) + return (0); + + WT_ASSERT(session, F_ISSET(conn, WT_CONN_RECOVERING)); + WT_ASSERT(session, !F_ISSET(conn, WT_CONN_READONLY)); + /* + * We know we're single threaded and called from recovery only when + * toggling logging back on. Therefore the only log files we have are + * old and outdated and the new one created when logging opened before + * recovery. We have to remove all old log files first and then create + * the new one so that log file numbers are contiguous in the file + * system. + */ + WT_RET(__wt_close(session, &log->log_fh)); + WT_RET(__log_get_files(session, WT_LOG_FILENAME, &logfiles, &logcount)); + for (i = 0; i < logcount; i++) { + WT_ERR(__wt_log_extract_lognum( + session, logfiles[i], &old_lognum)); + WT_ASSERT(session, old_lognum < lognum || lognum == 1); + WT_ERR(__wt_log_remove(session, WT_LOG_FILENAME, old_lognum)); + } + log->fileid = lognum; + + /* Send in true to update connection creation LSNs. */ + WT_WITH_SLOT_LOCK(session, log, + ret = __log_newfile(session, true, NULL)); + WT_ERR(__wt_log_slot_init(session, false)); +err: WT_TRET(__wt_fs_directory_list_free(session, &logfiles, logcount)); + return (ret); +} + +/* * __log_zero -- * Zero a log file. */ @@ -880,18 +928,16 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) __wt_yield(); } /* - * Note, the file server worker thread has code that knows that - * the file handle is set before the LSN. Do not reorder without - * also reviewing that code. + * Note, the file server worker thread requires the LSN be set once the + * close file handle is set, force that ordering. */ - log->log_close_fh = log->log_fh; - if (log->log_close_fh != NULL) + if (log->log_fh == NULL) + log->log_close_fh = NULL; + else { log->log_close_lsn = log->alloc_lsn; + WT_PUBLISH(log->log_close_fh, log->log_fh); + } log->fileid++; - /* - * Make sure everything we set above is visible. - */ - WT_FULL_BARRIER(); /* * If pre-allocating log files look for one; otherwise, or if we don't @@ -1101,8 +1147,7 @@ __log_truncate(WT_SESSION_IMPL *session, WT_ERR(__log_get_files(session, WT_LOG_FILENAME, &logfiles, &logcount)); for (i = 0; i < logcount; i++) { WT_ERR(__wt_log_extract_lognum(session, logfiles[i], &lognum)); - if (lognum > lsn->l.file && - lognum < log->trunc_lsn.l.file) { + if (lognum > lsn->l.file && lognum < log->trunc_lsn.l.file) { WT_ERR(__log_openfile(session, &log_fh, file_prefix, lognum, 0)); /* @@ -1468,7 +1513,7 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) * be holes in the log file. */ WT_STAT_CONN_INCR(session, log_release_write_lsn); - WT_ERR(__log_wait_for_earlier_slot(session, slot)); + __log_wait_for_earlier_slot(session, slot); log->write_start_lsn = slot->slot_start_lsn; log->write_lsn = slot->slot_end_lsn; @@ -1489,7 +1534,6 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) * current fsync completes and advance log->sync_lsn. */ while (F_ISSET(slot, WT_SLOT_SYNC | WT_SLOT_SYNC_DIR)) { - WT_ERR(WT_SESSION_CHECK_PANIC(session)); /* * We have to wait until earlier log files have finished their * sync operations. The most recent one will set the LSN to the @@ -1631,10 +1675,40 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, WT_RET_MSG(session, WT_ERROR, "choose either a start LSN or a start flag"); - /* Offsets must be on allocation boundaries. */ - if (lsnp->l.offset % allocsize != 0 || - lsnp->l.file > log->fileid) - return (WT_NOTFOUND); + /* + * Offsets must be on allocation boundaries. + * An invalid LSN from a user should just return + * WT_NOTFOUND. It is not an error. But if it is + * from recovery, we expect valid LSNs so give more + * information about that. + */ + if (lsnp->l.offset % allocsize != 0) { + if (LF_ISSET(WT_LOGSCAN_RECOVER)) + WT_RET_MSG(session, WT_NOTFOUND, + "__wt_log_scan unaligned LSN %" + PRIu32 "/%" PRIu32, + lsnp->l.file, lsnp->l.offset); + else + return (WT_NOTFOUND); + } + /* + * If the file is in the future it doesn't exist. + * An invalid LSN from a user should just return + * WT_NOTFOUND. It is not an error. But if it is + * from recovery, we expect valid LSNs so give more + * information about that. + */ + if (lsnp->l.file > log->fileid) { + if (LF_ISSET(WT_LOGSCAN_RECOVER)) + WT_RET_MSG(session, WT_NOTFOUND, + "__wt_log_scan LSN %" PRIu32 "/%" + PRIu32 + " larger than biggest log file %" + PRIu32, lsnp->l.file, + lsnp->l.offset, log->fileid); + else + return (WT_NOTFOUND); + } /* * Log cursors may not know the starting LSN. If an @@ -1873,8 +1947,7 @@ advance: /* Truncate if we're in recovery. */ if (LF_ISSET(WT_LOGSCAN_RECOVER) && __wt_log_cmp(&rd_lsn, &log->trunc_lsn) < 0) - WT_ERR(__log_truncate(session, - &rd_lsn, WT_LOG_FILENAME, 0)); + WT_ERR(__log_truncate(session, &rd_lsn, WT_LOG_FILENAME, 0)); err: WT_STAT_CONN_INCR(session, log_scans); /* @@ -2008,8 +2081,7 @@ __wt_log_write(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, if (compression_failed || result_len / log->allocsize >= record->size / log->allocsize) - WT_STAT_CONN_INCR(session, - log_compress_write_fails); + WT_STAT_CONN_INCR(session, log_compress_write_fails); else { WT_STAT_CONN_INCR(session, log_compress_writes); WT_STAT_CONN_INCRV(session, log_compress_mem, @@ -2129,7 +2201,7 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, * The only time joining a slot should ever return an error is if it * detects a panic. */ - WT_ERR(__wt_log_slot_join(session, rdup_len, flags, &myslot)); + __wt_log_slot_join(session, rdup_len, flags, &myslot); /* * If the addition of this record crosses the buffer boundary, * switch in a new slot. @@ -2141,8 +2213,7 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, ret = __wt_log_slot_switch(session, &myslot, true, false, NULL); if (ret == 0) ret = __log_fill(session, &myslot, false, record, &lsn); - release_size = __wt_log_slot_release( - session, &myslot, (int64_t)rdup_len); + release_size = __wt_log_slot_release(&myslot, (int64_t)rdup_len); /* * If we get an error we still need to do proper accounting in * the slot fields. @@ -2171,19 +2242,15 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, if (LF_ISSET(WT_LOG_FLUSH)) { /* Wait for our writes to reach the OS */ while (__wt_log_cmp(&log->write_lsn, &lsn) <= 0 && - myslot.slot->slot_error == 0) { - WT_ERR(WT_SESSION_CHECK_PANIC(session)); + myslot.slot->slot_error == 0) __wt_cond_wait( session, log->log_write_cond, 10000, NULL); - } } else if (LF_ISSET(WT_LOG_FSYNC)) { /* Wait for our writes to reach disk */ while (__wt_log_cmp(&log->sync_lsn, &lsn) <= 0 && - myslot.slot->slot_error == 0) { - WT_ERR(WT_SESSION_CHECK_PANIC(session)); + myslot.slot->slot_error == 0) __wt_cond_wait( session, log->log_sync_cond, 10000, NULL); - } } /* diff --git a/src/log/log_slot.c b/src/log/log_slot.c index 97e317ce68c..5bd3d53a973 100644 --- a/src/log/log_slot.c +++ b/src/log/log_slot.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -126,13 +126,17 @@ retry: * processed by another closing thread. Only return 0 when we * actually closed the slot. */ - if (WT_LOG_SLOT_CLOSED(old_state)) + if (WT_LOG_SLOT_CLOSED(old_state)) { + WT_STAT_CONN_INCR(session, log_slot_close_race); return (WT_NOTFOUND); + } /* * If someone completely processed this slot, we're done. */ - if (FLD64_ISSET((uint64_t)slot->slot_state, WT_LOG_SLOT_RESERVED)) + if (FLD64_ISSET((uint64_t)slot->slot_state, WT_LOG_SLOT_RESERVED)) { + WT_STAT_CONN_INCR(session, log_slot_close_race); return (WT_NOTFOUND); + } new_state = (old_state | WT_LOG_SLOT_CLOSE); /* * Close this slot. If we lose the race retry. @@ -160,7 +164,7 @@ retry: #endif if (WT_LOG_SLOT_UNBUFFERED_ISSET(old_state)) { while (slot->slot_unbuffered == 0) { - WT_RET(WT_SESSION_CHECK_PANIC(session)); + WT_STAT_CONN_INCR(session, log_slot_close_unbuf); __wt_yield(); #ifdef HAVE_DIAGNOSTIC ++count; @@ -250,8 +254,6 @@ __log_slot_new(WT_SESSION_IMPL *session) * We have a new, initialized slot to use. * Set it as the active slot. */ - WT_STAT_CONN_INCR(session, - log_slot_transitions); log->active_slot = slot; log->pool_index = pool_i; return (0); @@ -318,7 +320,6 @@ __log_slot_switch_internal( *did_work = false; return (0); } - WT_RET(WT_SESSION_CHECK_PANIC(session)); /* * We may come through here multiple times if we were not able to @@ -401,7 +402,7 @@ __wt_log_slot_switch(WT_SESSION_IMPL *session, * Initialize the slot array. */ int -__wt_log_slot_init(WT_SESSION_IMPL *session) +__wt_log_slot_init(WT_SESSION_IMPL *session, bool alloc) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; @@ -423,15 +424,17 @@ __wt_log_slot_init(WT_SESSION_IMPL *session) * switch log files very aggressively. Scale back the buffer for * small log file sizes. */ - log->slot_buf_size = (uint32_t)WT_MIN( - (size_t)conn->log_file_max / 10, WT_LOG_SLOT_BUF_SIZE); - for (i = 0; i < WT_SLOT_POOL; i++) { - WT_ERR(__wt_buf_init(session, - &log->slot_pool[i].slot_buf, log->slot_buf_size)); - F_SET(&log->slot_pool[i], WT_SLOT_INIT_FLAGS); + if (alloc) { + log->slot_buf_size = (uint32_t)WT_MIN( + (size_t)conn->log_file_max / 10, WT_LOG_SLOT_BUF_SIZE); + for (i = 0; i < WT_SLOT_POOL; i++) { + WT_ERR(__wt_buf_init(session, + &log->slot_pool[i].slot_buf, log->slot_buf_size)); + F_SET(&log->slot_pool[i], WT_SLOT_INIT_FLAGS); + } + WT_STAT_CONN_SET(session, + log_buffer_size, log->slot_buf_size * WT_SLOT_POOL); } - WT_STAT_CONN_SET(session, - log_buffer_size, log->slot_buf_size * WT_SLOT_POOL); /* * Set up the available slot from the pool the first time. */ @@ -492,16 +495,18 @@ __wt_log_slot_destroy(WT_SESSION_IMPL *session) * __wt_log_slot_join -- * Join a consolidated logging slot. */ -int +void __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, uint32_t flags, WT_MYSLOT *myslot) { + struct timespec start, stop; WT_CONNECTION_IMPL *conn; WT_LOG *log; WT_LOGSLOT *slot; + uint64_t usecs; int64_t flag_state, new_state, old_state, released; - int32_t join_offset, new_join; - bool unbuffered, yld; + int32_t join_offset, new_join, wait_cnt; + bool closed, diag_yield, raced, slept, unbuffered, yielded; conn = S2C(session); log = conn->log; @@ -512,13 +517,15 @@ __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, /* * There should almost always be a slot open. */ - unbuffered = false; + unbuffered = yielded = false; + closed = raced = slept = false; + wait_cnt = 0; #ifdef HAVE_DIAGNOSTIC - yld = (++log->write_calls % 7) == 0; + diag_yield = (++log->write_calls % 7) == 0; if ((log->write_calls % WT_THOUSAND) == 0 || mysize > WT_LOG_SLOT_BUF_MAX) { #else - yld = false; + diag_yield = false; if (mysize > WT_LOG_SLOT_BUF_MAX) { #endif unbuffered = true; @@ -526,7 +533,6 @@ __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, } for (;;) { WT_BARRIER(); - WT_RET(WT_SESSION_CHECK_PANIC(session)); slot = log->active_slot; old_state = slot->slot_state; if (WT_LOG_SLOT_OPEN(old_state)) { @@ -548,7 +554,7 @@ __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, /* * Braces used due to potential empty body warning. */ - if (yld) { + if (diag_yield) { WT_DIAGNOSTIC_YIELD; } /* @@ -558,19 +564,44 @@ __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, &slot->slot_state, old_state, new_state)) break; WT_STAT_CONN_INCR(session, log_slot_races); - } else + raced = true; + } else { WT_STAT_CONN_INCR(session, log_slot_active_closed); + closed = true; + ++wait_cnt; + } + if (!yielded) + __wt_epoch(session, &start); + yielded = true; /* * The slot is no longer open or we lost the race to * update it. Yield and try again. */ - __wt_yield(); + if (wait_cnt < WT_THOUSAND) + __wt_yield(); + else { + __wt_sleep(0, WT_THOUSAND); + slept = true; + } } /* * We joined this slot. Fill in our information to return to * the caller. */ - WT_STAT_CONN_INCR(session, log_slot_joins); + if (!yielded) + WT_STAT_CONN_INCR(session, log_slot_immediate); + else { + WT_STAT_CONN_INCR(session, log_slot_yield); + __wt_epoch(session, &stop); + usecs = WT_TIMEDIFF_US(stop, start); + WT_STAT_CONN_INCRV(session, log_slot_yield_duration, usecs); + if (closed) + WT_STAT_CONN_INCR(session, log_slot_yield_close); + if (raced) + WT_STAT_CONN_INCR(session, log_slot_yield_race); + if (slept) + WT_STAT_CONN_INCR(session, log_slot_yield_sleep); + } if (LF_ISSET(WT_LOG_DSYNC | WT_LOG_FSYNC)) F_SET(slot, WT_SLOT_SYNC_DIR); if (LF_ISSET(WT_LOG_FLUSH)) @@ -585,7 +616,6 @@ __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, myslot->slot = slot; myslot->offset = join_offset; myslot->end_offset = (wt_off_t)((uint64_t)join_offset + mysize); - return (0); } /* @@ -595,7 +625,7 @@ __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, * the memory buffer. */ int64_t -__wt_log_slot_release(WT_SESSION_IMPL *session, WT_MYSLOT *myslot, int64_t size) +__wt_log_slot_release(WT_MYSLOT *myslot, int64_t size) { WT_LOGSLOT *slot; wt_off_t cur_offset, my_start; @@ -609,7 +639,6 @@ __wt_log_slot_release(WT_SESSION_IMPL *session, WT_MYSLOT *myslot, int64_t size) * was written rather than the beginning record of the slot. */ while ((cur_offset = slot->slot_last_offset) < my_start) { - WT_RET(WT_SESSION_CHECK_PANIC(session)); /* * Set our offset if we are larger. */ diff --git a/src/lsm/lsm_cursor.c b/src/lsm/lsm_cursor.c index 52265f02e62..1d15ed793a2 100644 --- a/src/lsm/lsm_cursor.c +++ b/src/lsm/lsm_cursor.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -486,7 +486,7 @@ __clsm_open_cursors( * cursor, take a copy before closing cursors. */ if (F_ISSET(c, WT_CURSTD_KEY_INT)) - WT_CURSOR_NEEDKEY(c); + WT_ERR(__cursor_needkey(c)); F_CLR(clsm, WT_CLSM_ITERATE_NEXT | WT_CLSM_ITERATE_PREV); @@ -844,8 +844,8 @@ __clsm_compare(WT_CURSOR *a, WT_CURSOR *b, int *cmpp) WT_ERR_MSG(session, EINVAL, "comparison method cursors must reference the same object"); - WT_CURSOR_NEEDKEY(a); - WT_CURSOR_NEEDKEY(b); + WT_ERR(__cursor_needkey(a)); + WT_ERR(__cursor_needkey(b)); WT_ERR(__wt_compare( session, alsm->lsm_tree->collator, &a->key, &b->key, cmpp)); @@ -871,7 +871,7 @@ __clsm_next(WT_CURSOR *cursor) clsm = (WT_CURSOR_LSM *)cursor; CURSOR_API_CALL(cursor, session, next, NULL); - WT_CURSOR_NOVALUE(cursor); + __cursor_novalue(cursor); WT_ERR(__clsm_enter(clsm, false, false)); /* If we aren't positioned for a forward scan, get started. */ @@ -997,7 +997,7 @@ __clsm_next_random(WT_CURSOR *cursor) clsm = (WT_CURSOR_LSM *)cursor; CURSOR_API_CALL(cursor, session, next, NULL); - WT_CURSOR_NOVALUE(cursor); + __cursor_novalue(cursor); WT_ERR(__clsm_enter(clsm, false, false)); for (;;) { @@ -1051,7 +1051,7 @@ __clsm_prev(WT_CURSOR *cursor) clsm = (WT_CURSOR_LSM *)cursor; CURSOR_API_CALL(cursor, session, prev, NULL); - WT_CURSOR_NOVALUE(cursor); + __cursor_novalue(cursor); WT_ERR(__clsm_enter(clsm, false, false)); /* If we aren't positioned for a reverse scan, get started. */ @@ -1268,8 +1268,8 @@ __clsm_search(WT_CURSOR *cursor) clsm = (WT_CURSOR_LSM *)cursor; CURSOR_API_CALL(cursor, session, search, NULL); - WT_CURSOR_NEEDKEY(cursor); - WT_CURSOR_NOVALUE(cursor); + WT_ERR(__cursor_needkey(cursor)); + __cursor_novalue(cursor); WT_ERR(__clsm_enter(clsm, true, false)); ret = __clsm_lookup(clsm, &cursor->value); @@ -1301,8 +1301,8 @@ __clsm_search_near(WT_CURSOR *cursor, int *exactp) exact = 0; CURSOR_API_CALL(cursor, session, search_near, NULL); - WT_CURSOR_NEEDKEY(cursor); - WT_CURSOR_NOVALUE(cursor); + WT_ERR(__cursor_needkey(cursor)); + __cursor_novalue(cursor); WT_ERR(__clsm_enter(clsm, true, false)); F_CLR(clsm, WT_CLSM_ITERATE_NEXT | WT_CLSM_ITERATE_PREV); @@ -1438,11 +1438,12 @@ err: __clsm_leave(clsm); */ static inline int __clsm_put(WT_SESSION_IMPL *session, WT_CURSOR_LSM *clsm, - const WT_ITEM *key, const WT_ITEM *value, bool position) + const WT_ITEM *key, const WT_ITEM *value, bool position, bool reserve) { WT_CURSOR *c, *primary; WT_LSM_TREE *lsm_tree; u_int i, slot; + int (*func)(WT_CURSOR *); lsm_tree = clsm->lsm_tree; @@ -1473,8 +1474,12 @@ __clsm_put(WT_SESSION_IMPL *session, WT_CURSOR_LSM *clsm, c = clsm->chunks[slot]->cursor; c->set_key(c, key); - c->set_value(c, value); - WT_RET((position && i == 0) ? c->update(c) : c->insert(c)); + func = c->insert; + if (i == 0 && position) + func = reserve ? c->reserve : c->update; + if (func != c->reserve) + c->set_value(c, value); + WT_RET(func(c)); } /* @@ -1520,11 +1525,16 @@ __clsm_insert(WT_CURSOR *cursor) clsm = (WT_CURSOR_LSM *)cursor; - CURSOR_UPDATE_API_CALL(cursor, session, insert, NULL); - WT_CURSOR_NEEDKEY(cursor); - WT_CURSOR_NEEDVALUE(cursor); + CURSOR_UPDATE_API_CALL(cursor, session, insert); + WT_ERR(__cursor_needkey(cursor)); + WT_ERR(__cursor_needvalue(cursor)); WT_ERR(__clsm_enter(clsm, false, true)); + /* + * It isn't necessary to copy the key out after the lookup in this + * case because any non-failed lookup results in an error, and a + * failed lookup leaves the original key intact. + */ if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE) && (ret = __clsm_lookup(clsm, &value)) != WT_NOTFOUND) { if (ret == 0) @@ -1533,7 +1543,7 @@ __clsm_insert(WT_CURSOR *cursor) } WT_ERR(__clsm_deleted_encode(session, &cursor->value, &value, &buf)); - WT_ERR(__clsm_put(session, clsm, &cursor->key, &value, false)); + WT_ERR(__clsm_put(session, clsm, &cursor->key, &value, false, false)); /* * WT_CURSOR.insert doesn't leave the cursor positioned, and the @@ -1564,15 +1574,21 @@ __clsm_update(WT_CURSOR *cursor) clsm = (WT_CURSOR_LSM *)cursor; - CURSOR_UPDATE_API_CALL(cursor, session, update, NULL); - WT_CURSOR_NEEDKEY(cursor); - WT_CURSOR_NEEDVALUE(cursor); + CURSOR_UPDATE_API_CALL(cursor, session, update); + WT_ERR(__cursor_needkey(cursor)); + WT_ERR(__cursor_needvalue(cursor)); WT_ERR(__clsm_enter(clsm, false, true)); - if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE)) + if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE)) { WT_ERR(__clsm_lookup(clsm, &value)); + /* + * Copy the key out, since the insert resets non-primary chunk + * cursors which our lookup may have landed on. + */ + WT_ERR(__cursor_needkey(cursor)); + } WT_ERR(__clsm_deleted_encode(session, &cursor->value, &value, &buf)); - WT_ERR(__clsm_put(session, clsm, &cursor->key, &value, true)); + WT_ERR(__clsm_put(session, clsm, &cursor->key, &value, true, false)); /* * Set the cursor to reference the internal key/value of the positioned @@ -1612,14 +1628,20 @@ __clsm_remove(WT_CURSOR *cursor) positioned = F_ISSET(cursor, WT_CURSTD_KEY_INT); CURSOR_REMOVE_API_CALL(cursor, session, NULL); - WT_CURSOR_NEEDKEY(cursor); - WT_CURSOR_NOVALUE(cursor); + WT_ERR(__cursor_needkey(cursor)); + __cursor_novalue(cursor); WT_ERR(__clsm_enter(clsm, false, true)); - if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE)) + if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE)) { WT_ERR(__clsm_lookup(clsm, &value)); + /* + * Copy the key out, since the insert resets non-primary chunk + * cursors which our lookup may have landed on. + */ + WT_ERR(__cursor_needkey(cursor)); + } WT_ERR(__clsm_put( - session, clsm, &cursor->key, &__tombstone, positioned)); + session, clsm, &cursor->key, &__tombstone, positioned, false)); /* * If the cursor was positioned, it stays positioned with a key but no @@ -1639,6 +1661,48 @@ err: __clsm_leave(clsm); } /* + * __clsm_reserve -- + * WT_CURSOR->reserve method for the LSM cursor type. + */ +static int +__clsm_reserve(WT_CURSOR *cursor) +{ + WT_CURSOR_LSM *clsm; + WT_DECL_RET; + WT_ITEM value; + WT_SESSION_IMPL *session; + + clsm = (WT_CURSOR_LSM *)cursor; + + CURSOR_UPDATE_API_CALL(cursor, session, reserve); + WT_ERR(__cursor_needkey(cursor)); + __cursor_novalue(cursor); + WT_ERR(__wt_txn_context_check(session, true)); + WT_ERR(__clsm_enter(clsm, false, true)); + + WT_ERR(__clsm_lookup(clsm, &value)); + /* + * Copy the key out, since the insert resets non-primary chunk cursors + * which our lookup may have landed on. + */ + WT_ERR(__cursor_needkey(cursor)); + ret = __clsm_put(session, clsm, &cursor->key, NULL, true, true); + +err: __clsm_leave(clsm); + CURSOR_UPDATE_API_END(session, ret); + + /* + * The application might do a WT_CURSOR.get_value call when we return, + * so we need a value and the underlying functions didn't set one up. + * For various reasons, those functions may not have done a search and + * any previous value in the cursor might race with WT_CURSOR.reserve + * (and in cases like LSM, the reserve never encountered the original + * key). For simplicity, repeat the search here. + */ + return (ret == 0 ? cursor->search(cursor) : ret); +} + +/* * __wt_clsm_close -- * WT_CURSOR->close method for the LSM cursor type. */ @@ -1661,8 +1725,6 @@ __wt_clsm_close(WT_CURSOR *cursor) /* In case we were somehow left positioned, clear that. */ __clsm_leave(clsm); - /* The WT_LSM_TREE owns the URI. */ - cursor->uri = NULL; if (clsm->lsm_tree != NULL) __wt_lsm_tree_release(session, clsm->lsm_tree); WT_TRET(__wt_cursor_close(cursor)); @@ -1692,8 +1754,10 @@ __wt_clsm_open(WT_SESSION_IMPL *session, __clsm_search, /* search */ __clsm_search_near, /* search-near */ __clsm_insert, /* insert */ + __wt_cursor_modify_notsup, /* modify */ __clsm_update, /* update */ __clsm_remove, /* remove */ + __clsm_reserve, /* reserve */ __wt_cursor_reconfigure, /* reconfigure */ __wt_clsm_close); /* close */ WT_CURSOR *cursor; @@ -1744,7 +1808,7 @@ __wt_clsm_open(WT_SESSION_IMPL *session, cursor = &clsm->iface; *cursor = iface; cursor->session = &session->iface; - cursor->uri = lsm_tree->name; + WT_ERR(__wt_strdup(session, lsm_tree->name, &cursor->uri)); cursor->key_format = lsm_tree->key_format; cursor->value_format = lsm_tree->value_format; diff --git a/src/lsm/lsm_cursor_bulk.c b/src/lsm/lsm_cursor_bulk.c index 7a6a40e380f..ba5f04c7697 100644 --- a/src/lsm/lsm_cursor_bulk.c +++ b/src/lsm/lsm_cursor_bulk.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/lsm/lsm_manager.c b/src/lsm/lsm_manager.c index e33e119aa41..b1f775a275e 100644 --- a/src/lsm/lsm_manager.c +++ b/src/lsm/lsm_manager.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -284,12 +284,8 @@ __wt_lsm_manager_destroy(WT_SESSION_IMPL *session) manager = &conn->lsm_manager; removed = 0; - /* - * Clear the LSM server flag and flush to ensure running threads see - * the state change. - */ + /* Clear the LSM server flag. */ F_CLR(conn, WT_CONN_SERVER_LSM); - WT_FULL_BARRIER(); WT_ASSERT(session, !F_ISSET(conn, WT_CONN_READONLY) || manager->lsm_workers == 0); @@ -334,7 +330,7 @@ __wt_lsm_manager_destroy(WT_SESSION_IMPL *session) __wt_spin_destroy(session, &manager->switch_lock); __wt_spin_destroy(session, &manager->app_lock); __wt_spin_destroy(session, &manager->manager_lock); - WT_TRET(__wt_cond_destroy(session, &manager->work_cond)); + __wt_cond_destroy(session, &manager->work_cond); return (ret); } @@ -388,7 +384,7 @@ __lsm_manager_run_server(WT_SESSION_IMPL *session) __wt_readlock(session, &conn->dhandle_lock); F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST_READ); dhandle_locked = true; - TAILQ_FOREACH(lsm_tree, &S2C(session)->lsmqh, q) { + TAILQ_FOREACH(lsm_tree, &conn->lsmqh, q) { if (!lsm_tree->active) continue; __wt_epoch(session, &now); @@ -500,7 +496,7 @@ void __wt_lsm_manager_clear_tree(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) { WT_LSM_MANAGER *manager; - WT_LSM_WORK_UNIT *current, *next; + WT_LSM_WORK_UNIT *current, *tmp; uint64_t removed; manager = &S2C(session)->lsm_manager; @@ -508,11 +504,7 @@ __wt_lsm_manager_clear_tree(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) /* Clear out the tree from the switch queue */ __wt_spin_lock(session, &manager->switch_lock); - - /* Structure the loop so that it's safe to free as we iterate */ - for (current = TAILQ_FIRST(&manager->switchqh); - current != NULL; current = next) { - next = TAILQ_NEXT(current, q); + TAILQ_FOREACH_SAFE(current, &manager->switchqh, q, tmp) { if (current->lsm_tree != lsm_tree) continue; ++removed; @@ -522,9 +514,7 @@ __wt_lsm_manager_clear_tree(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) __wt_spin_unlock(session, &manager->switch_lock); /* Clear out the tree from the application queue */ __wt_spin_lock(session, &manager->app_lock); - for (current = TAILQ_FIRST(&manager->appqh); - current != NULL; current = next) { - next = TAILQ_NEXT(current, q); + TAILQ_FOREACH_SAFE(current, &manager->appqh, q, tmp) { if (current->lsm_tree != lsm_tree) continue; ++removed; @@ -534,9 +524,7 @@ __wt_lsm_manager_clear_tree(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) __wt_spin_unlock(session, &manager->app_lock); /* Clear out the tree from the manager queue */ __wt_spin_lock(session, &manager->manager_lock); - for (current = TAILQ_FIRST(&manager->managerqh); - current != NULL; current = next) { - next = TAILQ_NEXT(current, q); + TAILQ_FOREACH_SAFE(current, &manager->managerqh, q, tmp) { if (current->lsm_tree != lsm_tree) continue; ++removed; diff --git a/src/lsm/lsm_merge.c b/src/lsm/lsm_merge.c index 8838638f388..882dfa86a18 100644 --- a/src/lsm/lsm_merge.c +++ b/src/lsm/lsm_merge.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/lsm/lsm_meta.c b/src/lsm/lsm_meta.c index fc4dde82470..66ad24dee5b 100644 --- a/src/lsm/lsm_meta.c +++ b/src/lsm/lsm_meta.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/lsm/lsm_stat.c b/src/lsm/lsm_stat.c index 411655878af..63b9e6c6d14 100644 --- a/src/lsm/lsm_stat.c +++ b/src/lsm/lsm_stat.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index a9275976023..62ec44764e7 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -134,11 +134,12 @@ int __wt_lsm_tree_close_all(WT_SESSION_IMPL *session) { WT_DECL_RET; - WT_LSM_TREE *lsm_tree; + WT_LSM_TREE *lsm_tree, *lsm_tree_tmp; /* We are shutting down: the handle list lock isn't required. */ - while ((lsm_tree = TAILQ_FIRST(&S2C(session)->lsmqh)) != NULL) { + WT_TAILQ_SAFE_REMOVE_BEGIN(lsm_tree, + &S2C(session)->lsmqh, q, lsm_tree_tmp) { /* * Tree close assumes that we have a reference to the tree * so it can tell when it's safe to do the close. We could @@ -149,7 +150,7 @@ __wt_lsm_tree_close_all(WT_SESSION_IMPL *session) (void)__wt_atomic_add32(&lsm_tree->refcnt, 1); __lsm_tree_close(session, lsm_tree, true); WT_TRET(__lsm_tree_discard(session, lsm_tree, true)); - } + } WT_TAILQ_SAFE_REMOVE_END return (ret); } @@ -471,7 +472,7 @@ __lsm_tree_open(WT_SESSION_IMPL *session, /* Try to open the tree. */ WT_RET(__wt_calloc_one(session, &lsm_tree)); - __wt_rwlock_init(session, &lsm_tree->rwlock); + WT_ERR(__wt_rwlock_init(session, &lsm_tree->rwlock)); WT_ERR(__lsm_tree_set_name(session, lsm_tree, uri)); @@ -499,7 +500,7 @@ __lsm_tree_open(WT_SESSION_IMPL *session, __wt_epoch(session, &lsm_tree->last_flush_ts); /* Now the tree is setup, make it visible to others. */ - TAILQ_INSERT_HEAD(&S2C(session)->lsmqh, lsm_tree, q); + TAILQ_INSERT_HEAD(&conn->lsmqh, lsm_tree, q); if (!exclusive) lsm_tree->active = true; F_SET(lsm_tree, WT_LSM_TREE_OPEN); @@ -767,13 +768,13 @@ __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_ERR(__wt_lsm_meta_write(session, lsm_tree, NULL)); lsm_tree->need_switch = false; - ++lsm_tree->dsk_gen; - lsm_tree->modified = true; + /* * Ensure the updated disk generation is visible to all other threads * before updating the transaction ID. */ + ++lsm_tree->dsk_gen; WT_FULL_BARRIER(); /* diff --git a/src/lsm/lsm_work_unit.c b/src/lsm/lsm_work_unit.c index e6a29666094..ec55de31e0d 100644 --- a/src/lsm/lsm_work_unit.c +++ b/src/lsm/lsm_work_unit.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -328,8 +328,9 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, */ saved_isolation = session->txn.isolation; session->txn.isolation = WT_ISO_READ_UNCOMMITTED; - WT_ERR(__wt_cache_op(session, WT_SYNC_WRITE_LEAVES)); + ret = __wt_cache_op(session, WT_SYNC_WRITE_LEAVES); session->txn.isolation = saved_isolation; + WT_ERR(ret); __wt_verbose(session, WT_VERB_LSM, "LSM worker checkpointing %s", chunk->uri); diff --git a/src/lsm/lsm_worker.c b/src/lsm/lsm_worker.c index 1cabbd4888d..ba6adf37cce 100644 --- a/src/lsm/lsm_worker.c +++ b/src/lsm/lsm_worker.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/meta/meta_apply.c b/src/meta/meta_apply.c index dc93180a5e5..9fb70dac081 100644 --- a/src/meta/meta_apply.c +++ b/src/meta/meta_apply.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/meta/meta_ckpt.c b/src/meta/meta_ckpt.c index 151bbe0e081..0e96c4ee6ca 100644 --- a/src/meta/meta_ckpt.c +++ b/src/meta/meta_ckpt.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/meta/meta_ext.c b/src/meta/meta_ext.c index aa1ea8b974d..b1d1d2be28f 100644 --- a/src/meta/meta_ext.c +++ b/src/meta/meta_ext.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/meta/meta_table.c b/src/meta/meta_table.c index aca69d0e6a2..326ad12bd33 100644 --- a/src/meta/meta_table.c +++ b/src/meta/meta_table.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -62,9 +62,10 @@ __wt_metadata_cursor_open( * first update is safe because it's single-threaded from * wiredtiger_open). */ +#define WT_EVICT_META_SKEW 10000 if (btree->evict_priority == 0) WT_WITH_BTREE(session, btree, - __wt_evict_priority_set(session, WT_EVICT_INT_SKEW)); + __wt_evict_priority_set(session, WT_EVICT_META_SKEW)); if (F_ISSET(btree, WT_BTREE_NO_LOGGING)) F_CLR(btree, WT_BTREE_NO_LOGGING); @@ -266,7 +267,9 @@ __wt_metadata_search(WT_SESSION_IMPL *session, const char *key, char **valuep) * that Coverity complains a lot, add an error check to get some * peace and quiet. */ - if ((ret = __wt_turtle_read(session, key, valuep)) != 0) + WT_WITH_TURTLE_LOCK(session, + ret = __wt_turtle_read(session, key, valuep)); + if (ret != 0) __wt_free(session, *valuep); return (ret); } diff --git a/src/meta/meta_track.c b/src/meta/meta_track.c index 460b615b267..fe7b467c199 100644 --- a/src/meta/meta_track.c +++ b/src/meta/meta_track.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/meta/meta_turtle.c b/src/meta/meta_turtle.c index 5a089471059..362a3aa2bbe 100644 --- a/src/meta/meta_turtle.c +++ b/src/meta/meta_turtle.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -246,6 +246,9 @@ __wt_turtle_read(WT_SESSION_IMPL *session, const char *key, char **valuep) *valuep = NULL; + /* Require single-threading. */ + WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TURTLE)); + /* * Open the turtle file; there's one case where we won't find the turtle * file, yet still succeed. We create the metadata file before creating @@ -302,6 +305,9 @@ __wt_turtle_update(WT_SESSION_IMPL *session, const char *key, const char *value) fs = NULL; + /* Require single-threading. */ + WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TURTLE)); + /* * Create the turtle setup file: we currently re-write it from scratch * every time. diff --git a/src/os_common/filename.c b/src/os_common/filename.c index d5695f63d91..16825410dc3 100644 --- a/src/os_common/filename.c +++ b/src/os_common/filename.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_common/os_abort.c b/src/os_common/os_abort.c index 034eedcfbf8..905f3160acf 100644 --- a/src/os_common/os_abort.c +++ b/src/os_common/os_abort.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_common/os_alloc.c b/src/os_common/os_alloc.c index ef96ed09ea7..388c9c8c18b 100644 --- a/src/os_common/os_alloc.c +++ b/src/os_common/os_alloc.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -266,6 +266,8 @@ __wt_strndup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp) WT_RET(__wt_malloc(session, len + 1, &p)); + WT_ASSERT(session, p != NULL); /* quiet clang scan-build */ + /* * Don't change this to strncpy, we rely on this function to duplicate * "strings" that contain nul bytes. diff --git a/src/os_common/os_errno.c b/src/os_common/os_errno.c index 7ac89536e79..d88d06d7610 100644 --- a/src/os_common/os_errno.c +++ b/src/os_common/os_errno.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_common/os_fhandle.c b/src/os_common/os_fhandle.c index 3fd5b5db773..0dcc6bc00ef 100644 --- a/src/os_common/os_fhandle.c +++ b/src/os_common/os_fhandle.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -281,6 +281,42 @@ err: if (open_called) } /* + * __handle_close -- + * Final close of a handle. + */ +static int +__handle_close(WT_SESSION_IMPL *session, WT_FH *fh, bool locked) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + uint64_t bucket; + + conn = S2C(session); + + if (fh->ref != 0) { + __wt_errx(session, + "Closing a file handle with open references: %s", fh->name); + WT_TRET(EBUSY); + } + + /* Remove from the list. */ + bucket = fh->name_hash % WT_HASH_ARRAY_SIZE; + WT_FILE_HANDLE_REMOVE(conn, fh, bucket); + (void)__wt_atomic_sub32(&conn->open_file_count, 1); + + if (locked) + __wt_spin_unlock(session, &conn->fh_lock); + + /* Discard underlying resources. */ + WT_TRET(fh->handle->close(fh->handle, (WT_SESSION *)session)); + + __wt_free(session, fh->name); + __wt_free(session, fh); + + return (ret); +} + +/* * __wt_close -- * Close a file handle. */ @@ -288,9 +324,7 @@ int __wt_close(WT_SESSION_IMPL *session, WT_FH **fhp) { WT_CONNECTION_IMPL *conn; - WT_DECL_RET; WT_FH *fh; - uint64_t bucket; conn = S2C(session); @@ -315,20 +349,7 @@ __wt_close(WT_SESSION_IMPL *session, WT_FH **fhp) return (0); } - /* Remove from the list. */ - bucket = fh->name_hash % WT_HASH_ARRAY_SIZE; - WT_FILE_HANDLE_REMOVE(conn, fh, bucket); - (void)__wt_atomic_sub32(&conn->open_file_count, 1); - - __wt_spin_unlock(session, &conn->fh_lock); - - /* Discard underlying resources. */ - ret = fh->handle->close(fh->handle, (WT_SESSION *)session); - - __wt_free(session, fh->name); - __wt_free(session, fh); - - return (ret); + return (__handle_close(session, fh, true)); } /* @@ -339,21 +360,10 @@ int __wt_close_connection_close(WT_SESSION_IMPL *session) { WT_DECL_RET; - WT_FH *fh; - WT_CONNECTION_IMPL *conn; + WT_FH *fh, *fh_tmp; - conn = S2C(session); - - while ((fh = TAILQ_FIRST(&conn->fhqh)) != NULL) { - if (fh->ref != 0) { - ret = EBUSY; - __wt_errx(session, - "Connection has open file handles: %s", fh->name); - } - - fh->ref = 1; - - WT_TRET(__wt_close(session, &fh)); - } + WT_TAILQ_SAFE_REMOVE_BEGIN(fh, &S2C(session)->fhqh, q, fh_tmp) { + WT_TRET(__handle_close(session, fh, false)); + } WT_TAILQ_SAFE_REMOVE_END return (ret); } diff --git a/src/os_common/os_fs_inmemory.c b/src/os_common/os_fs_inmemory.c index 1670e97be45..e669ea2802d 100644 --- a/src/os_common/os_fs_inmemory.c +++ b/src/os_common/os_fs_inmemory.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -52,7 +52,7 @@ __im_handle_search(WT_FILE_SYSTEM *file_system, const char *name) */ static int __im_handle_remove(WT_SESSION_IMPL *session, - WT_FILE_SYSTEM *file_system, WT_FILE_HANDLE_INMEM *im_fh) + WT_FILE_SYSTEM *file_system, WT_FILE_HANDLE_INMEM *im_fh, bool force) { WT_FILE_HANDLE *fhp; WT_FILE_SYSTEM_INMEM *im_fs; @@ -60,9 +60,11 @@ __im_handle_remove(WT_SESSION_IMPL *session, im_fs = (WT_FILE_SYSTEM_INMEM *)file_system; - if (im_fh->ref != 0) - WT_RET_MSG(session, EBUSY, - "%s: file-remove", im_fh->iface.name); + if (im_fh->ref != 0) { + __wt_err(session, EBUSY, "%s: file-remove", im_fh->iface.name); + if (!force) + return (EBUSY); + } bucket = im_fh->name_hash % WT_HASH_ARRAY_SIZE; WT_FILE_HANDLE_REMOVE(im_fs, im_fh, bucket); @@ -205,7 +207,7 @@ __im_fs_remove(WT_FILE_SYSTEM *file_system, ret = ENOENT; if ((im_fh = __im_handle_search(file_system, name)) != NULL) - ret = __im_handle_remove(session, file_system, im_fh); + ret = __im_handle_remove(session, file_system, im_fh, false); __wt_spin_unlock(session, &im_fs->lock); return (ret); @@ -511,15 +513,16 @@ static int __im_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session) { WT_DECL_RET; - WT_FILE_HANDLE_INMEM *im_fh; + WT_FILE_HANDLE_INMEM *im_fh, *im_fh_tmp; WT_FILE_SYSTEM_INMEM *im_fs; WT_SESSION_IMPL *session; session = (WT_SESSION_IMPL *)wt_session; im_fs = (WT_FILE_SYSTEM_INMEM *)file_system; - while ((im_fh = TAILQ_FIRST(&im_fs->fhqh)) != NULL) - WT_TRET(__im_handle_remove(session, file_system, im_fh)); + WT_TAILQ_SAFE_REMOVE_BEGIN(im_fh, &im_fs->fhqh, q, im_fh_tmp) { + WT_TRET(__im_handle_remove(session, file_system, im_fh, true)); + } WT_TAILQ_SAFE_REMOVE_END __wt_spin_destroy(session, &im_fs->lock); __wt_free(session, im_fs); diff --git a/src/os_common/os_fstream.c b/src/os_common/os_fstream.c index 744da732d84..2fe11b92dd0 100644 --- a/src/os_common/os_fstream.c +++ b/src/os_common/os_fstream.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_common/os_fstream_stdio.c b/src/os_common/os_fstream_stdio.c index 0cc75e109a1..82e82b5f3e5 100644 --- a/src/os_common/os_fstream_stdio.c +++ b/src/os_common/os_fstream_stdio.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_common/os_getopt.c b/src/os_common/os_getopt.c index 960776c3999..ca516ca62e5 100644 --- a/src/os_common/os_getopt.c +++ b/src/os_common/os_getopt.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -59,13 +59,17 @@ #include "wt_internal.h" -extern int __wt_opterr, __wt_optind, __wt_optopt, __wt_optreset; +extern int __wt_opterr WT_ATTRIBUTE_LIBRARY_VISIBLE; +extern int __wt_optind WT_ATTRIBUTE_LIBRARY_VISIBLE; +extern int __wt_optopt WT_ATTRIBUTE_LIBRARY_VISIBLE; +extern int __wt_optreset WT_ATTRIBUTE_LIBRARY_VISIBLE; + int __wt_opterr = 1, /* if error message should be printed */ __wt_optind = 1, /* index into parent argv vector */ __wt_optopt, /* character checked for validity */ __wt_optreset; /* reset getopt */ -extern char *__wt_optarg; +extern char *__wt_optarg WT_ATTRIBUTE_LIBRARY_VISIBLE; char *__wt_optarg; /* argument associated with option */ #define BADCH (int)'?' diff --git a/src/os_common/os_strtouq.c b/src/os_common/os_strtouq.c index cb4da0de058..1cedfbdcb08 100644 --- a/src/os_common/os_strtouq.c +++ b/src/os_common/os_strtouq.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_posix/os_dir.c b/src/os_posix/os_dir.c index 627278540d1..8f77aba5f96 100644 --- a/src/os_posix/os_dir.c +++ b/src/os_posix/os_dir.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -37,7 +37,13 @@ __wt_posix_directory_list(WT_FILE_SYSTEM *file_system, dirallocsz = 0; entries = NULL; + /* + * If opendir fails, we should have a NULL pointer with an error value, + * but various static analysis programs remain unconvinced, check both. + */ WT_SYSCALL_RETRY(((dirp = opendir(directory)) == NULL ? -1 : 0), ret); + if (dirp == NULL && ret == 0) + ret = EINVAL; if (ret != 0) WT_RET_MSG(session, ret, "%s: directory-list: opendir", directory); diff --git a/src/os_posix/os_dlopen.c b/src/os_posix/os_dlopen.c index ad1fcc90150..154b15a886c 100644 --- a/src/os_posix/os_dlopen.c +++ b/src/os_posix/os_dlopen.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_posix/os_fallocate.c b/src/os_posix/os_fallocate.c index 111f6558816..5c57c5964b5 100644 --- a/src/os_posix/os_fallocate.c +++ b/src/os_posix/os_fallocate.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_posix/os_fs.c b/src/os_posix/os_fs.c index bc8cbf67025..d0391537543 100644 --- a/src/os_posix/os_fs.c +++ b/src/os_posix/os_fs.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/src/os_posix/os_getenv.c b/src/os_posix/os_getenv.c index f779f90acee..5b5a52cb273 100644 --- a/src/os_posix/os_getenv.c +++ b/src/os_posix/os_getenv.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_posix/os_map.c b/src/os_posix/os_map.c index 91ccc04ff7e..d8aaf5f591f 100644 --- a/src/os_posix/os_map.c +++ b/src/os_posix/os_map.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_posix/os_mtx_cond.c b/src/os_posix/os_mtx_cond.c index a5ee78f9e3e..1018bf860d6 100644 --- a/src/os_posix/os_mtx_cond.c +++ b/src/os_posix/os_mtx_cond.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -153,7 +153,7 @@ err: * __wt_cond_destroy -- * Destroy a condition variable. */ -int +void __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) { WT_CONDVAR *cond; @@ -161,11 +161,15 @@ __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) cond = *condp; if (cond == NULL) - return (0); + return; - ret = pthread_cond_destroy(&cond->cond); - WT_TRET(pthread_mutex_destroy(&cond->mtx)); - __wt_free(session, *condp); + if ((ret = pthread_cond_destroy(&cond->cond)) != 0) + WT_PANIC_MSG( + session, ret, "pthread_cond_destroy: %s", cond->name); - return (ret); + if ((ret = pthread_mutex_destroy(&cond->mtx)) != 0) + WT_PANIC_MSG( + session, ret, "pthread_mutex_destroy: %s", cond->name); + + __wt_free(session, *condp); } diff --git a/src/os_posix/os_once.c b/src/os_posix/os_once.c index 8d900042330..d2913997711 100644 --- a/src/os_posix/os_once.c +++ b/src/os_posix/os_once.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_posix/os_pagesize.c b/src/os_posix/os_pagesize.c index 4a7e7084cc6..09c52c41fe5 100644 --- a/src/os_posix/os_pagesize.c +++ b/src/os_posix/os_pagesize.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_posix/os_path.c b/src/os_posix/os_path.c index 6dc54675eb8..fc1a0fd4910 100644 --- a/src/os_posix/os_path.c +++ b/src/os_posix/os_path.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_posix/os_priv.c b/src/os_posix/os_priv.c index 5ffbbf7a1f2..0e0f5dfb190 100644 --- a/src/os_posix/os_priv.c +++ b/src/os_posix/os_priv.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_posix/os_setvbuf.c b/src/os_posix/os_setvbuf.c index ac3958be22f..a916ef79311 100644 --- a/src/os_posix/os_setvbuf.c +++ b/src/os_posix/os_setvbuf.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_posix/os_sleep.c b/src/os_posix/os_sleep.c index 2c60987ced7..67c0aaa375c 100644 --- a/src/os_posix/os_sleep.c +++ b/src/os_posix/os_sleep.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -18,6 +18,14 @@ __wt_sleep(uint64_t seconds, uint64_t micro_seconds) { struct timeval t; + /* + * Sleeping isn't documented as a memory barrier, and it's a reasonable + * expectation to have. There's no reason not to explicitly include a + * barrier since we're giving up the CPU, and ensures callers are never + * surprised. + */ + WT_FULL_BARRIER(); + t.tv_sec = (time_t)(seconds + micro_seconds / WT_MILLION); t.tv_usec = (suseconds_t)(micro_seconds % WT_MILLION); diff --git a/src/os_posix/os_snprintf.c b/src/os_posix/os_snprintf.c index 390e2e0334a..3ac0183f5ec 100644 --- a/src/os_posix/os_snprintf.c +++ b/src/os_posix/os_snprintf.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_posix/os_thread.c b/src/os_posix/os_thread.c index 18e4c347436..8af672dd0d4 100644 --- a/src/os_posix/os_thread.c +++ b/src/os_posix/os_thread.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -15,6 +15,7 @@ int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_DECL_RET; @@ -26,9 +27,11 @@ __wt_thread_create(WT_SESSION_IMPL *session, WT_FULL_BARRIER(); /* Spawn a new thread of control. */ - WT_SYSCALL_RETRY(pthread_create(tidret, NULL, func, arg), ret); - if (ret == 0) + WT_SYSCALL_RETRY(pthread_create(&tidret->id, NULL, func, arg), ret); + if (ret == 0) { + tidret->created = true; return (0); + } WT_RET_MSG(session, ret, "pthread_create"); } @@ -38,9 +41,14 @@ __wt_thread_create(WT_SESSION_IMPL *session, */ int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_DECL_RET; + /* Only attempt to join if thread was created successfully */ + if (!tid.created) + return (0); + /* * Joining a thread isn't a memory barrier, but WiredTiger commonly * sets flags and or state and then expects worker threads to halt. @@ -48,9 +56,11 @@ __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) */ WT_FULL_BARRIER(); - WT_SYSCALL(pthread_join(tid, NULL), ret); - if (ret == 0) + WT_SYSCALL(pthread_join(tid.id, NULL), ret); + if (ret == 0) { + tid.created = false; return (0); + } WT_RET_MSG(session, ret, "pthread_join"); } diff --git a/src/os_posix/os_time.c b/src/os_posix/os_time.c index 6f150ee8ffe..cc9516468aa 100644 --- a/src/os_posix/os_time.c +++ b/src/os_posix/os_time.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -16,6 +16,7 @@ void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { + struct timespec tmp; WT_DECL_RET; /* @@ -27,21 +28,34 @@ __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) tsp->tv_sec = 0; tsp->tv_nsec = 0; + /* + * Read into a local variable so that we're comparing the correct + * value when we check for monotonic increasing time. There are + * many places we read into an unlocked global variable. + */ #if defined(HAVE_CLOCK_GETTIME) - WT_SYSCALL_RETRY(clock_gettime(CLOCK_REALTIME, tsp), ret); - if (ret == 0) + WT_SYSCALL_RETRY(clock_gettime(CLOCK_REALTIME, &tmp), ret); + if (ret == 0) { + __wt_time_check_monotonic(session, &tmp); + tsp->tv_sec = tmp.tv_sec; + tsp->tv_nsec = tmp.tv_nsec; return; + } WT_PANIC_MSG(session, ret, "clock_gettime"); #elif defined(HAVE_GETTIMEOFDAY) + { struct timeval v; WT_SYSCALL_RETRY(gettimeofday(&v, NULL), ret); if (ret == 0) { - tsp->tv_sec = v.tv_sec; - tsp->tv_nsec = v.tv_usec * WT_THOUSAND; + tmp.tv_sec = v.tv_sec; + tmp.tv_nsec = v.tv_usec * WT_THOUSAND; + __wt_time_check_monotonic(session, &tmp); + *tsp = tmp; return; } WT_PANIC_MSG(session, ret, "gettimeofday"); + } #else NO TIME-OF-DAY IMPLEMENTATION: see src/os_posix/os_time.c #endif diff --git a/src/os_posix/os_yield.c b/src/os_posix/os_yield.c index f7c43aae746..3190e9e7062 100644 --- a/src/os_posix/os_yield.c +++ b/src/os_posix/os_yield.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_win/os_dir.c b/src/os_win/os_dir.c index 47d4f95b793..69235659f04 100644 --- a/src/os_win/os_dir.c +++ b/src/os_win/os_dir.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_win/os_dlopen.c b/src/os_win/os_dlopen.c index 6857be2a05e..9ee4d703c7a 100644 --- a/src/os_win/os_dlopen.c +++ b/src/os_win/os_dlopen.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_win/os_fs.c b/src/os_win/os_fs.c index 5cf47ea5763..1410a7bad03 100644 --- a/src/os_win/os_fs.c +++ b/src/os_win/os_fs.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -55,10 +55,11 @@ __win_fs_remove(WT_FILE_SYSTEM *file_system, if (DeleteFileW(name_wide->data) == FALSE) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s: file-remove: DeleteFileW: %s", name, __wt_formatmessage(session, windows_error)); - WT_ERR(__wt_map_windows_error(windows_error)); + WT_ERR(ret); } err: __wt_scr_free(session, &name_wide); @@ -74,9 +75,9 @@ __win_fs_rename(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *from, const char *to, uint32_t flags) { DWORD windows_error; - WT_DECL_RET; WT_DECL_ITEM(from_wide); WT_DECL_ITEM(to_wide); + WT_DECL_RET; WT_SESSION_IMPL *session; WT_UNUSED(file_system); @@ -98,10 +99,11 @@ __win_fs_rename(WT_FILE_SYSTEM *file_system, if (MoveFileExW(from_wide->data, to_wide->data, MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) == FALSE) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s to %s: file-rename: MoveFileExW: %s", from, to, __wt_formatmessage(session, windows_error)); - WT_ERR(__wt_map_windows_error(windows_error)); + WT_ERR(ret); } err: __wt_scr_free(session, &from_wide); @@ -118,9 +120,9 @@ __wt_win_fs_size(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *name, wt_off_t *sizep) { DWORD windows_error; - WT_DECL_RET; WIN32_FILE_ATTRIBUTE_DATA data; WT_DECL_ITEM(name_wide); + WT_DECL_RET; WT_SESSION_IMPL *session; WT_UNUSED(file_system); @@ -131,10 +133,11 @@ __wt_win_fs_size(WT_FILE_SYSTEM *file_system, if (GetFileAttributesExW( name_wide->data, GetFileExInfoStandard, &data) == 0) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s: file-size: GetFileAttributesEx: %s", name, __wt_formatmessage(session, windows_error)); - WT_ERR(__wt_map_windows_error(windows_error)); + WT_ERR(ret); } *sizep = ((int64_t)data.nFileSizeHigh << 32) | data.nFileSizeLow; @@ -168,21 +171,21 @@ __win_file_close(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session) if (win_fh->filehandle != INVALID_HANDLE_VALUE && CloseHandle(win_fh->filehandle) == 0) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s: handle-close: CloseHandle: %s", file_handle->name, __wt_formatmessage(session, windows_error)); - ret = __wt_map_windows_error(windows_error); } if (win_fh->filehandle_secondary != INVALID_HANDLE_VALUE && CloseHandle(win_fh->filehandle_secondary) == 0) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s: handle-close: secondary: CloseHandle: %s", file_handle->name, __wt_formatmessage(session, windows_error)); - ret = __wt_map_windows_error(windows_error); } __wt_free(session, file_handle->name); @@ -199,6 +202,7 @@ __win_file_lock( WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, bool lock) { DWORD windows_error; + WT_DECL_RET; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; @@ -218,22 +222,22 @@ __win_file_lock( if (lock) { if (LockFile(win_fh->filehandle, 0, 0, 1, 0) == FALSE) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s: handle-lock: LockFile: %s", file_handle->name, __wt_formatmessage(session, windows_error)); - return (__wt_map_windows_error(windows_error)); } } else if (UnlockFile(win_fh->filehandle, 0, 0, 1, 0) == FALSE) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s: handle-lock: UnlockFile: %s", file_handle->name, __wt_formatmessage(session, windows_error)); - return (__wt_map_windows_error(windows_error)); } - return (0); + return (ret); } /* @@ -245,10 +249,11 @@ __win_file_read(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, wt_off_t offset, size_t len, void *buf) { DWORD chunk, nr, windows_error; - uint8_t *addr; OVERLAPPED overlapped = { 0 }; + WT_DECL_RET; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; + uint8_t *addr; win_fh = (WT_FILE_HANDLE_WIN *)file_handle; session = (WT_SESSION_IMPL *)wt_session; @@ -273,12 +278,13 @@ __win_file_read(WT_FILE_HANDLE *file_handle, if (!ReadFile( win_fh->filehandle, addr, chunk, &nr, &overlapped)) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s: handle-read: ReadFile: failed to read %lu " "bytes at offset %" PRIuMAX ": %s", file_handle->name, chunk, (uintmax_t)offset, __wt_formatmessage(session, windows_error)); - return (__wt_map_windows_error(windows_error)); + return (ret); } } return (0); @@ -293,9 +299,10 @@ __win_file_size( WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, wt_off_t *sizep) { DWORD windows_error; + LARGE_INTEGER size; + WT_DECL_RET; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; - LARGE_INTEGER size; win_fh = (WT_FILE_HANDLE_WIN *)file_handle; session = (WT_SESSION_IMPL *)wt_session; @@ -306,10 +313,11 @@ __win_file_size( } windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s: handle-size: GetFileSizeEx: %s", file_handle->name, __wt_formatmessage(session, windows_error)); - return (__wt_map_windows_error(windows_error)); + return (ret); } /* @@ -320,6 +328,7 @@ static int __win_file_sync(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session) { DWORD windows_error; + WT_DECL_RET; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; @@ -337,11 +346,12 @@ __win_file_sync(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session) if (FlushFileBuffers(win_fh->filehandle) == FALSE) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s handle-sync: FlushFileBuffers: %s", file_handle->name, __wt_formatmessage(session, windows_error)); - return (__wt_map_windows_error(windows_error)); + return (ret); } return (0); } @@ -355,9 +365,10 @@ __win_file_set_end( WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, wt_off_t len) { DWORD windows_error; + LARGE_INTEGER largeint; + WT_DECL_RET; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; - LARGE_INTEGER largeint; win_fh = (WT_FILE_HANDLE_WIN *)file_handle; session = (WT_SESSION_IMPL *)wt_session; @@ -372,22 +383,24 @@ __win_file_set_end( if (SetFilePointerEx(win_fh->filehandle_secondary, largeint, NULL, FILE_BEGIN) == FALSE) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s: handle-set-end: SetFilePointerEx: %s", file_handle->name, __wt_formatmessage(session, windows_error)); - return (__wt_map_windows_error(windows_error)); + return (ret); } if (SetEndOfFile(win_fh->filehandle_secondary) == FALSE) { if (GetLastError() == ERROR_USER_MAPPED_FILE) return (EBUSY); windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s: handle-set-end: SetEndOfFile: %s", file_handle->name, __wt_formatmessage(session, windows_error)); - return (__wt_map_windows_error(windows_error)); + return (ret); } return (0); } @@ -401,10 +414,11 @@ __win_file_write(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, wt_off_t offset, size_t len, const void *buf) { DWORD chunk, nw, windows_error; - const uint8_t *addr; OVERLAPPED overlapped = { 0 }; + WT_DECL_RET; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; + const uint8_t *addr; win_fh = (WT_FILE_HANDLE_WIN *)file_handle; session = (WT_SESSION_IMPL *)wt_session; @@ -429,12 +443,13 @@ __win_file_write(WT_FILE_HANDLE *file_handle, if (!WriteFile( win_fh->filehandle, addr, chunk, &nw, &overlapped)) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s: handle-write: WriteFile: failed to write %lu " "bytes at offset %" PRIuMAX ": %s", file_handle->name, chunk, (uintmax_t)offset, __wt_formatmessage(session, windows_error)); - return (__wt_map_windows_error(windows_error)); + return (ret); } } return (0); @@ -451,8 +466,8 @@ __win_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, { DWORD dwCreationDisposition, windows_error; WT_CONNECTION_IMPL *conn; - WT_DECL_RET; WT_DECL_ITEM(name_wide); + WT_DECL_RET; WT_FILE_HANDLE *file_handle; WT_FILE_HANDLE_WIN *win_fh; WT_SESSION_IMPL *session; @@ -538,14 +553,15 @@ __win_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, NULL, OPEN_EXISTING, f, NULL); if (win_fh->filehandle == INVALID_HANDLE_VALUE) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, win_fh->direct_io ? "%s: handle-open: CreateFileW: failed with direct " "I/O configured, some filesystem types do not " "support direct I/O: %s" : "%s: handle-open: CreateFileW: %s", name, __wt_formatmessage(session, windows_error)); - WT_ERR(__wt_map_windows_error(windows_error)); + WT_ERR(ret); } } @@ -560,10 +576,11 @@ __win_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, NULL, OPEN_EXISTING, f, NULL); if (win_fh->filehandle_secondary == INVALID_HANDLE_VALUE) { windows_error = __wt_getlasterror(); - __wt_errx(session, + ret = __wt_map_windows_error(windows_error); + __wt_err(session, ret, "%s: handle-open: Creatively: secondary: %s", name, __wt_formatmessage(session, windows_error)); - WT_ERR(__wt_map_windows_error(windows_error)); + WT_ERR(ret); } } diff --git a/src/os_win/os_getenv.c b/src/os_win/os_getenv.c index fe228328ee6..b7b7f765656 100644 --- a/src/os_win/os_getenv.c +++ b/src/os_win/os_getenv.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_win/os_map.c b/src/os_win/os_map.c index a03e6cc3e52..c0aa6dac28f 100644 --- a/src/os_win/os_map.c +++ b/src/os_win/os_map.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_win/os_mtx_cond.c b/src/os_win/os_mtx_cond.c index 0001c6c2322..9d4339c8731 100644 --- a/src/os_win/os_mtx_cond.c +++ b/src/os_win/os_mtx_cond.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -163,18 +163,16 @@ __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond) * __wt_cond_destroy -- * Destroy a condition variable. */ -int +void __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) { WT_CONDVAR *cond; cond = *condp; if (cond == NULL) - return (0); + return; /* Do nothing to delete Condition Variable */ DeleteCriticalSection(&cond->mtx); __wt_free(session, *condp); - - return (0); } diff --git a/src/os_win/os_once.c b/src/os_win/os_once.c index 347d1883cca..dd21c58b8af 100644 --- a/src/os_win/os_once.c +++ b/src/os_win/os_once.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_win/os_pagesize.c b/src/os_win/os_pagesize.c index 648105c0e7c..07b1c3afc5c 100644 --- a/src/os_win/os_pagesize.c +++ b/src/os_win/os_pagesize.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_win/os_path.c b/src/os_win/os_path.c index 74050600417..78ad3bda509 100644 --- a/src/os_win/os_path.c +++ b/src/os_win/os_path.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_win/os_priv.c b/src/os_win/os_priv.c index 8c1f3893920..acc3793255a 100644 --- a/src/os_win/os_priv.c +++ b/src/os_win/os_priv.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_win/os_setvbuf.c b/src/os_win/os_setvbuf.c index b38ab1ebee2..78e42ecf4b5 100644 --- a/src/os_win/os_setvbuf.c +++ b/src/os_win/os_setvbuf.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_win/os_sleep.c b/src/os_win/os_sleep.c index 1cb61f7c4aa..477474e0665 100644 --- a/src/os_win/os_sleep.c +++ b/src/os_win/os_sleep.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -18,8 +18,15 @@ __wt_sleep(uint64_t seconds, uint64_t micro_seconds) DWORD dwMilliseconds; /* - * If the caller wants a small pause, set to our - * smallest granularity. + * Sleeping isn't documented as a memory barrier, and it's a reasonable + * expectation to have. There's no reason not to explicitly include a + * barrier since we're giving up the CPU, and ensures callers are never + * surprised. + */ + WT_FULL_BARRIER(); + + /* + * If the caller wants a small pause, set to our smallest granularity. */ if (seconds == 0 && micro_seconds < WT_THOUSAND) micro_seconds = WT_THOUSAND; diff --git a/src/os_win/os_snprintf.c b/src/os_win/os_snprintf.c index f3025b12a60..20231b468c6 100644 --- a/src/os_win/os_snprintf.c +++ b/src/os_win/os_snprintf.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_win/os_thread.c b/src/os_win/os_thread.c index 4c8f212bb4f..1ecf53e382e 100644 --- a/src/os_win/os_thread.c +++ b/src/os_win/os_thread.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -24,9 +24,11 @@ __wt_thread_create(WT_SESSION_IMPL *session, WT_FULL_BARRIER(); /* Spawn a new thread of control. */ - *tidret = (HANDLE)_beginthreadex(NULL, 0, func, arg, 0, NULL); - if (*tidret != 0) + tidret->id = (HANDLE)_beginthreadex(NULL, 0, func, arg, 0, NULL); + if (tidret->id != 0) { + tidret->created = true; return (0); + } WT_RET_MSG(session, __wt_errno(), "thread create: _beginthreadex"); } @@ -40,6 +42,10 @@ __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) { DWORD windows_error; + /* Only attempt to join if thread was created successfully */ + if (!tid.created) + return (0); + /* * Joining a thread isn't a memory barrier, but WiredTiger commonly * sets flags and or state and then expects worker threads to halt. @@ -48,7 +54,7 @@ __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_FULL_BARRIER(); if ((windows_error = - WaitForSingleObject(tid, INFINITE)) != WAIT_OBJECT_0) { + WaitForSingleObject(tid.id, INFINITE)) != WAIT_OBJECT_0) { if (windows_error == WAIT_FAILED) windows_error = __wt_getlasterror(); __wt_errx(session, "thread join: WaitForSingleObject: %s", @@ -58,13 +64,14 @@ __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) return (WT_PANIC); } - if (CloseHandle(tid) == 0) { + if (CloseHandle(tid.id) == 0) { windows_error = __wt_getlasterror(); __wt_errx(session, "thread join: CloseHandle: %s", __wt_formatmessage(session, windows_error)); return (__wt_map_windows_error(windows_error)); } + tid.created = false; return (0); } diff --git a/src/os_win/os_time.c b/src/os_win/os_time.c index 6aa5b3719f6..038c1d78d21 100644 --- a/src/os_win/os_time.c +++ b/src/os_win/os_time.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -15,17 +15,18 @@ void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) { + struct timespec tmp; FILETIME time; uint64_t ns100; - WT_UNUSED(session); - GetSystemTimeAsFileTime(&time); ns100 = (((int64_t)time.dwHighDateTime << 32) + time.dwLowDateTime) - 116444736000000000LL; - tsp->tv_sec = ns100 / 10000000; - tsp->tv_nsec = (long)((ns100 % 10000000) * 100); + tmp.tv_sec = ns100 / 10000000; + tmp.tv_nsec = (long)((ns100 % 10000000) * 100); + __wt_time_check_monotonic(session, &tmp); + *tsp = tmp; } /* diff --git a/src/os_win/os_utf8.c b/src/os_win/os_utf8.c index ccd8321aecf..f7bab41c81f 100644 --- a/src/os_win/os_utf8.c +++ b/src/os_win/os_utf8.c @@ -1,7 +1,7 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. - * All rights reserved. + * All rights reserved. * * See the file LICENSE for redistribution information. */ diff --git a/src/os_win/os_winerr.c b/src/os_win/os_winerr.c index 70499580c48..c7748d80fb2 100644 --- a/src/os_win/os_winerr.c +++ b/src/os_win/os_winerr.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/os_win/os_yield.c b/src/os_win/os_yield.c index 038f2efe162..e38fc21e16b 100644 --- a/src/os_win/os_yield.c +++ b/src/os_win/os_yield.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/packing/pack_api.c b/src/packing/pack_api.c index 4c65406cd64..ee7ce6c4c0d 100644 --- a/src/packing/pack_api.c +++ b/src/packing/pack_api.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/packing/pack_impl.c b/src/packing/pack_impl.c index 5dbb0f33842..d40043fc13c 100644 --- a/src/packing/pack_impl.c +++ b/src/packing/pack_impl.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/packing/pack_stream.c b/src/packing/pack_stream.c index 1393eb9a9c1..dc2925acaf3 100644 --- a/src/packing/pack_stream.c +++ b/src/packing/pack_stream.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/reconcile/rec_track.c b/src/reconcile/rec_track.c index 5bf425b1b21..a431465661f 100644 --- a/src/reconcile/rec_track.c +++ b/src/reconcile/rec_track.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/reconcile/rec_write.c b/src/reconcile/rec_write.c index 6f95b84d292..1c266496ec8 100644 --- a/src/reconcile/rec_write.c +++ b/src/reconcile/rec_write.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -25,12 +25,25 @@ typedef struct { WT_PAGE *page; uint32_t flags; /* Caller's configuration */ - WT_ITEM disk_image; /* Temporary disk-image buffer */ /* - * Temporary buffer used to write out a disk image when managing two - * chunks worth of data in memory + * Reconciliation can end up requiring two temporary disk image buffers + * if a page split is involved. These two disk images are pointed to by + * current and the previous image pointers. During initialization the + * first image is allocated and pointed to by the current image pointer. + * If and when a split is involved the second image gets allocated and + * is pointed to by the current image pointer. The previous image + * pointer is made to refer the first image at this point. Two images + * are kept in memory to redistribute data among them in case the last + * split chunk ends up being smaller than the minimum required. As + * reconciliation generates more split chunks, the image referred to by + * the previous image pointer is written to the disk, the current and + * the previous image pointers are swapped, making space for another + * split chunk to be reconciled in the buffer that was just written out + * to the disk. */ - WT_ITEM *interim_buf; + WT_ITEM disk_image[2]; /* Temporary disk-image buffers */ + WT_ITEM *cur_img_ptr; + WT_ITEM *prev_img_ptr; /* * Track start/stop write generation to decide if all changes to the @@ -48,9 +61,9 @@ typedef struct { /* Track the page's maximum transaction ID. */ uint64_t max_txn; - /* Track if all updates were skipped. */ - uint64_t update_cnt; - uint64_t update_skip_cnt; + uint64_t update_mem_all; /* Total update memory size */ + uint64_t update_mem_saved; /* Saved update memory size */ + uint64_t update_mem_uncommitted;/* Uncommitted update memory size */ /* * When we can't mark the page clean (for example, checkpoint found some @@ -146,17 +159,6 @@ typedef struct { * that references all of our split pages. */ struct __rec_boundary { - /* - * Offset is the byte offset in the initial split buffer of the - * first byte of the split chunk, recorded before we decide to - * split the page; the difference between chunk[1]'s offset and - * chunk[0]'s offset is chunk[0]'s length. - * - * Once we split a page, we stop filling in offset values, we're - * writing the split chunks as we find them. - */ - size_t offset; /* Split's first byte */ - WT_ADDR addr; /* Split's written location */ uint32_t size; /* Split's size */ uint32_t checksum; /* Split's checksum */ @@ -338,7 +340,8 @@ static int __rec_split_write(WT_SESSION_IMPL *, WT_RECONCILE *, WT_BOUNDARY *, WT_ITEM *, bool); static int __rec_update_las( WT_SESSION_IMPL *, WT_RECONCILE *, uint32_t, WT_BOUNDARY *); -static int __rec_write_check_complete(WT_SESSION_IMPL *, WT_RECONCILE *); +static int __rec_write_check_complete( + WT_SESSION_IMPL *, WT_RECONCILE *, bool *); static int __rec_write_init(WT_SESSION_IMPL *, WT_REF *, uint32_t, WT_SALVAGE_COOKIE *, void *); static void __rec_write_page_status(WT_SESSION_IMPL *, WT_RECONCILE *); @@ -351,6 +354,7 @@ static int __rec_dictionary_init(WT_SESSION_IMPL *, WT_RECONCILE *, u_int); static int __rec_dictionary_lookup( WT_SESSION_IMPL *, WT_RECONCILE *, WT_KV *, WT_DICTIONARY **); static void __rec_dictionary_reset(WT_RECONCILE *); +static void __rec_verbose_lookaside_write(WT_SESSION_IMPL *); /* * __wt_reconcile -- @@ -386,7 +390,7 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, * In-memory splits: reconciliation of an internal page cannot handle * a child page splitting during the reconciliation. */ - __wt_writelock(session, &page->page_lock); + WT_PAGE_LOCK(session, page); oldest_id = __wt_txn_oldest_id(session); if (LF_ISSET(WT_EVICTING)) @@ -405,7 +409,7 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, /* Initialize the reconciliation structure for each new run. */ if ((ret = __rec_write_init( session, ref, flags, salvage, &session->reconcile)) != 0) { - __wt_writeunlock(session, &page->page_lock); + WT_PAGE_UNLOCK(session, page); return (ret); } r = session->reconcile; @@ -437,7 +441,7 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, /* Checks for a successful reconciliation. */ if (ret == 0) - ret = __rec_write_check_complete(session, r); + ret = __rec_write_check_complete(session, r, lookaside_retryp); /* Wrap up the page reconciliation. */ if (ret == 0 && (ret = __rec_write_wrapup(session, r, page)) == 0) @@ -446,15 +450,7 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_TRET(__rec_write_wrapup_err(session, r, page)); /* Release the reconciliation lock. */ - __wt_writeunlock(session, &page->page_lock); - - /* - * If our caller can configure lookaside table reconciliation, flag if - * that's worth trying. The lookaside table doesn't help if we skipped - * updates, it can only help with older readers preventing eviction. - */ - if (lookaside_retryp != NULL && r->update_cnt == r->update_skip_cnt) - *lookaside_retryp = true; + WT_PAGE_UNLOCK(session, page); /* Update statistics. */ WT_STAT_CONN_INCR(session, rec_pages); @@ -526,10 +522,8 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, static inline bool __rec_las_checkpoint_test(WT_SESSION_IMPL *session, WT_RECONCILE *r) { - WT_CONNECTION_IMPL *conn; WT_BTREE *btree; - conn = S2C(session); btree = S2BT(session); /* @@ -550,7 +544,8 @@ __rec_las_checkpoint_test(WT_SESSION_IMPL *session, WT_RECONCILE *r) if (F_ISSET(btree, WT_BTREE_NO_CHECKPOINT)) return (false); if (r->orig_btree_checkpoint_gen == btree->checkpoint_gen && - r->orig_txn_checkpoint_gen == conn->txn_global.checkpoint_gen && + r->orig_txn_checkpoint_gen == + __wt_gen(session, WT_GEN_CHECKPOINT) && r->orig_btree_checkpoint_gen == r->orig_txn_checkpoint_gen) return (false); return (true); @@ -558,13 +553,21 @@ __rec_las_checkpoint_test(WT_SESSION_IMPL *session, WT_RECONCILE *r) /* * __rec_write_check_complete -- - * Check that reconciliation should complete + * Check that reconciliation should complete. */ static int -__rec_write_check_complete(WT_SESSION_IMPL *session, WT_RECONCILE *r) +__rec_write_check_complete( + WT_SESSION_IMPL *session, WT_RECONCILE *r, bool *lookaside_retryp) { - WT_BOUNDARY *bnd; - size_t i; + /* + * Tests in this function are lookaside tests and tests to decide if + * rewriting a page in memory is worth doing. In-memory configurations + * can't use a lookaside table, and we ignore page rewrite desirability + * checks for in-memory eviction because a small cache can force us to + * rewrite every possible page. + */ + if (F_ISSET(r, WT_EVICT_IN_MEMORY)) + return (0); /* * If we have used the lookaside table, check for a lookaside table and @@ -574,19 +577,62 @@ __rec_write_check_complete(WT_SESSION_IMPL *session, WT_RECONCILE *r) return (EBUSY); /* - * If we are doing update/restore based eviction, confirm part of the - * page is being discarded, or at least 10% of the updates won't have - * to be re-instantiated. Otherwise, it isn't progress, don't bother. + * Eviction can configure lookaside table reconciliation, consider if + * it's worth giving up this reconciliation attempt and falling back to + * using the lookaside table. We continue with evict/restore if + * switching to the lookaside doesn't make sense for any reason: we + * won't retry an evict/restore reconciliation until/unless the + * transactional system moves forward, so at worst it's a single wasted + * effort. + * + * First, check if the lookaside table is a possible alternative. */ - if (F_ISSET(r, WT_EVICT_UPDATE_RESTORE)) { - for (bnd = r->bnd, i = 0; i < r->bnd_entries; ++bnd, ++i) - if (bnd->supd == NULL) - break; - if (i == r->bnd_entries && - r->update_cnt / 10 >= r->update_skip_cnt) - return (EBUSY); - } - return (0); + if (lookaside_retryp == NULL) + return (0); + + /* + * We only suggest lookaside if currently in an evict/restore attempt + * and some updates were saved. Our caller sets the evict/restore flag + * based on various conditions (like if this is a leaf page), which is + * why we're testing that flag instead of a set of other conditions. + * If no updates were saved, eviction will succeed without needing to + * restore anything. + */ + if (!F_ISSET(r, WT_EVICT_UPDATE_RESTORE) || r->bnd->supd == NULL) + return (0); + + /* + * Check if this reconciliation attempt is making progress. If there's + * any sign of progress, don't fall back to the lookaside table. + * + * Check if the current reconciliation split, in which case we'll + * likely get to write at least one of the blocks. If that page is + * empty, that's also progress. + */ + if (r->bnd_next != 1) + return (0); + + /* + * Check if the current reconciliation applied some updates, in which + * case evict/restore should gain us some space. + */ + if (r->update_mem_saved != r->update_mem_all) + return (0); + + /* + * Check if lookaside eviction is possible. If any of the updates we + * saw were uncommitted, the lookaside table cannot be used: it only + * helps with older readers preventing eviction. + */ + if (r->update_mem_uncommitted != 0) + return (0); + + /* + * The current evict/restore approach shows no signs of being useful, + * lookaside is possible, suggest the lookaside table. + */ + *lookaside_retryp = true; + return (EBUSY); } /* @@ -810,12 +856,10 @@ __rec_write_init(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags, WT_SALVAGE_COOKIE *salvage, void *reconcilep) { WT_BTREE *btree; - WT_CONNECTION_IMPL *conn; WT_PAGE *page; WT_RECONCILE *r; btree = S2BT(session); - conn = S2C(session); page = ref->page; if ((r = *(WT_RECONCILE **)reconcilep) == NULL) { @@ -829,7 +873,8 @@ __rec_write_init(WT_SESSION_IMPL *session, r->last = &r->_last; /* Disk buffers need to be aligned for writing. */ - F_SET(&r->disk_image, WT_ITEM_ALIGNED); + F_SET(&r->disk_image[0], WT_ITEM_ALIGNED); + F_SET(&r->disk_image[1], WT_ITEM_ALIGNED); } /* Reconciliation is not re-entrant, make sure that doesn't happen. */ @@ -845,7 +890,7 @@ __rec_write_init(WT_SESSION_IMPL *session, * These are all ordered reads, but we only need one. */ r->orig_btree_checkpoint_gen = btree->checkpoint_gen; - r->orig_txn_checkpoint_gen = conn->txn_global.checkpoint_gen; + r->orig_txn_checkpoint_gen = __wt_gen(session, WT_GEN_CHECKPOINT); WT_ORDERED_READ(r->orig_write_gen, page->modify->write_gen); /* @@ -891,7 +936,7 @@ __rec_write_init(WT_SESSION_IMPL *session, r->max_txn = WT_TXN_NONE; /* Track if all updates were skipped. */ - r->update_cnt = r->update_skip_cnt = 0; + r->update_mem_all = r->update_mem_saved = r->update_mem_uncommitted = 0; /* Track if the page can be marked clean. */ r->leave_dirty = false; @@ -974,8 +1019,8 @@ __rec_destroy(WT_SESSION_IMPL *session, void *reconcilep) return; *(WT_RECONCILE **)reconcilep = NULL; - __wt_buf_free(session, &r->disk_image); - __wt_scr_free(session, &r->interim_buf); + __wt_buf_free(session, &r->disk_image[0]); + __wt_buf_free(session, &r->disk_image[1]); __wt_free(session, r->raw_entries); __wt_free(session, r->raw_offsets); @@ -1115,7 +1160,7 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_DECL_ITEM(tmp); WT_PAGE *page; WT_UPDATE *append, *upd, *upd_list; - size_t notused; + size_t notused, update_mem; uint64_t max_txn, min_txn, txnid; bool append_origv, skipped; @@ -1136,36 +1181,62 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, } else upd_list = ins->upd; - ++r->update_cnt; - for (skipped = false, - max_txn = WT_TXN_NONE, min_txn = UINT64_MAX, - upd = upd_list; upd != NULL; upd = upd->next) { - if ((txnid = upd->txnid) == WT_TXN_ABORTED) - continue; + skipped = false; + update_mem = 0; + max_txn = WT_TXN_NONE; + min_txn = UINT64_MAX; - /* Track the largest/smallest transaction IDs on the list. */ - if (WT_TXNID_LT(max_txn, txnid)) - max_txn = txnid; - if (WT_TXNID_LT(txnid, min_txn)) - min_txn = txnid; + if (F_ISSET(r, WT_EVICTING)) { + /* Discard obsolete updates. */ + if ((upd = __wt_update_obsolete_check( + session, page, upd_list->next)) != NULL) + __wt_update_obsolete_free(session, page, upd); + + for (upd = upd_list; upd != NULL; upd = upd->next) { + /* Track the total memory in the update chain. */ + update_mem += WT_UPDATE_MEMSIZE(upd); + + if ((txnid = upd->txnid) == WT_TXN_ABORTED) + continue; - /* - * Find the first update we can use. - */ - if (F_ISSET(r, WT_EVICTING)) { /* + * Track the largest/smallest transaction IDs on the + * list. + */ + if (WT_TXNID_LT(max_txn, txnid)) + max_txn = txnid; + if (WT_TXNID_LT(txnid, min_txn)) + min_txn = txnid; + + /* + * Find the first update we can use. + * * Eviction can write any committed update. * * When reconciling for eviction, track whether any * uncommitted updates are found. + * + * When reconciling for eviction, track the memory held + * by the update chain. */ if (__wt_txn_committed(session, txnid)) { if (*updp == NULL) *updp = upd; } else skipped = true; - } else { + } + } else + for (upd = upd_list; upd != NULL; upd = upd->next) { + if ((txnid = upd->txnid) == WT_TXN_ABORTED) + continue; + + /* Track the largest transaction ID on the list. */ + if (WT_TXNID_LT(max_txn, txnid)) + max_txn = txnid; + /* + * Find the first update we can use. + * * Checkpoint can only write updates visible as of its * snapshot. * @@ -1180,7 +1251,12 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, skipped = true; } } - } + + /* Reconciliation should never see a reserved update. */ + WT_ASSERT(session, + *updp == NULL || (*updp)->type != WT_UPDATE_RESERVED); + + r->update_mem_all += update_mem; /* * If all of the updates were aborted, quit. This test is not strictly @@ -1227,12 +1303,6 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, txnid != S2C(session)->txn_global.checkpoint_txnid || WT_SESSION_IS_CHECKPOINT(session)); #endif - - /* - * Track how many update chains we saw vs. how many update - * chains had an entry we skipped. - */ - ++r->update_skip_cnt; return (0); } @@ -1276,6 +1346,23 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, if (skipped && !F_ISSET(r, WT_EVICT_UPDATE_RESTORE)) return (EBUSY); + /* + * Track the memory required by the update chain. + * + * A page with no uncommitted (skipped) updates, that can't be evicted + * because some updates aren't yet globally visible, can be evicted by + * writing previous versions of the updates to the lookaside file. That + * test is just checking if the skipped updates memory is zero. + * + * If that's not possible (there are skipped updates), we can rewrite + * the pages in-memory, but we don't want to unless there's memory to + * recover. That test is comparing the memory we'd recover to the memory + * we'd have to re-instantiate as part of the rewrite. + */ + r->update_mem_saved += update_mem; + if (skipped) + r->update_mem_uncommitted += update_mem; + append_origv = false; if (F_ISSET(r, WT_EVICT_UPDATE_RESTORE)) { /* @@ -1353,14 +1440,14 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, * place a deleted record at the end of the update list. */ if (vpack == NULL || vpack->type == WT_CELL_DEL) - WT_RET(__wt_update_alloc( - session, NULL, &append, ¬used)); + WT_RET(__wt_update_alloc(session, + NULL, &append, ¬used, WT_UPDATE_DELETED)); else { WT_RET(__wt_scr_alloc(session, 0, &tmp)); if ((ret = __wt_page_cell_data_ref( session, page, vpack, tmp)) == 0) - ret = __wt_update_alloc( - session, tmp, &append, ¬used); + ret = __wt_update_alloc(session, + tmp, &append, ¬used, WT_UPDATE_STANDARD); __wt_scr_free(session, &tmp); WT_RET(ret); } @@ -1721,7 +1808,7 @@ __rec_incr(WT_SESSION_IMPL *session, WT_RECONCILE *r, uint32_t v, size_t size) */ WT_ASSERT(session, r->space_avail >= size); WT_ASSERT(session, WT_BLOCK_FITS( - r->first_free, size, r->disk_image.mem, r->disk_image.memsize)); + r->first_free, size, r->cur_img_ptr->mem, r->cur_img_ptr->memsize)); r->entries += v; r->space_avail -= size; @@ -1808,7 +1895,7 @@ __rec_dict_replace( * copy cell instead. */ if (dp->offset == 0) - dp->offset = WT_PTRDIFF32(r->first_free, r->disk_image.mem); + dp->offset = WT_PTRDIFF32(r->first_free, r->cur_img_ptr->mem); else { /* * The offset is the byte offset from this cell to the previous, @@ -1816,7 +1903,7 @@ __rec_dict_replace( * page. */ offset = (uint64_t)WT_PTRDIFF(r->first_free, - (uint8_t *)r->disk_image.mem + dp->offset); + (uint8_t *)r->cur_img_ptr->mem + dp->offset); val->len = val->cell_len = __wt_cell_pack_copy(&val->cell, rle, offset); val->buf.data = NULL; @@ -1952,7 +2039,6 @@ __rec_leaf_page_max(WT_SESSION_IMPL *session, WT_RECONCILE *r) static void __rec_split_bnd_init(WT_SESSION_IMPL *session, WT_BOUNDARY *bnd) { - bnd->offset = 0; bnd->max_bnd_recno = WT_RECNO_OOB; bnd->max_bnd_entries = 0; @@ -2105,8 +2191,8 @@ __rec_split_init(WT_SESSION_IMPL *session, r->page_size = r->page_size_orig = max; if (r->raw_compression) r->max_raw_page_size = r->page_size = - (uint32_t)WT_MIN(r->page_size * 10, - WT_MAX(r->page_size, btree->maxmempage / 2)); + (uint32_t)WT_MIN((uint64_t)r->page_size * 10, + WT_MAX((uint64_t)r->page_size, btree->maxmempage / 2)); /* * If we have to split, we want to choose a smaller page size for the * split pages, because otherwise we could end up splitting one large @@ -2165,15 +2251,14 @@ __rec_split_init(WT_SESSION_IMPL *session, * Ensure the disk image buffer is large enough for the max object, as * corrected by the underlying block manager. * - * The buffer that we build disk image in, needs to hold two chunks - * worth of data. Since we want to support split_size more than the page - * size (to allow for adjustments based on the compression), this buffer - * should be greater of twice of split_size and page_size. + * Since we want to support split_size more than the page size (to allow + * for adjustments based on the compression), this buffer should be + * greater of split_size and page_size. */ corrected_page_size = r->page_size; - disk_img_buf_size = 2 * WT_MAX(corrected_page_size, r->split_size); WT_RET(bm->write_size(bm, session, &corrected_page_size)); - WT_RET(__wt_buf_init(session, &r->disk_image, disk_img_buf_size)); + disk_img_buf_size = WT_MAX(corrected_page_size, r->split_size); + WT_RET(__wt_buf_init(session, &r->disk_image[0], disk_img_buf_size)); /* * Clear the disk page header to ensure all of it is initialized, even @@ -2183,15 +2268,17 @@ __rec_split_init(WT_SESSION_IMPL *session, * fixed-length column-store sets bits in bytes, where the bytes are * assumed to initially be 0. */ - memset(r->disk_image.mem, 0, page->type == WT_PAGE_COL_FIX ? + memset(r->disk_image[0].mem, 0, page->type == WT_PAGE_COL_FIX ? disk_img_buf_size : WT_PAGE_HEADER_SIZE); /* * Set the page type (the type doesn't change, and setting it later * would require additional code in a few different places). */ - dsk = r->disk_image.mem; + dsk = r->disk_image[0].mem; dsk->type = page->type; + r->cur_img_ptr = &r->disk_image[0]; + r->prev_img_ptr = NULL; r->first_free = WT_PAGE_HEADER_BYTE(btree, dsk); @@ -2200,7 +2287,6 @@ __rec_split_init(WT_SESSION_IMPL *session, WT_RET(__rec_split_bnd_grow(session, r)); __rec_split_bnd_init(session, &r->bnd[0]); r->bnd[0].max_bnd_recno = recno; - r->bnd[0].offset = WT_PAGE_HEADER_BYTE_SIZE(btree); /* Initialize the entry counter. */ r->entries = 0; @@ -2406,21 +2492,18 @@ __rec_split_grow(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t add_len) { WT_BM *bm; WT_BTREE *btree; - size_t corrected_page_size, inuse, len; + size_t corrected_page_size, inuse; btree = S2BT(session); bm = btree->bm; - len = WT_PTRDIFF(r->first_free, r->disk_image.mem); - inuse = (len - r->bnd[r->bnd_next].offset) + - WT_PAGE_HEADER_BYTE_SIZE(btree); + inuse = WT_PTRDIFF(r->first_free, r->cur_img_ptr->mem); corrected_page_size = inuse + add_len; WT_RET(bm->write_size(bm, session, &corrected_page_size)); - /* Need to account for buffer carrying two chunks worth of data */ - WT_RET(__wt_buf_grow(session, &r->disk_image, 2 * corrected_page_size)); + WT_RET(__wt_buf_grow(session, r->cur_img_ptr, corrected_page_size)); - r->first_free = (uint8_t *)r->disk_image.mem + len; + r->first_free = (uint8_t *)r->cur_img_ptr->mem + inuse; WT_ASSERT(session, corrected_page_size >= inuse); r->space_avail = corrected_page_size - inuse; WT_ASSERT(session, r->space_avail >= add_len); @@ -2429,89 +2512,55 @@ __rec_split_grow(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t add_len) } /* - * __rec_split_write_prev_and_shift_cur -- - * Write the previous split chunk to the disk as a page. Shift the contents - * of the current chunk to the start of the buffer, making space for a new - * chunk to be written. - * If the caller asks for a chunk resizing, the boundary between the two - * chunks is readjusted to the minimum split size boundary details stored - * in the previous chunk, letting the current chunk grow at the cost of the - * previous chunk. + * __rec_split_write_prev_and_swap_buf -- + * If there is a previous split chunk held in the memory, write it to the + * disk as a page. If there isn't one, this is the first time we are + * splitting and need to initialize a second buffer. Also, swap the + * previous and the current buffer pointers. */ static int -__rec_split_write_prev_and_shift_cur( - WT_SESSION_IMPL *session, WT_RECONCILE *r, bool resize_chunks) +__rec_split_write_prev_and_swap_buf(WT_SESSION_IMPL *session, WT_RECONCILE *r) { - WT_BM *bm; - WT_BOUNDARY *bnd_cur, *bnd_prev; - WT_BTREE *btree; - WT_PAGE_HEADER *dsk, *dsk_tmp; - size_t cur_len, len; - uint8_t *dsk_start; - - WT_ASSERT(session, r->bnd_next != 0); - - btree = S2BT(session); - bm = btree->bm; - bnd_cur = &r->bnd[r->bnd_next]; - bnd_prev = bnd_cur - 1; - dsk = r->disk_image.mem; - cur_len = WT_PTRDIFF(r->first_free, dsk) - bnd_cur->offset; - - /* - * Resize chunks if the current is smaller than the minimum, and there - * are details on the minimum split size boundary available in the - * previous boundary details. - * - * There is a possibility that we do not have a minimum boundary set, in - * such a case we skip chunk resizing. Such a condition is possible for - * instance when we are building the image in the buffer and the first - * K/V pair is large enough that it surpasses both the minimum split - * size and the split size the application has set. In such a case we - * split the chunk without saving any minimum boundary. - */ - if (resize_chunks && - cur_len < r->min_split_size && bnd_prev->min_bnd_offset != 0) { - bnd_cur->offset = bnd_prev->min_bnd_offset; - bnd_cur->max_bnd_entries += - bnd_prev->max_bnd_entries - bnd_prev->min_bnd_entries; - bnd_prev->max_bnd_entries = bnd_prev->min_bnd_entries; - bnd_cur->max_bnd_recno = bnd_prev->min_bnd_recno; - - WT_RET(__wt_buf_set(session, &bnd_cur->max_bnd_key, - bnd_prev->min_bnd_key.data, bnd_prev->min_bnd_key.size)); - - /* Update current chunk's length */ - cur_len = WT_PTRDIFF(r->first_free, dsk) - bnd_cur->offset; + WT_BOUNDARY *bnd_prev; + WT_ITEM *tmp_img_ptr; + WT_PAGE_HEADER *dsk; + size_t disk_img_size; + + WT_ASSERT(session, (r->prev_img_ptr == NULL && r->bnd_next == 0) || + (r->prev_img_ptr != NULL && r->bnd_next != 0)); + + /* Write previous chunk, if there is one */ + if (r->prev_img_ptr != NULL) { + bnd_prev = &r->bnd[r->bnd_next - 1]; + dsk = r->prev_img_ptr->mem; + dsk->recno = bnd_prev->max_bnd_recno; + dsk->u.entries = bnd_prev->max_bnd_entries; + dsk->mem_size = (uint32_t)bnd_prev->size; + r->prev_img_ptr->size = dsk->mem_size; + WT_RET(__rec_split_write(session, + r, bnd_prev, r->prev_img_ptr, false)); + } else { + /* + * If we do not have a previous buffer, we should initialize the + * second buffer before proceeding. We will create the second + * buffer of the same size as the current buffer. + */ + disk_img_size = r->cur_img_ptr->memsize; + WT_RET(__wt_buf_init(session, + &r->disk_image[1], disk_img_size)); + r->prev_img_ptr = &r->disk_image[1]; + dsk = r->prev_img_ptr->mem; + memset(dsk, 0, + r->page->type == WT_PAGE_COL_FIX ? + disk_img_size : WT_PAGE_HEADER_SIZE); + dsk->type = r->page->type; } - /* - * Create an interim buffer if not already done to prepare the previous - * chunk's disk image. - */ - len = bnd_cur->offset; - WT_RET(bm->write_size(bm, session, &len)); - if (r->interim_buf == NULL) - WT_RET(__wt_scr_alloc(session, len, &r->interim_buf)); - else - WT_RET(__wt_buf_init(session, r->interim_buf, len)); - - dsk_tmp = r->interim_buf->mem; - memcpy(dsk_tmp, dsk, bnd_cur->offset); - dsk_tmp->recno = bnd_prev->max_bnd_recno; - dsk_tmp->u.entries = bnd_prev->max_bnd_entries; - dsk_tmp->mem_size = WT_STORE_SIZE(bnd_cur->offset); - r->interim_buf->size = dsk_tmp->mem_size; - WT_RET(__rec_split_write(session, r, bnd_prev, r->interim_buf, false)); - - /* Shift the current chunk to the start of the buffer */ - dsk_start = WT_PAGE_HEADER_BYTE(btree, dsk); - (void)memmove(dsk_start, (uint8_t *)dsk + bnd_cur->offset, cur_len); - - /* Fix boundary offset */ - bnd_cur->offset = WT_PAGE_HEADER_BYTE_SIZE(btree); - /* Fix where free points */ - r->first_free = dsk_start + cur_len; + /* swap previous and current buffers */ + tmp_img_ptr = r->prev_img_ptr; + r->prev_img_ptr = r->cur_img_ptr; + r->cur_img_ptr = tmp_img_ptr; + return (0); } @@ -2529,7 +2578,7 @@ __rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) size_t inuse; btree = S2BT(session); - dsk = r->disk_image.mem; + dsk = r->cur_img_ptr->mem; /* Fixed length col store can call with next_len 0 */ WT_ASSERT(session, next_len == 0 || r->space_avail < next_len); @@ -2543,9 +2592,7 @@ __rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) "%s page too large, attempted split during salvage", __wt_page_type_string(r->page->type)); - last = &r->bnd[r->bnd_next]; - inuse = (WT_PTRDIFF(r->first_free, dsk) - last->offset) + - WT_PAGE_HEADER_BYTE_SIZE(btree); + inuse = WT_PTRDIFF(r->first_free, dsk); /* * We can get here if the first key/value pair won't fit. @@ -2558,8 +2605,10 @@ __rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) /* All page boundaries reset the dictionary. */ __rec_dictionary_reset(r); - /* Set the number of entries for the just finished chunk. */ + /* Set the number of entries and size for the just finished chunk. */ + last = &r->bnd[r->bnd_next]; last->max_bnd_entries = r->entries; + last->size = (uint32_t)inuse; /* * In case of bulk load, write out chunks as we get them. Otherwise we @@ -2571,19 +2620,22 @@ __rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) dsk->recno = last->max_bnd_recno; dsk->u.entries = last->max_bnd_entries; dsk->mem_size = (uint32_t)inuse; - r->disk_image.size = dsk->mem_size; - WT_RET(__rec_split_write( - session, r, last, &r->disk_image, false)); - /* Fix where free points */ - r->first_free = WT_PAGE_HEADER_BYTE(btree, dsk); - } else if (r->bnd_next != 0) - WT_RET(__rec_split_write_prev_and_shift_cur(session, r, false)); + r->cur_img_ptr->size = dsk->mem_size; + WT_RET(__rec_split_write(session, + r, last, r->cur_img_ptr, false)); + } else { + WT_RET(__rec_split_write_prev_and_swap_buf(session, r)); + /* current image we are writing to has changed */ + dsk = r->cur_img_ptr->mem; + } + + /* Fix where free points */ + r->first_free = WT_PAGE_HEADER_BYTE(btree, dsk); /* Prepare the next boundary */ WT_RET(__rec_split_bnd_grow(session, r)); r->bnd_next++; next = &r->bnd[r->bnd_next]; - next->offset = WT_PTRDIFF(r->first_free, dsk); /* Set the key for the next chunk. */ next->max_bnd_recno = r->recno; if (dsk->type == WT_PAGE_ROW_INT || dsk->type == WT_PAGE_ROW_LEAF) @@ -2642,9 +2694,8 @@ __rec_split_crossing_bnd( !WT_CROSSING_SPLIT_BND(r, next_len)) { btree = S2BT(session); bnd = &r->bnd[r->bnd_next]; - dsk = r->disk_image.mem; - min_bnd_offset = (WT_PTRDIFF(r->first_free, dsk) - - bnd->offset) + WT_PAGE_HEADER_BYTE_SIZE(btree); + dsk = r->cur_img_ptr->mem; + min_bnd_offset = WT_PTRDIFF(r->first_free, dsk); if (min_bnd_offset == WT_PAGE_HEADER_BYTE_SIZE(btree)) /* * This is possible if the first record doesn't fit in @@ -2705,7 +2756,7 @@ __rec_split_raw_worker(WT_SESSION_IMPL *session, unpack = &_unpack; compressor = btree->compressor; dst = &r->raw_destination; - dsk = r->disk_image.mem; + dsk = r->cur_img_ptr->mem; WT_RET(__rec_split_bnd_grow(session, r)); last = &r->bnd[r->bnd_next]; @@ -3021,7 +3072,7 @@ no_slots: r->first_free = dsk_start + len; r->space_avail += r->raw_offsets[result_slots]; WT_ASSERT(session, r->first_free + r->space_avail <= - (uint8_t *)r->disk_image.mem + r->disk_image.memsize); + (uint8_t *)r->cur_img_ptr->mem + r->cur_img_ptr->memsize); /* * Set the key for the next block (before writing the block, a @@ -3060,13 +3111,13 @@ no_slots: dsk->recno = last->max_bnd_recno; dsk->mem_size = WT_PTRDIFF32(r->first_free, dsk); dsk->u.entries = r->entries; - r->disk_image.size = dsk->mem_size; + r->cur_img_ptr->size = dsk->mem_size; r->entries = 0; r->first_free = WT_PAGE_HEADER_BYTE(btree, dsk); r->space_avail = r->page_size - WT_PAGE_HEADER_BYTE_SIZE(btree); - write_ref = &r->disk_image; + write_ref = r->cur_img_ptr; last->already_compressed = false; } else { /* @@ -3094,7 +3145,7 @@ no_slots: last_block && __rec_is_checkpoint(session, r, last)) { if (write_ref == dst) WT_RET(__wt_buf_set( - session, &r->disk_image, dst->mem, dst->size)); + session, r->cur_img_ptr, dst->mem, dst->size)); } else WT_RET( __rec_split_write(session, r, last, write_ref, last_block)); @@ -3128,15 +3179,120 @@ __rec_split_raw(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) } /* + * __rec_split_finish_process_prev -- + * If the two split chunks together fit in a single page, merge them into + * one. If they do not fit in a single page but the last is smaller than + * the minimum desired, move some data from the penultimate chunk to the + * last chunk and write out the previous/penultimate. Finally, update the + * pointer to the current image buffer. After this function exits, we will + * have one (last) buffer in memory, pointed to by the current image + * pointer. + */ +static int +__rec_split_finish_process_prev( + WT_SESSION_IMPL *session, WT_RECONCILE *r, bool *chunks_merged) +{ + WT_BOUNDARY *bnd_cur, *bnd_prev; + WT_BTREE *btree; + WT_PAGE_HEADER *dsk; + size_t len_to_move; + uint32_t combined_size; + uint8_t *cur_dsk_start; + + WT_ASSERT(session, r->prev_img_ptr != NULL); + + btree = S2BT(session); + bnd_cur = &r->bnd[r->bnd_next]; + bnd_prev = bnd_cur - 1; + *chunks_merged = false; + /* + * The sizes referred to in the boundary structure include the header, + * so when calculating the combined size, make sure not to include the + * header twice. + */ + combined_size = bnd_prev->size + + (bnd_cur->size - WT_PAGE_HEADER_BYTE_SIZE(btree)); + + if (combined_size <= r->page_size) { + /* + * We have two boundaries, but the data in the buffers can fit a + * single page. Merge the boundaries and create a single chunk. + */ + dsk = r->cur_img_ptr->mem; + memcpy((uint8_t *)r->prev_img_ptr->mem + bnd_prev->size, + WT_PAGE_HEADER_BYTE(btree, dsk), + bnd_cur->size - WT_PAGE_HEADER_BYTE_SIZE(btree)); + bnd_prev->size = combined_size; + bnd_prev->max_bnd_entries += bnd_cur->max_bnd_entries; + r->bnd_next--; + *chunks_merged = true; + } else { + if (bnd_cur->size < r->min_split_size && + bnd_prev->min_bnd_offset != 0 ) { + /* + * The last chunk, pointed to by the current image + * pointer, has less than the minimum data. Let's move + * any data more than the minimum from the previous + * image into the current. + */ + len_to_move = bnd_prev->size - bnd_prev->min_bnd_offset; + /* Grow current buffer if it is not large enough */ + if (r->space_avail < len_to_move) + WT_RET(__rec_split_grow(session, + r, len_to_move)); + cur_dsk_start = WT_PAGE_HEADER_BYTE(btree, + r->cur_img_ptr->mem); + + /* + * Shift the contents of the current buffer to make + * space for the data that will be prepended into the + * current buffer + */ + memmove(cur_dsk_start + len_to_move, + cur_dsk_start, bnd_cur->size - + WT_PAGE_HEADER_BYTE_SIZE(btree)); + /* + * copy any data more than the minimum, from the + * previous buffer to the start of the current. + */ + memcpy(cur_dsk_start, (uint8_t *)r->prev_img_ptr->mem + + bnd_prev->min_bnd_offset, len_to_move); + + /* Update boundary information */ + bnd_cur->size += (uint32_t)len_to_move; + bnd_prev->size -= (uint32_t)len_to_move; + bnd_cur->max_bnd_entries += bnd_prev->max_bnd_entries - + bnd_prev->min_bnd_entries; + bnd_prev->max_bnd_entries = bnd_prev->min_bnd_entries; + bnd_cur->max_bnd_recno = bnd_prev->min_bnd_recno; + WT_RET(__wt_buf_set(session, + &bnd_cur->max_bnd_key, bnd_prev->min_bnd_key.data, + bnd_prev->min_bnd_key.size)); + } + + /* Write out the previous image */ + WT_RET(__rec_split_write_prev_and_swap_buf(session, r)); + } + + /* + * At this point, there is only one disk image in the memory, pointed to + * by the previous image pointer. Update the current image pointer to + * this image. + */ + r->cur_img_ptr = r->prev_img_ptr; + return (0); +} + +/* * __rec_split_finish_std -- * Finish processing a page, standard version. */ static int __rec_split_finish_std(WT_SESSION_IMPL *session, WT_RECONCILE *r) { - WT_BOUNDARY *bnd_cur, *bnd_prev; + WT_BOUNDARY *bnd_cur; WT_PAGE_HEADER *dsk; - bool grow_bnd; + bool chunks_merged; /* * We may arrive here with no entries to write if the page was entirely @@ -3163,50 +3319,22 @@ __rec_split_finish_std(WT_SESSION_IMPL *session, WT_RECONCILE *r) return (EBUSY); } - dsk = r->disk_image.mem; - - /* Set the number of entries for the just finished chunk. */ + /* Set the number of entries and size for the just finished chunk. */ bnd_cur = &r->bnd[r->bnd_next]; bnd_cur->max_bnd_entries = r->entries; + bnd_cur->size = WT_PTRDIFF32(r->first_free, r->cur_img_ptr->mem); - grow_bnd = true; - /* - * We can reach here even with raw_compression when the last split chunk - * is too small to be sent for raw compression. - */ - if (!r->is_bulk_load && !r->raw_compression) { - if (WT_PTRDIFF(r->first_free, dsk) > r->page_size && - r->bnd_next != 0) { - /* - * We hold two boundaries worth of data in the buffer, - * and this data doesn't fit in a single page. If the - * last chunk is too small, readjust the boundary to a - * pre-computed minimum. - * Write out the penultimate chunk to the disk as a page - */ - WT_RET(__rec_split_write_prev_and_shift_cur( - session, r, true)); - } else - if (r->bnd_next != 0) { - /* - * We have two boundaries, but the data in the - * buffer can fit a single page. Merge the - * boundaries to create a single chunk. - */ - bnd_prev = bnd_cur - 1; - bnd_prev->max_bnd_entries += - bnd_cur->max_bnd_entries; - r->bnd_next--; - grow_bnd = false; - } - } + chunks_merged = false; + if (r->prev_img_ptr != NULL) + WT_RET(__rec_split_finish_process_prev(session, + r, &chunks_merged)); /* * We already have space for an extra boundary if we merged two * boundaries above, in that case we do not need to grow the boundary * structure. */ - if (grow_bnd) + if (!chunks_merged) WT_RET(__rec_split_bnd_grow(session, r)); bnd_cur = &r->bnd[r->bnd_next]; r->bnd_next++; @@ -3215,14 +3343,15 @@ __rec_split_finish_std(WT_SESSION_IMPL *session, WT_RECONCILE *r) * Current boundary now has all the remaining data/last page now. * Let's write it to the disk */ + dsk = r->cur_img_ptr->mem; dsk->recno = bnd_cur->max_bnd_recno; dsk->u.entries = bnd_cur->max_bnd_entries; - dsk->mem_size = WT_PTRDIFF32(r->first_free, dsk); - r->disk_image.size = dsk->mem_size; + dsk->mem_size = bnd_cur->size; + r->cur_img_ptr->size = dsk->mem_size; /* If this is a checkpoint, we're done, otherwise write the page. */ return (__rec_is_checkpoint(session, r, bnd_cur) ? - 0 : __rec_split_write(session, r, bnd_cur, &r->disk_image, true)); + 0 : __rec_split_write(session, r, bnd_cur, r->cur_img_ptr, true)); } /* @@ -3244,7 +3373,7 @@ __rec_split_finish(WT_SESSION_IMPL *session, WT_RECONCILE *r) if (r->raw_compression && r->entries != 0) { while (r->entries != 0) { data_size = - WT_PTRDIFF(r->first_free, r->disk_image.mem); + WT_PTRDIFF(r->first_free, r->cur_img_ptr->mem); if (data_size <= btree->allocsize) break; WT_RET(__rec_split_raw_worker(session, r, 0, true)); @@ -3523,8 +3652,7 @@ __rec_update_las(WT_SESSION_IMPL *session, WT_PAGE *page; WT_SAVE_UPD *list; WT_UPDATE *upd; - uint64_t las_counter; - int64_t insert_cnt; + uint64_t insert_cnt, las_counter; uint32_t i, session_flags, slot; uint8_t *p; @@ -3613,20 +3741,24 @@ __rec_update_las(WT_SESSION_IMPL *session, /* * Walk the list of updates, storing each key/value pair into - * the lookaside table. + * the lookaside table. Skipped reserved items, they're never + * restored, obviously. */ do { + if (upd->type == WT_UPDATE_RESERVED) + continue; + cursor->set_key(cursor, btree_id, &las_addr, ++las_counter, list->onpage_txn, key); - if (WT_UPDATE_DELETED_ISSET(upd)) + if (upd->type == WT_UPDATE_DELETED) las_value.size = 0; else { las_value.data = WT_UPDATE_DATA(upd); las_value.size = upd->size; } cursor->set_value( - cursor, upd->txnid, upd->size, &las_value); + cursor, upd->txnid, upd->type, &las_value); WT_ERR(cursor->insert(cursor)); ++insert_cnt; @@ -3635,9 +3767,11 @@ __rec_update_las(WT_SESSION_IMPL *session, err: WT_TRET(__wt_las_cursor_close(session, &cursor, session_flags)); - if (insert_cnt > 0) - (void)__wt_atomic_addi64( + if (insert_cnt > 0) { + (void)__wt_atomic_add64( &S2C(session)->las_record_cnt, insert_cnt); + __rec_verbose_lookaside_write(session); + } __wt_scr_free(session, &key); return (ret); @@ -4389,8 +4523,7 @@ __rec_col_var_helper(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_RET(__rec_split_raw(session, r, val->len)); } else if (WT_CHECK_CROSSING_BND(r, val->len)) - WT_RET(__rec_split_crossing_bnd( - session, r, val->len)); + WT_RET(__rec_split_crossing_bnd(session, r, val->len)); /* Copy the value onto the page. */ if (!deleted && !overflow_type && btree->dictionary) @@ -4553,7 +4686,7 @@ record_loop: /* update_no_copy = true; /* No data copy */ repeat_count = 1; /* Single record */ - deleted = WT_UPDATE_DELETED_ISSET(upd); + deleted = upd->type == WT_UPDATE_DELETED; if (!deleted) { data = WT_UPDATE_DATA(upd); size = upd->size; @@ -4788,7 +4921,7 @@ compare: /* } } else { deleted = upd == NULL || - WT_UPDATE_DELETED_ISSET(upd); + upd->type == WT_UPDATE_DELETED; if (!deleted) { data = WT_UPDATE_DATA(upd); size = upd->size; @@ -5333,7 +5466,7 @@ __rec_row_leaf(WT_SESSION_IMPL *session, __wt_ovfl_cache(session, page, rip, vpack)); /* If this key/value pair was deleted, we're done. */ - if (WT_UPDATE_DELETED_ISSET(upd)) { + if (upd->type == WT_UPDATE_DELETED) { /* * Overflow keys referencing discarded values * are no longer useful, discard the backing @@ -5543,7 +5676,7 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) for (; ins != NULL; ins = WT_SKIP_NEXT(ins)) { /* Look for an update. */ WT_RET(__rec_txn_read(session, r, ins, NULL, NULL, &upd)); - if (upd == NULL || WT_UPDATE_DELETED_ISSET(upd)) + if (upd == NULL || upd->type == WT_UPDATE_DELETED) continue; if (upd->size == 0) /* Build value cell. */ @@ -5833,7 +5966,7 @@ __rec_write_wrapup(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) * write the buffer so we know what to do here. */ if (bnd->addr.addr == NULL) - WT_RET(__wt_bt_write(session, &r->disk_image, + WT_RET(__wt_bt_write(session, r->cur_img_ptr, NULL, NULL, true, F_ISSET(r, WT_CHECKPOINTING), bnd->already_compressed)); else { @@ -6497,7 +6630,7 @@ __rec_dictionary_lookup( for (dp = __rec_dictionary_skip_search(r->dictionary_head, hash); dp != NULL && dp->hash == hash; dp = dp->next[0]) { WT_RET(__wt_cell_pack_data_match( - (WT_CELL *)((uint8_t *)r->disk_image.mem + dp->offset), + (WT_CELL *)((uint8_t *)r->cur_img_ptr->mem + dp->offset), &val->cell, val->buf.data, &match)); if (match) { WT_STAT_DATA_INCR(session, rec_dictionary); @@ -6530,3 +6663,51 @@ __rec_dictionary_lookup( *dpp = next; return (0); } + +/* + * __rec_verbose_lookaside_write -- + * Create a verbose message to display once per checkpoint with details + * about the cache state when performing a lookaside table write. + */ +static void +__rec_verbose_lookaside_write(WT_SESSION_IMPL *session) +{ +#ifdef HAVE_VERBOSE + WT_CONNECTION_IMPL *conn; + uint64_t ckpt_gen_current, ckpt_gen_last; + uint32_t pct_dirty, pct_full; + + if (!WT_VERBOSE_ISSET(session, WT_VERB_LOOKASIDE)) return; + + conn = S2C(session); + ckpt_gen_current = __wt_gen(session, WT_GEN_CHECKPOINT); + ckpt_gen_last = conn->las_verb_gen_write; + + /* + * This message is throttled to one per checkpoint. To do this we + * track the generation of the last checkpoint for which the message + * was printed and check against the current checkpoint generation. + */ + if (ckpt_gen_current > ckpt_gen_last) { + /* + * Attempt to atomically replace the last checkpoint generation + * for which this message was printed. If the atomic swap fails + * we have raced and the winning thread will print the message. + */ + if (__wt_atomic_casv64(&conn->las_verb_gen_write, + ckpt_gen_last, ckpt_gen_current)) { + (void)__wt_eviction_clean_needed(session, &pct_full); + (void)__wt_eviction_dirty_needed(session, &pct_dirty); + + __wt_verbose(session, WT_VERB_LOOKASIDE, + "Page reconciliation triggered lookaside write. " + "Entries now in lookaside file: %" PRIu64 ", " + "cache dirty: %" PRIu32 "%% , " + "cache use: %" PRIu32 "%%", + conn->las_record_cnt, pct_dirty, pct_full); + } + } +#else + WT_UNUSED(session); +#endif +} diff --git a/src/schema/schema_alter.c b/src/schema/schema_alter.c index 26d800aa98e..346f09a1a64 100644 --- a/src/schema/schema_alter.c +++ b/src/schema/schema_alter.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -172,7 +172,7 @@ __wt_schema_alter(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) ret = ENOENT; /* Bump the schema generation so that stale data is ignored. */ - ++S2C(session)->schema_gen; + (void)__wt_gen_next(session, WT_GEN_SCHEMA); WT_TRET(__wt_meta_track_off(session, true, ret != 0)); diff --git a/src/schema/schema_create.c b/src/schema/schema_create.c index 0677fa711a5..1ba0961cced 100644 --- a/src/schema/schema_create.c +++ b/src/schema/schema_create.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/schema/schema_drop.c b/src/schema/schema_drop.c index 49801e4e5f9..ec12ec3752f 100644 --- a/src/schema/schema_drop.c +++ b/src/schema/schema_drop.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -201,7 +201,7 @@ __wt_schema_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) ret = force ? 0 : ENOENT; /* Bump the schema generation so that stale data is ignored. */ - ++S2C(session)->schema_gen; + (void)__wt_gen_next(session, WT_GEN_SCHEMA); WT_TRET(__wt_meta_track_off(session, true, ret != 0)); diff --git a/src/schema/schema_list.c b/src/schema/schema_list.c index 74ef5135a4a..20e65d5acc9 100644 --- a/src/schema/schema_list.c +++ b/src/schema/schema_list.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -66,7 +66,8 @@ restart: * between checking the generation and opening the * first column group. */ - if (table->schema_gen != S2C(session)->schema_gen) { + if (table->schema_gen != + __wt_gen(session, WT_GEN_SCHEMA)) { if (table->refcnt == 0) { WT_RET(__wt_schema_remove_table( session, table)); @@ -243,9 +244,11 @@ int __wt_schema_close_tables(WT_SESSION_IMPL *session) { WT_DECL_RET; - WT_TABLE *table; + WT_TABLE *table, *table_tmp; - while ((table = TAILQ_FIRST(&session->tables)) != NULL) + WT_TAILQ_SAFE_REMOVE_BEGIN(table, &session->tables, q, table_tmp) { WT_TRET(__wt_schema_remove_table(session, table)); + } WT_TAILQ_SAFE_REMOVE_END + return (ret); } diff --git a/src/schema/schema_open.c b/src/schema/schema_open.c index 44bd66e011a..d765882a3b6 100644 --- a/src/schema/schema_open.c +++ b/src/schema/schema_open.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -502,7 +502,7 @@ __schema_open_table(WT_SESSION_IMPL *session, table->name); /* Copy the schema generation into the new table. */ - table->schema_gen = S2C(session)->schema_gen; + table->schema_gen = __wt_gen(session, WT_GEN_SCHEMA); *tablep = table; diff --git a/src/schema/schema_plan.c b/src/schema/schema_plan.c index 475902be887..cef8260d265 100644 --- a/src/schema/schema_plan.c +++ b/src/schema/schema_plan.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/schema/schema_project.c b/src/schema/schema_project.c index fd59539ae89..9ea8afc8580 100644 --- a/src/schema/schema_project.c +++ b/src/schema/schema_project.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/schema/schema_rename.c b/src/schema/schema_rename.c index a374f4c2831..1868d907d00 100644 --- a/src/schema/schema_rename.c +++ b/src/schema/schema_rename.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -277,7 +277,7 @@ __wt_schema_rename(WT_SESSION_IMPL *session, ret = __wt_bad_object_type(session, uri); /* Bump the schema generation so that stale data is ignored. */ - ++S2C(session)->schema_gen; + (void)__wt_gen_next(session, WT_GEN_SCHEMA); WT_TRET(__wt_meta_track_off(session, true, ret != 0)); diff --git a/src/schema/schema_stat.c b/src/schema/schema_stat.c index 345f9164e9b..d2d61febc39 100644 --- a/src/schema/schema_stat.c +++ b/src/schema/schema_stat.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/schema/schema_truncate.c b/src/schema/schema_truncate.c index 563bafa8ffc..b3a69dd5abd 100644 --- a/src/schema/schema_truncate.c +++ b/src/schema/schema_truncate.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -138,9 +138,9 @@ __wt_schema_range_truncate( uri = start->internal_uri; if (WT_PREFIX_MATCH(uri, "file:")) { - WT_CURSOR_NEEDKEY(start); + WT_ERR(__cursor_needkey(start)); if (stop != NULL) - WT_CURSOR_NEEDKEY(stop); + WT_ERR(__cursor_needkey(stop)); WT_WITH_BTREE(session, ((WT_CURSOR_BTREE *)start)->btree, ret = __wt_btcur_range_truncate( (WT_CURSOR_BTREE *)start, (WT_CURSOR_BTREE *)stop)); diff --git a/src/schema/schema_util.c b/src/schema/schema_util.c index 9de4b916a79..da58d4d7104 100644 --- a/src/schema/schema_util.c +++ b/src/schema/schema_util.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/schema/schema_worker.c b/src/schema/schema_worker.c index 62cdd7d367b..7655456b243 100644 --- a/src/schema/schema_worker.c +++ b/src/schema/schema_worker.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/session/session_api.c b/src/session/session_api.c index b7daf0e2e02..592d6835809 100644 --- a/src/session/session_api.c +++ b/src/session/session_api.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -72,11 +72,7 @@ __wt_session_copy_values(WT_SESSION_IMPL *session) (WT_PREFIX_MATCH(cursor->uri, "file:") && F_ISSET((WT_CURSOR_BTREE *)cursor, WT_CBT_NO_TXN))); #endif - - F_CLR(cursor, WT_CURSTD_VALUE_INT); - WT_RET(__wt_buf_set(session, &cursor->value, - cursor->value.data, cursor->value.size)); - F_SET(cursor, WT_CURSTD_VALUE_EXT); + WT_RET(__cursor_localvalue(cursor)); } return (0); @@ -99,6 +95,9 @@ __wt_session_release_resources(WT_SESSION_IMPL *session) if (session->reconcile_cleanup != NULL) WT_TRET(session->reconcile_cleanup(session)); + /* Stashed memory. */ + __wt_stash_discard(session); + /* * Discard scratch buffers, error memory; last, just in case a cleanup * routine uses scratch buffers. @@ -180,7 +179,7 @@ static int __session_close(WT_SESSION *wt_session, const char *config) { WT_CONNECTION_IMPL *conn; - WT_CURSOR *cursor; + WT_CURSOR *cursor, *cursor_tmp; WT_DECL_RET; WT_SESSION_IMPL *session; @@ -202,7 +201,7 @@ __session_close(WT_SESSION *wt_session, const char *config) __wt_txn_release_snapshot(session); /* Close all open cursors. */ - while ((cursor = TAILQ_FIRST(&session->cursors)) != NULL) { + WT_TAILQ_SAFE_REMOVE_BEGIN(cursor, &session->cursors, q, cursor_tmp) { /* * Notify the user that we are closing the cursor handle * via the registered close callback. @@ -212,7 +211,7 @@ __session_close(WT_SESSION *wt_session, const char *config) WT_TRET(session->event_handler->handle_close( session->event_handler, wt_session, cursor)); WT_TRET(cursor->close(cursor)); - } + } WT_TAILQ_SAFE_REMOVE_END WT_ASSERT(session, session->ncursors == 0); @@ -290,8 +289,7 @@ __session_reconfigure(WT_SESSION *wt_session, const char *config) */ WT_UNUSED(cfg); - if (F_ISSET(&session->txn, WT_TXN_RUNNING)) - WT_ERR_MSG(session, EINVAL, "transaction in progress"); + WT_ERR(__wt_txn_context_check(session, false)); WT_ERR(__wt_session_reset_cursors(session, false)); @@ -813,8 +811,7 @@ __session_reset(WT_SESSION *wt_session) SESSION_API_CALL_NOCONF(session, reset); - if (F_ISSET(&session->txn, WT_TXN_RUNNING)) - WT_ERR_MSG(session, EINVAL, "transaction in progress"); + WT_ERR(__wt_txn_context_check(session, false)); WT_TRET(__wt_session_reset_cursors(session, true)); @@ -1105,7 +1102,6 @@ int __wt_session_range_truncate(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *start, WT_CURSOR *stop) { - WT_CURSOR *cursor; WT_DECL_RET; int cmp; bool local_start; @@ -1134,12 +1130,13 @@ __wt_session_range_truncate(WT_SESSION_IMPL *session, } /* - * Cursor truncate is only supported for some objects, check for the - * supporting methods we need, range_truncate and compare. + * Cursor truncate is only supported for some objects, check for a + * supporting compare method. */ - cursor = start == NULL ? stop : start; - if (cursor->compare == NULL) - WT_ERR(__wt_bad_object_type(session, cursor->uri)); + if (start != NULL && start->compare == NULL) + WT_ERR(__wt_bad_object_type(session, start->uri)); + if (stop != NULL && stop->compare == NULL) + WT_ERR(__wt_bad_object_type(session, stop->uri)); /* * If both cursors set, check they're correctly ordered with respect to @@ -1150,6 +1147,9 @@ __wt_session_range_truncate(WT_SESSION_IMPL *session, * reference the same object and the keys are set. */ if (start != NULL && stop != NULL) { + /* quiet clang scan-build */ + WT_ASSERT(session, start->compare != NULL); + WT_ERR(start->compare(start, stop, &cmp)); if (cmp > 0) WT_ERR_MSG(session, EINVAL, @@ -1185,8 +1185,11 @@ __wt_session_range_truncate(WT_SESSION_IMPL *session, * data structures can move through pages faster forward than backward. * If we don't have a start cursor, create one and position it at the * first record. + * + * If start is NULL, stop must not be NULL, but static analyzers have + * a hard time with that, test explicitly. */ - if (start == NULL) { + if (start == NULL && stop != NULL) { WT_ERR(__session_open_cursor( (WT_SESSION *)session, stop->uri, NULL, NULL, &start)); local_start = true; @@ -1400,8 +1403,7 @@ __session_begin_transaction(WT_SESSION *wt_session, const char *config) SESSION_API_CALL(session, begin_transaction, config, cfg); WT_STAT_CONN_INCR(session, txn_begin); - if (F_ISSET(&session->txn, WT_TXN_RUNNING)) - WT_ERR_MSG(session, EINVAL, "Transaction already running"); + WT_ERR(__wt_txn_context_check(session, false)); ret = __wt_txn_begin(session, cfg); @@ -1423,6 +1425,8 @@ __session_commit_transaction(WT_SESSION *wt_session, const char *config) SESSION_API_CALL(session, commit_transaction, config, cfg); WT_STAT_CONN_INCR(session, txn_commit); + WT_ERR(__wt_txn_context_check(session, true)); + txn = &session->txn; if (F_ISSET(txn, WT_TXN_ERROR) && txn->mod_count != 0) WT_ERR_MSG(session, EINVAL, @@ -1452,6 +1456,8 @@ __session_rollback_transaction(WT_SESSION *wt_session, const char *config) SESSION_API_CALL(session, rollback_transaction, config, cfg); WT_STAT_CONN_INCR(session, txn_rollback); + WT_ERR(__wt_txn_context_check(session, true)); + WT_TRET(__wt_session_reset_cursors(session, false)); WT_TRET(__wt_txn_rollback(session, cfg)); @@ -1517,7 +1523,6 @@ __session_transaction_sync(WT_SESSION *wt_session, const char *config) WT_DECL_RET; WT_LOG *log; WT_SESSION_IMPL *session; - WT_TXN *txn; struct timespec now, start; uint64_t remaining_usec, timeout_ms, waited_ms; bool forever; @@ -1527,9 +1532,7 @@ __session_transaction_sync(WT_SESSION *wt_session, const char *config) WT_STAT_CONN_INCR(session, txn_sync); conn = S2C(session); - txn = &session->txn; - if (F_ISSET(txn, WT_TXN_RUNNING)) - WT_ERR_MSG(session, EINVAL, "transaction in progress"); + WT_ERR(__wt_txn_context_check(session, false)); /* * If logging is not enabled there is nothing to do. @@ -1620,7 +1623,6 @@ __session_checkpoint(WT_SESSION *wt_session, const char *config) { WT_DECL_RET; WT_SESSION_IMPL *session; - WT_TXN *txn; session = (WT_SESSION_IMPL *)wt_session; @@ -1645,10 +1647,7 @@ __session_checkpoint(WT_SESSION *wt_session, const char *config) * from evicting anything newer than this because we track the oldest * transaction ID in the system that is not visible to all readers. */ - txn = &session->txn; - if (F_ISSET(txn, WT_TXN_RUNNING)) - WT_ERR_MSG(session, EINVAL, - "Checkpoint not permitted in a transaction"); + WT_ERR(__wt_txn_context_check(session, false)); ret = __wt_txn_checkpoint(session, cfg, true); diff --git a/src/session/session_compact.c b/src/session/session_compact.c index 72c072e0fb8..c4710dbb1a5 100644 --- a/src/session/session_compact.c +++ b/src/session/session_compact.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -225,10 +225,13 @@ __compact_checkpoint(WT_SESSION_IMPL *session) * generation number changes, the checkpoint blocking us has completed. */ txn_global = &S2C(session)->txn_global; - for (txn_gen = txn_global->checkpoint_gen;;) { - WT_READ_BARRIER(); + for (txn_gen = __wt_gen(session, WT_GEN_CHECKPOINT);;) { + /* + * This loop only checks objects that are declared volatile, + * therefore no barriers are needed. + */ if (!txn_global->checkpoint_running || - txn_gen != txn_global->checkpoint_gen) + txn_gen != __wt_gen(session, WT_GEN_CHECKPOINT)) break; WT_RET(__wt_session_compact_check_timeout(session)); @@ -316,7 +319,6 @@ __wt_session_compact( WT_DATA_SOURCE *dsrc; WT_DECL_RET; WT_SESSION_IMPL *session; - WT_TXN *txn; u_int i; session = (WT_SESSION_IMPL *)wt_session; @@ -332,10 +334,7 @@ __wt_session_compact( * reason for LSM to allow this, possible or not), and check now so the * error message isn't confusing. */ - txn = &session->txn; - if (F_ISSET(txn, WT_TXN_RUNNING)) - WT_ERR_MSG(session, EINVAL, - "compaction not permitted in a transaction"); + WT_ERR(__wt_txn_context_check(session, false)); /* Disallow objects in the WiredTiger name space. */ WT_ERR(__wt_str_name_check(session, uri)); diff --git a/src/session/session_dhandle.c b/src/session/session_dhandle.c index 95fb6a6f90e..dd2b6ef30ff 100644 --- a/src/session/session_dhandle.c +++ b/src/session/session_dhandle.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -229,7 +229,8 @@ __wt_session_lock_dhandle( WT_ASSERT(session, !F_ISSET(dhandle, WT_DHANDLE_DEAD)); return (0); } - if (ret != EBUSY || (is_open && want_exclusive)) + if (ret != EBUSY || (is_open && want_exclusive) || + LF_ISSET(WT_DHANDLE_LOCK_ONLY)) return (ret); lock_busy = true; @@ -261,8 +262,8 @@ __wt_session_release_btree(WT_SESSION_IMPL *session) * can get a handle without special flags. */ if (F_ISSET(dhandle, WT_DHANDLE_DISCARD | WT_DHANDLE_DISCARD_FORCE)) { - __session_find_dhandle(session, - dhandle->name, dhandle->checkpoint, &dhandle_cache); + WT_SAVE_DHANDLE(session, __session_find_dhandle(session, + dhandle->name, dhandle->checkpoint, &dhandle_cache)); if (dhandle_cache != NULL) __session_discard_dhandle(session, dhandle_cache); } @@ -369,10 +370,12 @@ retry: WT_RET(__wt_meta_checkpoint_last_name( void __wt_session_close_cache(WT_SESSION_IMPL *session) { - WT_DATA_HANDLE_CACHE *dhandle_cache; + WT_DATA_HANDLE_CACHE *dhandle_cache, *dhandle_cache_tmp; - while ((dhandle_cache = TAILQ_FIRST(&session->dhandles)) != NULL) + WT_TAILQ_SAFE_REMOVE_BEGIN(dhandle_cache, + &session->dhandles, q, dhandle_cache_tmp) { __session_discard_dhandle(session, dhandle_cache); + } WT_TAILQ_SAFE_REMOVE_END } /* @@ -384,7 +387,7 @@ __session_dhandle_sweep(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle; - WT_DATA_HANDLE_CACHE *dhandle_cache, *dhandle_cache_next; + WT_DATA_HANDLE_CACHE *dhandle_cache, *dhandle_cache_tmp; time_t now; conn = S2C(session); @@ -400,9 +403,8 @@ __session_dhandle_sweep(WT_SESSION_IMPL *session) WT_STAT_CONN_INCR(session, dh_session_sweeps); - dhandle_cache = TAILQ_FIRST(&session->dhandles); - while (dhandle_cache != NULL) { - dhandle_cache_next = TAILQ_NEXT(dhandle_cache, q); + TAILQ_FOREACH_SAFE(dhandle_cache, + &session->dhandles, q, dhandle_cache_tmp) { dhandle = dhandle_cache->dhandle; if (dhandle != session->dhandle && dhandle->session_inuse == 0 && @@ -414,7 +416,6 @@ __session_dhandle_sweep(WT_SESSION_IMPL *session) WT_ASSERT(session, !WT_IS_METADATA(dhandle)); __session_discard_dhandle(session, dhandle_cache); } - dhandle_cache = dhandle_cache_next; } } diff --git a/src/session/session_salvage.c b/src/session/session_salvage.c index 12ce71cdbb0..5a67bd1f7ac 100644 --- a/src/session/session_salvage.c +++ b/src/session/session_salvage.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/support/cond_auto.c b/src/support/cond_auto.c index 600e5eab0ff..2d43eb3bf79 100644 --- a/src/support/cond_auto.c +++ b/src/support/cond_auto.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/support/crypto.c b/src/support/crypto.c index cce0d228832..6208d83b0f2 100644 --- a/src/support/crypto.c +++ b/src/support/crypto.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/support/err.c b/src/support/err.c index 57efde72b23..5ec995d8f65 100644 --- a/src/support/err.c +++ b/src/support/err.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -144,6 +144,8 @@ __wt_event_handler_set(WT_SESSION_IMPL *session, WT_EVENT_HANDLER *handler) handler->handle_message = __handle_message_default; if (handler->handle_progress == NULL) handler->handle_progress = __handle_progress_default; + if (handler->handle_close == NULL) + handler->handle_close = __handle_close_default; } session->event_handler = handler; @@ -500,8 +502,12 @@ __wt_panic(WT_SESSION_IMPL *session) #if defined(HAVE_DIAGNOSTIC) __wt_abort(session); /* Drop core if testing. */ /* NOTREACHED */ -#else +#endif +#if !defined(HAVE_DIAGNOSTIC) || defined(_WIN32) /* + * Confusing #ifdef structure because gcc knows we can't get here and + * Visual Studio doesn't. + * * Chaos reigns within. * Reflect, repent, and reboot. * Order shall return. @@ -523,12 +529,7 @@ __wt_illegal_value(WT_SESSION_IMPL *session, const char *name) name == NULL ? "" : name, name == NULL ? "" : ": ", "encountered an illegal file format or internal value"); -#if defined(HAVE_DIAGNOSTIC) - __wt_abort(session); /* Drop core if testing. */ - /* NOTREACHED */ -#else return (__wt_panic(session)); -#endif } /* diff --git a/src/support/generation.c b/src/support/generation.c new file mode 100644 index 00000000000..6e16d7e57fe --- /dev/null +++ b/src/support/generation.c @@ -0,0 +1,350 @@ +/*- + * Copyright (c) 2014-2017 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * WiredTiger uses generations to manage various resources. Threads publish an + * a current generation before accessing a resource, and clear it when they are + * done. For example, a thread wanting to replace an object in memory replaces + * the object and increments the object's generation. Once no threads have the + * previous generation published, it is safe to discard the previous version of + * the object. + */ + +/* + * __wt_gen_init -- + * Initialize the connection's generations. + */ +void +__wt_gen_init(WT_SESSION_IMPL *session) +{ + int i; + + /* + * All generations start at 1, a session with a generation of 0 isn't + * using the resource. + */ + for (i = 0; i < WT_GENERATIONS; ++i) + S2C(session)->generations[i] = 1; + + /* Ensure threads see the state change. */ + WT_WRITE_BARRIER(); +} + +/* + * __wt_gen -- + * Return the resource's generation. + */ +uint64_t +__wt_gen(WT_SESSION_IMPL *session, int which) +{ + return (S2C(session)->generations[which]); +} + +/* + * __wt_gen_next -- + * Switch the resource to its next generation. + */ +uint64_t +__wt_gen_next(WT_SESSION_IMPL *session, int which) +{ + return (__wt_atomic_addv64(&S2C(session)->generations[which], 1)); +} + +/* + * __wt_gen_next_drain -- + * Switch the resource to its next generation, then wait for it to drain. + */ +uint64_t +__wt_gen_next_drain(WT_SESSION_IMPL *session, int which) +{ + uint64_t v; + + v = __wt_atomic_addv64(&S2C(session)->generations[which], 1); + + __wt_gen_drain(session, which, v); + + return (v); +} + +/* + * __wt_gen_drain -- + * Wait for the resource to drain. + */ +void +__wt_gen_drain(WT_SESSION_IMPL *session, int which, uint64_t generation) +{ + WT_CONNECTION_IMPL *conn; + WT_SESSION_IMPL *s; + uint64_t v; + uint32_t i, session_cnt; + int pause_cnt; + + conn = S2C(session); + + /* + * No lock is required because the session array is fixed size, but it + * may contain inactive entries. We must review any active session, so + * insert a read barrier after reading the active session count. That + * way, no matter what sessions come or go, we'll check the slots for + * all of the sessions that could have been active when we started our + * check. + */ + WT_ORDERED_READ(session_cnt, conn->session_cnt); + for (pause_cnt = 0, + s = conn->sessions, i = 0; i < session_cnt; ++s, ++i) { + if (!s->active) + continue; + + for (;;) { + /* Ensure we only read the value once. */ + WT_ORDERED_READ(v, s->generations[which]); + + /* + * The generation argument is newer than the limit. Wait + * for threads in generations older than the argument + * generation, threads in argument generations are OK. + * + * The thread's generation may be 0 (that is, not set). + */ + if (v == 0 || v >= generation) + break; + + /* + * The pause count is cumulative, quit spinning if it's + * not doing us any good, that can happen in generations + * that don't move quickly. + */ + if (++pause_cnt < WT_THOUSAND) + WT_PAUSE(); + else + __wt_sleep(0, 10); + } + } +} + +/* + * __wt_gen_oldest -- + * Return the oldest generation in use for the resource. + */ +uint64_t +__wt_gen_oldest(WT_SESSION_IMPL *session, int which) +{ + WT_CONNECTION_IMPL *conn; + WT_SESSION_IMPL *s; + uint64_t oldest, v; + uint32_t i, session_cnt; + + conn = S2C(session); + + /* + * No lock is required because the session array is fixed size, but it + * may contain inactive entries. We must review any active session, so + * insert a read barrier after reading the active session count. That + * way, no matter what sessions come or go, we'll check the slots for + * all of the sessions that could have been active when we started our + * check. + */ + WT_ORDERED_READ(session_cnt, conn->session_cnt); + for (oldest = conn->generations[which] + 1, + s = conn->sessions, i = 0; i < session_cnt; ++s, ++i) { + if (!s->active) + continue; + + /* Ensure we only read the value once. */ + WT_ORDERED_READ(v, s->generations[which]); + + if (v != 0 && v < oldest) + oldest = v; + } + + return (oldest); +} + +/* + * __wt_session_gen -- + * Return the thread's resource generation. + */ +uint64_t +__wt_session_gen(WT_SESSION_IMPL *session, int which) +{ + return (session->generations[which]); +} + +/* + * __wt_session_gen_enter -- + * Publish a thread's resource generation. + */ +void +__wt_session_gen_enter(WT_SESSION_IMPL *session, int which) +{ + /* + * Assign the thread's resource generation and publish it, ensuring + * threads waiting on a resource to drain see the new value. Check we + * haven't raced with a generation update after publishing, we rely on + * the published value not being missed when scanning for the oldest + * generation. + */ + do { + session->generations[which] = __wt_gen(session, which); + WT_WRITE_BARRIER(); + } while (session->generations[which] != __wt_gen(session, which)); +} + +/* + * __wt_session_gen_leave -- + * Leave a thread's resource generation. + */ +void +__wt_session_gen_leave(WT_SESSION_IMPL *session, int which) +{ + /* Ensure writes made by this thread are visible. */ + WT_PUBLISH(session->generations[which], 0); + + /* Let threads waiting for the resource to drain proceed quickly. */ + WT_FULL_BARRIER(); +} + +/* + * __stash_discard -- + * Discard any memory from a session stash that we can. + */ +static void +__stash_discard(WT_SESSION_IMPL *session, int which) +{ + WT_CONNECTION_IMPL *conn; + WT_SESSION_STASH *session_stash; + WT_STASH *stash; + uint64_t oldest; + size_t i; + + conn = S2C(session); + session_stash = &session->stash[which]; + + /* Get the resource's oldest generation. */ + oldest = __wt_gen_oldest(session, which); + + for (i = 0, + stash = session_stash->list; i < session_stash->cnt; ++i, ++stash) { + if (stash->p == NULL) + continue; + /* + * The list is expected to be in generation-sorted order, quit + * as soon as we find a object we can't discard. + */ + if (stash->gen >= oldest) + break; + + (void)__wt_atomic_sub64(&conn->stashed_bytes, stash->len); + (void)__wt_atomic_sub64(&conn->stashed_objects, 1); + + /* + * It's a bad thing if another thread is in this memory after + * we free it, make sure nothing good happens to that thread. + */ + __wt_overwrite_and_free_len(session, stash->p, stash->len); + } + + /* + * If there are enough free slots at the beginning of the list, shuffle + * everything down. + */ + if (i > 100 || i == session_stash->cnt) + if ((session_stash->cnt -= i) > 0) + memmove(session_stash->list, stash, + session_stash->cnt * sizeof(*stash)); +} + +/* + * __wt_stash_discard -- + * Discard any memory from a session stash that we can. + */ +void +__wt_stash_discard(WT_SESSION_IMPL *session) +{ + WT_SESSION_STASH *session_stash; + int which; + + for (which = 0; which < WT_GENERATIONS; ++which) { + session_stash = &session->stash[which]; + if (session_stash->cnt >= 1) + __stash_discard(session, which); + } +} + +/* + * __wt_stash_add -- + * Add a new entry into a session stash list. + */ +int +__wt_stash_add(WT_SESSION_IMPL *session, + int which, uint64_t generation, void *p, size_t len) +{ + WT_CONNECTION_IMPL *conn; + WT_SESSION_STASH *session_stash; + WT_STASH *stash; + + conn = S2C(session); + session_stash = &session->stash[which]; + + /* Grow the list as necessary. */ + WT_RET(__wt_realloc_def(session, &session_stash->alloc, + session_stash->cnt + 1, &session_stash->list)); + + /* + * If no caller stashes memory with a lower generation than a previously + * stashed object, the list is in generation-sorted order and discarding + * can be faster. (An error won't cause problems other than we might not + * discard stashed objects as soon as we otherwise would have.) + */ + stash = session_stash->list + session_stash->cnt++; + stash->p = p; + stash->len = len; + stash->gen = generation; + + (void)__wt_atomic_add64(&conn->stashed_bytes, len); + (void)__wt_atomic_add64(&conn->stashed_objects, 1); + + /* See if we can free any previous entries. */ + if (session_stash->cnt > 1) + __stash_discard(session, which); + + return (0); +} + +/* + * __wt_stash_discard_all -- + * Discard all memory from a session's stash. + */ +void +__wt_stash_discard_all(WT_SESSION_IMPL *session_safe, WT_SESSION_IMPL *session) +{ + WT_SESSION_STASH *session_stash; + WT_STASH *stash; + int which; + size_t i; + + /* + * This function is called during WT_CONNECTION.close to discard any + * memory that remains. For that reason, we take two WT_SESSION_IMPL + * arguments: session_safe is still linked to the WT_CONNECTION and + * can be safely used for calls to other WiredTiger functions, while + * session is the WT_SESSION_IMPL we're cleaning up. + */ + for (which = 0; which < WT_GENERATIONS; ++which) { + session_stash = &session->stash[which]; + + for (i = 0, stash = session_stash->list; + i < session_stash->cnt; ++i, ++stash) + __wt_free(session_safe, stash->p); + + __wt_free(session_safe, session_stash->list); + session_stash->cnt = session_stash->alloc = 0; + } +} diff --git a/src/support/global.c b/src/support/global.c index aa69e0db9d6..6525fe21809 100644 --- a/src/support/global.c +++ b/src/support/global.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/support/hash_city.c b/src/support/hash_city.c index 8354532e820..e14368d3529 100644 --- a/src/support/hash_city.c +++ b/src/support/hash_city.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/src/support/hash_fnv.c b/src/support/hash_fnv.c index 83dd2574099..aad698229fd 100644 --- a/src/support/hash_fnv.c +++ b/src/support/hash_fnv.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/src/support/hazard.c b/src/support/hazard.c index 7e88ad183fe..6a1b7149a91 100644 --- a/src/support/hazard.c +++ b/src/support/hazard.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -22,6 +22,7 @@ hazard_grow(WT_SESSION_IMPL *session) WT_HAZARD *nhazard; size_t size; void *ohazard; + uint64_t hazard_gen; /* * Allocate a new, larger hazard pointer array and copy the contents of @@ -40,10 +41,6 @@ hazard_grow(WT_SESSION_IMPL *session) ohazard = session->hazard; WT_PUBLISH(session->hazard, nhazard); - __wt_spin_lock(session, &S2C(session)->api_lock); - __wt_conn_foc_add(session, ohazard); - __wt_spin_unlock(session, &S2C(session)->api_lock); - /* * Increase the size of the session's pointer array after swapping it * into place (the session's reference must be updated before eviction @@ -51,6 +48,15 @@ hazard_grow(WT_SESSION_IMPL *session) */ WT_PUBLISH(session->hazard_size, (uint32_t)(size * 2)); + /* + * Threads using the hazard pointer array from now on will use the new + * one. Increment the hazard pointer generation number, and schedule a + * future free of the old memory. Ignore any failure, leak the memory. + */ + hazard_gen = __wt_gen_next(session, WT_GEN_HAZARD); + WT_IGNORE_RET( + __wt_stash_add(session, WT_GEN_HAZARD, hazard_gen, ohazard, 0)); + return (0); } @@ -325,6 +331,13 @@ __wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref) WT_STAT_CONN_INCR(session, cache_hazard_checks); /* + * Hazard pointer arrays might grow and be freed underneath us; enter + * the current hazard resource generation for the duration of the walk + * to ensure that doesn't happen. + */ + __wt_session_gen_enter(session, WT_GEN_HAZARD); + + /* * No lock is required because the session array is fixed size, but it * may contain inactive entries. We must review any active session * that might contain a hazard pointer, so insert a read barrier after @@ -350,12 +363,17 @@ __wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref) if (hp->ref == ref) { WT_STAT_CONN_INCRV(session, cache_hazard_walks, walk_cnt); - return (hp); + goto done; } } } WT_STAT_CONN_INCRV(session, cache_hazard_walks, walk_cnt); - return (NULL); + hp = NULL; + +done: /* Leave the current resource generation. */ + __wt_session_gen_leave(session, WT_GEN_HAZARD); + + return (hp); } /* diff --git a/src/support/hex.c b/src/support/hex.c index b54a08dd8f3..e0b1b6de1ea 100644 --- a/src/support/hex.c +++ b/src/support/hex.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/support/huffman.c b/src/support/huffman.c index afc785b39a9..17342c53ced 100644 --- a/src/support/huffman.c +++ b/src/support/huffman.c @@ -1,5 +1,5 @@ -/* - * Copyright (c) 2014-2016 MongoDB, Inc. +/*- + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -483,7 +483,7 @@ __wt_huffman_open(WT_SESSION_IMPL *session, set_codes(node, huffman->codes, 0, 0); WT_ERR(__wt_calloc_def( - session, 1U << huffman->max_depth, &huffman->code2symbol)); + session, (size_t)1U << huffman->max_depth, &huffman->code2symbol)); make_table(session, huffman->code2symbol, huffman->max_depth, huffman->codes, huffman->numSymbols); diff --git a/src/support/mtx_rw.c b/src/support/mtx_rw.c index 35ad5da23f2..eeb9c6b72a2 100644 --- a/src/support/mtx_rw.c +++ b/src/support/mtx_rw.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -27,7 +27,7 @@ */ /* - * Based on "Spinlocks and Read-Write Locks" by Dr. Steven Fuerst: + * Inspired by "Spinlocks and Read-Write Locks" by Dr. Steven Fuerst: * http://locklessinc.com/articles/locks/ * * Dr. Fuerst further credits: @@ -39,77 +39,46 @@ * by John Mellor-Crummey and Michael Scott in their landmark paper "Scalable * Reader-Writer Synchronization for Shared-Memory Multiprocessors". * - * The following is an explanation of this code. First, the underlying lock - * structure. + * The following is an explanation of our interpretation and implementation. + * First, the underlying lock structure. * + * volatile union { + * uint64_t v; // Full 64-bit value * struct { - * uint16_t writers; Now serving for writers - * uint16_t readers; Now serving for readers - * uint16_t next; Next available ticket number - * uint16_t __notused; Padding - * } + * uint8_t current; // Current ticket + * uint8_t next; // Next available ticket + * uint8_t reader; // Read queue ticket + * uint8_t __notused; // Padding + * uint16_t readers_active; // Count of active readers + * uint16_t readers_queued; // Count of queued readers + * } s; + * } u; * * First, imagine a store's 'take a number' ticket algorithm. A customer takes * a unique ticket number and customers are served in ticket order. In the data - * structure, 'writers' is the next writer to be served, 'readers' is the next - * reader to be served, and 'next' is the next available ticket number. + * structure, 'next' is the ticket that will be allocated next, and 'current' + * is the ticket being served. * - * Next, consider exclusive (write) locks. The 'now serving' number for writers - * is 'writers'. To lock, 'take a number' and wait until that number is being - * served; more specifically, atomically copy and increment the current value of - * 'next', and then wait until 'writers' equals that copied number. + * Next, consider exclusive (write) locks. To lock, 'take a number' and wait + * until that number is being served; more specifically, atomically increment + * 'next', and then wait until 'current' equals that allocated ticket. * - * Shared (read) locks are similar. Like writers, readers atomically get the - * next number available. However, instead of waiting for 'writers' to equal - * their number, they wait for 'readers' to equal their number. + * Shared (read) locks are similar, except that readers can share a ticket + * (both with each other and with a single writer). Readers with a given + * ticket execute before the writer with that ticket. In other words, writers + * wait for both their ticket to become current and for all readers to exit + * the lock. * - * This has the effect of queuing lock requests in the order they arrive - * (incidentally avoiding starvation). + * If there are no active writers (indicated by 'current' == 'next'), readers + * can immediately enter the lock by atomically incrementing 'readers_active'. + * When there are writers active, readers form a new queue by first setting + * 'reader' to 'next' (i.e. readers are scheduled after any queued writers, + * avoiding starvation), then atomically incrementing 'readers_queued'. * - * Each lock/unlock pair requires incrementing both 'readers' and 'writers'. - * In the case of a reader, the 'readers' increment happens when the reader - * acquires the lock (to allow read-lock sharing), and the 'writers' increment - * happens when the reader releases the lock. In the case of a writer, both - * 'readers' and 'writers' are incremented when the writer releases the lock. - * - * For example, consider the following read (R) and write (W) lock requests: - * - * writers readers next - * 0 0 0 - * R: ticket 0, readers match OK 0 1 1 - * R: ticket 1, readers match OK 0 2 2 - * R: ticket 2, readers match OK 0 3 3 - * W: ticket 3, writers no match block 0 3 4 - * R: ticket 2, unlock 1 3 4 - * R: ticket 0, unlock 2 3 4 - * R: ticket 1, unlock 3 3 4 - * W: ticket 3, writers match OK 3 3 4 - * - * Note the writer blocks until 'writers' equals its ticket number and it does - * not matter if readers unlock in order or not. - * - * Readers or writers entering the system after the write lock is queued block, - * and the next ticket holder (reader or writer) will unblock when the writer - * unlocks. An example, continuing from the last line of the above example: - * - * writers readers next - * W: ticket 3, writers match OK 3 3 4 - * R: ticket 4, readers no match block 3 3 5 - * R: ticket 5, readers no match block 3 3 6 - * W: ticket 6, writers no match block 3 3 7 - * W: ticket 3, unlock 4 4 7 - * R: ticket 4, readers match OK 4 5 7 - * R: ticket 5, readers match OK 4 6 7 - * - * The 'next' field is a 2-byte value so the available ticket number wraps at - * 64K requests. If a thread's lock request is not granted until the 'next' - * field cycles and the same ticket is taken by another thread, we could grant - * a lock to two separate threads at the same time, and bad things happen: two - * writer threads or a reader thread and a writer thread would run in parallel, - * and lock waiters could be skipped if the unlocks race. This is unlikely, it - * only happens if a lock request is blocked by 64K other requests. The fix is - * to grow the lock structure fields, but the largest atomic instruction we have - * is 8 bytes, the structure has no room to grow. + * The 'next' field is a 1-byte value so the available ticket number wraps + * after 256 requests. If a thread's write lock request would cause the 'next' + * field to catch up with 'current', instead it waits to avoid the same ticket + * being allocated to multiple threads. */ #include "wt_internal.h" @@ -118,12 +87,16 @@ * __wt_rwlock_init -- * Initialize a read/write lock. */ -void +int __wt_rwlock_init(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - WT_UNUSED(session); + l->u.v = 0; + l->stat_read_count_off = l->stat_write_count_off = -1; + l->stat_app_usecs_off = l->stat_int_usecs_off = -1; - l->u = 0; + WT_RET(__wt_cond_alloc(session, "rwlock wait", &l->cond_readers)); + WT_RET(__wt_cond_alloc(session, "rwlock wait", &l->cond_writers)); + return (0); } /* @@ -133,9 +106,10 @@ __wt_rwlock_init(WT_SESSION_IMPL *session, WT_RWLOCK *l) void __wt_rwlock_destroy(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - WT_UNUSED(session); + l->u.v = 0; - l->u = 0; + __wt_cond_destroy(session, &l->cond_readers); + __wt_cond_destroy(session, &l->cond_writers); } /* @@ -146,49 +120,42 @@ int __wt_try_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { WT_RWLOCK new, old; + int64_t **stats; WT_STAT_CONN_INCR(session, rwlock_read); + if (l->stat_read_count_off != -1 && WT_STAT_ENABLED(session)) { + stats = (int64_t **)S2C(session)->stats; + stats[session->stat_bucket][l->stat_read_count_off]++; + } - new = old = *l; + old.u.v = l->u.v; - /* - * This read lock can only be granted if the lock was last granted to - * a reader and there are no readers or writers blocked on the lock, - * that is, if this thread's ticket would be the next ticket granted. - * Do the cheap test to see if this can possibly succeed (and confirm - * the lock is in the correct state to grant this read lock). - */ - if (old.s.readers != old.s.next) + /* This read lock can only be granted if there are no active writers. */ + if (old.u.s.current != old.u.s.next) return (EBUSY); /* - * The replacement lock value is a result of allocating a new ticket and - * incrementing the reader value to match it. + * The replacement lock value is a result of adding an active reader. + * Check for overflow: if the maximum number of readers are already + * active, no new readers can enter the lock. */ - new.s.readers = new.s.next = old.s.next + 1; - return (__wt_atomic_cas64(&l->u, old.u, new.u) ? 0 : EBUSY); + new.u.v = old.u.v; + if (++new.u.s.readers_active == 0) + return (EBUSY); + + /* We rely on this atomic operation to provide a barrier. */ + return (__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v) ? 0 : EBUSY); } /* - * __wt_readlock_spin -- - * Spin to get a read lock: only yield the CPU if the lock is held - * exclusive. + * __read_blocked -- + * Check whether the current read lock request should keep waiting. */ -void -__wt_readlock_spin(WT_SESSION_IMPL *session, WT_RWLOCK *l) +static bool +__read_blocked(WT_SESSION_IMPL *session) { - /* - * Try to get the lock in a single operation if it is available to - * readers. This avoids the situation where multiple readers arrive - * concurrently and have to line up in order to enter the lock. For - * read-heavy workloads it can make a significant difference. - */ - while (__wt_try_readlock(session, l) != 0) { - if (l->s.writers_active > 0) - __wt_yield(); - else - WT_PAUSE(); - } + return (session->current_rwticket != + session->current_rwlock->u.s.current); } /* @@ -198,43 +165,113 @@ __wt_readlock_spin(WT_SESSION_IMPL *session, WT_RWLOCK *l) void __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - uint16_t ticket; + WT_RWLOCK new, old; + struct timespec enter, leave; + int64_t **stats; int pause_cnt; + int16_t writers_active; + uint8_t ticket; + bool set_stats; WT_STAT_CONN_INCR(session, rwlock_read); + stats = (int64_t **)S2C(session)->stats; + set_stats = (l->stat_read_count_off != -1 && WT_STAT_ENABLED(session)); + if (set_stats) + stats[session->stat_bucket][l->stat_read_count_off]++; WT_DIAGNOSTIC_YIELD; - /* - * Possibly wrap: if we have more than 64K lockers waiting, the ticket - * value will wrap and two lockers will simultaneously be granted the - * lock. - */ - ticket = __wt_atomic_fetch_add16(&l->s.next, 1); - for (pause_cnt = 0; ticket != l->s.readers;) { + for (;;) { /* - * We failed to get the lock; pause before retrying and if we've - * paused enough, yield so we don't burn CPU to no purpose. This - * situation happens if there are more threads than cores in the - * system and we're thrashing on shared resources. + * Fast path: if there is no active writer, join the current + * group. */ - if (++pause_cnt < WT_THOUSAND) + for (old.u.v = l->u.v; + old.u.s.current == old.u.s.next; + old.u.v = l->u.v) { + new.u.v = old.u.v; + /* + * Check for overflow: if the maximum number of readers + * are already active, no new readers can enter the + * lock. + */ + if (++new.u.s.readers_active == 0) + goto stall; + if (__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v)) + return; WT_PAUSE(); - else - __wt_yield(); + } + + /* + * There is an active writer: join the next group. + * + * Limit how many readers can queue: don't allow more readers + * to queue than there are active writers (calculated as + * `next - current`): otherwise, in write-heavy workloads, + * readers can keep queuing up in front of writers and + * throughput is unstable. + * + * If the maximum number of readers are already queued, wait + * until we can get a valid ticket. + */ + writers_active = old.u.s.next - old.u.s.current; + if (old.u.s.readers_queued > writers_active) { +stall: __wt_cond_wait(session, + l->cond_readers, 10 * WT_THOUSAND, NULL); + continue; + } + + /* + * If we are the first reader to queue, set the next read + * group. Note: don't re-read from the lock or we could race + * with a writer unlocking. + */ + new.u.v = old.u.v; + if (new.u.s.readers_queued++ == 0) + new.u.s.reader = new.u.s.next; + ticket = new.u.s.reader; + + if (__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v)) + break; } - /* - * We're the only writer of the readers field, so the update does not - * need to be atomic. - */ - ++l->s.readers; + if (set_stats) + __wt_epoch(session, &enter); + /* Wait for our group to start. */ + for (pause_cnt = 0; ticket != l->u.s.current; pause_cnt++) { + if (pause_cnt < 1000) + WT_PAUSE(); + else if (pause_cnt < 1200) + __wt_yield(); + else { + session->current_rwlock = l; + session->current_rwticket = ticket; + __wt_cond_wait(session, + l->cond_readers, 10 * WT_THOUSAND, __read_blocked); + } + } + if (set_stats) { + __wt_epoch(session, &leave); + if (F_ISSET(session, WT_SESSION_INTERNAL)) + stats[session->stat_bucket][l->stat_int_usecs_off] += + (int64_t)WT_TIMEDIFF_US(leave, enter); + else + stats[session->stat_bucket][l->stat_app_usecs_off] += + (int64_t)WT_TIMEDIFF_US(leave, enter); + } /* * Applications depend on a barrier here so that operations holding the - * lock see consistent data. + * lock see consistent data. The atomic operation above isn't + * sufficient here because we don't own the lock until our ticket comes + * up and whatever data we are protecting may have changed in the + * meantime. */ WT_READ_BARRIER(); + + /* Sanity check that we (still) have the lock. */ + WT_ASSERT(session, + ticket == l->u.s.current && l->u.s.readers_active > 0); } /* @@ -244,13 +281,22 @@ __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) void __wt_readunlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - WT_UNUSED(session); + WT_RWLOCK new, old; - /* - * Increment the writers value (other readers are doing the same, make - * sure we don't race). - */ - (void)__wt_atomic_add16(&l->s.writers, 1); + do { + old.u.v = l->u.v; + WT_ASSERT(session, old.u.s.readers_active > 0); + + /* + * Decrement the active reader count (other readers are doing + * the same, make sure we don't race). + */ + new.u.v = old.u.v; + --new.u.s.readers_active; + } while (!__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v)); + + if (new.u.s.readers_active == 0 && new.u.s.current != new.u.s.next) + __wt_cond_signal(session, l->cond_writers); } /* @@ -261,25 +307,52 @@ int __wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { WT_RWLOCK new, old; + int64_t **stats; WT_STAT_CONN_INCR(session, rwlock_write); - - old = new = *l; + if (l->stat_write_count_off != -1 && WT_STAT_ENABLED(session)) { + stats = (int64_t **)S2C(session)->stats; + stats[session->stat_bucket][l->stat_write_count_off]++; + } /* - * This write lock can only be granted if the lock was last granted to - * a writer and there are no readers or writers blocked on the lock, - * that is, if this thread's ticket would be the next ticket granted. - * Do the cheap test to see if this can possibly succeed (and confirm - * the lock is in the correct state to grant this write lock). + * This write lock can only be granted if no readers or writers blocked + * on the lock, that is, if this thread's ticket would be the next + * ticket granted. Check if this can possibly succeed (and confirm the + * lock is in the correct state to grant this write lock). */ - if (old.s.writers != old.s.next) + old.u.v = l->u.v; + if (old.u.s.current != old.u.s.next || old.u.s.readers_active != 0) return (EBUSY); - /* The replacement lock value is a result of allocating a new ticket. */ - ++new.s.next; - ++new.s.writers_active; - return (__wt_atomic_cas64(&l->u, old.u, new.u) ? 0 : EBUSY); + /* + * We've checked above that there is no writer active (since + * `current == next`), so there should be no readers queued. + */ + WT_ASSERT(session, old.u.s.readers_queued == 0); + + /* + * The replacement lock value is a result of allocating a new ticket. + * + * We rely on this atomic operation to provide a barrier. + */ + new.u.v = old.u.v; + new.u.s.next++; + return (__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v) ? 0 : EBUSY); +} + +/* + * __write_blocked -- + * Check whether the current write lock request should keep waiting. + */ +static bool +__write_blocked(WT_SESSION_IMPL *session) +{ + WT_RWLOCK *l; + + l = session->current_rwlock; + return (session->current_rwticket != l->u.s.current || + l->u.s.readers_active != 0); } /* @@ -289,36 +362,86 @@ __wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) void __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - uint16_t ticket; + WT_RWLOCK new, old; + struct timespec enter, leave; + int64_t **stats; int pause_cnt; + uint8_t ticket; + bool set_stats; WT_STAT_CONN_INCR(session, rwlock_write); + stats = (int64_t **)S2C(session)->stats; + set_stats = (l->stat_write_count_off != -1 && WT_STAT_ENABLED(session)); + if (set_stats) + stats[session->stat_bucket][l->stat_write_count_off]++; + + for (;;) { + old.u.v = l->u.v; + + /* Allocate a ticket. */ + new.u.v = old.u.v; + ticket = new.u.s.next++; - /* - * Possibly wrap: if we have more than 64K lockers waiting, the ticket - * value will wrap and two lockers will simultaneously be granted the - * lock. - */ - ticket = __wt_atomic_fetch_add16(&l->s.next, 1); - (void)__wt_atomic_add16(&l->s.writers_active, 1); - for (pause_cnt = 0; ticket != l->s.writers;) { /* - * We failed to get the lock; pause before retrying and if we've - * paused enough, sleep so we don't burn CPU to no purpose. This - * situation happens if there are more threads than cores in the - * system and we're thrashing on shared resources. + * Check for overflow: if the next ticket is allowed to catch + * up with the current batch, two writers could be granted the + * lock simultaneously. */ - if (++pause_cnt < WT_THOUSAND) + if (new.u.s.current == new.u.s.next) { + __wt_cond_wait(session, + l->cond_writers, 10 * WT_THOUSAND, NULL); + continue; + } + if (__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v)) + break; + } + + /* + * Wait for our group to start and any readers to drain. + * + * We take care here to do an atomic read of the full 64-bit lock + * value. Otherwise, reads are not guaranteed to be ordered and we + * could see no readers active from a different batch and decide that + * we have the lock. + */ + if (set_stats) + __wt_epoch(session, &enter); + for (pause_cnt = 0, old.u.v = l->u.v; + ticket != old.u.s.current || old.u.s.readers_active != 0; + pause_cnt++, old.u.v = l->u.v) { + if (pause_cnt < 1000) WT_PAUSE(); + else if (pause_cnt < 1200) + __wt_yield(); + else { + session->current_rwlock = l; + session->current_rwticket = ticket; + __wt_cond_wait(session, + l->cond_writers, 10 * WT_THOUSAND, __write_blocked); + } + } + if (set_stats) { + __wt_epoch(session, &leave); + if (F_ISSET(session, WT_SESSION_INTERNAL)) + stats[session->stat_bucket][l->stat_int_usecs_off] += + (int64_t)WT_TIMEDIFF_US(leave, enter); else - __wt_sleep(0, 10); + stats[session->stat_bucket][l->stat_app_usecs_off] += + (int64_t)WT_TIMEDIFF_US(leave, enter); } /* * Applications depend on a barrier here so that operations holding the - * lock see consistent data. + * lock see consistent data. The atomic operation above isn't + * sufficient here because we don't own the lock until our ticket comes + * up and whatever data we are protecting may have changed in the + * meantime. */ WT_READ_BARRIER(); + + /* Sanity check that we (still) have the lock. */ + WT_ASSERT(session, + ticket == l->u.s.current && l->u.s.readers_active == 0); } /* @@ -328,29 +451,35 @@ __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) void __wt_writeunlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - WT_RWLOCK new; - - WT_UNUSED(session); - - (void)__wt_atomic_sub16(&l->s.writers_active, 1); + WT_RWLOCK new, old; - /* - * Ensure that all updates made while the lock was held are visible to - * the next thread to acquire the lock. - */ - WT_WRITE_BARRIER(); + do { + old.u.v = l->u.v; - new = *l; + /* + * We're holding the lock exclusive, there shouldn't be any + * active readers. + */ + WT_ASSERT(session, old.u.s.readers_active == 0); - /* - * We're the only writer of the writers/readers fields, so the update - * does not need to be atomic; we have to update both values at the - * same time though, otherwise we'd potentially race with the thread - * next granted the lock. - */ - ++new.s.writers; - ++new.s.readers; - l->i.wr = new.i.wr; + /* + * Allow the next batch to start. + * + * If there are readers in the next group, swap queued readers + * to active: this could race with new readlock requests, so we + * have to spin. + */ + new.u.v = old.u.v; + if (++new.u.s.current == new.u.s.reader) { + new.u.s.readers_active = new.u.s.readers_queued; + new.u.s.readers_queued = 0; + } + } while (!__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v)); + + if (new.u.s.readers_active != 0) + __wt_cond_signal(session, l->cond_readers); + else if (new.u.s.current != new.u.s.next) + __wt_cond_signal(session, l->cond_writers); WT_DIAGNOSTIC_YIELD; } @@ -363,8 +492,11 @@ __wt_writeunlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) bool __wt_rwlock_islocked(WT_SESSION_IMPL *session, WT_RWLOCK *l) { + WT_RWLOCK old; + WT_UNUSED(session); - return (l->s.writers != l->s.next || l->s.readers != l->s.next); + old.u.v = l->u.v; + return (old.u.s.current != old.u.s.next || old.u.s.readers_active != 0); } #endif diff --git a/src/support/pow.c b/src/support/pow.c index 028263581d3..cd770a514b2 100644 --- a/src/support/pow.c +++ b/src/support/pow.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/src/support/rand.c b/src/support/rand.c index 4fae43edc8e..8083b8801c1 100644 --- a/src/support/rand.c +++ b/src/support/rand.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/src/support/scratch.c b/src/support/scratch.c index 485cea90e89..c0e4cfe6ab7 100644 --- a/src/support/scratch.c +++ b/src/support/scratch.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/support/stat.c b/src/support/stat.c index 2c2217f8c20..2dc006da827 100644 --- a/src/support/stat.c +++ b/src/support/stat.c @@ -97,9 +97,11 @@ static const char * const __stats_dsrc_desc[] = { "cursor: cursor-remove key bytes removed", "cursor: cursor-update value bytes updated", "cursor: insert calls", + "cursor: modify calls", "cursor: next calls", "cursor: prev calls", "cursor: remove calls", + "cursor: reserve calls", "cursor: reset calls", "cursor: restarted searches", "cursor: search calls", @@ -259,9 +261,11 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->cursor_remove_bytes = 0; stats->cursor_update_bytes = 0; stats->cursor_insert = 0; + stats->cursor_modify = 0; stats->cursor_next = 0; stats->cursor_prev = 0; stats->cursor_remove = 0; + stats->cursor_reserve = 0; stats->cursor_reset = 0; stats->cursor_restart = 0; stats->cursor_search = 0; @@ -410,9 +414,11 @@ __wt_stat_dsrc_aggregate_single( to->cursor_remove_bytes += from->cursor_remove_bytes; to->cursor_update_bytes += from->cursor_update_bytes; to->cursor_insert += from->cursor_insert; + to->cursor_modify += from->cursor_modify; to->cursor_next += from->cursor_next; to->cursor_prev += from->cursor_prev; to->cursor_remove += from->cursor_remove; + to->cursor_reserve += from->cursor_reserve; to->cursor_reset += from->cursor_reset; to->cursor_restart += from->cursor_restart; to->cursor_search += from->cursor_search; @@ -588,9 +594,11 @@ __wt_stat_dsrc_aggregate( to->cursor_remove_bytes += WT_STAT_READ(from, cursor_remove_bytes); to->cursor_update_bytes += WT_STAT_READ(from, cursor_update_bytes); to->cursor_insert += WT_STAT_READ(from, cursor_insert); + to->cursor_modify += WT_STAT_READ(from, cursor_modify); to->cursor_next += WT_STAT_READ(from, cursor_next); to->cursor_prev += WT_STAT_READ(from, cursor_prev); to->cursor_remove += WT_STAT_READ(from, cursor_remove); + to->cursor_reserve += WT_STAT_READ(from, cursor_reserve); to->cursor_reset += WT_STAT_READ(from, cursor_reset); to->cursor_restart += WT_STAT_READ(from, cursor_restart); to->cursor_search += WT_STAT_READ(from, cursor_search); @@ -682,7 +690,8 @@ static const char * const __stats_connection_desc[] = { "cache: eviction worker thread evicting pages", "cache: eviction worker thread removed", "cache: eviction worker thread stable number", - "cache: failed eviction of pages that exceeded the in-memory maximum", + "cache: failed eviction of pages that exceeded the in-memory maximum count", + "cache: failed eviction of pages that exceeded the in-memory maximum time (usecs)", "cache: files with active eviction walks", "cache: files with new eviction walks started", "cache: force re-tuning of eviction workers once in a while", @@ -706,8 +715,10 @@ static const char * const __stats_connection_desc[] = { "cache: page split during eviction deepened the tree", "cache: page written requiring lookaside records", "cache: pages currently held in the cache", - "cache: pages evicted because they exceeded the in-memory maximum", - "cache: pages evicted because they had chains of deleted items", + "cache: pages evicted because they exceeded the in-memory maximum count", + "cache: pages evicted because they exceeded the in-memory maximum time (usecs)", + "cache: pages evicted because they had chains of deleted items count", + "cache: pages evicted because they had chains of deleted items time (usecs)", "cache: pages evicted by application threads", "cache: pages queued for eviction", "cache: pages queued for urgent eviction", @@ -728,6 +739,7 @@ static const char * const __stats_connection_desc[] = { "cache: unmodified pages evicted", "connection: auto adjusting condition resets", "connection: auto adjusting condition wait calls", + "connection: detected system time went backwards", "connection: files currently open", "connection: memory allocations", "connection: memory frees", @@ -740,9 +752,11 @@ static const char * const __stats_connection_desc[] = { "connection: total write I/Os", "cursor: cursor create calls", "cursor: cursor insert calls", + "cursor: cursor modify calls", "cursor: cursor next calls", "cursor: cursor prev calls", "cursor: cursor remove calls", + "cursor: cursor reserve calls", "cursor: cursor reset calls", "cursor: cursor restarted searches", "cursor: cursor search calls", @@ -760,24 +774,21 @@ static const char * const __stats_connection_desc[] = { "lock: checkpoint lock acquisitions", "lock: checkpoint lock application thread wait time (usecs)", "lock: checkpoint lock internal thread wait time (usecs)", - "lock: handle-list lock eviction thread wait time (usecs)", + "lock: dhandle lock application thread time waiting for the dhandle lock (usecs)", + "lock: dhandle lock internal thread time waiting for the dhandle lock (usecs)", + "lock: dhandle read lock acquisitions", + "lock: dhandle write lock acquisitions", "lock: metadata lock acquisitions", "lock: metadata lock application thread wait time (usecs)", "lock: metadata lock internal thread wait time (usecs)", "lock: schema lock acquisitions", "lock: schema lock application thread wait time (usecs)", "lock: schema lock internal thread wait time (usecs)", - "lock: table lock acquisitions", "lock: table lock application thread time waiting for the table lock (usecs)", "lock: table lock internal thread time waiting for the table lock (usecs)", + "lock: table read lock acquisitions", + "lock: table write lock acquisitions", "log: busy returns attempting to switch slots", - "log: consolidated slot closures", - "log: consolidated slot join active slot closed", - "log: consolidated slot join races", - "log: consolidated slot join transitions", - "log: consolidated slot joins", - "log: consolidated slot transitions unable to find free slot", - "log: consolidated slot unbuffered writes", "log: log bytes of payload data", "log: log bytes written", "log: log files manually zero-filled", @@ -804,6 +815,19 @@ static const char * const __stats_connection_desc[] = { "log: pre-allocated log files prepared", "log: pre-allocated log files used", "log: records processed by log scan", + "log: slot close lost race", + "log: slot close unbuffered waits", + "log: slot closures", + "log: slot join atomic update races", + "log: slot join calls atomic updates raced", + "log: slot join calls did not yield", + "log: slot join calls found active slot closed", + "log: slot join calls slept", + "log: slot join calls yielded", + "log: slot join found active slot closed", + "log: slot joins yield time (usecs)", + "log: slot transitions unable to find free slot", + "log: slot unbuffered writes", "log: total in-memory size of compressed records", "log: total log buffer size", "log: total size of compressed records", @@ -868,6 +892,7 @@ static const char * const __stats_connection_desc[] = { "transaction: transaction sync calls", "transaction: transactions committed", "transaction: transactions rolled back", + "transaction: update conflicts", }; int @@ -969,6 +994,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_eviction_worker_removed = 0; /* not clearing cache_eviction_stable_state_workers */ stats->cache_eviction_force_fail = 0; + stats->cache_eviction_force_fail_time = 0; /* not clearing cache_eviction_walks_active */ stats->cache_eviction_walks_started = 0; stats->cache_eviction_force_retune = 0; @@ -993,7 +1019,9 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_write_lookaside = 0; /* not clearing cache_pages_inuse */ stats->cache_eviction_force = 0; + stats->cache_eviction_force_time = 0; stats->cache_eviction_force_delete = 0; + stats->cache_eviction_force_delete_time = 0; stats->cache_eviction_app = 0; stats->cache_eviction_pages_queued = 0; stats->cache_eviction_pages_queued_urgent = 0; @@ -1014,6 +1042,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_eviction_clean = 0; stats->cond_auto_wait_reset = 0; stats->cond_auto_wait = 0; + stats->time_travel = 0; /* not clearing file_open */ stats->memory_allocation = 0; stats->memory_free = 0; @@ -1026,9 +1055,11 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->write_io = 0; stats->cursor_create = 0; stats->cursor_insert = 0; + stats->cursor_modify = 0; stats->cursor_next = 0; stats->cursor_prev = 0; stats->cursor_remove = 0; + stats->cursor_reserve = 0; stats->cursor_reset = 0; stats->cursor_restart = 0; stats->cursor_search = 0; @@ -1046,24 +1077,21 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->lock_checkpoint_count = 0; stats->lock_checkpoint_wait_application = 0; stats->lock_checkpoint_wait_internal = 0; - stats->lock_handle_list_wait_eviction = 0; + stats->lock_dhandle_wait_application = 0; + stats->lock_dhandle_wait_internal = 0; + stats->lock_dhandle_read_count = 0; + stats->lock_dhandle_write_count = 0; stats->lock_metadata_count = 0; stats->lock_metadata_wait_application = 0; stats->lock_metadata_wait_internal = 0; stats->lock_schema_count = 0; stats->lock_schema_wait_application = 0; stats->lock_schema_wait_internal = 0; - stats->lock_table_count = 0; stats->lock_table_wait_application = 0; stats->lock_table_wait_internal = 0; + stats->lock_table_read_count = 0; + stats->lock_table_write_count = 0; stats->log_slot_switch_busy = 0; - stats->log_slot_closes = 0; - stats->log_slot_active_closed = 0; - stats->log_slot_races = 0; - stats->log_slot_transitions = 0; - stats->log_slot_joins = 0; - stats->log_slot_no_free_slots = 0; - stats->log_slot_unbuffered = 0; stats->log_bytes_payload = 0; stats->log_bytes_written = 0; stats->log_zero_fills = 0; @@ -1090,6 +1118,19 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->log_prealloc_files = 0; stats->log_prealloc_used = 0; stats->log_scan_records = 0; + stats->log_slot_close_race = 0; + stats->log_slot_close_unbuf = 0; + stats->log_slot_closes = 0; + stats->log_slot_races = 0; + stats->log_slot_yield_race = 0; + stats->log_slot_immediate = 0; + stats->log_slot_yield_close = 0; + stats->log_slot_yield_sleep = 0; + stats->log_slot_yield = 0; + stats->log_slot_active_closed = 0; + /* not clearing log_slot_yield_duration */ + stats->log_slot_no_free_slots = 0; + stats->log_slot_unbuffered = 0; stats->log_compress_mem = 0; /* not clearing log_buffer_size */ stats->log_compress_len = 0; @@ -1154,6 +1195,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->txn_sync = 0; stats->txn_commit = 0; stats->txn_rollback = 0; + stats->txn_update_conflict = 0; } void @@ -1254,6 +1296,8 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, cache_eviction_stable_state_workers); to->cache_eviction_force_fail += WT_STAT_READ(from, cache_eviction_force_fail); + to->cache_eviction_force_fail_time += + WT_STAT_READ(from, cache_eviction_force_fail_time); to->cache_eviction_walks_active += WT_STAT_READ(from, cache_eviction_walks_active); to->cache_eviction_walks_started += @@ -1293,8 +1337,12 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, cache_write_lookaside); to->cache_pages_inuse += WT_STAT_READ(from, cache_pages_inuse); to->cache_eviction_force += WT_STAT_READ(from, cache_eviction_force); + to->cache_eviction_force_time += + WT_STAT_READ(from, cache_eviction_force_time); to->cache_eviction_force_delete += WT_STAT_READ(from, cache_eviction_force_delete); + to->cache_eviction_force_delete_time += + WT_STAT_READ(from, cache_eviction_force_delete_time); to->cache_eviction_app += WT_STAT_READ(from, cache_eviction_app); to->cache_eviction_pages_queued += WT_STAT_READ(from, cache_eviction_pages_queued); @@ -1320,6 +1368,7 @@ __wt_stat_connection_aggregate( to->cache_eviction_clean += WT_STAT_READ(from, cache_eviction_clean); to->cond_auto_wait_reset += WT_STAT_READ(from, cond_auto_wait_reset); to->cond_auto_wait += WT_STAT_READ(from, cond_auto_wait); + to->time_travel += WT_STAT_READ(from, time_travel); to->file_open += WT_STAT_READ(from, file_open); to->memory_allocation += WT_STAT_READ(from, memory_allocation); to->memory_free += WT_STAT_READ(from, memory_free); @@ -1332,9 +1381,11 @@ __wt_stat_connection_aggregate( to->write_io += WT_STAT_READ(from, write_io); to->cursor_create += WT_STAT_READ(from, cursor_create); to->cursor_insert += WT_STAT_READ(from, cursor_insert); + to->cursor_modify += WT_STAT_READ(from, cursor_modify); to->cursor_next += WT_STAT_READ(from, cursor_next); to->cursor_prev += WT_STAT_READ(from, cursor_prev); to->cursor_remove += WT_STAT_READ(from, cursor_remove); + to->cursor_reserve += WT_STAT_READ(from, cursor_reserve); to->cursor_reset += WT_STAT_READ(from, cursor_reset); to->cursor_restart += WT_STAT_READ(from, cursor_restart); to->cursor_search += WT_STAT_READ(from, cursor_search); @@ -1355,8 +1406,14 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, lock_checkpoint_wait_application); to->lock_checkpoint_wait_internal += WT_STAT_READ(from, lock_checkpoint_wait_internal); - to->lock_handle_list_wait_eviction += - WT_STAT_READ(from, lock_handle_list_wait_eviction); + to->lock_dhandle_wait_application += + WT_STAT_READ(from, lock_dhandle_wait_application); + to->lock_dhandle_wait_internal += + WT_STAT_READ(from, lock_dhandle_wait_internal); + to->lock_dhandle_read_count += + WT_STAT_READ(from, lock_dhandle_read_count); + to->lock_dhandle_write_count += + WT_STAT_READ(from, lock_dhandle_write_count); to->lock_metadata_count += WT_STAT_READ(from, lock_metadata_count); to->lock_metadata_wait_application += WT_STAT_READ(from, lock_metadata_wait_application); @@ -1367,21 +1424,15 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, lock_schema_wait_application); to->lock_schema_wait_internal += WT_STAT_READ(from, lock_schema_wait_internal); - to->lock_table_count += WT_STAT_READ(from, lock_table_count); to->lock_table_wait_application += WT_STAT_READ(from, lock_table_wait_application); to->lock_table_wait_internal += WT_STAT_READ(from, lock_table_wait_internal); + to->lock_table_read_count += + WT_STAT_READ(from, lock_table_read_count); + to->lock_table_write_count += + WT_STAT_READ(from, lock_table_write_count); to->log_slot_switch_busy += WT_STAT_READ(from, log_slot_switch_busy); - to->log_slot_closes += WT_STAT_READ(from, log_slot_closes); - to->log_slot_active_closed += - WT_STAT_READ(from, log_slot_active_closed); - to->log_slot_races += WT_STAT_READ(from, log_slot_races); - to->log_slot_transitions += WT_STAT_READ(from, log_slot_transitions); - to->log_slot_joins += WT_STAT_READ(from, log_slot_joins); - to->log_slot_no_free_slots += - WT_STAT_READ(from, log_slot_no_free_slots); - to->log_slot_unbuffered += WT_STAT_READ(from, log_slot_unbuffered); to->log_bytes_payload += WT_STAT_READ(from, log_bytes_payload); to->log_bytes_written += WT_STAT_READ(from, log_bytes_written); to->log_zero_fills += WT_STAT_READ(from, log_zero_fills); @@ -1412,6 +1463,22 @@ __wt_stat_connection_aggregate( to->log_prealloc_files += WT_STAT_READ(from, log_prealloc_files); to->log_prealloc_used += WT_STAT_READ(from, log_prealloc_used); to->log_scan_records += WT_STAT_READ(from, log_scan_records); + to->log_slot_close_race += WT_STAT_READ(from, log_slot_close_race); + to->log_slot_close_unbuf += WT_STAT_READ(from, log_slot_close_unbuf); + to->log_slot_closes += WT_STAT_READ(from, log_slot_closes); + to->log_slot_races += WT_STAT_READ(from, log_slot_races); + to->log_slot_yield_race += WT_STAT_READ(from, log_slot_yield_race); + to->log_slot_immediate += WT_STAT_READ(from, log_slot_immediate); + to->log_slot_yield_close += WT_STAT_READ(from, log_slot_yield_close); + to->log_slot_yield_sleep += WT_STAT_READ(from, log_slot_yield_sleep); + to->log_slot_yield += WT_STAT_READ(from, log_slot_yield); + to->log_slot_active_closed += + WT_STAT_READ(from, log_slot_active_closed); + to->log_slot_yield_duration += + WT_STAT_READ(from, log_slot_yield_duration); + to->log_slot_no_free_slots += + WT_STAT_READ(from, log_slot_no_free_slots); + to->log_slot_unbuffered += WT_STAT_READ(from, log_slot_unbuffered); to->log_compress_mem += WT_STAT_READ(from, log_compress_mem); to->log_buffer_size += WT_STAT_READ(from, log_buffer_size); to->log_compress_len += WT_STAT_READ(from, log_compress_len); @@ -1515,6 +1582,7 @@ __wt_stat_connection_aggregate( to->txn_sync += WT_STAT_READ(from, txn_sync); to->txn_commit += WT_STAT_READ(from, txn_commit); to->txn_rollback += WT_STAT_READ(from, txn_rollback); + to->txn_update_conflict += WT_STAT_READ(from, txn_update_conflict); } static const char * const __stats_join_desc[] = { diff --git a/src/support/thread_group.c b/src/support/thread_group.c index 2b4b7ad4e61..59caaedf5cf 100644 --- a/src/support/thread_group.c +++ b/src/support/thread_group.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -9,11 +9,11 @@ #include "wt_internal.h" /* - * __wt_thread_run -- + * __thread_run -- * General wrapper for any thread. */ -WT_THREAD_RET -__wt_thread_run(void *arg) +static WT_THREAD_RET +__thread_run(void *arg) { WT_DECL_RET; WT_SESSION_IMPL *session; @@ -22,7 +22,20 @@ __wt_thread_run(void *arg) thread = (WT_THREAD*)arg; session = thread->session; - ret = thread->run_func(session, thread); + for (;;) { + if (!F_ISSET(thread, WT_THREAD_RUN)) + break; + if (!F_ISSET(thread, WT_THREAD_ACTIVE)) + __wt_cond_wait(session, thread->pause_cond, + WT_THREAD_PAUSE * WT_MILLION, thread->chk_func); + WT_ERR(thread->run_func(session, thread)); + } + + /* + * If a thread is stopping it may have subsystem cleanup to do. + */ +err: if (thread->stop_func != NULL) + ret = thread->stop_func(session, thread); if (ret != 0 && F_ISSET(thread, WT_THREAD_PANIC_FAIL)) WT_PANIC_MSG(session, ret, @@ -41,42 +54,13 @@ __wt_thread_run(void *arg) } /* - * __thread_group_grow -- - * Increase the number of running threads in the group. - */ -static int -__thread_group_grow( - WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, uint32_t new_count) -{ - WT_THREAD *thread; - - WT_ASSERT(session, __wt_rwlock_islocked(session, &group->lock)); - - /* - * Any bounds checking is done by the caller so we know that - * there is space in the array for new threads. - */ - while (group->current_threads < new_count) { - thread = group->threads[group->current_threads++]; - __wt_verbose(session, WT_VERB_THREAD_GROUP, - "Starting utility thread: %p:%" PRIu32, - (void *)group, thread->id); - F_SET(thread, WT_THREAD_RUN); - WT_ASSERT(session, thread->session != NULL); - WT_RET(__wt_thread_create(thread->session, - &thread->tid, __wt_thread_run, thread)); - } - return (0); -} - -/* * __thread_group_shrink -- - * Decrease the number of running threads in the group. Optionally free any - * memory associated with slots larger than the new count. + * Decrease the number of threads in the group and free memory + * associated with slots larger than the new count. */ static int -__thread_group_shrink(WT_SESSION_IMPL *session, - WT_THREAD_GROUP *group, uint32_t new_count, bool free_thread) +__thread_group_shrink( + WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, uint32_t new_count) { WT_DECL_RET; WT_SESSION *wt_session; @@ -95,29 +79,47 @@ __thread_group_shrink(WT_SESSION_IMPL *session, if (thread == NULL) continue; - /* Wake threads to ensure they notice the state change */ - if (thread->tid != 0) { - __wt_verbose(session, WT_VERB_THREAD_GROUP, - "Stopping utility thread: %p:%" PRIu32, - (void *)group, thread->id); - F_CLR(thread, WT_THREAD_RUN); - __wt_cond_signal(session, group->wait_cond); - WT_TRET(__wt_thread_join(session, thread->tid)); - thread->tid = 0; - } - if (free_thread) { - if (thread->session != NULL) { - wt_session = (WT_SESSION *)thread->session; - WT_TRET(wt_session->close(wt_session, NULL)); - thread->session = NULL; - } - __wt_free(session, thread); - group->threads[current_slot] = NULL; - } + WT_ASSERT(session, thread->tid.created); + __wt_verbose(session, WT_VERB_THREAD_GROUP, + "Stopping utility thread: %p:%" PRIu32, + (void *)group, thread->id); + if (F_ISSET(thread, WT_THREAD_ACTIVE)) + --group->current_threads; + F_CLR(thread, WT_THREAD_ACTIVE | WT_THREAD_RUN); + /* + * Signal the thread in case it is in a long timeout. + */ + __wt_cond_signal(session, thread->pause_cond); + __wt_cond_signal(session, group->wait_cond); + } + + /* + * We have to perform the join without holding the lock because + * the threads themselves may be waiting on the lock. + */ + __wt_writeunlock(session, &group->lock); + for (current_slot = group->alloc; current_slot > new_count; ) { + thread = group->threads[--current_slot]; + + if (thread == NULL) + continue; + WT_TRET(__wt_thread_join(session, thread->tid)); + __wt_cond_destroy(session, &thread->pause_cond); + } + __wt_writelock(session, &group->lock); + for (current_slot = group->alloc; current_slot > new_count; ) { + thread = group->threads[--current_slot]; + + if (thread == NULL) + continue; + WT_ASSERT(session, thread->session != NULL); + wt_session = (WT_SESSION *)thread->session; + WT_TRET(wt_session->close(wt_session, NULL)); + thread->session = NULL; + __wt_free(session, thread); + group->threads[current_slot] = NULL; } - /* Update the thread group state to match our changes */ - group->current_threads = current_slot; return (ret); } @@ -132,13 +134,20 @@ __thread_group_resize( { WT_CONNECTION_IMPL *conn; WT_DECL_RET; + WT_SESSION *wt_session; WT_THREAD *thread; size_t alloc; uint32_t i, session_flags; conn = S2C(session); + thread = NULL; session_flags = 0; + __wt_verbose(session, WT_VERB_THREAD_GROUP, + "Resize thread group: %p, from min: %" PRIu32 " -> %" PRIu32 + " from max: %" PRIu32 " -> %" PRIu32, + (void *)group, group->min, new_min, group->max, new_max); + WT_ASSERT(session, group->current_threads <= group->alloc && __wt_rwlock_islocked(session, &group->lock)); @@ -153,7 +162,7 @@ __thread_group_resize( * Call shrink to reduce the number of thread structures and running * threads if required by the change in group size. */ - WT_RET(__thread_group_shrink(session, group, new_max, true)); + WT_RET(__thread_group_shrink(session, group, new_max)); /* * Only reallocate the thread array if it is the largest ever, since @@ -187,30 +196,57 @@ __thread_group_resize( if (LF_ISSET(WT_THREAD_PANIC_FAIL)) F_SET(thread, WT_THREAD_PANIC_FAIL); thread->id = i; + thread->chk_func = group->chk_func; thread->run_func = group->run_func; + thread->stop_func = group->stop_func; + WT_ERR(__wt_cond_alloc( + session, "Thread cond", &thread->pause_cond)); + + /* + * Start thread as inactive. We'll activate the needed + * number later. + */ + __wt_verbose(session, WT_VERB_THREAD_GROUP, + "Starting utility thread: %p:%" PRIu32, + (void *)group, thread->id); + F_SET(thread, WT_THREAD_RUN); + WT_ERR(__wt_thread_create(thread->session, + &thread->tid, __thread_run, thread)); + WT_ASSERT(session, group->threads[i] == NULL); group->threads[i] = thread; + thread = NULL; } - if (group->current_threads < new_min) - WT_ERR(__thread_group_grow(session, group, new_min)); + group->max = new_max; + group->min = new_min; + while (group->current_threads < new_min) + __wt_thread_group_start_one(session, group, true); + return (0); err: /* + * An error resizing a thread array is currently fatal, it should only + * happen in an out of memory situation. Do real cleanup just in case + * that changes in the future. + */ + if (thread != NULL) { + if (thread->session != NULL) { + wt_session = (WT_SESSION *)thread->session; + WT_TRET(wt_session->close(wt_session, NULL)); + } + __wt_cond_destroy(session, &thread->pause_cond); + __wt_free(session, thread); + } + + /* * Update the thread group information even on failure to improve our * chances of cleaning up properly. */ group->max = new_max; group->min = new_min; + WT_TRET(__wt_thread_group_destroy(session, group)); - /* - * An error resizing a thread array is fatal, it should only happen - * in an out of memory situation. - */ - if (ret != 0) { - WT_TRET(__wt_thread_group_destroy(session, group)); - WT_PANIC_RET(session, ret, "Error while resizing thread group"); - } - return (ret); + WT_PANIC_RET(session, ret, "Error while resizing thread group"); } /* @@ -224,11 +260,6 @@ __wt_thread_group_resize( { WT_DECL_RET; - __wt_verbose(session, WT_VERB_THREAD_GROUP, - "Resize thread group: %p, from min: %" PRIu32 " -> %" PRIu32 - " from max: %" PRIu32 " -> %" PRIu32, - (void *)group, group->min, new_min, group->max, new_max); - __wt_writelock(session, &group->lock); WT_TRET(__thread_group_resize(session, group, new_min, new_max, flags)); __wt_writeunlock(session, &group->lock); @@ -244,7 +275,9 @@ int __wt_thread_group_create( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, const char *name, uint32_t min, uint32_t max, uint32_t flags, - int (*run_func)(WT_SESSION_IMPL *session, WT_THREAD *context)) + bool (*chk_func)(WT_SESSION_IMPL *session), + int (*run_func)(WT_SESSION_IMPL *session, WT_THREAD *context), + int (*stop_func)(WT_SESSION_IMPL *session, WT_THREAD *context)) { WT_DECL_RET; bool cond_alloced; @@ -257,13 +290,15 @@ __wt_thread_group_create( __wt_verbose(session, WT_VERB_THREAD_GROUP, "Creating thread group: %p", (void *)group); - __wt_rwlock_init(session, &group->lock); + WT_RET(__wt_rwlock_init(session, &group->lock)); WT_ERR(__wt_cond_alloc( session, "thread group cond", &group->wait_cond)); cond_alloced = true; __wt_writelock(session, &group->lock); + group->chk_func = chk_func; group->run_func = run_func; + group->stop_func = stop_func; group->name = name; WT_TRET(__thread_group_resize(session, group, min, max, flags)); @@ -272,7 +307,7 @@ __wt_thread_group_create( /* Cleanup on error to avoid leaking resources */ err: if (ret != 0) { if (cond_alloced) - WT_TRET(__wt_cond_destroy(session, &group->wait_cond)); + __wt_cond_destroy(session, &group->wait_cond); __wt_rwlock_destroy(session, &group->lock); } return (ret); @@ -293,11 +328,11 @@ __wt_thread_group_destroy(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group) WT_ASSERT(session, __wt_rwlock_islocked(session, &group->lock)); /* Shut down all threads and free associated resources. */ - WT_TRET(__thread_group_shrink(session, group, 0, true)); + WT_TRET(__thread_group_shrink(session, group, 0)); __wt_free(session, group->threads); - WT_TRET(__wt_cond_destroy(session, &group->wait_cond)); + __wt_cond_destroy(session, &group->wait_cond); __wt_rwlock_destroy(session, &group->lock); /* @@ -314,52 +349,55 @@ __wt_thread_group_destroy(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group) * __wt_thread_group_start_one -- * Start a new thread if possible. */ -int +void __wt_thread_group_start_one( - WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, bool wait) + WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, bool is_locked) { - WT_DECL_RET; + WT_THREAD *thread; if (group->current_threads >= group->max) - return (0); + return; - if (wait) + if (!is_locked) __wt_writelock(session, &group->lock); - else - WT_RET(__wt_try_writelock(session, &group->lock)); /* Recheck the bounds now that we hold the lock */ - if (group->current_threads < group->max) - WT_TRET(__thread_group_grow( - session, group, group->current_threads + 1)); - __wt_writeunlock(session, &group->lock); - - return (ret); + if (group->current_threads < group->max) { + thread = group->threads[group->current_threads++]; + WT_ASSERT(session, thread != NULL); + __wt_verbose(session, WT_VERB_THREAD_GROUP, + "Activating utility thread: %p:%" PRIu32, + (void *)group, thread->id); + WT_ASSERT(session, !F_ISSET(thread, WT_THREAD_ACTIVE)); + F_SET(thread, WT_THREAD_ACTIVE); + __wt_cond_signal(session, thread->pause_cond); + } + if (!is_locked) + __wt_writeunlock(session, &group->lock); } /* * __wt_thread_group_stop_one -- - * Stop one thread if possible. + * Pause one thread if possible. */ -int -__wt_thread_group_stop_one( - WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, bool wait) +void +__wt_thread_group_stop_one(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group) { - WT_DECL_RET; + WT_THREAD *thread; if (group->current_threads <= group->min) - return (0); - - if (wait) - __wt_writelock(session, &group->lock); - else - WT_RET(__wt_try_writelock(session, &group->lock)); + return; + __wt_writelock(session, &group->lock); /* Recheck the bounds now that we hold the lock */ - if (group->current_threads > group->min) - WT_TRET(__thread_group_shrink( - session, group, group->current_threads - 1, false)); + if (group->current_threads > group->min) { + thread = group->threads[--group->current_threads]; + __wt_verbose(session, WT_VERB_THREAD_GROUP, + "Pausing utility thread: %p:%" PRIu32, + (void *)group, thread->id); + WT_ASSERT(session, F_ISSET(thread, WT_THREAD_ACTIVE)); + F_CLR(thread, WT_THREAD_ACTIVE); + __wt_cond_signal(session, thread->pause_cond); + } __wt_writeunlock(session, &group->lock); - - return (ret); } diff --git a/src/txn/txn.c b/src/txn/txn.c index 6eebf5ecf9f..fb77ab4e860 100644 --- a/src/txn/txn.c +++ b/src/txn/txn.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -126,7 +126,7 @@ __wt_txn_get_snapshot(WT_SESSION_IMPL *session) n = 0; /* We're going to scan the table: wait for the lock. */ - __wt_readlock_spin(session, &txn_global->scan_rwlock); + __wt_readlock(session, &txn_global->scan_rwlock); current_id = pinned_id = txn_global->current; prev_oldest_id = txn_global->oldest_id; @@ -293,7 +293,7 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) /* First do a read-only scan. */ if (wait) - __wt_readlock_spin(session, &txn_global->scan_rwlock); + __wt_readlock(session, &txn_global->scan_rwlock); else if ((ret = __wt_try_readlock(session, &txn_global->scan_rwlock)) != 0) return (ret == EBUSY ? 0 : ret); @@ -477,10 +477,9 @@ __wt_txn_release(WT_SESSION_IMPL *session) /* Free the scratch buffer allocated for logging. */ __wt_logrec_free(session, &txn->logrec); - /* Discard any memory from the session's split stash that we can. */ - WT_ASSERT(session, session->split_gen == 0); - if (session->split_stash_cnt > 0) - __wt_split_stash_discard(session); + /* Discard any memory from the session's stash that we can. */ + WT_ASSERT(session, __wt_session_gen(session, WT_GEN_SPLIT) == 0); + __wt_stash_discard(session); /* * Reset the transaction state to not running and release the snapshot. @@ -510,10 +509,9 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) txn = &session->txn; conn = S2C(session); did_update = txn->mod_count != 0; - WT_ASSERT(session, !F_ISSET(txn, WT_TXN_ERROR) || !did_update); - if (!F_ISSET(txn, WT_TXN_RUNNING)) - WT_RET_MSG(session, EINVAL, "No transaction is active"); + WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING)); + WT_ASSERT(session, !F_ISSET(txn, WT_TXN_ERROR) || !did_update); /* * The default sync setting is inherited from the connection, but can @@ -594,9 +592,26 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) return (ret); } - /* Free memory associated with updates. */ - for (i = 0, op = txn->mod; i < txn->mod_count; i++, op++) + for (i = 0, op = txn->mod; i < txn->mod_count; i++, op++) { + switch (op->type) { + case WT_TXN_OP_BASIC: + case WT_TXN_OP_INMEM: + /* + * Switch reserved operations to abort to simplify + * obsolete update list truncation. + */ + if (op->u.upd->type == WT_UPDATE_RESERVED) + op->u.upd->txnid = WT_TXN_ABORTED; + break; + case WT_TXN_OP_REF: + case WT_TXN_OP_TRUNCATE_COL: + case WT_TXN_OP_TRUNCATE_ROW: + break; + } + + /* Free memory associated with updates. */ __wt_txn_op_free(session, op); + } txn->mod_count = 0; __wt_txn_release(session); @@ -618,8 +633,7 @@ __wt_txn_rollback(WT_SESSION_IMPL *session, const char *cfg[]) WT_UNUSED(cfg); txn = &session->txn; - if (!F_ISSET(txn, WT_TXN_RUNNING)) - WT_RET_MSG(session, EINVAL, "No transaction is active"); + WT_ASSERT(session, F_ISSET(txn, WT_TXN_RUNNING)); /* Rollback notification. */ if (txn->notify != NULL) @@ -768,8 +782,8 @@ __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_spin_init(session, &txn_global->id_lock, "transaction id lock")); - __wt_rwlock_init(session, &txn_global->scan_rwlock); - __wt_rwlock_init(session, &txn_global->nsnap_rwlock); + WT_RET(__wt_rwlock_init(session, &txn_global->scan_rwlock)); + WT_RET(__wt_rwlock_init(session, &txn_global->nsnap_rwlock)); txn_global->nsnap_oldest_id = WT_TXN_NONE; TAILQ_INIT(&txn_global->nsnaph); @@ -836,7 +850,8 @@ __wt_verbose_dump_txn(WT_SESSION_IMPL *session) WT_RET(__wt_msg(session, "checkpoint running? %s", txn_global->checkpoint_running ? "yes" : "no")); WT_RET(__wt_msg(session, - "checkpoint generation: %" PRIu64, txn_global->checkpoint_gen)); + "checkpoint generation: %" PRIu64, + __wt_gen(session, WT_GEN_CHECKPOINT))); WT_RET(__wt_msg(session, "checkpoint pinned ID: %" PRIu64, txn_global->checkpoint_pinned)); WT_RET(__wt_msg(session, diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c index f4ccf5eacd0..82163f471b8 100644 --- a/src/txn/txn_ckpt.c +++ b/src/txn/txn_ckpt.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -110,8 +110,7 @@ __checkpoint_update_generation(WT_SESSION_IMPL *session) if (WT_IS_METADATA(session->dhandle)) return; - WT_PUBLISH(btree->checkpoint_gen, - S2C(session)->txn_global.checkpoint_gen); + WT_PUBLISH(btree->checkpoint_gen, __wt_gen(session, WT_GEN_CHECKPOINT)); WT_STAT_DATA_SET(session, btree_checkpoint_generation, btree->checkpoint_gen); } @@ -533,7 +532,7 @@ __checkpoint_verbose_track(WT_SESSION_IMPL *session, __wt_verbose(session, WT_VERB_CHECKPOINT, "time: %" PRIu64 " us, gen: %" PRIu64 ": Full database checkpoint %s", - msec, S2C(session)->txn_global.checkpoint_gen, msg); + msec, __wt_gen(session, WT_GEN_CHECKPOINT), msg); /* Update the timestamp so we are reporting intervals. */ memcpy(start, &stop, sizeof(*start)); @@ -667,7 +666,7 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_TXN_ISOLATION saved_isolation; void *saved_meta_next; u_int i; - uint64_t fsync_duration_usecs; + uint64_t fsync_duration_usecs, generation; bool failed, full, idle, logging, tracking; conn = S2C(session); @@ -733,9 +732,8 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) * of the transaction table, or a thread evicting in a tree could * ignore the checkpoint's transaction. */ - (void)__wt_atomic_addv64(&txn_global->checkpoint_gen, 1); - WT_STAT_CONN_SET(session, - txn_checkpoint_generation, txn_global->checkpoint_gen); + generation = __wt_gen_next(session, WT_GEN_CHECKPOINT); + WT_STAT_CONN_SET(session, txn_checkpoint_generation, generation); /* Keep track of handles acquired for locking. */ WT_ERR(__wt_meta_track_on(session)); @@ -945,13 +943,11 @@ __txn_checkpoint_wrapper(WT_SESSION_IMPL *session, const char *cfg[]) WT_STAT_CONN_SET(session, txn_checkpoint_running, 1); txn_global->checkpoint_running = true; - WT_FULL_BARRIER(); ret = __txn_checkpoint(session, cfg); WT_STAT_CONN_SET(session, txn_checkpoint_running, 0); txn_global->checkpoint_running = false; - WT_FULL_BARRIER(); return (ret); } @@ -1448,8 +1444,7 @@ __checkpoint_tree( * the checkpoint start, which might not be included, will re-set the * modified flag. The "unless reconciliation skips updates" problem is * handled in the reconciliation code: if reconciliation skips updates, - * it sets the modified flag itself. Use a full barrier so we get the - * store done quickly, this isn't a performance path. + * it sets the modified flag itself. */ btree->modified = false; WT_FULL_BARRIER(); @@ -1527,7 +1522,7 @@ err: /* */ if (ret != 0) { btree->modified = true; - S2C(session)->modified = true; + conn->modified = true; } __wt_meta_ckptlist_free(session, &btree->ckpt); @@ -1549,8 +1544,8 @@ __checkpoint_presync(WT_SESSION_IMPL *session, const char *cfg[]) WT_UNUSED(cfg); btree = S2BT(session); - WT_ASSERT(session, btree->checkpoint_gen == - S2C(session)->txn_global.checkpoint_gen); + WT_ASSERT(session, + btree->checkpoint_gen == __wt_gen(session, WT_GEN_CHECKPOINT)); btree->evict_walk_period = btree->evict_walk_saved; return (0); } diff --git a/src/txn/txn_ext.c b/src/txn/txn_ext.c index 9ea1af6c4f8..625f970cca8 100644 --- a/src/txn/txn_ext.c +++ b/src/txn/txn_ext.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/txn/txn_log.c b/src/txn/txn_log.c index 2931dc1ce82..74dc679a6ef 100644 --- a/src/txn/txn_log.c +++ b/src/txn/txn_log.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -13,6 +13,51 @@ typedef struct { uint32_t flags; } WT_TXN_PRINTLOG_ARGS; +#ifdef HAVE_DIAGNOSTIC +/* + * __txn_op_log_row_key_check -- + * Confirm the cursor references the correct key. + */ +static void +__txn_op_log_row_key_check(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) +{ + WT_CURSOR *cursor; + WT_ITEM key; + WT_PAGE *page; + WT_ROW *rip; + + cursor = &cbt->iface; + WT_ASSERT(session, F_ISSET(cursor, WT_CURSTD_KEY_SET)); + + memset(&key, 0, sizeof(key)); + + /* + * We used to take the key for row-store logging from the page + * referenced by the cursor, when we switched to taking it from the + * cursor itself. Check that they are the same. + * + * If the cursor references a WT_INSERT item, take the key from there, + * else take the key from the original page. + */ + if (cbt->ins == NULL) { + session = (WT_SESSION_IMPL *)cbt->iface.session; + page = cbt->ref->page; + rip = &page->pg_row[cbt->slot]; + WT_ASSERT(session, + __wt_row_leaf_key(session, page, rip, &key, false) == 0); + } else { + key.data = WT_INSERT_KEY(cbt->ins); + key.size = WT_INSERT_KEY_SIZE(cbt->ins); + } + + WT_ASSERT(session, + key.size == cursor->key.size && + memcmp(key.data, cursor->key.data, key.size) == 0); + + __wt_buf_free(session, &key); +} +#endif + /* * __txn_op_log -- * Log an operation for the current transaction. @@ -21,46 +66,46 @@ static int __txn_op_log(WT_SESSION_IMPL *session, WT_ITEM *logrec, WT_TXN_OP *op, WT_CURSOR_BTREE *cbt) { - WT_DECL_RET; - WT_ITEM key, value; + WT_CURSOR *cursor; + WT_ITEM value; WT_UPDATE *upd; uint64_t recno; - WT_CLEAR(key); + cursor = &cbt->iface; + upd = op->u.upd; value.data = WT_UPDATE_DATA(upd); value.size = upd->size; /* - * Log the operation. It must be one of the following: - * 1) column store remove; - * 2) column store insert/update; - * 3) row store remove; or - * 4) row store insert/update. + * Log the operation. It must be a row- or column-store insert, remove + * or update, all of which require log records. We shouldn't ever log + * reserve operations. */ + WT_ASSERT(session, upd->type != WT_UPDATE_RESERVED); if (cbt->btree->type == BTREE_ROW) { - WT_ERR(__wt_cursor_row_leaf_key(cbt, &key)); - - if (WT_UPDATE_DELETED_ISSET(upd)) - WT_ERR(__wt_logop_row_remove_pack(session, logrec, - op->fileid, &key)); +#ifdef HAVE_DIAGNOSTIC + __txn_op_log_row_key_check(session, cbt); +#endif + if (upd->type == WT_UPDATE_DELETED) + WT_RET(__wt_logop_row_remove_pack( + session, logrec, op->fileid, &cursor->key)); else - WT_ERR(__wt_logop_row_put_pack(session, logrec, - op->fileid, &key, &value)); + WT_RET(__wt_logop_row_put_pack( + session, logrec, op->fileid, &cursor->key, &value)); } else { recno = WT_INSERT_RECNO(cbt->ins); WT_ASSERT(session, recno != WT_RECNO_OOB); - if (WT_UPDATE_DELETED_ISSET(upd)) - WT_ERR(__wt_logop_col_remove_pack(session, logrec, - op->fileid, recno)); + if (upd->type == WT_UPDATE_DELETED) + WT_RET(__wt_logop_col_remove_pack( + session, logrec, op->fileid, recno)); else - WT_ERR(__wt_logop_col_put_pack(session, logrec, - op->fileid, recno, &value)); + WT_RET(__wt_logop_col_put_pack( + session, logrec, op->fileid, recno, &value)); } -err: __wt_buf_free(session, &key); - return (ret); + return (0); } /* @@ -289,6 +334,7 @@ int __wt_txn_checkpoint_log( WT_SESSION_IMPL *session, bool full, uint32_t flags, WT_LSN *lsnp) { + WT_CONNECTION_IMPL *conn; WT_DECL_ITEM(logrec); WT_DECL_RET; WT_ITEM *ckpt_snapshot, empty; @@ -299,6 +345,7 @@ __wt_txn_checkpoint_log( uint32_t i, rectype = WT_LOGREC_CHECKPOINT; const char *fmt = WT_UNCHECKED_STRING(IIIIu); + conn = S2C(session); txn = &session->txn; ckpt_lsn = &txn->ckpt_lsn; @@ -363,20 +410,20 @@ __wt_txn_checkpoint_log( txn->ckpt_nsnapshot, ckpt_snapshot)); logrec->size += (uint32_t)recsize; WT_ERR(__wt_log_write(session, logrec, lsnp, - F_ISSET(S2C(session), WT_CONN_CKPT_SYNC) ? + F_ISSET(conn, WT_CONN_CKPT_SYNC) ? WT_LOG_FSYNC : 0)); /* * If this full checkpoint completed successfully and there is - * no hot backup in progress and this is not recovery, tell - * the logging subsystem the checkpoint LSN so that it can - * archive. Do not update the logging checkpoint LSN if this - * is during a clean connection close, only during a full - * checkpoint. A clean close may not update any metadata LSN - * and we do not want to archive in that case. + * no hot backup in progress and this is not an unclean + * recovery, tell the logging subsystem the checkpoint LSN so + * that it can archive. Do not update the logging checkpoint + * LSN if this is during a clean connection close, only during + * a full checkpoint. A clean close may not update any + * metadata LSN and we do not want to archive in that case. */ - if (!S2C(session)->hot_backup && - !F_ISSET(S2C(session), WT_CONN_RECOVERING) && + if (!conn->hot_backup && + !FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_DIRTY) && txn->full_ckpt) __wt_log_ckpt(session, ckpt_lsn); diff --git a/src/txn/txn_nsnap.c b/src/txn/txn_nsnap.c index 659570dbcd9..601d9492566 100644 --- a/src/txn/txn_nsnap.c +++ b/src/txn/txn_nsnap.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/txn/txn_recover.c b/src/txn/txn_recover.c index 30932195b1e..58f4f0750d7 100644 --- a/src/txn/txn_recover.c +++ b/src/txn/txn_recover.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -20,6 +20,7 @@ typedef struct { } *files; size_t file_alloc; /* Allocated size of files array. */ u_int max_fileid; /* Maximum file ID seen. */ + WT_LSN max_lsn; /* Maximum checkpoint LSN seen. */ u_int nfiles; /* Number of files in the metadata. */ WT_LSN ckpt_lsn; /* Start LSN for main recovery loop. */ @@ -342,6 +343,10 @@ __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config) "Recovering %s with id %" PRIu32 " @ (%" PRIu32 ", %" PRIu32 ")", uri, fileid, lsn.l.file, lsn.l.offset); + if ((!WT_IS_MAX_LSN(&lsn) && !WT_IS_INIT_LSN(&lsn)) && + (WT_IS_MAX_LSN(&r->max_lsn) || __wt_log_cmp(&lsn, &r->max_lsn) > 0)) + r->max_lsn = lsn; + return (0); } @@ -428,6 +433,7 @@ __wt_txn_recover(WT_SESSION_IMPL *session) WT_RET(__wt_open_internal_session(conn, "txn-recover", false, WT_SESSION_NO_LOGGING, &session)); r.session = session; + WT_MAX_LSN(&r.max_lsn); F_SET(conn, WT_CONN_RECOVERING); WT_ERR(__wt_metadata_search(session, WT_METAFILE_URI, &config)); @@ -441,11 +447,31 @@ __wt_txn_recover(WT_SESSION_IMPL *session) * last checkpoint was done with logging disabled, recovery should not * run. Scan the metadata to figure out the largest file ID. */ - if (!FLD_ISSET(S2C(session)->log_flags, WT_CONN_LOG_EXISTED) || + if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_EXISTED) || WT_IS_MAX_LSN(&metafile->ckpt_lsn)) { + /* + * Detect if we're going from logging disabled to enabled. + * We need to know this to verify LSNs and start at the correct + * log file later. If someone ran with logging, then disabled + * it and removed all the log files and then turned logging back + * on, we have to start logging in the log file number that is + * larger than any checkpoint LSN we have from the earlier time. + */ WT_ERR(__recovery_file_scan(&r)); + /* + * The array can be re-allocated in recovery_file_scan. Reset + * our pointer after scanning all the files. + */ + metafile = &r.files[WT_METAFILE_ID]; conn->next_file_id = r.max_fileid; - goto done; + + if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED) && + WT_IS_MAX_LSN(&metafile->ckpt_lsn) && + !WT_IS_MAX_LSN(&r.max_lsn)) { + WT_ERR(__wt_log_reset(session, r.max_lsn.l.file)); + goto ckpt; + } else + goto done; } /* @@ -488,6 +514,11 @@ __wt_txn_recover(WT_SESSION_IMPL *session) /* Scan the metadata to find the live files and their IDs. */ WT_ERR(__recovery_file_scan(&r)); + /* + * Clear this out. We no longer need it and it could have been + * re-allocated when scanning the files. + */ + metafile = NULL; /* * We no longer need the metadata cursor: close it to avoid pinning any @@ -535,6 +566,8 @@ __wt_txn_recover(WT_SESSION_IMPL *session) * this is not a read-only connection. * We can consider skipping it in the future. */ + if (needs_rec) + FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_DIRTY); if (WT_IS_INIT_LSN(&r.ckpt_lsn)) WT_ERR(__wt_log_scan(session, NULL, WT_LOGSCAN_FIRST | WT_LOGSCAN_RECOVER, @@ -554,11 +587,12 @@ __wt_txn_recover(WT_SESSION_IMPL *session) * open is fast and keep the metadata up to date with the checkpoint * LSN and archiving. */ - WT_ERR(session->iface.checkpoint(&session->iface, "force=1")); +ckpt: WT_ERR(session->iface.checkpoint(&session->iface, "force=1")); done: FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_DONE); err: WT_TRET(__recovery_free(&r)); __wt_free(session, config); + FLD_CLR(conn->log_flags, WT_CONN_LOG_RECOVER_DIRTY); if (ret != 0) __wt_err(session, ret, "Recovery failed"); diff --git a/src/utilities/util.h b/src/utilities/util.h index 93a96d44219..0238915df07 100644 --- a/src/utilities/util.h +++ b/src/utilities/util.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_alter.c b/src/utilities/util_alter.c index ef01a1ed826..da6316b2364 100644 --- a/src/utilities/util_alter.c +++ b/src/utilities/util_alter.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_backup.c b/src/utilities/util_backup.c index f1b31f7621a..7d809c2a624 100644 --- a/src/utilities/util_backup.c +++ b/src/utilities/util_backup.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_compact.c b/src/utilities/util_compact.c index e469b4dce6e..c8963a8fda6 100644 --- a/src/utilities/util_compact.c +++ b/src/utilities/util_compact.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_cpyright.c b/src/utilities/util_cpyright.c index 7de0eab6dc6..0cfba056387 100644 --- a/src/utilities/util_cpyright.c +++ b/src/utilities/util_cpyright.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * @@ -11,7 +11,7 @@ void util_copyright(void) { - printf("%s\n", "Copyright (c) 2008-2016 MongoDB, Inc."); + printf("%s\n", "Copyright (c) 2008-2017 MongoDB, Inc."); printf("%s\n\n", "All rights reserved."); printf("%s\n\n", diff --git a/src/utilities/util_create.c b/src/utilities/util_create.c index 7c22a67792b..2c7a87fd406 100644 --- a/src/utilities/util_create.c +++ b/src/utilities/util_create.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_drop.c b/src/utilities/util_drop.c index 456005d445d..460c9a6de57 100644 --- a/src/utilities/util_drop.c +++ b/src/utilities/util_drop.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_dump.c b/src/utilities/util_dump.c index 955148b7d46..15200d70f7e 100644 --- a/src/utilities/util_dump.c +++ b/src/utilities/util_dump.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_dump.h b/src/utilities/util_dump.h index e3fd8e6a501..7f037cc659a 100644 --- a/src/utilities/util_dump.h +++ b/src/utilities/util_dump.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_list.c b/src/utilities/util_list.c index f19ba4d1f97..72888e03183 100644 --- a/src/utilities/util_list.c +++ b/src/utilities/util_list.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_load.c b/src/utilities/util_load.c index d2f00402217..7f5c9529f2c 100644 --- a/src/utilities/util_load.c +++ b/src/utilities/util_load.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_load.h b/src/utilities/util_load.h index 710b18bfe83..53fce665ddc 100644 --- a/src/utilities/util_load.h +++ b/src/utilities/util_load.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_load_json.c b/src/utilities/util_load_json.c index c693e2b7651..8bc643f8556 100644 --- a/src/utilities/util_load_json.c +++ b/src/utilities/util_load_json.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_loadtext.c b/src/utilities/util_loadtext.c index 7602d43f8c9..1053ab89694 100644 --- a/src/utilities/util_loadtext.c +++ b/src/utilities/util_loadtext.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_main.c b/src/utilities/util_main.c index c6f225bb667..010af63ea30 100644 --- a/src/utilities/util_main.c +++ b/src/utilities/util_main.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_misc.c b/src/utilities/util_misc.c index e26185a0096..d0fe35ff370 100644 --- a/src/utilities/util_misc.c +++ b/src/utilities/util_misc.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_printlog.c b/src/utilities/util_printlog.c index 5f3ed43905b..2e5ae3aa926 100644 --- a/src/utilities/util_printlog.c +++ b/src/utilities/util_printlog.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_read.c b/src/utilities/util_read.c index 393949b6a1c..a27e8454780 100644 --- a/src/utilities/util_read.c +++ b/src/utilities/util_read.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_rebalance.c b/src/utilities/util_rebalance.c index c188ea17d22..f58f086e777 100644 --- a/src/utilities/util_rebalance.c +++ b/src/utilities/util_rebalance.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_rename.c b/src/utilities/util_rename.c index bb2d40cd103..51e936100ff 100644 --- a/src/utilities/util_rename.c +++ b/src/utilities/util_rename.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_salvage.c b/src/utilities/util_salvage.c index 6cc2278b846..dc311b5ee9a 100644 --- a/src/utilities/util_salvage.c +++ b/src/utilities/util_salvage.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_stat.c b/src/utilities/util_stat.c index 0692afe2819..baabaaeff01 100644 --- a/src/utilities/util_stat.c +++ b/src/utilities/util_stat.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_truncate.c b/src/utilities/util_truncate.c index 35de02345c8..101fa23c0cb 100644 --- a/src/utilities/util_truncate.c +++ b/src/utilities/util_truncate.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_upgrade.c b/src/utilities/util_upgrade.c index f89bd46e133..1ad95bf3cf2 100644 --- a/src/utilities/util_upgrade.c +++ b/src/utilities/util_upgrade.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_verbose.c b/src/utilities/util_verbose.c index e568ec0a414..979cd7451c1 100644 --- a/src/utilities/util_verbose.c +++ b/src/utilities/util_verbose.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_verify.c b/src/utilities/util_verify.c index ace1be7a5de..f0c51596ca4 100644 --- a/src/utilities/util_verify.c +++ b/src/utilities/util_verify.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/src/utilities/util_write.c b/src/utilities/util_write.c index 1d3e6937f8d..ca4203e1e51 100644 --- a/src/utilities/util_write.c +++ b/src/utilities/util_write.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2014-2017 MongoDB, Inc. * Copyright (c) 2008-2014 WiredTiger, Inc. * All rights reserved. * diff --git a/test/bloom/test_bloom.c b/test/bloom/test_bloom.c index b6299bbbadc..e12a0ed1550 100644 --- a/test/bloom/test_bloom.c +++ b/test/bloom/test_bloom.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/checkpoint/checkpointer.c b/test/checkpoint/checkpointer.c index 84d2765843a..3135caa8cad 100644 --- a/test/checkpoint/checkpointer.c +++ b/test/checkpoint/checkpointer.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -28,7 +28,7 @@ #include "test_checkpoint.h" -static void *checkpointer(void *); +static WT_THREAD_RET checkpointer(void *); static int compare_cursors( WT_CURSOR *, const char *, WT_CURSOR *, const char *); static int diagnose_key_error(WT_CURSOR *, int, WT_CURSOR *, int); @@ -39,35 +39,28 @@ static int verify_checkpoint(WT_SESSION *); * start_checkpoints -- * Responsible for creating the checkpoint thread. */ -int +void start_checkpoints(void) { - int ret; - - if ((ret = pthread_create( - &g.checkpoint_thread, NULL, checkpointer, NULL)) != 0) - return (log_print_err("pthread_create", ret, 1)); - return (0); + testutil_check(__wt_thread_create(NULL, + &g.checkpoint_thread, checkpointer, NULL)); } /* * end_checkpoints -- * Responsible for cleanly shutting down the checkpoint thread. */ -int +void end_checkpoints(void) { - void *thread_ret; - - return (pthread_join(g.checkpoint_thread, &thread_ret)); - + testutil_check(__wt_thread_join(NULL, g.checkpoint_thread)); } /* * checkpointer -- * Checkpoint thread start function. */ -static void * +static WT_THREAD_RET checkpointer(void *arg) { char tid[128]; @@ -78,7 +71,7 @@ checkpointer(void *arg) printf("checkpointer thread starting: tid: %s\n", tid); (void)real_checkpointer(); - return (NULL); + return (WT_THREAD_RET_VALUE); } /* diff --git a/test/checkpoint/test_checkpoint.c b/test/checkpoint/test_checkpoint.c index e7e1a0b81a5..cfe5ef1bad4 100644 --- a/test/checkpoint/test_checkpoint.c +++ b/test/checkpoint/test_checkpoint.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -150,20 +150,14 @@ main(int argc, char *argv[]) break; } - if ((ret = start_checkpoints()) != 0) { - (void)log_print_err("Start checkpoints failed", ret, 1); - break; - } + start_checkpoints(); if ((ret = start_workers(ttype)) != 0) { (void)log_print_err("Start workers failed", ret, 1); break; } g.running = 0; - if ((ret = end_checkpoints()) != 0) { - (void)log_print_err("Start workers failed", ret, 1); - break; - } + end_checkpoints(); free(g.cookies); g.cookies = NULL; diff --git a/test/checkpoint/test_checkpoint.h b/test/checkpoint/test_checkpoint.h index 347bd2c6e89..36551211b7e 100644 --- a/test/checkpoint/test_checkpoint.h +++ b/test/checkpoint/test_checkpoint.h @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -64,12 +64,12 @@ typedef struct { int running; /* Whether to stop */ int status; /* Exit status */ COOKIE *cookies; /* Per-thread info */ - pthread_t checkpoint_thread; /* Checkpoint thread */ + wt_thread_t checkpoint_thread; /* Checkpoint thread */ } GLOBAL; extern GLOBAL g; -int end_checkpoints(void); -int log_print_err(const char *, int, int); -int start_checkpoints(void); -int start_workers(table_type); +void end_checkpoints(void); +int log_print_err(const char *, int, int); +void start_checkpoints(void); +int start_workers(table_type); const char *type_to_string(table_type); diff --git a/test/checkpoint/workers.c b/test/checkpoint/workers.c index 82d1b8685c4..724475926ee 100644 --- a/test/checkpoint/workers.c +++ b/test/checkpoint/workers.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -29,7 +29,7 @@ #include "test_checkpoint.h" static int real_worker(void); -static void *worker(void *); +static WT_THREAD_RET worker(void *); /* * create_table -- @@ -64,9 +64,8 @@ start_workers(table_type type) WT_SESSION *session; struct timeval start, stop; double seconds; - pthread_t *tids; + wt_thread_t *tids; int i, ret; - void *thread_ret; ret = 0; @@ -98,17 +97,13 @@ start_workers(table_type type) (void)gettimeofday(&start, NULL); /* Create threads. */ - for (i = 0; i < g.nworkers; ++i) { - if ((ret = pthread_create( - &tids[i], NULL, worker, &g.cookies[i])) != 0) { - (void)log_print_err("pthread_create", ret, 1); - goto err; - } - } + for (i = 0; i < g.nworkers; ++i) + testutil_check(__wt_thread_create( + NULL, &tids[i], worker, &g.cookies[i])); /* Wait for the threads. */ for (i = 0; i < g.nworkers; ++i) - (void)pthread_join(tids[i], &thread_ret); + testutil_check(__wt_thread_join(NULL, tids[i])); (void)gettimeofday(&stop, NULL); seconds = (stop.tv_sec - start.tv_sec) + @@ -146,7 +141,7 @@ worker_op(WT_CURSOR *cursor, uint64_t keyno, u_int new_val) * worker -- * Worker thread start function. */ -static void * +static WT_THREAD_RET worker(void *arg) { char tid[128]; @@ -157,7 +152,7 @@ worker(void *arg) printf("worker thread starting: tid: %s\n", tid); (void)real_worker(); - return (NULL); + return (WT_THREAD_RET_VALUE); } /* @@ -215,7 +210,7 @@ real_worker(void) } } else if (ret == WT_ROLLBACK) { if ((ret = session->rollback_transaction( - session, NULL)) != 0) { + session, NULL)) != 0) { (void)log_print_err( "real_worker:rollback_transaction", ret, 1); goto err; diff --git a/test/csuite/Makefile.am b/test/csuite/Makefile.am index 10ab890f2f5..f2b4fcacdc8 100644 --- a/test/csuite/Makefile.am +++ b/test/csuite/Makefile.am @@ -57,6 +57,9 @@ noinst_PROGRAMS += test_wt3135_search_near_collator test_wt3184_dup_index_collator_SOURCES = wt3184_dup_index_collator/main.c noinst_PROGRAMS += test_wt3184_dup_index_collator +test_rwlock_SOURCES = rwlock/main.c +noinst_PROGRAMS += test_rwlock + # Run this during a "make check" smoke test. TESTS = $(noinst_PROGRAMS) LOG_COMPILER = $(TEST_WRAPPER) diff --git a/test/csuite/rwlock/main.c b/test/csuite/rwlock/main.c new file mode 100644 index 00000000000..04813182478 --- /dev/null +++ b/test/csuite/rwlock/main.c @@ -0,0 +1,184 @@ +/*- + * Public Domain 2014-2017 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * JIRA ticket reference: HELP-4355 + * Test rwlock collapse under load. + */ +#define MAX_THREADS 1000 +#define READS_PER_WRITE 10000 +//#define READS_PER_WRITE 1000000 +//#define READS_PER_WRITE 100 + +#define CHECK_CORRECTNESS 1 +//#define USE_POSIX 1 + +static WT_RWLOCK rwlock; +static pthread_rwlock_t p_rwlock; +static bool running; +static uint64_t shared_counter; + +void *thread_rwlock(void *); +void *thread_dump(void *); + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + struct timespec te, ts; + pthread_t dump_id, id[MAX_THREADS]; + int i; + + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + opts->nthreads = 100; + opts->nops = 1000000; /* per thread */ + testutil_check(testutil_parse_opts(argc, argv, opts)); + running = true; + + testutil_make_work_dir(opts->home); + testutil_check(wiredtiger_open(opts->home, NULL, + "create,session_max=1000,statistics=(fast)", &opts->conn)); + + testutil_check(__wt_rwlock_init(NULL, &rwlock)); + testutil_check(pthread_rwlock_init(&p_rwlock, NULL)); + + testutil_check(pthread_create( + &dump_id, NULL, thread_dump, (void *)opts)); + + __wt_epoch(NULL, &ts); + for (i = 0; i < (int)opts->nthreads; ++i) + testutil_check(pthread_create( + &id[i], NULL, thread_rwlock, (void *)opts)); + + while (--i >= 0) + testutil_check(pthread_join(id[i], NULL)); + __wt_epoch(NULL, &te); + printf("%.2lf\n", WT_TIMEDIFF_MS(te, ts) / 1000.0); + + running = false; + testutil_check(pthread_join(dump_id, NULL)); + + testutil_check(pthread_rwlock_destroy(&p_rwlock)); + testutil_cleanup(opts); + return (EXIT_SUCCESS); +} + +/* + * Acquire a rwlock, every Nth operation, acquire exclusive. + */ +void * +thread_rwlock(void *arg) +{ + TEST_OPTS *opts; + WT_SESSION *wt_session; + WT_SESSION_IMPL *session; + uint64_t i, counter; + bool writelock; + + opts = (TEST_OPTS *)arg; + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &wt_session)); + session = (WT_SESSION_IMPL *)wt_session; + + printf("Running rwlock thread\n"); + for (i = 1; i <= opts->nops; ++i) { + writelock = (i % READS_PER_WRITE == 0); + +#ifdef USE_POSIX + if (writelock) + testutil_check(pthread_rwlock_wrlock(&p_rwlock)); + else + testutil_check(pthread_rwlock_rdlock(&p_rwlock)); +#else + if (writelock) + __wt_writelock(session, &rwlock); + else + __wt_readlock(session, &rwlock); +#endif + + /* + * Do a tiny amount of work inside the lock so the compiler + * can't optimize everything away. + */ + (void)__wt_atomic_add64(&counter, 1); + +#ifdef CHECK_CORRECTNESS + if (writelock) + counter = ++shared_counter; + else + counter = shared_counter; + + __wt_yield(); + + testutil_assert(counter == shared_counter); +#endif + +#ifdef USE_POSIX + testutil_check(pthread_rwlock_unlock(&p_rwlock)); +#else + if (writelock) + __wt_writeunlock(session, &rwlock); + else + __wt_readunlock(session, &rwlock); +#endif + + if (i % 10000 == 0) { + printf("%s", session->id == 20 ? ".\n" : "."); + fflush(stdout); + } + } + + opts->running = false; + + return (NULL); +} + +void * +thread_dump(void *arg) { + WT_UNUSED(arg); + + while (running) { + sleep(1); + printf("\n" + "rwlock { current %" PRIu8 ", next %" PRIu8 + ", reader %" PRIu8 ", readers_active %" PRIu16 + ", readers_queued %" PRIu16 " }\n", + rwlock.u.s.current, + rwlock.u.s.next, + rwlock.u.s.reader, + rwlock.u.s.readers_active, + rwlock.u.s.readers_queued); + } + + return (NULL); +} diff --git a/test/csuite/scope/main.c b/test/csuite/scope/main.c index 15dabd97c40..83d6bd479d9 100644 --- a/test/csuite/scope/main.c +++ b/test/csuite/scope/main.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -28,7 +28,7 @@ #include "test_util.h" #define KEY "key" -#define VALUE "value" +#define VALUE "value,value,value" static int ignore_errors; @@ -63,46 +63,55 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) { struct { const char *op; - enum { INSERT, SEARCH, SEARCH_NEAR, + enum { INSERT, MODIFY, SEARCH, SEARCH_NEAR, REMOVE, REMOVE_POS, RESERVE, UPDATE } func; const char *config; } *op, ops[] = { /* - * The ops order is fixed and shouldn't change, that is, insert - * has to happen first so search, update and remove operations - * are possible, and remove has to be last. + * The ops order is specific: insert has to happen first so + * other operations are possible, and remove has to be last. */ { "insert", INSERT, NULL, }, { "search", SEARCH, NULL, }, { "search", SEARCH_NEAR, NULL, }, -#if 0 { "reserve", RESERVE, NULL, }, -#endif + { "insert", MODIFY, NULL, }, { "update", UPDATE, NULL, }, { "remove", REMOVE, NULL, }, { "remove", REMOVE_POS, NULL, }, { NULL, INSERT, NULL } }; WT_CURSOR *cursor; +#define MODIFY_ENTRIES 2 + WT_MODIFY entries[MODIFY_ENTRIES]; + WT_ITEM vu; uint64_t keyr; - const char *key, *value; + const char *key, *vs; char keybuf[100], valuebuf[100]; int exact; - bool recno; + bool recno, vstring; /* Reserve requires a running transaction. */ testutil_check(session->begin_transaction(session, NULL)); cursor = NULL; for (op = ops; op->op != NULL; op++) { - key = value = NULL; + key = vs = NULL; + memset(&vu, 0, sizeof(vu)); /* Open a cursor. */ if (cursor != NULL) testutil_check(cursor->close(cursor)); testutil_check(session->open_cursor( session, uri, NULL, op->config, &cursor)); + + /* Operations change based on the key/value formats. */ recno = strcmp(cursor->key_format, "r") == 0; + vstring = strcmp(cursor->value_format, "S") == 0; + + /* Modify is only possible with "item" values. */ + if (vstring && op->func == MODIFY) + continue; /* * Set up application buffers so we can detect overwrites @@ -116,7 +125,12 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) cursor->set_key(cursor, keybuf); } strcpy(valuebuf, VALUE); - cursor->set_value(cursor, valuebuf); + if (vstring) + cursor->set_value(cursor, valuebuf); + else { + vu.size = strlen(vu.data = valuebuf); + cursor->set_value(cursor, &vu); + } /* * The application must keep key and value memory valid until @@ -129,6 +143,20 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) case INSERT: testutil_check(cursor->insert(cursor)); break; + case MODIFY: + /* Modify, but don't really change anything. */ + entries[0].data.data = &VALUE[0]; + entries[0].data.size = 2; + entries[0].offset = 0; + entries[0].size = 2; + entries[1].data.data = &VALUE[3]; + entries[1].data.size = 5; + entries[1].offset = 3; + entries[1].size = 5; + + testutil_check( + cursor->modify(cursor, entries, MODIFY_ENTRIES)); + break; case SEARCH: testutil_check(cursor->search(cursor)); break; @@ -148,9 +176,7 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) testutil_check(cursor->remove(cursor)); break; case RESERVE: -#if 0 testutil_check(cursor->reserve(cursor)); -#endif break; case UPDATE: testutil_check(cursor->update(cursor)); @@ -184,7 +210,12 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) else testutil_assert( cursor->get_key(cursor, &key) != 0); - testutil_assert(cursor->get_value(cursor, &value) != 0); + if (vstring) + testutil_assert( + cursor->get_value(cursor, &vs) != 0); + else + testutil_assert( + cursor->get_value(cursor, &vu) != 0); testutil_assert(ignore_errors == 0); break; case REMOVE_POS: @@ -205,16 +236,22 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) testutil_assert(strcmp(key, KEY) == 0); } ignore_errors = 1; - testutil_assert(cursor->get_value(cursor, &value) != 0); + if (vstring) + testutil_assert( + cursor->get_value(cursor, &vs) != 0); + else + testutil_assert( + cursor->get_value(cursor, &vu) != 0); testutil_assert(ignore_errors == 0); break; + case MODIFY: case RESERVE: case SEARCH: case SEARCH_NEAR: case UPDATE: /* - * Reserve, search, search-near and update position the - * cursor and have both a key and value. + * Modify, reserve, search, search-near and update all + * position the cursor and have both a key and value. * * Any key/value should not reference application * memory. @@ -229,9 +266,19 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) testutil_assert(key != keybuf); testutil_assert(strcmp(key, KEY) == 0); } - testutil_assert(cursor->get_value(cursor, &value) == 0); - testutil_assert(value != valuebuf); - testutil_assert(strcmp(value, VALUE) == 0); + if (vstring) { + testutil_assert( + cursor->get_value(cursor, &vs) == 0); + testutil_assert(vs != valuebuf); + testutil_assert(strcmp(vs, VALUE) == 0); + } else { + testutil_assert( + cursor->get_value(cursor, &vu) == 0); + testutil_assert(vu.data != valuebuf); + testutil_assert(vu.size == strlen(VALUE)); + testutil_assert( + memcmp(vu.data, VALUE, strlen(VALUE)) == 0); + } break; } @@ -243,9 +290,16 @@ cursor_scope_ops(WT_SESSION *session, const char *uri) if (recno) cursor->set_key(cursor, (uint64_t)1); else { - cursor->set_key(cursor, KEY); + strcpy(keybuf, KEY); + cursor->set_key(cursor, keybuf); + } + strcpy(valuebuf, VALUE); + if (vstring) + cursor->set_value(cursor, valuebuf); + else { + vu.size = strlen(vu.data = valuebuf); + cursor->set_value(cursor, &vu); } - cursor->set_value(cursor, VALUE); testutil_check(cursor->insert(cursor)); } } @@ -276,11 +330,19 @@ main(int argc, char *argv[]) wiredtiger_open(opts->home, &event_handler, "create", &opts->conn)); run(opts->conn, "file:file.SS", "key_format=S,value_format=S"); + run(opts->conn, "file:file.Su", "key_format=S,value_format=u"); run(opts->conn, "file:file.rS", "key_format=r,value_format=S"); + run(opts->conn, "file:file.ru", "key_format=r,value_format=u"); + run(opts->conn, "lsm:lsm.SS", "key_format=S,value_format=S"); + run(opts->conn, "lsm:lsm.Su", "key_format=S,value_format=S"); run(opts->conn, "lsm:lsm.rS", "key_format=r,value_format=S"); + run(opts->conn, "lsm:lsm.ru", "key_format=r,value_format=S"); + run(opts->conn, "table:table.SS", "key_format=S,value_format=S"); + run(opts->conn, "table:table.Su", "key_format=S,value_format=u"); run(opts->conn, "table:table.rS", "key_format=r,value_format=S"); + run(opts->conn, "table:table.ru", "key_format=r,value_format=u"); testutil_cleanup(opts); diff --git a/test/csuite/wt1965_col_efficiency/main.c b/test/csuite/wt1965_col_efficiency/main.c index e5b73d5e642..e6801d8d37e 100644 --- a/test/csuite/wt1965_col_efficiency/main.c +++ b/test/csuite/wt1965_col_efficiency/main.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/csuite/wt2246_col_append/main.c b/test/csuite/wt2246_col_append/main.c index 9876582fffa..de7916b6859 100644 --- a/test/csuite/wt2246_col_append/main.c +++ b/test/csuite/wt2246_col_append/main.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/csuite/wt2323_join_visibility/main.c b/test/csuite/wt2323_join_visibility/main.c index 617490fec4d..a3ab8c153ed 100644 --- a/test/csuite/wt2323_join_visibility/main.c +++ b/test/csuite/wt2323_join_visibility/main.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/csuite/wt2403_lsm_workload/main.c b/test/csuite/wt2403_lsm_workload/main.c index 0c287484b9e..214276bda21 100644 --- a/test/csuite/wt2403_lsm_workload/main.c +++ b/test/csuite/wt2403_lsm_workload/main.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/csuite/wt2447_join_main_table/main.c b/test/csuite/wt2447_join_main_table/main.c index 656cea04145..8ad721c8d51 100644 --- a/test/csuite/wt2447_join_main_table/main.c +++ b/test/csuite/wt2447_join_main_table/main.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/csuite/wt2535_insert_race/main.c b/test/csuite/wt2535_insert_race/main.c index ba17d485e07..6ea599fc118 100644 --- a/test/csuite/wt2535_insert_race/main.c +++ b/test/csuite/wt2535_insert_race/main.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/csuite/wt2592_join_schema/main.c b/test/csuite/wt2592_join_schema/main.c index be3eff6136c..04fe954c427 100644 --- a/test/csuite/wt2592_join_schema/main.c +++ b/test/csuite/wt2592_join_schema/main.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/csuite/wt2695_checksum/main.c b/test/csuite/wt2695_checksum/main.c index db4fed5dc53..3bd9bfca3c0 100644 --- a/test/csuite/wt2695_checksum/main.c +++ b/test/csuite/wt2695_checksum/main.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/csuite/wt2719_reconfig/main.c b/test/csuite/wt2719_reconfig/main.c index 0942cfc73b2..cef95490c7e 100644 --- a/test/csuite/wt2719_reconfig/main.c +++ b/test/csuite/wt2719_reconfig/main.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/csuite/wt2834_join_bloom_fix/main.c b/test/csuite/wt2834_join_bloom_fix/main.c index e128df29f41..74128406a8e 100644 --- a/test/csuite/wt2834_join_bloom_fix/main.c +++ b/test/csuite/wt2834_join_bloom_fix/main.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/csuite/wt2853_perf/main.c b/test/csuite/wt2853_perf/main.c index 46ba71372e5..096bc64cf82 100644 --- a/test/csuite/wt2853_perf/main.c +++ b/test/csuite/wt2853_perf/main.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/csuite/wt2909_checkpoint_integrity/main.c b/test/csuite/wt2909_checkpoint_integrity/main.c index ce7bd72fa3f..c93da4c1068 100644 --- a/test/csuite/wt2909_checkpoint_integrity/main.c +++ b/test/csuite/wt2909_checkpoint_integrity/main.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -445,6 +445,31 @@ run_process(TEST_OPTS *opts, const char *prog, char *argv[], int *status) } /* +* subtest_error_handler -- +* Error event handler. +*/ +static int +subtest_error_handler(WT_EVENT_HANDLER *handler, + WT_SESSION *session, int error, const char *message) +{ + (void)(handler); + (void)(session); + (void)(message); + + /* Exit on panic, there's no checking to be done. */ + if (error == WT_PANIC) + exit (1); + return (0); +} + +static WT_EVENT_HANDLER event_handler = { + subtest_error_handler, + NULL, /* Message handler */ + NULL, /* Progress handler */ + NULL /* Close handler */ +}; + +/* * subtest_main -- * The main program for the subtest */ @@ -478,7 +503,8 @@ subtest_main(int argc, char *argv[], bool close_test) WT_FAIL_FS_LIB "=(early_load,config={environment=true,verbose=true})]")); - testutil_check(wiredtiger_open(opts->home, NULL, config, &opts->conn)); + testutil_check( + wiredtiger_open(opts->home, &event_handler, config, &opts->conn)); testutil_check( opts->conn->open_session(opts->conn, NULL, NULL, &session)); diff --git a/test/csuite/wt2999_join_extractor/main.c b/test/csuite/wt2999_join_extractor/main.c index 646a7077af1..194ff143610 100644 --- a/test/csuite/wt2999_join_extractor/main.c +++ b/test/csuite/wt2999_join_extractor/main.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/csuite/wt3135_search_near_collator/main.c b/test/csuite/wt3135_search_near_collator/main.c index 8783034a7d8..103a502f808 100644 --- a/test/csuite/wt3135_search_near_collator/main.c +++ b/test/csuite/wt3135_search_near_collator/main.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/csuite/wt3184_dup_index_collator/main.c b/test/csuite/wt3184_dup_index_collator/main.c index c969e7a1d7e..cd166780c6b 100644 --- a/test/csuite/wt3184_dup_index_collator/main.c +++ b/test/csuite/wt3184_dup_index_collator/main.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/cursor_order/cursor_order.c b/test/cursor_order/cursor_order.c index d3c64b54ab5..336ee54db63 100644 --- a/test/cursor_order/cursor_order.c +++ b/test/cursor_order/cursor_order.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -158,8 +158,7 @@ main(int argc, char *argv[]) wt_connect(cfg, config_open); /* WiredTiger connection */ - if (ops_start(cfg)) - return (EXIT_FAILURE); + ops_start(cfg); wt_shutdown(cfg); /* WiredTiger shut down */ } diff --git a/test/cursor_order/cursor_order.h b/test/cursor_order/cursor_order.h index 98a7d03c6f3..ab9f94850df 100644 --- a/test/cursor_order/cursor_order.h +++ b/test/cursor_order/cursor_order.h @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -50,5 +50,5 @@ typedef struct { } SHARED_CONFIG; void load(SHARED_CONFIG *, const char *); -int ops_start(SHARED_CONFIG *); +void ops_start(SHARED_CONFIG *); void verify(SHARED_CONFIG *, const char *); diff --git a/test/cursor_order/cursor_order_file.c b/test/cursor_order/cursor_order_file.c index 42d7af54de4..c1f69a65c88 100644 --- a/test/cursor_order/cursor_order_file.c +++ b/test/cursor_order/cursor_order_file.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/cursor_order/cursor_order_ops.c b/test/cursor_order/cursor_order_ops.c index 299f22684c9..cdd5af1a9ef 100644 --- a/test/cursor_order/cursor_order_ops.c +++ b/test/cursor_order/cursor_order_ops.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -28,9 +28,9 @@ #include "cursor_order.h" -static void *append_insert(void *); +static WT_THREAD_RET append_insert(void *); static void print_stats(SHARED_CONFIG *); -static void *reverse_scan(void *); +static WT_THREAD_RET reverse_scan(void *); typedef struct { char *name; /* object name */ @@ -45,15 +45,13 @@ typedef struct { static INFO *run_info; -int +void ops_start(SHARED_CONFIG *cfg) { struct timeval start, stop; double seconds; - pthread_t *tids; + wt_thread_t *tids; uint64_t i, name_index, offset, total_nops; - int ret; - void *thread_ret; tids = NULL; /* Keep GCC 4.1 happy. */ total_nops = 0; @@ -114,18 +112,15 @@ ops_start(SHARED_CONFIG *cfg) /* Create threads. */ for (i = 0; i < cfg->reverse_scanners; ++i) - if ((ret = pthread_create( - &tids[i], NULL, reverse_scan, (void *)(uintptr_t)i)) != 0) - testutil_die(ret, "pthread_create"); - for (; i < cfg->reverse_scanners + cfg->append_inserters; ++i) { - if ((ret = pthread_create( - &tids[i], NULL, append_insert, (void *)(uintptr_t)i)) != 0) - testutil_die(ret, "pthread_create"); - } + testutil_check(__wt_thread_create(NULL, + &tids[i], reverse_scan, (void *)(uintptr_t)i)); + for (; i < cfg->reverse_scanners + cfg->append_inserters; ++i) + testutil_check(__wt_thread_create(NULL, + &tids[i], append_insert, (void *)(uintptr_t)i)); /* Wait for the threads. */ for (i = 0; i < cfg->reverse_scanners + cfg->append_inserters; ++i) - (void)pthread_join(tids[i], &thread_ret); + testutil_check(__wt_thread_join(NULL, tids[i])); (void)gettimeofday(&stop, NULL); seconds = (stop.tv_sec - start.tv_sec) + @@ -154,8 +149,6 @@ ops_start(SHARED_CONFIG *cfg) free(run_info); free(tids); - - return (0); } /* @@ -217,7 +210,7 @@ reverse_scan_op( * reverse_scan -- * Reader thread start function. */ -static void * +static WT_THREAD_RET reverse_scan(void *arg) { INFO *s; @@ -260,7 +253,7 @@ reverse_scan(void *arg) /* Notify all other threads to finish once the first thread is done */ cfg->thread_finish = true; - return (NULL); + return (WT_THREAD_RET_VALUE); } /* @@ -307,7 +300,7 @@ append_insert_op( * append_insert -- * Writer thread start function. */ -static void * +static WT_THREAD_RET append_insert(void *arg) { INFO *s; @@ -347,7 +340,7 @@ append_insert(void *arg) /* Notify all other threads to finish once the first thread is done */ cfg->thread_finish = true; - return (NULL); + return (WT_THREAD_RET_VALUE); } /* diff --git a/test/fops/file.c b/test/fops/file.c index d1cd22ab391..1bb13f8a4de 100644 --- a/test/fops/file.c +++ b/test/fops/file.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/fops/fops.c b/test/fops/fops.c index 3c4de161423..911bfba55ad 100644 --- a/test/fops/fops.c +++ b/test/fops/fops.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -28,7 +28,7 @@ #include "thread.h" -static void *fop(void *); +static WT_THREAD_RET fop(void *); static void print_stats(u_int); typedef struct { @@ -46,15 +46,13 @@ typedef struct { static STATS *run_stats; -int +void fop_start(u_int nthreads) { struct timeval start, stop; double seconds; - pthread_t *tids; + wt_thread_t *tids; u_int i; - int ret; - void *thread_ret; tids = NULL; /* Silence GCC 4.1 warning. */ @@ -66,13 +64,12 @@ fop_start(u_int nthreads) /* Create threads. */ for (i = 0; i < nthreads; ++i) - if ((ret = pthread_create( - &tids[i], NULL, fop, (void *)(uintptr_t)i)) != 0) - testutil_die(ret, "pthread_create"); + testutil_check(__wt_thread_create( + NULL, &tids[i], fop, (void *)(uintptr_t)i)); /* Wait for the threads. */ for (i = 0; i < nthreads; ++i) - (void)pthread_join(tids[i], &thread_ret); + testutil_check(__wt_thread_join(NULL, tids[i])); (void)gettimeofday(&stop, NULL); seconds = (stop.tv_sec - start.tv_sec) + @@ -84,15 +81,13 @@ fop_start(u_int nthreads) free(run_stats); free(tids); - - return (0); } /* * fop -- * File operation function. */ -static void * +static WT_THREAD_RET fop(void *arg) { STATS *s; @@ -150,7 +145,7 @@ fop(void *arg) break; } - return (NULL); + return (WT_THREAD_RET_VALUE); } /* diff --git a/test/fops/t.c b/test/fops/t.c index 07ac07349e3..2357b170e49 100644 --- a/test/fops/t.c +++ b/test/fops/t.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -129,8 +129,7 @@ main(int argc, char *argv[]) wt_startup(config_open); - if (fop_start(nthreads)) - return (EXIT_FAILURE); + fop_start(nthreads); wt_shutdown(); printf("\n"); diff --git a/test/fops/thread.h b/test/fops/thread.h index 89b7984a166..f6b6bdffd63 100644 --- a/test/fops/thread.h +++ b/test/fops/thread.h @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -39,7 +39,7 @@ extern const char *config; /* Object config */ extern pthread_rwlock_t single; /* Single-thread */ -int fop_start(u_int); +void fop_start(u_int); void obj_bulk(void); void obj_bulk_unique(int); void obj_checkpoint(void); diff --git a/test/format/backup.c b/test/format/backup.c index 8aa614fa970..47f3c54325f 100644 --- a/test/format/backup.c +++ b/test/format/backup.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -83,7 +83,7 @@ copy_file(WT_SESSION *session, const char *name) * backup -- * Periodically do a backup and verify it. */ -void * +WT_THREAD_RET backup(void *arg) { WT_CONNECTION *conn; @@ -100,7 +100,7 @@ backup(void *arg) /* Backups aren't supported for non-standard data sources. */ if (DATASOURCE("helium") || DATASOURCE("kvsbdb")) - return (NULL); + return (WT_THREAD_RET_VALUE); /* Open a session. */ testutil_check(conn->open_session(conn, NULL, NULL, &session)); @@ -188,5 +188,5 @@ backup(void *arg) testutil_check(session->close(session, NULL)); - return (NULL); + return (WT_THREAD_RET_VALUE); } diff --git a/test/format/bdb.c b/test/format/bdb.c index 8b61573fdf9..6ee3e063cad 100644 --- a/test/format/bdb.c +++ b/test/format/bdb.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/format/bulk.c b/test/format/bulk.c index dab23bed404..0e7c54516e6 100644 --- a/test/format/bulk.c +++ b/test/format/bulk.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/format/compact.c b/test/format/compact.c index 240e5553697..f2fa7521946 100644 --- a/test/format/compact.c +++ b/test/format/compact.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -32,7 +32,7 @@ * compaction -- * Periodically do a compaction operation. */ -void * +WT_THREAD_RET compact(void *arg) { WT_CONNECTION *conn; @@ -44,7 +44,7 @@ compact(void *arg) /* Compaction isn't supported for all data sources. */ if (DATASOURCE("helium") || DATASOURCE("kvsbdb")) - return (NULL); + return (WT_THREAD_RET_VALUE); /* Open a session. */ conn = g.wts_conn; @@ -70,5 +70,5 @@ compact(void *arg) testutil_check(session->close(session, NULL)); - return (NULL); + return (WT_THREAD_RET_VALUE); } diff --git a/test/format/config.c b/test/format/config.c index 22b40f7164d..2685438af00 100644 --- a/test/format/config.c +++ b/test/format/config.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -489,6 +489,8 @@ config_pct(void) #define CONFIG_DELETE_ENTRY 0 { "delete_pct", &g.c_delete_pct, 0 }, { "insert_pct", &g.c_insert_pct, 0 }, +#define CONFIG_MODIFY_ENTRY 2 + { "modify_pct", &g.c_modify_pct, 0 }, { "read_pct", &g.c_read_pct, 0 }, { "write_pct", &g.c_write_pct, 0 }, }; @@ -508,6 +510,16 @@ config_pct(void) testutil_die(EINVAL, "operation percentages total to more than 100%%"); + /* Cursor modify isn't possible for fixed-length column store. */ + if (g.type == FIX) { + if (config_is_perm("modify_pct")) + testutil_die(EINVAL, + "WT_CURSOR.modify not supported by fixed-length " + "column store or LSM"); + list[CONFIG_MODIFY_ENTRY].order = 0; + *list[CONFIG_MODIFY_ENTRY].vp = 0; + } + /* * If the delete percentage isn't nailed down, periodically set it to * 0 so salvage gets run. Don't do it on the first run, all our smoke @@ -547,8 +559,9 @@ config_pct(void) list[max_slot].order = 0; pct -= *list[max_slot].vp; } - testutil_assert(g.c_delete_pct + - g.c_insert_pct + g.c_read_pct + g.c_write_pct == 100); + + testutil_assert(g.c_delete_pct + g.c_insert_pct + + g.c_modify_pct + g.c_read_pct + g.c_write_pct == 100); } /* diff --git a/test/format/config.h b/test/format/config.h index b5feb7a5321..3a41411e104 100644 --- a/test/format/config.h +++ b/test/format/config.h @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -238,6 +238,10 @@ static CONFIG c[] = { "configure for mmap operations", /* 90% */ C_BOOL, 90, 0, 0, &g.c_mmap, NULL }, + { "modify_pct", + "percent operations that are value modifications", + C_IGNORE, 0, 0, 100, &g.c_modify_pct, NULL }, + { "ops", "the number of modification operations done per run", 0x0, 0, M(2), M(100), &g.c_ops, NULL }, @@ -323,7 +327,7 @@ static CONFIG c[] = { C_IGNORE|C_STRING, 0, 0, 0, NULL, &g.c_config_open }, { "write_pct", - "percent operations that are writes", + "percent operations that are value updates", C_IGNORE, 0, 0, 100, &g.c_write_pct, NULL }, { NULL, NULL, 0x0, 0, 0, 0, NULL, NULL } diff --git a/test/format/format.h b/test/format/format.h index 41cc48c4278..602c1cc6d59 100644 --- a/test/format/format.h +++ b/test/format/format.h @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -78,6 +78,8 @@ #define FORMAT_OPERATION_REPS 3 /* 3 thread operations sets */ +#define MAX_MODIFY_ENTRIES 5 /* maximum change vectors */ + typedef struct { char *home; /* Home directory */ char *home_backup; /* Hot-backup directory */ @@ -147,28 +149,28 @@ typedef struct { uint32_t c_bloom_hash_count; uint32_t c_bloom_oldest; uint32_t c_cache; - uint32_t c_compact; uint32_t c_checkpoints; - char *c_checksum; + char *c_checksum; uint32_t c_chunk_size; - char *c_compression; - char *c_encryption; - char *c_config_open; + uint32_t c_compact; + char *c_compression; + char *c_config_open; uint32_t c_data_extend; - char *c_data_source; + char *c_data_source; uint32_t c_delete_pct; uint32_t c_dictionary; uint32_t c_direct_io; + char *c_encryption; uint32_t c_evict_max; + char *c_file_type; uint32_t c_firstfit; - char *c_file_type; uint32_t c_huffman_key; uint32_t c_huffman_value; uint32_t c_in_memory; uint32_t c_insert_pct; uint32_t c_internal_key_truncation; uint32_t c_intl_page_max; - char *c_isolation; + char *c_isolation; uint32_t c_key_gap; uint32_t c_key_max; uint32_t c_key_min; @@ -176,22 +178,23 @@ typedef struct { uint32_t c_leak_memory; uint32_t c_logging; uint32_t c_logging_archive; - char *c_logging_compression; + char *c_logging_compression; uint32_t c_logging_prealloc; uint32_t c_long_running_txn; uint32_t c_lsm_worker_threads; uint32_t c_merge_max; uint32_t c_mmap; + uint32_t c_modify_pct; uint32_t c_ops; - uint32_t c_quiet; uint32_t c_prefix_compression; uint32_t c_prefix_compression_min; + uint32_t c_quiet; + uint32_t c_read_pct; + uint32_t c_rebalance; uint32_t c_repeat_data_pct; uint32_t c_reverse; uint32_t c_rows; uint32_t c_runs; - uint32_t c_read_pct; - uint32_t c_rebalance; uint32_t c_salvage; uint32_t c_split_pct; uint32_t c_statistics; @@ -256,7 +259,7 @@ typedef struct { uint64_t deadlock; int id; /* simple thread ID */ - pthread_t tid; /* thread ID */ + wt_thread_t tid; /* thread ID */ int quit; /* thread should quit */ @@ -276,9 +279,9 @@ void bdb_remove(uint64_t, int *); void bdb_update(const void *, size_t, const void *, size_t); #endif -void *alter(void *); -void *backup(void *); -void *compact(void *); +WT_THREAD_RET alter(void *); +WT_THREAD_RET backup(void *); +WT_THREAD_RET compact(void *); void config_clear(void); void config_error(void); void config_file(const char *); @@ -290,7 +293,7 @@ void key_gen(WT_ITEM *, uint64_t); void key_gen_insert(WT_RAND_STATE *, WT_ITEM *, uint64_t); void key_gen_setup(WT_ITEM *); void key_len_setup(void); -void *lrt(void *); +WT_THREAD_RET lrt(void *); void path_setup(const char *); int read_row(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t); uint32_t rng(WT_RAND_STATE *); diff --git a/test/format/lrt.c b/test/format/lrt.c index 69d6b22d71f..b9622cdb635 100644 --- a/test/format/lrt.c +++ b/test/format/lrt.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -32,7 +32,7 @@ * lrt -- * Start a long-running transaction. */ -void * +WT_THREAD_RET lrt(void *arg) { WT_CONNECTION *conn; @@ -182,5 +182,5 @@ lrt(void *arg) free(value.mem); free(buf); - return (NULL); + return (WT_THREAD_RET_VALUE); } diff --git a/test/format/ops.c b/test/format/ops.c index 72e885bd0d6..a5e761d53a4 100644 --- a/test/format/ops.c +++ b/test/format/ops.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -29,14 +29,20 @@ #include "format.h" static int col_insert(TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t *); +static int col_modify( + TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t, bool); static int col_remove(WT_CURSOR *, WT_ITEM *, uint64_t, bool); +static int col_reserve(WT_CURSOR *, uint64_t, bool); static int col_update( TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t, bool); static int nextprev(WT_CURSOR *, int); -static void *ops(void *); +static WT_THREAD_RET ops(void *); static int row_insert( TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t, bool); +static int row_modify( + TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t, bool); static int row_remove(WT_CURSOR *, WT_ITEM *, uint64_t, bool); +static int row_reserve(WT_CURSOR *, WT_ITEM *, uint64_t, bool); static int row_update( TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t, bool); static void table_append_init(void); @@ -56,7 +62,7 @@ wts_ops(int lastrun) TINFO **tinfo_list, *tinfo, total; WT_CONNECTION *conn; WT_SESSION *session; - pthread_t alter_tid, backup_tid, compact_tid, lrt_tid; + wt_thread_t alter_tid, backup_tid, compact_tid, lrt_tid; int64_t fourths, thread_ops; uint32_t i; int running; @@ -115,7 +121,8 @@ wts_ops(int lastrun) tinfo_list[i] = tinfo = dcalloc(1, sizeof(TINFO)); tinfo->id = (int)i + 1; tinfo->state = TINFO_RUNNING; - testutil_check(pthread_create(&tinfo->tid, NULL, ops, tinfo)); + testutil_check( + __wt_thread_create(NULL, &tinfo->tid, ops, tinfo)); } /* @@ -123,14 +130,16 @@ wts_ops(int lastrun) * long-running reader threads. */ if (g.c_alter) - testutil_check(pthread_create(&alter_tid, NULL, alter, NULL)); + testutil_check( + __wt_thread_create(NULL, &alter_tid, alter, NULL)); if (g.c_backups) - testutil_check(pthread_create(&backup_tid, NULL, backup, NULL)); + testutil_check( + __wt_thread_create(NULL, &backup_tid, backup, NULL)); if (g.c_compact) testutil_check( - pthread_create(&compact_tid, NULL, compact, NULL)); + __wt_thread_create(NULL, &compact_tid, compact, NULL)); if (!SINGLETHREADED && g.c_long_running_txn) - testutil_check(pthread_create(&lrt_tid, NULL, lrt, NULL)); + testutil_check(__wt_thread_create(NULL, &lrt_tid, lrt, NULL)); /* Spin on the threads, calculating the totals. */ for (;;) { @@ -152,7 +161,8 @@ wts_ops(int lastrun) break; case TINFO_COMPLETE: tinfo->state = TINFO_JOINED; - (void)pthread_join(tinfo->tid, NULL); + testutil_check( + __wt_thread_join(NULL, tinfo->tid)); break; case TINFO_JOINED: break; @@ -190,13 +200,13 @@ wts_ops(int lastrun) /* Wait for the backup, compaction, long-running reader threads. */ g.workers_finished = 1; if (g.c_alter) - (void)pthread_join(alter_tid, NULL); + testutil_check(__wt_thread_join(NULL, alter_tid)); if (g.c_backups) - (void)pthread_join(backup_tid, NULL); + testutil_check(__wt_thread_join(NULL, backup_tid)); if (g.c_compact) - (void)pthread_join(compact_tid, NULL); + testutil_check(__wt_thread_join(NULL, compact_tid)); if (!SINGLETHREADED && g.c_long_running_txn) - (void)pthread_join(lrt_tid, NULL); + testutil_check(__wt_thread_join(NULL, lrt_tid)); g.workers_finished = 0; if (g.logging != 0) { @@ -398,10 +408,10 @@ snap_check(WT_CURSOR *cursor, * ops -- * Per-thread operations. */ -static void * +static WT_THREAD_RET ops(void *arg) { - enum { INSERT, READ, REMOVE, UPDATE } op; + enum { INSERT, MODIFY, READ, REMOVE, UPDATE } op; SNAP_OPS *snap, snap_list[64]; TINFO *tinfo; WT_CONNECTION *conn; @@ -608,11 +618,12 @@ skip_checkpoint: /* Pick the next checkpoint operation. */ op = REMOVE; else if (i < g.c_delete_pct + g.c_insert_pct) op = INSERT; - else if (i < - g.c_delete_pct + g.c_insert_pct + g.c_write_pct) + else if (i < g.c_delete_pct + + g.c_insert_pct + g.c_modify_pct) + op = MODIFY; + else if (i < g.c_delete_pct + + g.c_insert_pct + g.c_modify_pct + g.c_write_pct) op = UPDATE; - else - op = READ; } /* @@ -636,7 +647,7 @@ skip_checkpoint: /* Pick the next checkpoint operation. */ testutil_assert(ret == WT_NOTFOUND); } } -#if 0 + /* Optionally reserve a row. */ if (!readonly && intxn && mmrand(&tinfo->rnd, 0, 20) == 1) { switch (g.type) { @@ -659,7 +670,7 @@ skip_checkpoint: /* Pick the next checkpoint operation. */ testutil_assert(ret == WT_NOTFOUND); } } -#endif + /* Perform the operation. */ switch (op) { case INSERT: @@ -696,6 +707,30 @@ skip_checkpoint: /* Pick the next checkpoint operation. */ testutil_assert(ret == 0 || ret == WT_ROLLBACK); } break; + case MODIFY: + ++tinfo->update; + switch (g.type) { + case ROW: + ret = row_modify(tinfo, cursor, + key, value, keyno, positioned); + break; + case VAR: + ret = col_modify(tinfo, cursor, + key, value, keyno, positioned); + break; + } + if (ret == 0) { + positioned = true; + if (SNAP_TRACK) + snap_track(snap++, keyno, NULL, value); + } else { + positioned = false; + if (ret == WT_ROLLBACK && intxn) + goto deadlock; + testutil_assert(ret == 0 || + ret == WT_NOTFOUND || ret == WT_ROLLBACK); + } + break; case READ: ++tinfo->search; ret = read_row(cursor, key, value, keyno); @@ -740,17 +775,15 @@ skip_checkpoint: /* Pick the next checkpoint operation. */ case UPDATE: update_instead_of_insert: ++tinfo->update; - - /* Update the row. */ switch (g.type) { case ROW: - ret = row_update(tinfo, - cursor, key, value, keyno, positioned); + ret = row_update(tinfo, cursor, + key, value, keyno, positioned); break; case FIX: case VAR: - ret = col_update(tinfo, - cursor, key, value, keyno, positioned); + ret = col_update(tinfo, cursor, + key, value, keyno, positioned); break; } if (ret == 0) { @@ -835,7 +868,7 @@ deadlock: ++tinfo->deadlock; free(value->mem); tinfo->state = TINFO_COMPLETE; - return (NULL); + return (WT_THREAD_RET_VALUE); } /* @@ -1103,7 +1136,6 @@ nextprev(WT_CURSOR *cursor, int next) return (ret); } -#if 0 /* * row_reserve -- * Reserve a row in a row-store file. @@ -1166,7 +1198,235 @@ col_reserve(WT_CURSOR *cursor, uint64_t keyno, bool positioned) } return (0); } + +/* + * modify_build -- + * Generate a set of modify vectors, and copy what the final result + * should be into the value buffer. + */ +static bool +modify_build(TINFO *tinfo, + WT_CURSOR *cursor, WT_MODIFY *entries, int *nentriesp, WT_ITEM *value) +{ + static char repl[64]; + size_t len, size; + u_int i, nentries; + WT_ITEM *ta, _ta, *tb, _tb, *tmp; + + if (repl[0] == '\0') + memset(repl, '+', sizeof(repl)); + + ta = &_ta; + memset(ta, 0, sizeof(*ta)); + tb = &_tb; + memset(tb, 0, sizeof(*tb)); + + testutil_check(cursor->get_value(cursor, value)); + + /* + * Randomly select a number of byte changes, offsets and lengths. Start + * at least 11 bytes in so we skip the leading key information. + */ + nentries = mmrand(&tinfo->rnd, 1, MAX_MODIFY_ENTRIES); + for (i = 0; i < nentries; ++i) { + entries[i].data.data = repl; + entries[i].data.size = (size_t)mmrand(&tinfo->rnd, 0, 10); + entries[i].offset = (size_t)mmrand(&tinfo->rnd, 20, 40); + entries[i].size = (size_t)mmrand(&tinfo->rnd, 0, 10); + } + + /* + * Process the entries to figure out how large a buffer we need. This is + * a bit pessimistic because we're ignoring replacement bytes, but it's + * a simpler calculation. + */ + for (size = cursor->value.size, i = 0; i < nentries; ++i) { + if (entries[i].offset >= size) + size = entries[i].offset; + size += entries[i].data.size; + } + + /* If size is larger than the available buffer size, skip this one. */ + if (size >= value->memsize) + return (false); + + /* Allocate a pair of buffers. */ + ta->mem = dcalloc(size, sizeof(uint8_t)); + tb->mem = dcalloc(size, sizeof(uint8_t)); + + /* + * Use a brute-force process to create the value WiredTiger will create + * from this change vector. Don't do anything tricky to speed it up, we + * want to use a different algorithm from WiredTiger's, the idea is to + * bug-check the library. + */ + memcpy(ta->mem, value->data, value->size); + ta->size = value->size; + for (i = 0; i < nentries; ++i) { + /* Take leading bytes from the original, plus any gap bytes. */ + if (entries[i].offset >= ta->size) { + memcpy(tb->mem, ta->mem, ta->size); + if (entries[i].offset > ta->size) + memset((uint8_t *)tb->mem + ta->size, + '\0', entries[i].offset - ta->size); + } else + if (entries[i].offset > 0) + memcpy(tb->mem, ta->mem, entries[i].offset); + tb->size = entries[i].offset; + + /* Take replacement bytes. */ + if (entries[i].data.size > 0) { + memcpy((uint8_t *)tb->mem + tb->size, + entries[i].data.data, entries[i].data.size); + tb->size += entries[i].data.size; + } + + /* Take trailing bytes from the original. */ + len = entries[i].offset + entries[i].size; + if (ta->size > len) { + memcpy((uint8_t *)tb->mem + tb->size, + (uint8_t *)ta->mem + len, ta->size - len); + tb->size += ta->size - len; + } + testutil_assert(tb->size <= size); + + tmp = ta; + ta = tb; + tb = tmp; + } + + /* Copy the expected result into the value structure. */ + memcpy(value->mem, ta->mem, ta->size); + value->data = value->mem; + value->size = ta->size; + + free(ta->mem); + free(tb->mem); + + *nentriesp = (int)nentries; + return (true); +} + +/* + * row_modify -- + * Modify a row in a row-store file. + */ +static int +row_modify(TINFO *tinfo, WT_CURSOR *cursor, + WT_ITEM *key, WT_ITEM *value, uint64_t keyno, bool positioned) +{ + WT_DECL_RET; + WT_MODIFY entries[MAX_MODIFY_ENTRIES]; + int nentries; + + if (!positioned) { + key_gen(key, keyno); + cursor->set_key(cursor, key); + switch (ret = cursor->search(cursor)) { + case 0: + break; + case WT_CACHE_FULL: + case WT_ROLLBACK: + return (WT_ROLLBACK); + case WT_NOTFOUND: + return (WT_NOTFOUND); + default: + testutil_die(ret, + "row_modify: read row %" PRIu64 " by key", keyno); + } + } + + /* + * Generate a set of change vectors and copy the expected result into + * the value buffer. If the return value is non-zero, there wasn't a + * big enough value to work with, or for some reason we couldn't build + * a reasonable change vector. + */ + ret = WT_NOTFOUND; + if (modify_build(tinfo, cursor, entries, &nentries, value)) + ret = cursor->modify(cursor, entries, nentries); + switch (ret) { + case 0: + break; + case WT_CACHE_FULL: + case WT_ROLLBACK: + return (WT_ROLLBACK); + case WT_NOTFOUND: + return (WT_NOTFOUND); + default: + testutil_die(ret, + "row_modify: modify row %" PRIu64 " by key", keyno); + } + +#ifdef HAVE_BERKELEY_DB + if (!SINGLETHREADED) + return (0); + + bdb_update(key->data, key->size, value->data, value->size); +#endif + return (0); +} + +/* + * col_modify -- + * Modify a row in a column-store file. + */ +static int +col_modify(TINFO *tinfo, WT_CURSOR *cursor, + WT_ITEM *key, WT_ITEM *value, uint64_t keyno, bool positioned) +{ + WT_DECL_RET; + WT_MODIFY entries[MAX_MODIFY_ENTRIES]; + int nentries; + + if (!positioned) { + cursor->set_key(cursor, keyno); + switch (ret = cursor->search(cursor)) { + case 0: + break; + case WT_CACHE_FULL: + case WT_ROLLBACK: + return (WT_ROLLBACK); + case WT_NOTFOUND: + return (WT_NOTFOUND); + default: + testutil_die(ret, + "col_modify: read row %" PRIu64, keyno); + } + } + + /* + * Generate a set of change vectors and copy the expected result into + * the value buffer. If the return value is non-zero, there wasn't a + * big enough value to work with, or for some reason we couldn't build + * a reasonable change vector. + */ + ret = WT_NOTFOUND; + if (modify_build(tinfo, cursor, entries, &nentries, value)) + ret = cursor->modify(cursor, entries, nentries); + switch (ret) { + case 0: + break; + case WT_CACHE_FULL: + case WT_ROLLBACK: + return (WT_ROLLBACK); + case WT_NOTFOUND: + return (WT_NOTFOUND); + default: + testutil_die(ret, "col_modify: modify row %" PRIu64, keyno); + } + +#ifdef HAVE_BERKELEY_DB + if (!SINGLETHREADED) + return (0); + + key_gen(key, keyno); + bdb_update(key->data, key->size, value->data, value->size); +#else + (void)key; /* [-Wunused-variable] */ #endif + return (0); +} /* * row_update -- diff --git a/test/format/rebalance.c b/test/format/rebalance.c index e35c62e7255..195130cfa68 100644 --- a/test/format/rebalance.c +++ b/test/format/rebalance.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/format/salvage.c b/test/format/salvage.c index f82dc34dd5f..a7ac01eff15 100644 --- a/test/format/salvage.c +++ b/test/format/salvage.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/format/t.c b/test/format/t.c index c6686ae8b91..0cfe4e40421 100644 --- a/test/format/t.c +++ b/test/format/t.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/format/util.c b/test/format/util.c index 983d03e2525..f09bb160893 100644 --- a/test/format/util.c +++ b/test/format/util.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -472,7 +472,7 @@ fclose_and_clear(FILE **fpp) * alter -- * Periodically alter a table's metadata. */ -void * +WT_THREAD_RET alter(void *arg) { WT_CONNECTION *conn; @@ -510,5 +510,5 @@ alter(void *arg) } testutil_check(session->close(session, NULL)); - return (NULL); + return (WT_THREAD_RET_VALUE); } diff --git a/test/format/wts.c b/test/format/wts.c index 6aa4784d1c1..673b65794f5 100644 --- a/test/format/wts.c +++ b/test/format/wts.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -35,28 +35,40 @@ static const char * compressor(uint32_t compress_flag) { + const char *p; + + p = "unrecognized compressor flag"; switch (compress_flag) { case COMPRESS_NONE: - return ("none"); + p ="none"; + break; case COMPRESS_LZ4: - return ("lz4"); + p ="lz4"; + break; case COMPRESS_LZ4_NO_RAW: - return ("lz4-noraw"); + p ="lz4-noraw"; + break; case COMPRESS_LZO: - return ("LZO1B-6"); + p ="LZO1B-6"; + break; case COMPRESS_SNAPPY: - return ("snappy"); + p ="snappy"; + break; case COMPRESS_ZLIB: - return ("zlib"); + p ="zlib"; + break; case COMPRESS_ZLIB_NO_RAW: - return ("zlib-noraw"); + p ="zlib-noraw"; + break; case COMPRESS_ZSTD: - return ("zstd"); - default: + p ="zstd"; break; + default: + testutil_die(EINVAL, + "illegal compression flag: %#" PRIx32, compress_flag); + /* NOTREACHED */ } - testutil_die(EINVAL, - "illegal compression flag: %#" PRIx32, compress_flag); + return (p); } /* @@ -66,16 +78,22 @@ compressor(uint32_t compress_flag) static const char * encryptor(uint32_t encrypt_flag) { + const char *p; + + p = "unrecognized encryptor flag"; switch (encrypt_flag) { case ENCRYPT_NONE: - return ("none"); + p = "none"; + break; case ENCRYPT_ROTN_7: - return ("rotn,keyid=7"); - default: + p = "rotn,keyid=7"; break; + default: + testutil_die(EINVAL, + "illegal encryption flag: %#" PRIx32, encrypt_flag); + /* NOTREACHED */ } - testutil_die(EINVAL, - "illegal encryption flag: %#" PRIx32, encrypt_flag); + return (p); } static int @@ -276,8 +294,8 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) if ((ret = conn->load_extension( conn, HELIUM_PATH, helium_config)) != 0) testutil_die(ret, - "WT_CONNECTION.load_extension: %s:%s", - HELIUM_PATH, helium_config); + "WT_CONNECTION.load_extension: %s:%s", + HELIUM_PATH, helium_config); } *connp = conn; } diff --git a/test/huge/huge.c b/test/huge/huge.c index 2b0d5f498e3..a1fd45711a2 100644 --- a/test/huge/huge.c +++ b/test/huge/huge.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/java/com/wiredtiger/test/AsyncTest.java b/test/java/com/wiredtiger/test/AsyncTest.java index fc28e669313..11d98fb3b4e 100644 --- a/test/java/com/wiredtiger/test/AsyncTest.java +++ b/test/java/com/wiredtiger/test/AsyncTest.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/java/com/wiredtiger/test/AutoCloseTest.java b/test/java/com/wiredtiger/test/AutoCloseTest.java index d7304bb8a44..e4f720ede8b 100644 --- a/test/java/com/wiredtiger/test/AutoCloseTest.java +++ b/test/java/com/wiredtiger/test/AutoCloseTest.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/java/com/wiredtiger/test/BackupCursorTest.java b/test/java/com/wiredtiger/test/BackupCursorTest.java index dd25e4df7d6..af0a2784589 100644 --- a/test/java/com/wiredtiger/test/BackupCursorTest.java +++ b/test/java/com/wiredtiger/test/BackupCursorTest.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/java/com/wiredtiger/test/ConcurrentCloseTest.java b/test/java/com/wiredtiger/test/ConcurrentCloseTest.java index fead0b0bf38..3759057ef1f 100644 --- a/test/java/com/wiredtiger/test/ConcurrentCloseTest.java +++ b/test/java/com/wiredtiger/test/ConcurrentCloseTest.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/java/com/wiredtiger/test/ConfigTest.java b/test/java/com/wiredtiger/test/ConfigTest.java index 2afde7df2dc..432aa245afa 100644 --- a/test/java/com/wiredtiger/test/ConfigTest.java +++ b/test/java/com/wiredtiger/test/ConfigTest.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/java/com/wiredtiger/test/CursorTest.java b/test/java/com/wiredtiger/test/CursorTest.java index 4cd244e5b10..28c92dd8a8d 100644 --- a/test/java/com/wiredtiger/test/CursorTest.java +++ b/test/java/com/wiredtiger/test/CursorTest.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/java/com/wiredtiger/test/CursorTest02.java b/test/java/com/wiredtiger/test/CursorTest02.java index f107bf0b8f2..10705997352 100644 --- a/test/java/com/wiredtiger/test/CursorTest02.java +++ b/test/java/com/wiredtiger/test/CursorTest02.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/java/com/wiredtiger/test/CursorTest03.java b/test/java/com/wiredtiger/test/CursorTest03.java index 64f33f4d7b6..73c7a22f69d 100644 --- a/test/java/com/wiredtiger/test/CursorTest03.java +++ b/test/java/com/wiredtiger/test/CursorTest03.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/java/com/wiredtiger/test/ExceptionTest.java b/test/java/com/wiredtiger/test/ExceptionTest.java index 0c71ea4371b..99719225b05 100644 --- a/test/java/com/wiredtiger/test/ExceptionTest.java +++ b/test/java/com/wiredtiger/test/ExceptionTest.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/java/com/wiredtiger/test/PackTest.java b/test/java/com/wiredtiger/test/PackTest.java index f24ca6e2def..302313169cd 100644 --- a/test/java/com/wiredtiger/test/PackTest.java +++ b/test/java/com/wiredtiger/test/PackTest.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -184,6 +184,47 @@ public class PackTest { } @Test + public void pack08() + throws WiredTigerPackingException { + String format = "u"; + PackOutputStream packer = new PackOutputStream(format); + PackInputStream unpacker; + byte[] b0 = {}; + byte[] b1 = { 0x00 }; + byte[] packed; + + packer.addByteArray(b0); + packed = packer.getValue(); + unpacker = new PackInputStream(format, packed); + Assert.assertTrue(java.util.Arrays.equals( + unpacker.getByteArray(), b0)); + + packer = new PackOutputStream(format); + packer.addByteArray(b1); + packed = packer.getValue(); + unpacker = new PackInputStream(format, packed); + Assert.assertTrue(java.util.Arrays.equals( + unpacker.getByteArray(), b1)); + + format = "uu"; + for (int i = 0; i < 2; i++) { + byte[] arg0 = (i == 0 ? b0 : b1); + for (int j = 0; j < 2; j++) { + byte[] arg1 = (j == 0 ? b0 : b1); + packer = new PackOutputStream(format); + packer.addByteArray(arg0); + packer.addByteArray(arg1); + packed = packer.getValue(); + unpacker = new PackInputStream(format, packed); + Assert.assertTrue(java.util.Arrays.equals( + unpacker.getByteArray(), arg0)); + Assert.assertTrue(java.util.Arrays.equals( + unpacker.getByteArray(), arg1)); + } + } + } + + @Test public void packUnpackNumber01() throws WiredTigerPackingException { // Verify that we can pack and unpack single signed longs. diff --git a/test/java/com/wiredtiger/test/PackTest02.java b/test/java/com/wiredtiger/test/PackTest02.java index 847e3c4ab08..517afd1ec03 100644 --- a/test/java/com/wiredtiger/test/PackTest02.java +++ b/test/java/com/wiredtiger/test/PackTest02.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/java/com/wiredtiger/test/PackTest03.java b/test/java/com/wiredtiger/test/PackTest03.java index c3ae854dcaf..81e7987f987 100644 --- a/test/java/com/wiredtiger/test/PackTest03.java +++ b/test/java/com/wiredtiger/test/PackTest03.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/java/com/wiredtiger/test/WiredTigerSuite.java b/test/java/com/wiredtiger/test/WiredTigerSuite.java index 9322d30671a..c77ff5b3507 100644 --- a/test/java/com/wiredtiger/test/WiredTigerSuite.java +++ b/test/java/com/wiredtiger/test/WiredTigerSuite.java @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 42020d6ce9a..72ad6228006 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/mciproject.yml b/test/mciproject.yml index 6456475aa00..72022fe46ec 100644 --- a/test/mciproject.yml +++ b/test/mciproject.yml @@ -95,6 +95,16 @@ tasks: script: | set -o errexit set -o verbose + + # On 10.12, change the binary location with install_name_tool since DYLD_LIBRARY_PATH + # appears not to work for dynamic modules loaded by python. For wt, the libtool generated + # script has the wrong path for running on test machines. + if [ "$(uname -s)" == "Darwin" ]; then + WT_VERSION=$(m4 build_posix/aclocal/version.m4) + install_name_tool -change /usr/local/lib/libwiredtiger-$WT_VERSION.dylib $(pwd)/.libs/libwiredtiger-$WT_VERSION.dylib lang/python/_wiredtiger.so + install_name_tool -change /usr/local/lib/libwiredtiger-$WT_VERSION.dylib $(pwd)/.libs/libwiredtiger-$WT_VERSION.dylib .libs/wt + fi + ${test_env_vars|} python ./test/suite/run.py -v 2 ${smp_command|} 2>&1 - name: compile-windows-alt @@ -182,10 +192,10 @@ buildvariants: #- name: format - Enable when we have a solution for hangs and crashses - name: fops -- name: osx-1010 - display_name: OS X 10.10 +- name: macos-1012 + display_name: OS X 10.12 run_on: - - osx-1010 + - macos-1012 expansions: smp_command: -j $(sysctl -n hw.logicalcpu) configure_env_vars: PATH=/opt/local/bin:$PATH @@ -195,3 +205,4 @@ buildvariants: - name: compile - name: unit-test - name: fops + diff --git a/test/packing/intpack-test.c b/test/packing/intpack-test.c index c84823b741b..e7822015091 100644 --- a/test/packing/intpack-test.c +++ b/test/packing/intpack-test.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/packing/intpack-test2.c b/test/packing/intpack-test2.c index 4e612808a35..e216899cebb 100644 --- a/test/packing/intpack-test2.c +++ b/test/packing/intpack-test2.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/packing/intpack-test3.c b/test/packing/intpack-test3.c index 763b0255ecf..00fc80e24a2 100644 --- a/test/packing/intpack-test3.c +++ b/test/packing/intpack-test3.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/packing/packing-test.c b/test/packing/packing-test.c index 919b0622806..bd48ac7125c 100644 --- a/test/packing/packing-test.c +++ b/test/packing/packing-test.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/readonly/readonly.c b/test/readonly/readonly.c index 66c7a0ca692..6f1c34a1fc6 100644 --- a/test/readonly/readonly.c +++ b/test/readonly/readonly.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/recovery/random-abort.c b/test/recovery/random-abort.c index febe6530534..7e76f61bd12 100644 --- a/test/recovery/random-abort.c +++ b/test/recovery/random-abort.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -47,9 +47,9 @@ static bool inmem; #define RECORDS_FILE "records-%" PRIu32 #define ENV_CONFIG_DEF \ - "create,log=(file_max=10M,archive=false,enabled)" + "create,log=(file_max=10M,enabled)" #define ENV_CONFIG_TXNSYNC \ - "create,log=(file_max=10M,archive=false,enabled)," \ + "create,log=(file_max=10M,enabled)," \ "transaction_sync=(enabled,method=none)" #define ENV_CONFIG_REC "log=(recover=on)" #define MAX_VAL 4096 @@ -69,7 +69,7 @@ typedef struct { uint32_t id; } WT_THREAD_DATA; -static void * +static WT_THREAD_RET thread_run(void *arg) { FILE *fp; @@ -161,15 +161,15 @@ static void fill_db(uint32_t) static void fill_db(uint32_t nth) { - pthread_t *thr; WT_CONNECTION *conn; WT_SESSION *session; WT_THREAD_DATA *td; + wt_thread_t *thr; uint32_t i; int ret; const char *envconf; - thr = dcalloc(nth, sizeof(pthread_t)); + thr = dcalloc(nth, sizeof(*thr)); td = dcalloc(nth, sizeof(WT_THREAD_DATA)); if (chdir(home) != 0) testutil_die(errno, "Child chdir: %s", home); @@ -192,9 +192,8 @@ fill_db(uint32_t nth) td[i].conn = conn; td[i].start = (UINT64_MAX / nth) * i; td[i].id = i; - if ((ret = pthread_create( - &thr[i], NULL, thread_run, &td[i])) != 0) - testutil_die(ret, "pthread_create"); + testutil_check(__wt_thread_create( + NULL, &thr[i], thread_run, &td[i])); } printf("Spawned %" PRIu32 " writer threads\n", nth); fflush(stdout); @@ -203,7 +202,7 @@ fill_db(uint32_t nth) * it is killed. */ for (i = 0; i < nth; ++i) - testutil_assert(pthread_join(thr[i], NULL) == 0); + testutil_check(__wt_thread_join(NULL, thr[i])); /* * NOTREACHED */ diff --git a/test/recovery/truncated-log.c b/test/recovery/truncated-log.c index a127d8c1c63..c9d73e0cf48 100644 --- a/test/recovery/truncated-log.c +++ b/test/recovery/truncated-log.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -130,7 +130,7 @@ usage(void) /* * Child process creates the database and table, and then writes data into - * the table until it is killed by the parent. + * the table until it switches into log file 2. */ static void fill_db(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); @@ -246,7 +246,7 @@ fill_db(void) } if (fclose(fp) != 0) testutil_die(errno, "fclose"); - abort(); + exit(0); /* NOTREACHED */ } @@ -286,9 +286,7 @@ main(int argc, char *argv[]) testutil_make_work_dir(home); /* - * Fork a child to insert as many items. We will then randomly - * kill the child, run recovery and make sure all items we wrote - * exist after recovery runs. + * Fork a child to do its work. Wait for it to exit. */ if ((pid = fork()) < 0) testutil_die(errno, "fork"); diff --git a/test/salvage/salvage.c b/test/salvage/salvage.c index 83f9c6349bc..c19a529bcb8 100644 --- a/test/salvage/salvage.c +++ b/test/salvage/salvage.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -522,7 +522,7 @@ build(int ikey, int ivalue, int cnt) break; case WT_PAGE_ROW_LEAF: testutil_check(__wt_snprintf( - kbuf, sizeof(kbuf), "%010d KEY------", ikey)); + kbuf, sizeof(kbuf), "%010d KEY------", ikey)); key.data = kbuf; key.size = 20; cursor->set_key(cursor, &key); diff --git a/test/suite/helper.py b/test/suite/helper.py index d1f41f05e8b..2f9bbf8aa68 100644 --- a/test/suite/helper.py +++ b/test/suite/helper.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/run.py b/test/suite/run.py index 97c58bfdccf..8a936de584b 100644 --- a/test/suite/run.py +++ b/test/suite/run.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/suite_random.py b/test/suite/suite_random.py index fd580cec43b..16a8b89113c 100644 --- a/test/suite/suite_random.py +++ b/test/suite/suite_random.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/suite_subprocess.py b/test/suite/suite_subprocess.py index c56c8d8e933..86134db5f88 100644 --- a/test/suite/suite_subprocess.py +++ b/test/suite/suite_subprocess.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_alter01.py b/test/suite/test_alter01.py index dfdf6b7a17e..7a143afb32c 100644 --- a/test/suite/test_alter01.py +++ b/test/suite/test_alter01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_async01.py b/test/suite/test_async01.py index 158c16a9381..4faaad6b8f4 100644 --- a/test/suite/test_async01.py +++ b/test/suite/test_async01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_async02.py b/test/suite/test_async02.py index 28435fe85b2..fbd743fec29 100644 --- a/test/suite/test_async02.py +++ b/test/suite/test_async02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_async03.py b/test/suite/test_async03.py index 4859360924a..cf993071d73 100644 --- a/test/suite/test_async03.py +++ b/test/suite/test_async03.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_autoclose.py b/test/suite/test_autoclose.py index c5633d5a21e..ce152b24fe3 100644 --- a/test/suite/test_autoclose.py +++ b/test/suite/test_autoclose.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_backup01.py b/test/suite/test_backup01.py index 4e98b6d8e77..52d71ab53bb 100644 --- a/test/suite/test_backup01.py +++ b/test/suite/test_backup01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_backup02.py b/test/suite/test_backup02.py index d4089273be0..7d8f653feae 100644 --- a/test/suite/test_backup02.py +++ b/test/suite/test_backup02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_backup03.py b/test/suite/test_backup03.py index c1ed3cc9e1a..7d0bfd5eaaf 100644 --- a/test/suite/test_backup03.py +++ b/test/suite/test_backup03.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_backup04.py b/test/suite/test_backup04.py index be52a5e1e97..9f40ae2427b 100644 --- a/test/suite/test_backup04.py +++ b/test/suite/test_backup04.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_backup05.py b/test/suite/test_backup05.py index 4ecb782a0d5..fb44de04694 100644 --- a/test/suite/test_backup05.py +++ b/test/suite/test_backup05.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_backup06.py b/test/suite/test_backup06.py index 9f7a247f2b9..d416ba035b5 100644 --- a/test/suite/test_backup06.py +++ b/test/suite/test_backup06.py @@ -1,8 +1,8 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. - +# # This is free and unencumbered software released into the public domain. # # Anyone is free to copy, modify, publish, use, compile, sell, or diff --git a/test/suite/test_base01.py b/test/suite/test_base01.py index 2a5f96cbae2..f39ec3eb739 100644 --- a/test/suite/test_base01.py +++ b/test/suite/test_base01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_base02.py b/test/suite/test_base02.py index 2b51fe1b530..5e1140a5700 100644 --- a/test/suite/test_base02.py +++ b/test/suite/test_base02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_base03.py b/test/suite/test_base03.py index fe6fa53c288..ad1629db77e 100644 --- a/test/suite/test_base03.py +++ b/test/suite/test_base03.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_base04.py b/test/suite/test_base04.py index 973ee1327a5..f9fdddce157 100644 --- a/test/suite/test_base04.py +++ b/test/suite/test_base04.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_base05.py b/test/suite/test_base05.py index 4bee0efcfe2..5ba6d5eda4b 100644 --- a/test/suite/test_base05.py +++ b/test/suite/test_base05.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_baseconfig.py b/test/suite/test_baseconfig.py index 89b3b29544e..3a5778b3bb5 100644 --- a/test/suite/test_baseconfig.py +++ b/test/suite/test_baseconfig.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_bug001.py b/test/suite/test_bug001.py index 4c4a722285c..4353dad5e68 100644 --- a/test/suite/test_bug001.py +++ b/test/suite/test_bug001.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_bug003.py b/test/suite/test_bug003.py index 73d9cd13ab9..799c004e17d 100644 --- a/test/suite/test_bug003.py +++ b/test/suite/test_bug003.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_bug004.py b/test/suite/test_bug004.py index 464cc57e272..a47bdc6dd1e 100644 --- a/test/suite/test_bug004.py +++ b/test/suite/test_bug004.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_bug005.py b/test/suite/test_bug005.py index 69df175ae67..6d099bf2708 100644 --- a/test/suite/test_bug005.py +++ b/test/suite/test_bug005.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_bug006.py b/test/suite/test_bug006.py index c0f6055f720..505325de200 100644 --- a/test/suite/test_bug006.py +++ b/test/suite/test_bug006.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_bug007.py b/test/suite/test_bug007.py index 16cb5da903c..806d75d8394 100644 --- a/test/suite/test_bug007.py +++ b/test/suite/test_bug007.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_bug008.py b/test/suite/test_bug008.py index c54c92fc864..cb0bb390ad4 100644 --- a/test/suite/test_bug008.py +++ b/test/suite/test_bug008.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_bug009.py b/test/suite/test_bug009.py index 2bdfb7dec52..7f2af55f2d0 100644 --- a/test/suite/test_bug009.py +++ b/test/suite/test_bug009.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_bug010.py b/test/suite/test_bug010.py index 89f21e1da04..dfe317bf94e 100644 --- a/test/suite/test_bug010.py +++ b/test/suite/test_bug010.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_bug011.py b/test/suite/test_bug011.py index 5e0721b93f1..2c3fd831f93 100644 --- a/test/suite/test_bug011.py +++ b/test/suite/test_bug011.py @@ -1,6 +1,6 @@ -#!usr/bin/env python +#!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_bug012.py b/test/suite/test_bug012.py index 91f49d14b3f..ae80a9c7179 100644 --- a/test/suite/test_bug012.py +++ b/test/suite/test_bug012.py @@ -1,6 +1,6 @@ -#!usr/bin/env python +#!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_bug013.py b/test/suite/test_bug013.py index a42809aea5f..a15bd42c9da 100644 --- a/test/suite/test_bug013.py +++ b/test/suite/test_bug013.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_bug014.py b/test/suite/test_bug014.py index 1dee933e839..81e47bc331b 100644 --- a/test/suite/test_bug014.py +++ b/test/suite/test_bug014.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_bug015.py b/test/suite/test_bug015.py index 68cca49688f..5b2a64dc76e 100644 --- a/test/suite/test_bug015.py +++ b/test/suite/test_bug015.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_bug016.py b/test/suite/test_bug016.py index 4b8867e1e93..a2a40118008 100644 --- a/test/suite/test_bug016.py +++ b/test/suite/test_bug016.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_bug017.py b/test/suite/test_bug017.py index 03e7b2ba714..43aeee07bb6 100644 --- a/test/suite/test_bug017.py +++ b/test/suite/test_bug017.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_bulk01.py b/test/suite/test_bulk01.py index 8d5b6a04385..da399faba2d 100644 --- a/test/suite/test_bulk01.py +++ b/test/suite/test_bulk01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_bulk02.py b/test/suite/test_bulk02.py index fb9240e91e7..de9ebec5204 100644 --- a/test/suite/test_bulk02.py +++ b/test/suite/test_bulk02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_checkpoint01.py b/test/suite/test_checkpoint01.py index c0d004db78d..1964a94b31c 100644 --- a/test/suite/test_checkpoint01.py +++ b/test/suite/test_checkpoint01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_checkpoint02.py b/test/suite/test_checkpoint02.py index b5d20fb73b1..3a0a47d8163 100644 --- a/test/suite/test_checkpoint02.py +++ b/test/suite/test_checkpoint02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_colgap.py b/test/suite/test_colgap.py index 91df0fd6c1c..01e52ea1da5 100644 --- a/test/suite/test_colgap.py +++ b/test/suite/test_colgap.py @@ -1,6 +1,6 @@ -#!usr/bin/env python +#!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_collator.py b/test/suite/test_collator.py index 7ce135c8976..320c4e7d7b4 100644 --- a/test/suite/test_collator.py +++ b/test/suite/test_collator.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_compact01.py b/test/suite/test_compact01.py index 56ab6d39076..cfe5c909b1f 100644 --- a/test/suite/test_compact01.py +++ b/test/suite/test_compact01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_compact02.py b/test/suite/test_compact02.py index 803600eea14..eb1eb641191 100644 --- a/test/suite/test_compact02.py +++ b/test/suite/test_compact02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_compress01.py b/test/suite/test_compress01.py index ef1064d294e..1190a9dbe00 100644 --- a/test/suite/test_compress01.py +++ b/test/suite/test_compress01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_config01.py b/test/suite/test_config01.py index cbcb6835525..5252d805e07 100644 --- a/test/suite/test_config01.py +++ b/test/suite/test_config01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_config02.py b/test/suite/test_config02.py index 112a93ef2e0..441aa41d218 100644 --- a/test/suite/test_config02.py +++ b/test/suite/test_config02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_config03.py b/test/suite/test_config03.py index 89038d71319..810d399613a 100644 --- a/test/suite/test_config03.py +++ b/test/suite/test_config03.py @@ -1,6 +1,6 @@ -#!usr/bin/env python +#!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_config04.py b/test/suite/test_config04.py index db8a5f4a16a..b09189be8ea 100644 --- a/test/suite/test_config04.py +++ b/test/suite/test_config04.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_config05.py b/test/suite/test_config05.py index bee63d48da6..5960f01dc8e 100644 --- a/test/suite/test_config05.py +++ b/test/suite/test_config05.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_config06.py b/test/suite/test_config06.py index 55619e8774c..f39fe2d3a4f 100644 --- a/test/suite/test_config06.py +++ b/test/suite/test_config06.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_cursor01.py b/test/suite/test_cursor01.py index 8c66042eec0..99bdb6182c7 100644 --- a/test/suite/test_cursor01.py +++ b/test/suite/test_cursor01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. @@ -99,6 +99,7 @@ class test_cursor01(wttest.WiredTigerTestCase): self.pr('creating cursor') cursor = self.session.open_cursor(tablearg, None, None) self.assertCursorHasNoKeyValue(cursor) + self.assertEqual(cursor.uri, tablearg) for i in range(0, self.nentries): cursor[self.genkey(i)] = self.genvalue(i) diff --git a/test/suite/test_cursor02.py b/test/suite/test_cursor02.py index 0771a275cd2..35dc2587b1f 100644 --- a/test/suite/test_cursor02.py +++ b/test/suite/test_cursor02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_cursor03.py b/test/suite/test_cursor03.py index b4598483c12..8910dc741a4 100644 --- a/test/suite/test_cursor03.py +++ b/test/suite/test_cursor03.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_cursor04.py b/test/suite/test_cursor04.py index 8cbf922b5eb..b7457ec623d 100644 --- a/test/suite/test_cursor04.py +++ b/test/suite/test_cursor04.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_cursor05.py b/test/suite/test_cursor05.py index 4c276f06ff4..e0cce3dcb5e 100644 --- a/test/suite/test_cursor05.py +++ b/test/suite/test_cursor05.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_cursor06.py b/test/suite/test_cursor06.py index 117e29b0605..280d6f09171 100644 --- a/test/suite/test_cursor06.py +++ b/test/suite/test_cursor06.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_cursor07.py b/test/suite/test_cursor07.py index 19db718fd11..a31d0d401e0 100644 --- a/test/suite/test_cursor07.py +++ b/test/suite/test_cursor07.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_cursor08.py b/test/suite/test_cursor08.py index cc76f528aa9..82b4a3b7c7c 100644 --- a/test/suite/test_cursor08.py +++ b/test/suite/test_cursor08.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_cursor09.py b/test/suite/test_cursor09.py index 9a1fc06b617..de9ae5163b6 100644 --- a/test/suite/test_cursor09.py +++ b/test/suite/test_cursor09.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_cursor10.py b/test/suite/test_cursor10.py index 6cabfde9f1f..11fb43825ad 100644 --- a/test/suite/test_cursor10.py +++ b/test/suite/test_cursor10.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_cursor11.py b/test/suite/test_cursor11.py index e159ec499e6..1f3ea1555f2 100644 --- a/test/suite/test_cursor11.py +++ b/test/suite/test_cursor11.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_cursor12.py b/test/suite/test_cursor12.py new file mode 100644 index 00000000000..827f37cfcef --- /dev/null +++ b/test/suite/test_cursor12.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_cursor12.py +# Test cursor modify call +class test_cursor12(wttest.WiredTigerTestCase): + types = [ + ('file', dict(uri='file:modify')), + ('lsm', dict(uri='lsm:modify')), + ('table', dict(uri='table:modify')), + ] + scenarios = make_scenarios(types) + + # Smoke-test the modify API. + def test_modify_smoke(self): + # List with original value, final value, and modifications to get + # there. + list = [ + { + 'o' : 'ABCDEFGH', # no operation + 'f' : 'ABCDEFGH', + 'mods' : [['', 0, 0]] + },{ + 'o' : 'ABCDEFGH', # no operation with offset + 'f' : 'ABCDEFGH', + 'mods' : [['', 4, 0]] + },{ + 'o' : 'ABCDEFGH', # rewrite beginning + 'f' : '--CDEFGH', + 'mods' : [['--', 0, 2]] + },{ + 'o' : 'ABCDEFGH', # rewrite end + 'f' : 'ABCDEF--', + 'mods' : [['--', 6, 2]] + },{ + 'o' : 'ABCDEFGH', # append + 'f' : 'ABCDEFGH--', + 'mods' : [['--', 8, 2]] + },{ + 'o' : 'ABCDEFGH', # append with gap + 'f' : 'ABCDEFGH\00\00--', + 'mods' : [['--', 10, 2]] + },{ + 'o' : 'ABCDEFGH', # multiple replacements + 'f' : 'A-C-E-G-', + 'mods' : [['-', 1, 1], ['-', 3, 1], ['-', 5, 1], ['-', 7, 1]] + },{ + 'o' : 'ABCDEFGH', # multiple overlapping replacements + 'f' : 'A-CDEFGH', + 'mods' : [['+', 1, 1], ['+', 1, 1], ['+', 1, 1], ['-', 1, 1]] + },{ + 'o' : 'ABCDEFGH', # multiple overlapping gap replacements + 'f' : 'ABCDEFGH\00\00--', + 'mods' : [['+', 10, 1], ['+', 10, 1], ['+', 10, 1], ['--', 10, 2]] + },{ + 'o' : 'ABCDEFGH', # shrink beginning + 'f' : '--EFGH', + 'mods' : [['--', 0, 4]] + },{ + 'o' : 'ABCDEFGH', # shrink middle + 'f' : 'AB--GH', + 'mods' : [['--', 2, 4]] + },{ + 'o' : 'ABCDEFGH', # shrink end + 'f' : 'ABCD--', + 'mods' : [['--', 4, 4]] + },{ + 'o' : 'ABCDEFGH', # grow beginning + 'f' : '--ABCDEFGH', + 'mods' : [['--', 0, 0]] + },{ + 'o' : 'ABCDEFGH', # grow middle + 'f' : 'ABCD--EFGH', + 'mods' : [['--', 4, 0]] + },{ + 'o' : 'ABCDEFGH', # grow end + 'f' : 'ABCDEFGH--', + 'mods' : [['--', 8, 0]] + },{ + 'o' : 'ABCDEFGH', # discard beginning + 'f' : 'EFGH', + 'mods' : [['', 0, 4]] + },{ + 'o' : 'ABCDEFGH', # discard middle + 'f' : 'ABGH', + 'mods' : [['', 2, 4]] + },{ + 'o' : 'ABCDEFGH', # discard end + 'f' : 'ABCD', + 'mods' : [['', 4, 4]] + },{ + 'o' : 'ABCDEFGH', # overlap the end and append + 'f' : 'ABCDEF--XX', + 'mods' : [['--XX', 6, 2]] + },{ + 'o' : 'ABCDEFGH', # overlap the end with incorrect size + 'f' : 'ABCDEFG01234567', + 'mods' : [['01234567', 7, 2000]] + } + ] + + self.session.create(self.uri, 'key_format=S,value_format=u') + cursor = self.session.open_cursor(self.uri, None, None) + + # For each test in the list, set the original value, apply modifications + # in order, then confirm the final state. + for i in list: + cursor['ABC'] = i['o'] + + mods = [] + for j in i['mods']: + mod = wiredtiger.Modify(j[0], j[1], j[2]) + mods.append(mod) + + cursor.set_key('ABC') + cursor.modify(mods) + self.assertEquals(str(cursor['ABC']), i['f']) + + # Check that modify returns not-found after a delete. + def test_modify_delete(self): + self.session.create(self.uri, 'key_format=S,value_format=u') + cursor = self.session.open_cursor(self.uri, None, None) + cursor['ABC'] = 'ABCDEFGH' + cursor.set_key('ABC') + cursor.remove() + + mods = [] + mod = wiredtiger.Modify('ABCD', 3, 3) + mods.append(mod) + + cursor.set_key('ABC') + #self.assertEqual(cursor.modify(mods), wiredtiger.WT_NOTFOUND) + self.assertRaises( + wiredtiger.WiredTigerError, lambda:cursor.modify(mods)) + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_cursor_compare.py b/test/suite/test_cursor_compare.py index c0feb1d4867..7cf9ebfb0ca 100644 --- a/test/suite/test_cursor_compare.py +++ b/test/suite/test_cursor_compare.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_cursor_pin.py b/test/suite/test_cursor_pin.py index cb7045c7e41..91690ef6ed2 100644 --- a/test/suite/test_cursor_pin.py +++ b/test/suite/test_cursor_pin.py @@ -1,6 +1,6 @@ -#!usr/bin/env python +#!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_cursor_random.py b/test/suite/test_cursor_random.py index ee0f85a29ee..c7736e322e1 100644 --- a/test/suite/test_cursor_random.py +++ b/test/suite/test_cursor_random.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_cursor_random02.py b/test/suite/test_cursor_random02.py index d18d8efd94d..11ea8e1f489 100644 --- a/test/suite/test_cursor_random02.py +++ b/test/suite/test_cursor_random02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_cursor_tracker.py b/test/suite/test_cursor_tracker.py index a703e6cea70..dee3c6d1b45 100644 --- a/test/suite/test_cursor_tracker.py +++ b/test/suite/test_cursor_tracker.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_drop.py b/test/suite/test_drop.py index e241c05aa68..4be311b8bb2 100644 --- a/test/suite/test_drop.py +++ b/test/suite/test_drop.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_drop02.py b/test/suite/test_drop02.py index 017aa64e312..7ab891daf15 100644 --- a/test/suite/test_drop02.py +++ b/test/suite/test_drop02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_drop_create.py b/test/suite/test_drop_create.py index 654f054a583..eb851c3212f 100644 --- a/test/suite/test_drop_create.py +++ b/test/suite/test_drop_create.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_dump.py b/test/suite/test_dump.py index 3127c7aef00..37f4572b5c9 100644 --- a/test/suite/test_dump.py +++ b/test/suite/test_dump.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_dupc.py b/test/suite/test_dupc.py index c0cf6acc75e..6e35eb361a0 100644 --- a/test/suite/test_dupc.py +++ b/test/suite/test_dupc.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_durability01.py b/test/suite/test_durability01.py index 32cdd795914..97c89aabc4c 100644 --- a/test/suite/test_durability01.py +++ b/test/suite/test_durability01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_empty.py b/test/suite/test_empty.py index 578bec618c9..82a3bb406ee 100644 --- a/test/suite/test_empty.py +++ b/test/suite/test_empty.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_encrypt01.py b/test/suite/test_encrypt01.py index 317bed93246..5b4be01c861 100644 --- a/test/suite/test_encrypt01.py +++ b/test/suite/test_encrypt01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_encrypt02.py b/test/suite/test_encrypt02.py index d950be067e2..c62828cf607 100644 --- a/test/suite/test_encrypt02.py +++ b/test/suite/test_encrypt02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_encrypt03.py b/test/suite/test_encrypt03.py index 302572bd044..85be38a27ae 100644 --- a/test/suite/test_encrypt03.py +++ b/test/suite/test_encrypt03.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_encrypt04.py b/test/suite/test_encrypt04.py index 19c0b85d427..7bbc4c617f1 100644 --- a/test/suite/test_encrypt04.py +++ b/test/suite/test_encrypt04.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_encrypt05.py b/test/suite/test_encrypt05.py index d8862321821..d4653b2e9b6 100644 --- a/test/suite/test_encrypt05.py +++ b/test/suite/test_encrypt05.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_encrypt06.py b/test/suite/test_encrypt06.py index 72718e53b2b..62e32597f3d 100644 --- a/test/suite/test_encrypt06.py +++ b/test/suite/test_encrypt06.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_encrypt07.py b/test/suite/test_encrypt07.py index 81c9f1a49ea..4846a520b00 100644 --- a/test/suite/test_encrypt07.py +++ b/test/suite/test_encrypt07.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_env01.py b/test/suite/test_env01.py index 491ef9e8eac..c4ce7f69dd2 100644 --- a/test/suite/test_env01.py +++ b/test/suite/test_env01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_excl.py b/test/suite/test_excl.py index f8628d96ff7..539d599fe32 100644 --- a/test/suite/test_excl.py +++ b/test/suite/test_excl.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_hazard.py b/test/suite/test_hazard.py index f2891fce526..73c63099c85 100644 --- a/test/suite/test_hazard.py +++ b/test/suite/test_hazard.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_home.py b/test/suite/test_home.py index 48bf10d7618..667d466266b 100644 --- a/test/suite/test_home.py +++ b/test/suite/test_home.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_huffman01.py b/test/suite/test_huffman01.py index 8a880f7bae7..04a13210e40 100644 --- a/test/suite/test_huffman01.py +++ b/test/suite/test_huffman01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_huffman02.py b/test/suite/test_huffman02.py index d74704daf58..e009734ffb6 100644 --- a/test/suite/test_huffman02.py +++ b/test/suite/test_huffman02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_index01.py b/test/suite/test_index01.py index 5dfa5506277..bd3794bf730 100644 --- a/test/suite/test_index01.py +++ b/test/suite/test_index01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_index02.py b/test/suite/test_index02.py index 4f424e5d3d2..d2b7b66dfe3 100644 --- a/test/suite/test_index02.py +++ b/test/suite/test_index02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_inmem01.py b/test/suite/test_inmem01.py index 388485db29b..694bcabbe77 100644 --- a/test/suite/test_inmem01.py +++ b/test/suite/test_inmem01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. @@ -108,12 +108,15 @@ class test_inmem01(wttest.WiredTigerTestCase): cursor.reset() # Spin inserting to give eviction a chance to reclaim space + sleeps = 0 inserted = False for i in range(1, 1000): try: cursor[ds.key(1)] = ds.value(1) except wiredtiger.WiredTigerError: cursor.reset() + sleeps = sleeps + 1 + self.assertLess(sleeps, 60 * 5) sleep(1) continue inserted = True diff --git a/test/suite/test_inmem02.py b/test/suite/test_inmem02.py index b5e07fea967..f2340f6af69 100644 --- a/test/suite/test_inmem02.py +++ b/test/suite/test_inmem02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_intpack.py b/test/suite/test_intpack.py index ae391e68fca..215ebc8856a 100644 --- a/test/suite/test_intpack.py +++ b/test/suite/test_intpack.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_join01.py b/test/suite/test_join01.py index bdd86a06d4f..167f4793ce4 100644 --- a/test/suite/test_join01.py +++ b/test/suite/test_join01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_join02.py b/test/suite/test_join02.py index db11ed01039..7b85791f17a 100644 --- a/test/suite/test_join02.py +++ b/test/suite/test_join02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_join03.py b/test/suite/test_join03.py index dd8111f6ead..552e27632d2 100644 --- a/test/suite/test_join03.py +++ b/test/suite/test_join03.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_join04.py b/test/suite/test_join04.py index e65b8b53333..c5ba1ad8c79 100644 --- a/test/suite/test_join04.py +++ b/test/suite/test_join04.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_join05.py b/test/suite/test_join05.py index 7dcb3e08911..aedf7a04c24 100644 --- a/test/suite/test_join05.py +++ b/test/suite/test_join05.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_join06.py b/test/suite/test_join06.py index a6681cdccd0..c3d2aa2b9ca 100644 --- a/test/suite/test_join06.py +++ b/test/suite/test_join06.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_join07.py b/test/suite/test_join07.py index 8fae3539246..87bcc8040d3 100644 --- a/test/suite/test_join07.py +++ b/test/suite/test_join07.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_join08.py b/test/suite/test_join08.py index d344653717b..cdcd89a207a 100644 --- a/test/suite/test_join08.py +++ b/test/suite/test_join08.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_join09.py b/test/suite/test_join09.py index d48353b1580..0441349803e 100644 --- a/test/suite/test_join09.py +++ b/test/suite/test_join09.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_jsondump01.py b/test/suite/test_jsondump01.py index c7fa9cdf397..13eb7e7be26 100644 --- a/test/suite/test_jsondump01.py +++ b/test/suite/test_jsondump01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_jsondump02.py b/test/suite/test_jsondump02.py index 60863c4aa97..5c6bf810e08 100644 --- a/test/suite/test_jsondump02.py +++ b/test/suite/test_jsondump02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_las.py b/test/suite/test_las.py new file mode 100644 index 00000000000..d0bd1d108fa --- /dev/null +++ b/test/suite/test_las.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet + +# test_las.py +# Smoke tests to ensure lookaside tables are working. +class test_las(wttest.WiredTigerTestCase): + # Force a small cache. + def conn_config(self): + return 'cache_size=1GB' + + @wttest.longtest('lookaside table smoke test') + def test_las(self): + # Create a small table. + uri = "table:test_las" + nrows = 100 + ds = SimpleDataSet(self, uri, nrows, key_format="S") + ds.populate() + + # Take a snapshot. + self.session.snapshot("name=xxx") + + # Insert a large number of records, we'll hang if the lookaside table + # isn't doing its thing. + c = self.session.open_cursor(uri) + bigvalue = "abcde" * 100 + for i in range(1, 1000000): + c.set_key(ds.key(nrows + i)) + c.set_value(bigvalue) + self.assertEquals(c.insert(), 0) + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_lsm01.py b/test/suite/test_lsm01.py index f705b09b0a4..8a9972261fd 100644 --- a/test/suite/test_lsm01.py +++ b/test/suite/test_lsm01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_lsm02.py b/test/suite/test_lsm02.py index e9628139a97..c35dfa43646 100644 --- a/test/suite/test_lsm02.py +++ b/test/suite/test_lsm02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_lsm03.py b/test/suite/test_lsm03.py index d916db415da..0eb02d546f0 100644 --- a/test/suite/test_lsm03.py +++ b/test/suite/test_lsm03.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_metadata_cursor01.py b/test/suite/test_metadata_cursor01.py index 284e26bc936..f9476a06642 100644 --- a/test/suite/test_metadata_cursor01.py +++ b/test/suite/test_metadata_cursor01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_nsnap01.py b/test/suite/test_nsnap01.py index 4d5555277fe..ee97e4f9985 100644 --- a/test/suite/test_nsnap01.py +++ b/test/suite/test_nsnap01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_nsnap02.py b/test/suite/test_nsnap02.py index ed1c96ebe50..689c704c97e 100644 --- a/test/suite/test_nsnap02.py +++ b/test/suite/test_nsnap02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_nsnap03.py b/test/suite/test_nsnap03.py index 6964fb914c3..7be6557d458 100644 --- a/test/suite/test_nsnap03.py +++ b/test/suite/test_nsnap03.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_nsnap04.py b/test/suite/test_nsnap04.py index 8d491540d74..f53c9b5b3cd 100644 --- a/test/suite/test_nsnap04.py +++ b/test/suite/test_nsnap04.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_overwrite.py b/test/suite/test_overwrite.py index c894de99bd0..0e026235302 100644 --- a/test/suite/test_overwrite.py +++ b/test/suite/test_overwrite.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_pack.py b/test/suite/test_pack.py index 9d833f49e16..a24ef4fdfe1 100644 --- a/test/suite/test_pack.py +++ b/test/suite/test_pack.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. @@ -95,6 +95,11 @@ class test_pack(wttest.WiredTigerTestCase): self.check('3u', r"\x4") self.check('3uu', r"\x4", r"\x42" * 10) self.check('u3u', r"\x42" * 10, r"\x4") + self.check('u', '\x00') + self.check('u', '') + self.check('uu', '', '\x00') + self.check('uu', '\x00', '') + self.check('uu', '', '') self.check('s', "4") self.check("1s", "4") diff --git a/test/suite/test_perf001.py b/test/suite/test_perf001.py deleted file mode 100644 index 6331a3f64d6..00000000000 --- a/test/suite/test_perf001.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python -# -# Public Domain 2014-2016 MongoDB, Inc. -# Public Domain 2008-2014 WiredTiger, Inc. -# -# This is free and unencumbered software released into the public domain. -# -# Anyone is free to copy, modify, publish, use, compile, sell, or -# distribute this software, either in source code form or as a compiled -# binary, for any purpose, commercial or non-commercial, and by any -# means. -# -# In jurisdictions that recognize copyright laws, the author or authors -# of this software dedicate any and all copyright interest in the -# software to the public domain. We make this dedication for the benefit -# of the public at large and to the detriment of our heirs and -# successors. We intend this dedication to be an overt act of -# relinquishment in perpetuity of all present and future rights to this -# software under copyright law. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# test_perf001.py -# Test performance when inserting into a table with an index. - -import wiredtiger, wttest -import random -from time import clock, time -from wtscenario import make_scenarios - -# Test performance of inserting into a table with an index. -class test_perf001(wttest.WiredTigerTestCase): - table_name = 'test_perf001' - - scenarios = make_scenarios([ - #('file-file', dict(tabletype='file',indextype='file')), - ('file-lsm', dict(tabletype='file',indextype='lsm', cfg='', - conn_config="statistics=(fast),statistics_log=(wait=1)")), - #('lsm-file', dict(tabletype='lsm',indextype='file')), - #('lsm-lsm', dict(tabletype='lsm',indextype='lsm')), - ]) - conn_config = 'cache_size=512M' - - def test_performance_of_indices(self): - uri = 'table:' + self.table_name - create_args = 'key_format=i,value_format=ii,columns=(a,c,d),type=' + self.tabletype - self.session.create(uri, create_args) - self.session.create('index:' + self.table_name + ':ia', - 'columns=(d,c),type=' + self.indextype) - - c = self.session.open_cursor('table:' + self.table_name, None, None) - start_time = clock() - for i in xrange(750000): - # 100 operations should never take 5 seconds, sometimes they take - # 2 seconds when a page is being force-evicted. - if i % 100 == 0 and i != 0: - end_time = clock() - self.assertTrue(end_time - start_time < 5) - start_time = end_time - c[i] = (int(time()), random.randint(1,5)) - c.close() - -if __name__ == '__main__': - wttest.run() diff --git a/test/suite/test_readonly01.py b/test/suite/test_readonly01.py index f41280a3283..ee5f78294f4 100644 --- a/test/suite/test_readonly01.py +++ b/test/suite/test_readonly01.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -# Public Domain 2016-2016 MongoDB, Inc. -# Public Domain 2008-2016 WiredTiger, Inc. +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. # diff --git a/test/suite/test_readonly02.py b/test/suite/test_readonly02.py index 0df5465642d..3d3de8186d9 100644 --- a/test/suite/test_readonly02.py +++ b/test/suite/test_readonly02.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -# Public Domain 2016-2016 MongoDB, Inc. -# Public Domain 2008-2016 WiredTiger, Inc. +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. # diff --git a/test/suite/test_readonly03.py b/test/suite/test_readonly03.py index f30c591ca59..6fe2942ca18 100644 --- a/test/suite/test_readonly03.py +++ b/test/suite/test_readonly03.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -# Public Domain 2016-2016 MongoDB, Inc. -# Public Domain 2008-2016 WiredTiger, Inc. +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. # diff --git a/test/suite/test_rebalance.py b/test/suite/test_rebalance.py index 2d160bafec0..867d71b6d35 100644 --- a/test/suite/test_rebalance.py +++ b/test/suite/test_rebalance.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_reconfig01.py b/test/suite/test_reconfig01.py index cbc8bca5740..646b8622a72 100644 --- a/test/suite/test_reconfig01.py +++ b/test/suite/test_reconfig01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_reconfig02.py b/test/suite/test_reconfig02.py index 042d3bbe71f..3bdc19fb2f8 100644 --- a/test/suite/test_reconfig02.py +++ b/test/suite/test_reconfig02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_reconfig03.py b/test/suite/test_reconfig03.py index 0019bf4814e..3ab21735bf0 100644 --- a/test/suite/test_reconfig03.py +++ b/test/suite/test_reconfig03.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_reconfig04.py b/test/suite/test_reconfig04.py index 51d9b91c1f4..37288150d35 100644 --- a/test/suite/test_reconfig04.py +++ b/test/suite/test_reconfig04.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_rename.py b/test/suite/test_rename.py index 4e3af8e13e0..0e55a445cd4 100644 --- a/test/suite/test_rename.py +++ b/test/suite/test_rename.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_reserve.py b/test/suite/test_reserve.py new file mode 100644 index 00000000000..23159ed7f8a --- /dev/null +++ b/test/suite/test_reserve.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_reserve.py +# Reserve update tests. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet, SimpleIndexDataSet +from wtdataset import SimpleLSMDataSet, ComplexDataSet, ComplexLSMDataSet +from wtscenario import make_scenarios + +# Test WT_CURSOR.reserve. +class test_reserve(wttest.WiredTigerTestCase): + + keyfmt = [ + ('integer', dict(keyfmt='i')), + ('recno', dict(keyfmt='r')), + ('string', dict(keyfmt='S')), + ] + types = [ + ('file', dict(uri='file', ds=SimpleDataSet)), + ('lsm', dict(uri='lsm', ds=SimpleDataSet)), + ('table-complex', dict(uri='table', ds=ComplexDataSet)), + ('table-complex-lsm', dict(uri='table', ds=ComplexLSMDataSet)), + ('table-index', dict(uri='table', ds=SimpleIndexDataSet)), + ('table-simple', dict(uri='table', ds=SimpleDataSet)), + ('table-simple-lsm', dict(uri='table', ds=SimpleLSMDataSet)), + ] + scenarios = make_scenarios(types, keyfmt) + + def skip(self): + return self.keyfmt == 'r' and \ + (self.ds.is_lsm() or self.uri == 'lsm') + + def test_reserve(self): + if self.skip(): + return + + uri = self.uri + ':test_reserve' + + ds = self.ds(self, uri, 500, key_format=self.keyfmt) + ds.populate() + s = self.conn.open_session() + c = s.open_cursor(uri, None) + + # Repeatedly update a record. + for i in range(1, 5): + s.begin_transaction('isolation=snapshot') + c.set_key(ds.key(100)) + c.set_value(ds.value(100)) + self.assertEquals(c.update(), 0) + s.commit_transaction() + + # Confirm reserve fails if the record doesn't exist. + s.begin_transaction('isolation=snapshot') + c.set_key(ds.key(600)) + self.assertRaises(wiredtiger.WiredTigerError, lambda:c.reserve()) + s.rollback_transaction() + + # Repeatedly reserve a record and commit. + for i in range(1, 5): + s.begin_transaction('isolation=snapshot') + c.set_key(ds.key(100)) + self.assertEquals(c.reserve(), 0) + s.commit_transaction() + + # Repeatedly reserve a record and rollback. + for i in range(1, 5): + s.begin_transaction('isolation=snapshot') + c.set_key(ds.key(100)) + self.assertEquals(c.reserve(), 0) + s.rollback_transaction() + + # Repeatedly reserve, then update, a record, and commit. + for i in range(1, 5): + s.begin_transaction('isolation=snapshot') + c.set_key(ds.key(100)) + self.assertEquals(c.reserve(), 0) + c.set_value(ds.value(100)) + self.assertEquals(c.update(), 0) + s.commit_transaction() + + # Repeatedly reserve, then update, a record, and rollback. + for i in range(1, 5): + s.begin_transaction('isolation=snapshot') + c.set_key(ds.key(100)) + self.assertEquals(c.reserve(), 0) + c.set_value(ds.value(100)) + self.assertEquals(c.update(), 0) + s.commit_transaction() + + # Reserve a slot, repeatedly try and update a record from another + # transaction (which should fail), repeatedly update a record and + # commit. + s2 = self.conn.open_session() + c2 = s2.open_cursor(uri, None) + for i in range(1, 2): + s.begin_transaction('isolation=snapshot') + c.set_key(ds.key(100)) + self.assertEquals(c.reserve(), 0) + + s2.begin_transaction('isolation=snapshot') + c2.set_key(ds.key(100)) + c2.set_value(ds.value(100)) + self.assertRaises(wiredtiger.WiredTigerError, lambda:c2.update()) + s2.rollback_transaction() + + c.set_key(ds.key(100)) + c.set_value(ds.value(100)) + self.assertEquals(c.update(), 0) + s.commit_transaction() + + # Test cursor.reserve will fail if a key has not yet been set. + def test_reserve_without_key(self): + if self.skip(): + return + + uri = self.uri + ':test_reserve_without_key' + + ds = self.ds(self, uri, 10, key_format=self.keyfmt) + ds.populate() + s = self.conn.open_session() + c = s.open_cursor(uri, None) + s.begin_transaction('isolation=snapshot') + msg = "/requires key be set/" + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda:c.reserve(), msg) + + # Test cursor.reserve will fail if there's no running transaction. + def test_reserve_without_txn(self): + if self.skip(): + return + + uri = self.uri + ':test_reserve_without_txn' + + ds = self.ds(self, uri, 10, key_format=self.keyfmt) + ds.populate() + s = self.conn.open_session() + c = s.open_cursor(uri, None) + c.set_key(ds.key(5)) + msg = "/only permitted in a running transaction/" + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda:c.reserve(), msg) + + # Test cursor.reserve returns a value on success. + def test_reserve_returns_value(self): + if self.skip(): + return + + uri = self.uri + ':test_reserve_returns_value' + + ds = self.ds(self, uri, 10, key_format=self.keyfmt) + ds.populate() + s = self.conn.open_session() + c = s.open_cursor(uri, None) + s.begin_transaction('isolation=snapshot') + c.set_key(ds.key(5)) + self.assertEquals(c.reserve(), 0) + self.assertEqual(c.get_value(), ds.comparable_value(5)) + + # Test cursor.reserve fails on non-standard cursors. + def test_reserve_not_supported(self): + if self.skip(): + return + + uri = self.uri + ':test_reserve_not_supported' + s = self.conn.open_session() + s.create(uri, 'key_format=' + self.keyfmt + ",value_format=S") + + list = [ "bulk", "dump=json" ] + for l in list: + c = s.open_cursor(uri, None, l) + msg = "/Operation not supported/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(c.reserve(), 0), msg) + c.close() + + list = [ "backup:", "config:" "log:" "metadata:" "statistics:" ] + for l in list: + c = s.open_cursor(l, None, None) + msg = "/Operation not supported/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.assertEquals(c.reserve(), 0), msg) + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_salvage.py b/test/suite/test_salvage.py index 3b648a7f170..14045afa21e 100644 --- a/test/suite/test_salvage.py +++ b/test/suite/test_salvage.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_schema01.py b/test/suite/test_schema01.py index 52bff7a13ff..983593dad83 100644 --- a/test/suite/test_schema01.py +++ b/test/suite/test_schema01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_schema02.py b/test/suite/test_schema02.py index ffe710b7d3e..e34063aa66b 100644 --- a/test/suite/test_schema02.py +++ b/test/suite/test_schema02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_schema03.py b/test/suite/test_schema03.py index e5a6528914a..e5471a4de73 100644 --- a/test/suite/test_schema03.py +++ b/test/suite/test_schema03.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_schema04.py b/test/suite/test_schema04.py index 63c638b916c..765040ae73f 100644 --- a/test/suite/test_schema04.py +++ b/test/suite/test_schema04.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_schema05.py b/test/suite/test_schema05.py index d536a629373..f3a75447ee4 100644 --- a/test/suite/test_schema05.py +++ b/test/suite/test_schema05.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_schema06.py b/test/suite/test_schema06.py index e0eec189137..ef8434a1eaa 100644 --- a/test/suite/test_schema06.py +++ b/test/suite/test_schema06.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_schema07.py b/test/suite/test_schema07.py index 3e4b1d28a4d..8de0c477157 100644 --- a/test/suite/test_schema07.py +++ b/test/suite/test_schema07.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_shared_cache01.py b/test/suite/test_shared_cache01.py index c3bd946cc4b..5b348a0ca87 100644 --- a/test/suite/test_shared_cache01.py +++ b/test/suite/test_shared_cache01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_shared_cache02.py b/test/suite/test_shared_cache02.py index 67f9bf7c6b7..c6e5209ff8a 100644 --- a/test/suite/test_shared_cache02.py +++ b/test/suite/test_shared_cache02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_split.py b/test/suite/test_split.py index 411778f21ae..b3de91d3cdb 100644 --- a/test/suite/test_split.py +++ b/test/suite/test_split.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_stat01.py b/test/suite/test_stat01.py index 2b04a3cbcd5..03f0507ced4 100644 --- a/test/suite/test_stat01.py +++ b/test/suite/test_stat01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_stat02.py b/test/suite/test_stat02.py index 45af283ed02..d3bc18cb3e6 100644 --- a/test/suite/test_stat02.py +++ b/test/suite/test_stat02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_stat03.py b/test/suite/test_stat03.py index 7e5cf46ef13..d486cbda0b6 100644 --- a/test/suite/test_stat03.py +++ b/test/suite/test_stat03.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_stat04.py b/test/suite/test_stat04.py index b5309efff37..af5f0e282bf 100644 --- a/test/suite/test_stat04.py +++ b/test/suite/test_stat04.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_stat05.py b/test/suite/test_stat05.py index ef4d65e85e4..6478bb5e58c 100644 --- a/test/suite/test_stat05.py +++ b/test/suite/test_stat05.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_stat_log01.py b/test/suite/test_stat_log01.py index 65ce80dfe7d..8f17be042d6 100644 --- a/test/suite/test_stat_log01.py +++ b/test/suite/test_stat_log01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_sweep01.py b/test/suite/test_sweep01.py index 5559190caca..4d11942dc54 100644 --- a/test/suite/test_sweep01.py +++ b/test/suite/test_sweep01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_sweep02.py b/test/suite/test_sweep02.py index cff45e0d2f9..76931ecbfbd 100644 --- a/test/suite/test_sweep02.py +++ b/test/suite/test_sweep02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_sweep03.py b/test/suite/test_sweep03.py index 61078fa96b5..5ff747b1056 100644 --- a/test/suite/test_sweep03.py +++ b/test/suite/test_sweep03.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_truncate01.py b/test/suite/test_truncate01.py index 98b741ba6a4..88d29d8443a 100644 --- a/test/suite/test_truncate01.py +++ b/test/suite/test_truncate01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_truncate02.py b/test/suite/test_truncate02.py index 729825b26d4..06fa6bfc94f 100644 --- a/test/suite/test_truncate02.py +++ b/test/suite/test_truncate02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_truncate03.py b/test/suite/test_truncate03.py index 2b4628950b3..613ab772571 100644 --- a/test/suite/test_truncate03.py +++ b/test/suite/test_truncate03.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_txn01.py b/test/suite/test_txn01.py index d4ca2ac8d12..e0030909331 100644 --- a/test/suite/test_txn01.py +++ b/test/suite/test_txn01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_txn02.py b/test/suite/test_txn02.py index 01626057b9e..b61a9ed9f99 100644 --- a/test/suite/test_txn02.py +++ b/test/suite/test_txn02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. @@ -137,7 +137,10 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): self.check(self.session2, "isolation=read-uncommitted", current) # Opening a clone of the database home directory should run - # recovery and see the committed results. + # recovery and see the committed results. Flush the log because + # the backup may not get all the log records if we are running + # without a sync option. Use sync=off to force a write to the OS. + self.session.log_flush('sync=off') self.backup(self.backup_dir) backup_conn_params = 'log=(enabled,file_max=%s)' % self.logmax backup_conn = self.wiredtiger_open(self.backup_dir, backup_conn_params) @@ -169,7 +172,6 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): try: session = backup_conn.open_session() finally: - session.checkpoint("force") self.check(backup_conn.open_session(), None, committed) # Sleep long enough so that the archive thread is guaranteed # to run before we close the connection. diff --git a/test/suite/test_txn03.py b/test/suite/test_txn03.py index 18a0e096767..53e9b8e6206 100644 --- a/test/suite/test_txn03.py +++ b/test/suite/test_txn03.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_txn04.py b/test/suite/test_txn04.py index d8f6774ded1..470e37d6a9c 100644 --- a/test/suite/test_txn04.py +++ b/test/suite/test_txn04.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_txn05.py b/test/suite/test_txn05.py index 7aaff221ba4..6a5be0a5df4 100644 --- a/test/suite/test_txn05.py +++ b/test/suite/test_txn05.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. @@ -101,7 +101,10 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): self.check(self.session2, "isolation=read-uncommitted", current) # Opening a clone of the database home directory should run - # recovery and see the committed results. + # recovery and see the committed results. Flush the log because + # the backup may not get all the log records if we are running + # without a sync option. Use sync=off to force a write to the OS. + self.session.log_flush('sync=off') self.backup(self.backup_dir) backup_conn_params = 'log=(enabled,file_max=%s)' % self.logmax backup_conn = self.wiredtiger_open(self.backup_dir, backup_conn_params) @@ -134,12 +137,12 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): session = backup_conn.open_session() finally: self.check(session, None, committed) - # Force a checkpoint because we don't record the recovery - # checkpoint as available for archiving. - session.checkpoint("force") # Sleep long enough so that the archive thread is guaranteed # to run before we close the connection. time.sleep(1.0) + if count == 0: + first_logs = \ + fnmatch.filter(os.listdir(self.backup_dir), "*Log*") backup_conn.close() count += 1 # @@ -149,6 +152,11 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): # cur_logs = fnmatch.filter(os.listdir(self.backup_dir), "*Log*") for o in orig_logs: + # Creating the backup was effectively an unclean shutdown so + # even after sleeping, we should never archive log files + # because a checkpoint has not run. Later opens and runs of + # recovery will detect a clean shutdown and allow archiving. + self.assertEqual(True, o in first_logs) if self.archive == 'true': self.assertEqual(False, o in cur_logs) else: diff --git a/test/suite/test_txn06.py b/test/suite/test_txn06.py index c91dc6a623b..520c25f9b86 100644 --- a/test/suite/test_txn06.py +++ b/test/suite/test_txn06.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_txn07.py b/test/suite/test_txn07.py index e26cf5aaaea..fe1bdd346a0 100644 --- a/test/suite/test_txn07.py +++ b/test/suite/test_txn07.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. @@ -112,7 +112,10 @@ class test_txn07(wttest.WiredTigerTestCase, suite_subprocess): self.check(self.session2, "isolation=read-uncommitted", current) # Opening a clone of the database home directory should run - # recovery and see the committed results. + # recovery and see the committed results. Flush the log because + # the backup may not get all the log records if we are running + # without a sync option. Use sync=off to force a write to the OS. + self.session.log_flush('sync=off') self.backup(self.backup_dir) backup_conn_params = 'log=(enabled,file_max=%s,' % self.logmax + \ 'compressor=%s)' % self.compress + \ diff --git a/test/suite/test_txn08.py b/test/suite/test_txn08.py index 04faed9d45a..5ba0a529a31 100644 --- a/test/suite/test_txn08.py +++ b/test/suite/test_txn08.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_txn09.py b/test/suite/test_txn09.py index 768d714e248..cc5771ef681 100644 --- a/test/suite/test_txn09.py +++ b/test/suite/test_txn09.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. @@ -26,8 +26,8 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # -# test_txn02.py -# Transactions: commits and rollbacks +# test_txn09.py +# Transactions: recovery toggling logging # import fnmatch, os, shutil, time diff --git a/test/suite/test_txn10.py b/test/suite/test_txn10.py index a4745e60066..d27f83bf2e4 100644 --- a/test/suite/test_txn10.py +++ b/test/suite/test_txn10.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_txn11.py b/test/suite/test_txn11.py index 3c02b1e86e3..4b4db9ce315 100644 --- a/test/suite/test_txn11.py +++ b/test/suite/test_txn11.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_txn12.py b/test/suite/test_txn12.py index 32c058bea85..a0ecfb42bdb 100644 --- a/test/suite/test_txn12.py +++ b/test/suite/test_txn12.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_txn13.py b/test/suite/test_txn13.py index 2bf49486b3a..b9172662da0 100644 --- a/test/suite/test_txn13.py +++ b/test/suite/test_txn13.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_txn14.py b/test/suite/test_txn14.py index f9ccabaab8b..7579bbc8e54 100644 --- a/test/suite/test_txn14.py +++ b/test/suite/test_txn14.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_txn15.py b/test/suite/test_txn15.py index a2bfb626338..762c0613735 100644 --- a/test/suite/test_txn15.py +++ b/test/suite/test_txn15.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_txn16.py b/test/suite/test_txn16.py new file mode 100644 index 00000000000..929da2291c7 --- /dev/null +++ b/test/suite/test_txn16.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_txn16.py +# Recovery: Test that toggling between logging and not logging does not +# continue to generate more log files. +# + +import fnmatch, os, shutil, time +from suite_subprocess import suite_subprocess +import wttest + +class test_txn16(wttest.WiredTigerTestCase, suite_subprocess): + t1 = 'table:test_txn16_1' + t2 = 'table:test_txn16_2' + t3 = 'table:test_txn16_3' + nentries = 1000 + create_params = 'key_format=i,value_format=i' + # Set the log file size small so we generate checkpoints + # with LSNs in different files. + conn_config = 'config_base=false,' + \ + 'log=(archive=false,enabled,file_max=100K),' + \ + 'transaction_sync=(method=dsync,enabled)' + conn_on = 'config_base=false,' + \ + 'log=(archive=false,enabled,file_max=100K),' + \ + 'transaction_sync=(method=dsync,enabled)' + conn_off = 'config_base=false,log=(enabled=false)' + + def populate_table(self, uri): + self.session.create(uri, self.create_params) + c = self.session.open_cursor(uri, None, None) + # Populate with an occasional checkpoint to generate + # some varying LSNs. + for i in range(self.nentries): + c[i] = i + 1 + if i % 900 == 0: + self.session.checkpoint() + c.close() + + def copy_dir(self, olddir, newdir): + ''' Simulate a crash from olddir and restart in newdir. ''' + # with the connection still open, copy files to new directory + shutil.rmtree(newdir, ignore_errors=True) + os.mkdir(newdir) + for fname in os.listdir(olddir): + fullname = os.path.join(olddir, fname) + # Skip lock file on Windows since it is locked + if os.path.isfile(fullname) and \ + "WiredTiger.lock" not in fullname and \ + "Tmplog" not in fullname and \ + "Preplog" not in fullname: + shutil.copy(fullname, newdir) + # close the original connection. + self.close_conn() + + def run_toggle(self, homedir): + loop = 0 + # Record original log files. There should never be overlap + # with these even after they're removed. + orig_logs = fnmatch.filter(os.listdir(homedir), "*Log*") + while loop < 3: + # Reopen with logging on to run recovery first time + on_conn = self.wiredtiger_open(homedir, self.conn_on) + on_conn.close() + if loop > 0: + # Get current log files. + cur_logs = fnmatch.filter(os.listdir(homedir), "*Log*") + scur = set(cur_logs) + sorig = set(orig_logs) + # There should never be overlap with the log files that + # were there originally. Mostly this checks that after + # opening with logging disabled and then re-enabled, we + # don't see log file 1. + self.assertEqual(scur.isdisjoint(sorig), True) + if loop > 1: + # We should be creating the same log files each time. + for l in cur_logs: + self.assertEqual(l in last_logs, True) + for l in last_logs: + self.assertEqual(l in cur_logs, True) + last_logs = cur_logs + loop += 1 + # Remove all log files before opening without logging. + cur_logs = fnmatch.filter(os.listdir(homedir), "*Log*") + for l in cur_logs: + path=homedir + "/" + l + os.remove(path) + off_conn = self.wiredtiger_open(homedir, self.conn_off) + off_conn.close() + + def test_recovery(self): + ''' Check log file creation when toggling. ''' + + # Here's the strategy: + # - With logging populate 4 tables. Checkpoint + # them at different times. + # - Copy to a new directory to simulate a crash. + # - Close the original connection. + # On both a "copy" to simulate a crash and the original (3x): + # - Record log files existing. + # - Reopen with logging to run recovery. Close connection. + # - Record log files existing. + # - Remove all log files. + # - Open connection with logging disabled. + # - Record log files existing. Verify we don't keep adding. + # + self.populate_table(self.t1) + self.populate_table(self.t2) + self.populate_table(self.t3) + self.copy_dir(".", "RESTART") + self.run_toggle(".") + self.run_toggle("RESTART") + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_unicode01.py b/test/suite/test_unicode01.py index 0796abf4607..21d6a714268 100644 --- a/test/suite/test_unicode01.py +++ b/test/suite/test_unicode01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_upgrade.py b/test/suite/test_upgrade.py index 4eb6a9e6817..6672daf11d6 100644 --- a/test/suite/test_upgrade.py +++ b/test/suite/test_upgrade.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_util01.py b/test/suite/test_util01.py index 5795bb5b2e6..a181acd5568 100644 --- a/test/suite/test_util01.py +++ b/test/suite/test_util01.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_util02.py b/test/suite/test_util02.py index 7aa24605ed1..59c34e6ef0e 100644 --- a/test/suite/test_util02.py +++ b/test/suite/test_util02.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_util03.py b/test/suite/test_util03.py index ac93d04f799..e5e4f624991 100644 --- a/test/suite/test_util03.py +++ b/test/suite/test_util03.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_util04.py b/test/suite/test_util04.py index d165d350adb..cbfd63b6b65 100644 --- a/test/suite/test_util04.py +++ b/test/suite/test_util04.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_util07.py b/test/suite/test_util07.py index 1175ad8eb13..7d3d6ec5f37 100644 --- a/test/suite/test_util07.py +++ b/test/suite/test_util07.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_util08.py b/test/suite/test_util08.py index 456b68675c6..3c4561da263 100644 --- a/test/suite/test_util08.py +++ b/test/suite/test_util08.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_util09.py b/test/suite/test_util09.py index 4b514401478..3138ea087a2 100644 --- a/test/suite/test_util09.py +++ b/test/suite/test_util09.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_util11.py b/test/suite/test_util11.py index d5d1cda8c39..68cb751d364 100644 --- a/test/suite/test_util11.py +++ b/test/suite/test_util11.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_util12.py b/test/suite/test_util12.py index 3821139f266..6f4638e93aa 100644 --- a/test/suite/test_util12.py +++ b/test/suite/test_util12.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_util13.py b/test/suite/test_util13.py index 7890d4fdb1b..79dc232f5ef 100644 --- a/test/suite/test_util13.py +++ b/test/suite/test_util13.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_verify.py b/test/suite/test_verify.py index 46ae667464a..615b8e278ac 100644 --- a/test/suite/test_verify.py +++ b/test/suite/test_verify.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/test_version.py b/test/suite/test_version.py index 569f181acda..c854a393c2b 100644 --- a/test/suite/test_version.py +++ b/test/suite/test_version.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/wtdataset.py b/test/suite/wtdataset.py index 946b97d995f..3093f550e8b 100644 --- a/test/suite/wtdataset.py +++ b/test/suite/wtdataset.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. @@ -71,9 +71,9 @@ class BaseDataSet(object): # Create a key for a Simple or Complex data set. @staticmethod def key_by_format(i, key_format): - if key_format == 'i' or key_format == 'r' or key_format == 'u': + if key_format == 'i' or key_format == 'r': return i - elif key_format == 'S': + elif key_format == 'S' or key_format == 'u': return str('%015d' % i) else: raise AssertionError( @@ -82,9 +82,9 @@ class BaseDataSet(object): # Create a value for a Simple data set. @staticmethod def value_by_format(i, value_format): - if value_format == 'i' or value_format == 'r' or value_format == 'u': + if value_format == 'i' or value_format == 'r': return i - elif value_format == 'S': + elif value_format == 'S' or value_format == 'u': return str(i) + ': abcdefghijklmnopqrstuvwxyz' elif value_format == '8t': value = ( @@ -94,8 +94,7 @@ class BaseDataSet(object): return value[i % len(value)] else: raise AssertionError( - 'value: object has unexpected format: ' - + value_format) + 'value: object has unexpected format: ' + value_format) # Create a key for this data set. Simple and Complex data sets have # the same key space. @@ -119,6 +118,11 @@ class SimpleDataSet(BaseDataSet): def __init__(self, testcase, uri, rows, **kwargs): super(SimpleDataSet, self).__init__(testcase, uri, rows, **kwargs) + # A value suitable for checking the value returned by a cursor. + def comparable_value(self, i): + return BaseDataSet.value_by_format(i, self.value_format) + + # A value suitable for assigning to a cursor. def value(self, i): return BaseDataSet.value_by_format(i, self.value_format) @@ -260,9 +264,8 @@ class ComplexDataSet(BaseDataSet): str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%23], str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%18]] - # A value suitable for assigning to a cursor, as - # cursor.set_value() expects a tuple when there it is used with - # a single argument and the value is composite. + # A value suitable for assigning to a cursor, as cursor.set_value() expects + # a tuple when it is used with a single argument and the value is composite. def value(self, i): return tuple(self.comparable_value(i)) diff --git a/test/suite/wtscenario.py b/test/suite/wtscenario.py index 8576b3ac876..86faea330a3 100644 --- a/test/suite/wtscenario.py +++ b/test/suite/wtscenario.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/wttest.py b/test/suite/wttest.py index e91838544b9..1c95eb355ae 100644 --- a/test/suite/wttest.py +++ b/test/suite/wttest.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/suite/wtthread.py b/test/suite/wtthread.py index 046a915394d..54fc4a1961e 100644 --- a/test/suite/wtthread.py +++ b/test/suite/wtthread.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/syscall/syscall.py b/test/syscall/syscall.py index 59c2f347146..1caa718b4fc 100644 --- a/test/syscall/syscall.py +++ b/test/syscall/syscall.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/test/syscall/wt2336_base/base.run b/test/syscall/wt2336_base/base.run index 7d2c42ce64e..db455c97474 100644 --- a/test/syscall/wt2336_base/base.run +++ b/test/syscall/wt2336_base/base.run @@ -1,32 +1,35 @@ -// Public Domain 2014-2016 MongoDB, Inc. -// Public Domain 2008-2014 WiredTiger, Inc. -// -// This is free and unencumbered software released into the public domain. -// -// Anyone is free to copy, modify, publish, use, compile, sell, or -// distribute this software, either in source code form or as a compiled -// binary, for any purpose, commercial or non-commercial, and by any -// means. -// -// In jurisdictions that recognize copyright laws, the author or authors -// of this software dedicate any and all copyright interest in the -// software to the public domain. We make this dedication for the benefit -// of the public at large and to the detriment of our heirs and -// successors. We intend this dedication to be an overt act of -// relinquishment in perpetuity of all present and future rights to this -// software under copyright law. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -// IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -// OTHER DEALINGS IN THE SOFTWARE. -// -// base.run -// Command line syscall test runner -// +/*- + * Public Domain 2014-2017 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * base.run + * Command line syscall test runner + */ #ifdef __linux__ SYSTEM("Linux"); #define OPEN_EXISTING(name, flags) open(name, flags) diff --git a/test/syscall/wt2336_base/main.c b/test/syscall/wt2336_base/main.c index 22420371dd0..f22af235c19 100644 --- a/test/syscall/wt2336_base/main.c +++ b/test/syscall/wt2336_base/main.c @@ -1,3 +1,31 @@ +/*- + * Public Domain 2014-2017 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + #include <stdlib.h> #include <unistd.h> // TODO diff --git a/test/thread/file.c b/test/thread/file.c index 7a7d16c4cd6..66ee9dd8348 100644 --- a/test/thread/file.c +++ b/test/thread/file.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/thread/rw.c b/test/thread/rw.c index e8a2650ca51..3283f780b32 100644 --- a/test/thread/rw.c +++ b/test/thread/rw.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -29,8 +29,8 @@ #include "thread.h" static void print_stats(u_int); -static void *reader(void *); -static void *writer(void *); +static WT_THREAD_RET reader(void *); +static WT_THREAD_RET writer(void *); typedef struct { char *name; /* object name */ @@ -45,15 +45,13 @@ typedef struct { static INFO *run_info; -int +void rw_start(u_int readers, u_int writers) { struct timeval start, stop; + wt_thread_t *tids; double seconds; - pthread_t *tids; u_int i, name_index, offset, total_nops; - int ret; - void *thread_ret; tids = NULL; /* Keep GCC 4.1 happy. */ total_nops = 0; @@ -109,18 +107,15 @@ rw_start(u_int readers, u_int writers) /* Create threads. */ for (i = 0; i < readers; ++i) - if ((ret = pthread_create( - &tids[i], NULL, reader, (void *)(uintptr_t)i)) != 0) - testutil_die(ret, "pthread_create"); - for (; i < readers + writers; ++i) { - if ((ret = pthread_create( - &tids[i], NULL, writer, (void *)(uintptr_t)i)) != 0) - testutil_die(ret, "pthread_create"); - } + testutil_check(__wt_thread_create( + NULL, &tids[i], reader, (void *)(uintptr_t)i)); + for (; i < readers + writers; ++i) + testutil_check(__wt_thread_create( + NULL, &tids[i], writer, (void *)(uintptr_t)i)); /* Wait for the threads. */ for (i = 0; i < readers + writers; ++i) - (void)pthread_join(tids[i], &thread_ret); + testutil_check(__wt_thread_join(NULL, tids[i])); (void)gettimeofday(&stop, NULL); seconds = (stop.tv_sec - start.tv_sec) + @@ -147,8 +142,6 @@ rw_start(u_int readers, u_int writers) free(run_info); free(tids); - - return (0); } /* @@ -186,7 +179,7 @@ reader_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) * reader -- * Reader thread start function. */ -static void * +static WT_THREAD_RET reader(void *arg) { INFO *s; @@ -234,7 +227,7 @@ reader(void *arg) printf(" read thread %2d stopping: tid: %s, file: %s\n", id, tid, s->name); - return (NULL); + return (WT_THREAD_RET_VALUE); } /* @@ -291,7 +284,7 @@ writer_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) * writer -- * Writer thread start function. */ -static void * +static WT_THREAD_RET writer(void *arg) { INFO *s; @@ -339,7 +332,7 @@ writer(void *arg) printf("write thread %2d stopping: tid: %s, file: %s\n", id, tid, s->name); - return (NULL); + return (WT_THREAD_RET_VALUE); } /* diff --git a/test/thread/stats.c b/test/thread/stats.c index 839d65e8a4d..3950576a310 100644 --- a/test/thread/stats.c +++ b/test/thread/stats.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/thread/t.c b/test/thread/t.c index d2ed4c74bb7..c6ff9a95145 100644 --- a/test/thread/t.c +++ b/test/thread/t.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -160,8 +160,7 @@ main(int argc, char *argv[]) wt_connect(config_open); /* WiredTiger connection */ - if (rw_start(readers, writers)) /* Loop operations */ - return (EXIT_FAILURE); + rw_start(readers, writers); /* Loop operations */ stats(); /* Statistics */ diff --git a/test/thread/thread.h b/test/thread/thread.h index edcb919ec32..bcba442b4c1 100644 --- a/test/thread/thread.h +++ b/test/thread/thread.h @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -46,6 +46,6 @@ extern int vary_nops; /* Operations per thread */ extern int session_per_op; /* New session per operation */ void load(const char *); -int rw_start(u_int, u_int); +void rw_start(u_int, u_int); void stats(void); void verify(const char *); diff --git a/test/utility/misc.c b/test/utility/misc.c index 934dac86a7b..e119fef47f1 100644 --- a/test/utility/misc.c +++ b/test/utility/misc.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/utility/parse_opts.c b/test/utility/parse_opts.c index c3eff3360de..e5bd8ce0130 100644 --- a/test/utility/parse_opts.c +++ b/test/utility/parse_opts.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/utility/test_util.h b/test/utility/test_util.h index 406ed2c4961..9c67bde2457 100644 --- a/test/utility/test_util.h +++ b/test/utility/test_util.h @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -25,21 +25,21 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ -#include "wt_internal.h" /* For __wt_XXX */ +#include "wt_internal.h" #ifdef _WIN32 - #define DIR_DELIM '\\' - #define DIR_DELIM_STR "\\" - #define DIR_EXISTS_COMMAND "IF EXIST " - #define RM_COMMAND "rd /s /q " +#define DIR_DELIM '\\' +#define DIR_DELIM_STR "\\" +#define DIR_EXISTS_COMMAND "IF EXIST " +#define RM_COMMAND "rd /s /q " #else - #define DIR_DELIM '/' - #define DIR_DELIM_STR "/" - #define RM_COMMAND "rm -rf " +#define DIR_DELIM '/' +#define DIR_DELIM_STR "/" +#define RM_COMMAND "rm -rf " #endif -#define DEFAULT_DIR "WT_TEST" -#define MKDIR_COMMAND "mkdir " +#define DEFAULT_DIR "WT_TEST" +#define MKDIR_COMMAND "mkdir " #ifdef _WIN32 #include "windows_shim.h" diff --git a/test/utility/thread.c b/test/utility/thread.c index 122ad554442..08f49c54c5e 100644 --- a/test/utility/thread.c +++ b/test/utility/thread.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. diff --git a/test/windows/windows_shim.c b/test/windows/windows_shim.c index b161b29c2fa..8986c1a5ae1 100644 --- a/test/windows/windows_shim.c +++ b/test/windows/windows_shim.c @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -124,26 +124,3 @@ pthread_rwlock_wrlock(pthread_rwlock_t *rwlock) return (0); } - -#pragma warning( once : 4024 ) -#pragma warning( once : 4047 ) -int -pthread_create(pthread_t *tidret, const pthread_attr_t *ignored, - void *(*func)(void *), void * arg) -{ - ignored = ignored; - *tidret = CreateThread(NULL, 0, func, arg, 0, NULL); - - if (*tidret != NULL) - return (0); - - return (1); -} - -int -pthread_join(pthread_t thread, void **ignored) -{ - ignored = ignored; - WaitForSingleObject(thread, INFINITE); - return (0); -} diff --git a/test/windows/windows_shim.h b/test/windows/windows_shim.h index 8985904fb19..88b707f9ad9 100644 --- a/test/windows/windows_shim.h +++ b/test/windows/windows_shim.h @@ -1,5 +1,5 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2014-2017 MongoDB, Inc. * Public Domain 2008-2014 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. @@ -25,27 +25,13 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ - -#ifdef _WIN32 - -#define WIN32_LEAN_AND_MEAN -#include <windows.h> -#include <errno.h> -#include <stdint.h> -#include <direct.h> -#include <io.h> -#include <process.h> - #include "wt_internal.h" -#define inline __inline - -/* Define some POSIX types */ -typedef int u_int; +#include <direct.h> /* _mkdir */ /* Windows does not define constants for access() */ -#define R_OK 04 -#define X_OK R_OK +#define R_OK 04 +#define X_OK R_OK /* MSVC Doesn't provide __func__, it has __FUNCTION__ */ #ifdef _MSC_VER @@ -77,11 +63,13 @@ int gettimeofday(struct timeval* tp, void* tzp); */ typedef uint32_t useconds_t; -int -sleep(int seconds); +int sleep(int seconds); +int usleep(useconds_t useconds); -int -usleep(useconds_t useconds); +#define lseek(fd, offset, origin) \ + _lseek(fd, (long)(offset), origin) +#define write(fd, buffer, count) \ + _write(fd, buffer, (unsigned int)(count)) /* * Emulate the <pthread.h> support we need for tests and example code. @@ -102,16 +90,12 @@ typedef HANDLE pthread_t; typedef int pthread_rwlockattr_t; typedef int pthread_attr_t; -int pthread_rwlock_destroy(pthread_rwlock_t *); -int pthread_rwlock_init(pthread_rwlock_t *, - const pthread_rwlockattr_t *); -int pthread_rwlock_rdlock(pthread_rwlock_t *); -int pthread_rwlock_unlock(pthread_rwlock_t *); -int pthread_rwlock_trywrlock(pthread_rwlock_t *); -int pthread_rwlock_wrlock(pthread_rwlock_t *); - -int pthread_create(pthread_t *, const pthread_attr_t *, - void *(*)(void *), void *); -int pthread_join(pthread_t, void **); - -#endif +int pthread_create( + pthread_t *, const pthread_attr_t *, void *(*)(void *), void *); +int pthread_join(pthread_t, void **); +int pthread_rwlock_destroy(pthread_rwlock_t *); +int pthread_rwlock_init(pthread_rwlock_t *, const pthread_rwlockattr_t *); +int pthread_rwlock_rdlock(pthread_rwlock_t *); +int pthread_rwlock_trywrlock(pthread_rwlock_t *); +int pthread_rwlock_unlock(pthread_rwlock_t *); +int pthread_rwlock_wrlock(pthread_rwlock_t *); diff --git a/test/wtperf/test_conf_dump.py b/test/wtperf/test_conf_dump.py index ef7f276a1d0..bbfb8e819e9 100644 --- a/test/wtperf/test_conf_dump.py +++ b/test/wtperf/test_conf_dump.py @@ -1,3 +1,31 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + # Usage: python test_conf_dump.py <optional-wtperf-config> # # This script tests if the config file dumped in the test directory corresponds diff --git a/tools/wt_ckpt_decode.py b/tools/wt_ckpt_decode.py index f78bf8c34bf..0d45a652063 100644 --- a/tools/wt_ckpt_decode.py +++ b/tools/wt_ckpt_decode.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/tools/wtstats/stat_data.py b/tools/wtstats/stat_data.py index a94ce524ae3..09fca2b9525 100644 --- a/tools/wtstats/stat_data.py +++ b/tools/wtstats/stat_data.py @@ -26,6 +26,7 @@ no_scale_per_second_list = [ 'log: log sync_dir time duration (usecs)', 'log: maximum log file size', 'log: number of pre-allocated log files to create', + 'log: slot joins yield time (usecs)', 'log: total log buffer size', 'LSM: application work units currently queued', 'LSM: merge work units currently queued', @@ -145,6 +146,7 @@ no_clear_list = [ 'log: log sync_dir time duration (usecs)', 'log: maximum log file size', 'log: number of pre-allocated log files to create', + 'log: slot joins yield time (usecs)', 'log: total log buffer size', 'LSM: application work units currently queued', 'LSM: merge work units currently queued', diff --git a/tools/wtstats/test/test_wtstats.py b/tools/wtstats/test/test_wtstats.py index ac730c2fd4d..3d4e9dd1c49 100644 --- a/tools/wtstats/test/test_wtstats.py +++ b/tools/wtstats/test/test_wtstats.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. diff --git a/tools/wtstats/wtstats.py b/tools/wtstats/wtstats.py index bf5557d12f4..7d9e71b0360 100755 --- a/tools/wtstats/wtstats.py +++ b/tools/wtstats/wtstats.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2014-2017 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. |