diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2020-04-30 14:37:57 +1000 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-04-30 04:54:58 +0000 |
| commit | b6670adbbf280289cb3cf06edf30fe7290b0b350 (patch) | |
| tree | 0072a190228712696783197cfcc1a2e5975f0769 | |
| parent | 8dc854b5354ad2a2d236575b00c8e4a1c871952f (diff) | |
| download | mongo-b6670adbbf280289cb3cf06edf30fe7290b0b350.tar.gz | |
Import wiredtiger: a707df12a2503ad39ccdd82a84062faa6a07e082 from branch mongodb-4.4
ref: 7b994a862e..a707df12a2
for: 4.4.0-rc4
WT-3726 Add documentation specifying build pre-requisites
WT-5260 Restore assert that trees being discarded are not marked dirty
WT-5544 Fix cache stuck during salvage
WT-5573 Re-enable unit test test_util01.test_dump_process_timestamp_old
WT-5614 Coverity: Redundant NULL check
WT-5623 Support reading modify type updates in the history store for non-timestamp tables
WT-5661 Make WiredTiger utilities self documenting
WT-5676 Prepare support with durable history: add workgen stress tests
WT-5697 Dropping or renaming tables returns EBUSY in incremental backup test
WT-5728 Coverity analysis defect 114081: PW.SET_BUT_NOT_USED
WT-5749 Only pass a non-NULL WT_ITEM value when allocating a WT_UPDATE with associated data
WT-5764 Fix and reenable test_compact02
WT-5790 Disable test_wt2246_col_append
WT-5883 Add upgrade/downgrade testing to WiredTiger standalone tests
WT-5911 History store verification re-architecture
WT-5912 Change page deletion assert condition in __split_insert()
WT-5954 Turn off column store test and lower isolation levels in test_random_abort
WT-6010 Workgen changes - Add session config at Thread level
WT-6031 Remove legacy WT_PAGE_MODIFY.last_stable_timestamp
WT-6033 Clean up unused unittest imports
WT-6044 Coverity: Variable set but never used
WT-6045 Minor cleanup for test_random_abort
WT-6047 Coverity: Redundant NULL check
WT-6049 Fix eviction evicting uncommitted changes
WT-6050 Close the metadata cursor after bulk loading
WT-6053 fixed-width column-store macro can step past the end of allocated memory
WT-6073 Don't append the tombstone to the update chain if it is already there
WT-6078 Temporarily disable hs verify to unblock wiredtiger drop
WT-6081 Switch wtperf defaults to mirror MongoDB defaults
125 files changed, 1574 insertions, 1224 deletions
diff --git a/src/third_party/wiredtiger/INSTALL b/src/third_party/wiredtiger/INSTALL new file mode 100644 index 00000000000..9b4a6fc20a7 --- /dev/null +++ b/src/third_party/wiredtiger/INSTALL @@ -0,0 +1,11 @@ +WiredTiger 10.0.0: (April 20, 2020) + +This is version 10.0.0 of WiredTiger. + +Instructions for configuring, building, and installing WiredTiger are available online. + + For Linux, MacOS, and other POSIX systems: + http://source.wiredtiger.com/develop/build-posix.html + + For Windows: + http://source.wiredtiger.com/develop/build-windows.html diff --git a/src/third_party/wiredtiger/bench/workgen/runner/example_prepare.py b/src/third_party/wiredtiger/bench/workgen/runner/example_prepare.py index fe4bd4fa491..555a695a719 100644 --- a/src/third_party/wiredtiger/bench/workgen/runner/example_prepare.py +++ b/src/third_party/wiredtiger/bench/workgen/runner/example_prepare.py @@ -51,24 +51,28 @@ print('populate:') pop_workload.run(conn) opread = Operation(Operation.OP_SEARCH, table) -read_txn = txn(opread, 'read_timestamp') +read_txn = txn(opread * 5, 'read_timestamp') # read_timestamp_lag is the lag to the read_timestamp from current time read_txn.transaction.read_timestamp_lag = 2 treader = Thread(read_txn) opwrite = Operation(Operation.OP_INSERT, table) -write_txn = txn(opwrite * 10, 'isolation=snapshot') +write_txn = txn(opwrite * 5, 'isolation=snapshot') # use_prepare_timestamp - Commit the transaction with stable_timestamp. write_txn.transaction.use_prepare_timestamp = True twriter = Thread(write_txn) +# Thread.options.session_config - Session configuration. +twriter.options.session_config="isolation=snapshot" opupdate = Operation(Operation.OP_UPDATE, table) -update_txn = txn(opupdate, 'isolation=snapshot') +update_txn = txn(opupdate * 5, 'isolation=snapshot') # use_commit_timestamp - Commit the transaction with commit_timestamp. update_txn.transaction.use_commit_timestamp = True tupdate = Thread(update_txn) +# Thread.options.session_config - Session configuration. +tupdate.options.session_config="isolation=snapshot" -workload = Workload(context, 30 * twriter + 20 * tupdate + 10 * treader) +workload = Workload(context, 30 * twriter + 30 * tupdate + 30 * treader) workload.options.run_time = 50 workload.options.report_interval=500 # read_timestamp_lag - Number of seconds lag to the oldest_timestamp from current time. @@ -85,6 +89,6 @@ run_time = end_time - start_time print('Workload took %d minutes' %(run_time//60)) -latency_filename = homedir + "/latency.out" +latency_filename = os.path.join(context.args.home, "latency.out") latency.workload_latency(workload, latency_filename) conn.close() diff --git a/src/third_party/wiredtiger/bench/workgen/runner/prepare_stress.py b/src/third_party/wiredtiger/bench/workgen/runner/prepare_stress.py new file mode 100755 index 00000000000..a79edf71af8 --- /dev/null +++ b/src/third_party/wiredtiger/bench/workgen/runner/prepare_stress.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +# This file is derived from evict-btree-hs.py, see that file for its purpose and +# derivation. This benchmark is designed to stress the cache effects of prepared transactions, +# in particular eviction of data in general and prepared transactions in particular. +# These are the ways that this workload differs from evict-btree-hs.py: +# - Insert operations use a prepare call, and commit with a durable timestamp +# (see use_prepare_timestamp) +# - Update operations commit with a commit timestamp (see_commit_timestamp). +# - Read transactions use a "read_timestamp" that lags the current time (see read_timestamp_lag). +# - The system-wide "oldest_timestamp" and "stable_timestamp" advance periodically, but lag the +# current time (see oldest_timestamp_lag and stable_timestamp_lag and timestamp_advance). +# - Sessions and transactions use snapshot isolation. + +################################################################################################### +# These wtperf constants were used to originally generate this python file, which has been since +# edited. The table_count, icount, and other variables have been changed below. +''' +# wtperf options file: evict btree configuration +conn_config="cache_size=40G,checkpoint=(wait=60,log_size=2GB),eviction=(threads_min=12, +threads_max=12),log=(enabled=true),session_max=600,eviction_target=60,statistics=(fast), +statistics_log=(wait=1,json)" + +# 1B records * (key=12 + value=138) is about 150G total data size +key_sz=12 +value_sz=138 +log_like_table=true +table_config="type=file" +icount=1000000000 +report_interval=5 +run_time=3600 +# Scans every 10 minutes for all the scan specific tables. +# .4B records * (key=12 + value=138) is about 60G total data size for scan +# Running on a machine with 64G physical memory, this exhausts both the +# WT cache and the system cache. +scan_interval=600 +scan_pct=100 +scan_table_count=20 +scan_icount=400000000 +populate_threads=5 +table_count=100 +threads=((count=400,reads=1),(count=20,inserts=1,throttle=500),(count=10,updates=1,throttle=500)) +# Add throughput/latency monitoring +max_latency=50000 +sample_interval=5 +''' +################################################################################################### + +from runner import * +from wiredtiger import * +from workgen import * +import time + +context = Context() +conn_config = "cache_size=1G,checkpoint=(wait=60,log_size=2GB),\ + eviction=(threads_min=12,threads_max=12),log=(enabled=true),session_max=800,\ + debug_mode=(table_logging=true),\ + eviction_target=60,statistics=(fast),statistics_log=(wait=1,json)"# explicitly added +conn = context.wiredtiger_open("create," + conn_config) +s = conn.open_session("") + +wtperf_table_config = "key_format=S,value_format=S," +\ + "exclusive=true,allocation_size=4kb," +\ + "internal_page_max=64kb,leaf_page_max=4kb,split_pct=100," +compress_table_config = "" +table_config = "type=file" +tables = [] +table_count = 1 +for i in range(0, table_count): + tname = "table:test" + str(i) + table = Table(tname) + s.create(tname, wtperf_table_config +\ + compress_table_config + table_config) + table.options.key_size = 200 + table.options.value_size = 5000 + tables.append(table) + +populate_threads = 40 +icount = 500000 + +start_time = time.time() + +# If there are multiple tables to be filled during populate, +# the icount is split between them all. +pop_ops = Operation(Operation.OP_INSERT, tables[0]) +pop_ops = txn(pop_ops, 'isolation=snapshot') +pop_ops = op_multi_table(pop_ops, tables) +nops_per_thread = icount // (populate_threads * table_count) +pop_thread = Thread(pop_ops * nops_per_thread) +pop_thread.options.session_config="isolation=snapshot" +pop_workload = Workload(context, populate_threads * pop_thread) +pop_workload.run(conn) + +# Log like file, requires that logging be enabled in the connection config. +log_name = "table:log" +s.create(log_name, wtperf_table_config + "key_format=S,value_format=S," +\ + compress_table_config + table_config + ",log=(enabled=true)") +log_table = Table(log_name) + +# Read operation with read_timestamp_lag +ops = Operation(Operation.OP_SEARCH, tables[0],Key(Key.KEYGEN_PARETO, 0, ParetoOptions(1))) +ops = txn(ops, 'read_timestamp') +ops.transaction.read_timestamp_lag = 2 +ops = op_multi_table(ops, tables, False) +ops = op_log_like(ops, log_table, 0) +thread0 = Thread(ops) + +# Insert operations with snapshot isolation level and prepare_timestamp. +ops = Operation(Operation.OP_INSERT, tables[0]) +ops = txn(ops, 'isolation=snapshot') +# use_prepare_timestamp - Commit the transaction with prepare, commit and durable timestamp. +ops.transaction.use_prepare_timestamp = True +ops = op_multi_table(ops, tables, False) +ops = op_log_like(ops, log_table, 0) +thread1 = Thread(ops) +# Thread.options.session_config - Session configuration. +thread1.options.session_config="isolation=snapshot" +# These operations include log_like operations, which will increase the number +# of insert/update operations by a factor of 2.0. This may cause the +# actual operations performed to be above the throttle. + +# Insert operations with snapshot isolation level and sets commit timestamp. +ops = Operation(Operation.OP_UPDATE, tables[0]) +ops = txn(ops, 'isolation=snapshot') +# use_commit_timestamp - Commit the transaction with commit_timestamp. +ops.transaction.use_commit_timestamp = True +ops = op_multi_table(ops, tables, False) +ops = op_log_like(ops, log_table, 0) +thread2 = Thread(ops) +# Thread.options.session_config - Session configuration. +thread2.options.session_config="isolation=snapshot" +# These operations include log_like operations, which will increase the number +# of insert/update operations by a factor of 2.0. This may cause the +# actual operations performed to be above the throttle. +thread2.options.throttle=500 +thread2.options.throttle_burst=1.0 + +# Long running transactions. There is a 0.1 second sleep after a series of search and update +# operations. The sleep op is repeated 10000 times and this will make these transcations to at +# least run for ~17 minutes. +search_op = Operation(Operation.OP_SEARCH, tables[0], Key(Key.KEYGEN_PARETO, 0, ParetoOptions(1))) +update_op = Operation(Operation.OP_UPDATE, tables[0], Key(Key.KEYGEN_PARETO, 0, ParetoOptions(1))) +ops = txn(((search_op + update_op) * 1000 + sleep(0.1)) * 10000, 'isolation=snapshot') +ops.transaction.use_commit_timestamp = True +thread3 = Thread(ops) +thread3.options.session_config="isolation=snapshot" + +ops = Operation(Operation.OP_SLEEP, "0.1") + \ + Operation(Operation.OP_LOG_FLUSH, "") +logging_thread = Thread(ops) +logging_thread.options.session_config="isolation=snapshot" + +workload = Workload(context, 50 * thread0 + 50 * thread1 +\ + 10 * thread2 + 100 * thread3 + logging_thread) +workload.options.report_interval=5 +workload.options.run_time=500 +workload.options.max_latency=50000 +# oldest_timestamp_lag - Number of seconds lag to the oldest_timestamp from current time. +workload.options.oldest_timestamp_lag=30 +# stable_timestamp_lag - Number of seconds lag to the stable_timestamp from current time. +workload.options.stable_timestamp_lag=10 +# timestamp_advance is the number of seconds to wait before moving oldest and stable timestamp. +workload.options.timestamp_advance=1 +workload.run(conn) + +end_time = time.time() +run_time = end_time - start_time + +print('Workload took %d minutes' %(run_time//60)) + +latency_filename = os.path.join(context.args.home, "latency.out") +latency.workload_latency(workload, latency_filename) +conn.close() diff --git a/src/third_party/wiredtiger/bench/workgen/runner/runner/core.py b/src/third_party/wiredtiger/bench/workgen/runner/runner/core.py index 158a65d1fbd..5cf875a073c 100755 --- a/src/third_party/wiredtiger/bench/workgen/runner/runner/core.py +++ b/src/third_party/wiredtiger/bench/workgen/runner/runner/core.py @@ -34,7 +34,9 @@ from workgen import Key, Operation, OpList, Table, Transaction, Value # txn -- # Put the operation (and any suboperations) within a transaction. def txn(op, config=None): - t = Transaction(config) + t = Transaction() + if config != None: + t._begin_config = config op.transaction = t return op @@ -159,13 +161,39 @@ def _op_get_group_list(op): result.extend(grouplist) return result +# This function is used by op_copy to modify a "tree" of operations to change the table +# and/or key for each operation to a given value. It operates on the current operation, +# and recursively on any in its groiup list. +def _op_copy_mod(op, table, key): + if op._optype != Operation.OP_NONE: + if table != None: + op._table = table + if key != None: + op._key = key + if op._group != None: + newgroup = [] + for subop in _op_get_group_list(op): + newgroup.append(_op_copy_mod(subop, table, key)) + op._group = OpList(newgroup) + return op + +# This is a convenient function that copies an operation and all its +# "sub-operations", as well as any attached transaction. +def op_copy(src, table=None, key=None): + # Copy constructor does a deep copy, including subordinate + # operations and any attached transaction. + op = Operation(src) + if table != None or key != None: + _op_copy_mod(op, table, key) + return op + def _op_multi_table_as_list(ops_arg, tables, pareto_tables, multiplier): result = [] if ops_arg._optype != Operation.OP_NONE: if pareto_tables <= 0: for table in tables: for i in range(0, multiplier): - result.append(Operation(ops_arg._optype, table, ops_arg._key, ops_arg._value)) + result.append(op_copy(ops_arg, table=table)) else: # Use the multiplier unless the length of the list will be large. # In any case, make sure there's at least a multiplier of 3, to @@ -186,12 +214,21 @@ def _op_multi_table_as_list(ops_arg, tables, pareto_tables, multiplier): key = Key(ops_arg._key) key._pareto.range_low = (1.0 * i)/count key._pareto.range_high = (1.0 * (i + 1))/count - result.append(Operation(ops_arg._optype, table, key, ops_arg._value)) + result.append(op_copy(ops_arg, table=table, key=key)) else: - for op in _op_get_group_list(ops_arg): - for o in _op_multi_table_as_list(op, tables, pareto_tables, \ - multiplier): - result.append(Operation(o)) + copy = op_copy(ops_arg, table=tables[1]) + if ops_arg.transaction == None: + for op in _op_get_group_list(ops_arg): + for o in _op_multi_table_as_list(op, tables, pareto_tables, \ + multiplier): + result.append(Operation(o)) + elif pareto_tables <= 0: + entries = len(tables) * multiplier + for i in range(0, entries): + copy = op_copy(ops_arg, table=tables[i]) + result.append(copy) + else: + raise Exception('(pareto, range partition, transaction) combination not supported') return result # A convenient way to build a list of operations @@ -268,8 +305,14 @@ def _optype_is_write(optype): optype == Operation.OP_REMOVE # Emulate wtperf's log_like option. For all operations, add a second -# insert operation going to a log table. +# insert operation going to a log table. Ops_per_txn is only checked +# for zero vs non-zero, non-zero says don't add new transactions. +# If we have ops_per_txn, wtperf.py ensures that op_group_transactions was previous called +# to insert needed transactions. def op_log_like(op, log_table, ops_per_txn): + if op.transaction != None: + # Any non-zero number indicates that we already have a transaction around this. + ops_per_txn = 1 if op._optype != Operation.OP_NONE: if _optype_is_write(op._optype): op += _op_log_op(op, log_table) @@ -279,10 +322,13 @@ def op_log_like(op, log_table, ops_per_txn): oplist = [] for op2 in _op_get_group_list(op): if op2._optype == Operation.OP_NONE: - oplist.append(op_log_like(op2, log_table)) + oplist.append(op_log_like(op2, log_table, ops_per_txn)) elif ops_per_txn == 0 and _optype_is_write(op2._optype): op2 += _op_log_op(op2, log_table) - oplist.append(txn(op2)) # txn for each action. + if op2.transaction == None: + oplist.append(txn(op2)) # txn for each action. + else: + oplist.append(op2) # already have a txn else: oplist.append(op2) if _optype_is_write(op2._optype): diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.cxx b/src/third_party/wiredtiger/bench/workgen/workgen.cxx index ca0cd4b308d..739b50ebb27 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen.cxx +++ b/src/third_party/wiredtiger/bench/workgen/workgen.cxx @@ -513,7 +513,7 @@ int ThreadRunner::create_all(WT_CONNECTION *conn) { ASSERT(_session == NULL); if (_thread->options.synchronized) _thread->_op.synchronized_check(); - WT_RET(conn->open_session(conn, NULL, NULL, &_session)); + WT_RET(conn->open_session(conn, NULL, _thread->options.session_config.c_str(), &_session)); _table_usage.clear(); _stats.track_latency(_workload->options.sample_interval_ms > 0); WT_RET(workgen_random_alloc(_session, &_rand_state)); @@ -1096,9 +1096,10 @@ int Throttle::throttle(uint64_t op_count, uint64_t *op_limit) { return (0); } -ThreadOptions::ThreadOptions() : name(), throttle(0.0), throttle_burst(1.0), +ThreadOptions::ThreadOptions() : name(), session_config(), throttle(0.0), throttle_burst(1.0), synchronized(false), _options() { _options.add_string("name", name, "name of the thread"); + _options.add_string("session_config", session_config, "session config which is passed to open_session"); _options.add_double("throttle", throttle, "Limit to this number of operations per second"); _options.add_double("throttle_burst", throttle_burst, @@ -1106,7 +1107,7 @@ ThreadOptions::ThreadOptions() : name(), throttle(0.0), throttle_burst(1.0), "to having large bursts with lulls (10.0 or larger)"); } ThreadOptions::ThreadOptions(const ThreadOptions &other) : - name(other.name), throttle(other.throttle), + name(other.name), session_config(other.session_config), throttle(other.throttle), throttle_burst(other.throttle_burst), synchronized(other.synchronized), _options(other._options) {} ThreadOptions::~ThreadOptions() {} diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.h b/src/third_party/wiredtiger/bench/workgen/workgen.h index b963cf3d47e..61e2b76d5c7 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen.h +++ b/src/third_party/wiredtiger/bench/workgen/workgen.h @@ -232,7 +232,7 @@ struct ParetoOptions { ~ParetoOptions(); void describe(std::ostream &os) const { - os << "parameter " << param; + os << "Pareto: parameter " << param; if (range_low != 0.0 || range_high != 1.0) { os << "range [" << range_low << "-" << range_high << "]"; } @@ -266,7 +266,12 @@ struct Key { ~Key() {} void describe(std::ostream &os) const { - os << "Key: type " << _keytype << ", size " << _size; } + os << "Key: type " << _keytype << ", size " << _size; + if (_pareto.param != ParetoOptions::DEFAULT.param) { + os << ", "; + _pareto.describe(os); + } + } }; struct Value { @@ -330,6 +335,7 @@ struct Operation { // struct ThreadOptions { std::string name; + std::string session_config; double throttle; double throttle_burst; bool synchronized; @@ -342,6 +348,7 @@ struct ThreadOptions { os << "throttle " << throttle; os << ", throttle_burst " << throttle_burst; os << ", synchronized " << synchronized; + os << ", session_config " << session_config; } std::string help() const { return _options.help(); } @@ -392,23 +399,30 @@ struct Transaction { std::string _commit_config; double read_timestamp_lag; - Transaction(const char *_config = NULL) : _rollback(false), use_commit_timestamp(false), use_prepare_timestamp(false), _begin_config(_config == NULL ? "" : _config), _commit_config(), - read_timestamp_lag(0.0) - {} + Transaction() : _rollback(false), use_commit_timestamp(false), + use_prepare_timestamp(false), _begin_config(""), _commit_config(), read_timestamp_lag(0.0) + {} + + Transaction(const Transaction &other) : _rollback(other._rollback), + use_commit_timestamp(other.use_commit_timestamp), + use_prepare_timestamp(other.use_prepare_timestamp), + _begin_config(other._begin_config), _commit_config(other._commit_config), + read_timestamp_lag(other.read_timestamp_lag) + {} void describe(std::ostream &os) const { os << "Transaction: "; if (_rollback) os << "(rollback) "; + if (use_commit_timestamp) + os << "(use_commit_timestamp) "; + if (use_prepare_timestamp) + os << "(use_prepare_timestamp) "; os << "begin_config: " << _begin_config; if (!_commit_config.empty()) os << ", commit_config: " << _commit_config; - if (use_commit_timestamp) - os << "(use_commit_timestamp) "; - if (use_prepare_timestamp) - os << "(use_prepare_timestamp) "; - if (read_timestamp_lag) - os << "(read_timestamp_lag)"; + if (read_timestamp_lag != 0.0) + os << ", read_timestamp_lag: " << read_timestamp_lag; } }; diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u.wtperf index 74f835e78ff..a829c7aafa0 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u.wtperf @@ -9,6 +9,7 @@ conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000, create=false compression="snappy" sess_config="isolation=snapshot" +table_config="type=file" table_count=2 # close_conn as false allows this test to close/finish faster, but if running # as the set, the next test will need to run recovery. diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-80r20u.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-80r20u.wtperf index 8b56a86e022..f1e912448d0 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-80r20u.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-80r20u.wtperf @@ -12,6 +12,7 @@ compression="snappy" # as the set, the next test will need to run recovery. close_conn=false sess_config="isolation=snapshot" +table_config="type=file" table_count=2 key_sz=40 value_sz=120 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-rdonly.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-rdonly.wtperf index 8e25334ea07..eada818dec3 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-rdonly.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-rdonly.wtperf @@ -9,6 +9,7 @@ conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000, create=false compression="snappy" sess_config="isolation=snapshot" +table_config="type=file" table_count=2 key_sz=40 value_sz=120 diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i b/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i index cf310d33046..2706558c887 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i @@ -170,10 +170,10 @@ DEF_OPT_AS_UINT32(scan_table_count, 0, "that tables are shared with other operations") DEF_OPT_AS_CONFIG_STRING(sess_config, "", "session configuration string") DEF_OPT_AS_UINT32(session_count_idle, 0, "number of idle sessions to create. Default 0.") +/* The following table configuration is based on the configuration MongoDB uses for collections. */ DEF_OPT_AS_CONFIG_STRING(table_config, - "key_format=S,value_format=S,type=lsm,exclusive=true," - "allocation_size=4kb,internal_page_max=64kb,leaf_page_max=4kb," - "split_pct=100", + "key_format=S,value_format=S,type=file,exclusive=true," + "leaf_value_max=64MB,memory_page_max=10m,split_pct=90,checksum=on", "table configuration string") DEF_OPT_AS_UINT32(table_count, 1, "number of tables to run operations over. Keys are divided evenly " diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index a4858ddcdca..30a47fa3201 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -1034,6 +1034,10 @@ wiredtiger_open_common =\ @ref tune_durability for more information''', choices=['dsync', 'fsync', 'none']), ]), + Config('verify_metadata', 'false', r''' + open connection and verify any WiredTiger metadata. This API + allows verification and detection of corruption in WiredTiger metadata.''', + type='boolean'), Config('write_through', '', r''' Use \c FILE_FLAG_WRITE_THROUGH on Windows to write to files. Ignored on non-Windows systems. Options are given as a list, such as @@ -1389,9 +1393,6 @@ methods = { Display the contents of in-memory pages as they are verified, using the application's message handler, intended for debugging''', type='boolean'), - Config('history_store', 'false', r''' - Verify the history store.''', - type='boolean'), Config('stable_timestamp', 'false', r''' Ensure that no data has a start timestamp after the stable timestamp, to be run after rollback_to_stable.''', diff --git a/src/third_party/wiredtiger/dist/s_all b/src/third_party/wiredtiger/dist/s_all index 20a6a379c61..298fbed0fe2 100755 --- a/src/third_party/wiredtiger/dist/s_all +++ b/src/third_party/wiredtiger/dist/s_all @@ -72,6 +72,7 @@ run() # Non parallelizable scripts The following scripts either modify files or # already parallelize internally. run "sh ./s_readme $force" +run "sh ./s_install $force" run "python api_config.py" run "python api_err.py" run "python flags.py" diff --git a/src/third_party/wiredtiger/dist/s_install b/src/third_party/wiredtiger/dist/s_install new file mode 100755 index 00000000000..8b39a2f1e85 --- /dev/null +++ b/src/third_party/wiredtiger/dist/s_install @@ -0,0 +1,46 @@ +#! /bin/sh + +t=__wt.$$ +trap 'rm -f $t' 0 1 2 3 13 15 +f=../INSTALL + +. ../RELEASE_INFO + +force=no +while : + do case "$1" in + -f) # Force versions to be updated + force=yes + shift;; + *) + break;; + esac +done + +# If the version hasn't changed and we aren't forcing the issue, we're done. +# Don't generate a new INSTALL file just because the date changed unless forced: +# that happens all the time. +if test "$force" = no ; then + cnt=`(sed -e q < $f; echo "$WIREDTIGER_VERSION_STRING") | + sed -e 's/:.*//' | sort -u | wc -l` + test $cnt -eq 1 && exit 0 +fi + +cat << END_TEXT > $t +$WIREDTIGER_VERSION_STRING + +This is version $WIREDTIGER_VERSION of WiredTiger. + +Instructions for configuring, building, and installing WiredTiger are available online. + + For Linux, MacOS, and other POSIX systems: + http://source.wiredtiger.com/develop/build-posix.html + + For Windows: + http://source.wiredtiger.com/develop/build-windows.html +END_TEXT + +cmp $t $f > /dev/null 2>&1 || + (echo "Building $f" && rm -f $f && cp $t $f) + +exit 0 diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 5b20bd93c51..29f43d94ef7 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -220,7 +220,6 @@ LOGREC LOGSCAN LOOKASIDE LRU -LRrSVv LSB LSM LSN @@ -232,6 +231,7 @@ LZO LeafGreen LevelDB Levyx +LmRrSVv LmT LoadLoad LockFile diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index f367175962d..477909db5c1 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "7b994a862e899a12eb7c3ac814c9fada7d8d1ab9" + "commit": "a707df12a2503ad39ccdd82a84062faa6a07e082" } diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index 9087d643bbb..d6c89dacd33 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -425,7 +425,7 @@ restart_read_page: session, cbt, &cbt->iface.key, WT_RECNO_OOB, WT_ROW_UPDATE(page, rip), NULL, &upd)); if (upd == NULL) continue; - if (upd != NULL && upd->type == WT_UPDATE_TOMBSTONE) { + if (upd->type == WT_UPDATE_TOMBSTONE) { if (upd->txnid != WT_TXN_NONE && __wt_txn_upd_visible_all(session, upd)) ++cbt->page_deleted_count; if (F_ISSET(upd, WT_UPDATE_RESTORED_FROM_DISK)) diff --git a/src/third_party/wiredtiger/src/btree/bt_curprev.c b/src/third_party/wiredtiger/src/btree/bt_curprev.c index 6d187bd3057..0099d1ae594 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curprev.c +++ b/src/third_party/wiredtiger/src/btree/bt_curprev.c @@ -577,7 +577,7 @@ restart_read_page: session, cbt, &cbt->iface.key, WT_RECNO_OOB, WT_ROW_UPDATE(page, rip), NULL, &upd)); if (upd == NULL) continue; - if (upd != NULL && upd->type == WT_UPDATE_TOMBSTONE) { + if (upd->type == WT_UPDATE_TOMBSTONE) { if (upd->txnid != WT_TXN_NONE && __wt_txn_upd_visible_all(session, upd)) ++cbt->page_deleted_count; if (F_ISSET(upd, WT_UPDATE_RESTORED_FROM_DISK)) diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index 78437202d3d..61a0a2653f6 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -386,33 +386,13 @@ __cursor_row_search(WT_CURSOR_BTREE *cbt, bool insert, WT_REF *leaf, bool *leaf_ } /* - * __cursor_col_modify_v -- - * Column-store modify from a cursor, with a separate value. - */ -static inline int -__cursor_col_modify_v(WT_CURSOR_BTREE *cbt, WT_ITEM *value, u_int modify_type) -{ - return (__wt_col_modify(cbt, cbt->iface.recno, value, NULL, modify_type, false)); -} - -/* - * __cursor_row_modify_v -- - * Row-store modify from a cursor, with a separate value. - */ -static inline int -__cursor_row_modify_v(WT_CURSOR_BTREE *cbt, WT_ITEM *value, u_int modify_type) -{ - return (__wt_row_modify(cbt, &cbt->iface.key, value, NULL, modify_type, false)); -} - -/* * __cursor_col_modify -- * Column-store modify from a cursor. */ static inline int -__cursor_col_modify(WT_CURSOR_BTREE *cbt, u_int modify_type) +__cursor_col_modify(WT_CURSOR_BTREE *cbt, WT_ITEM *value, u_int modify_type) { - return (__wt_col_modify(cbt, cbt->iface.recno, &cbt->iface.value, NULL, modify_type, false)); + return (__wt_col_modify(cbt, cbt->iface.recno, value, NULL, modify_type, false)); } /* @@ -420,9 +400,9 @@ __cursor_col_modify(WT_CURSOR_BTREE *cbt, u_int modify_type) * Row-store modify from a cursor. */ static inline int -__cursor_row_modify(WT_CURSOR_BTREE *cbt, u_int modify_type) +__cursor_row_modify(WT_CURSOR_BTREE *cbt, WT_ITEM *value, u_int modify_type) { - return (__wt_row_modify(cbt, &cbt->iface.key, &cbt->iface.value, NULL, modify_type, false)); + return (__wt_row_modify(cbt, &cbt->iface.key, value, NULL, modify_type, false)); } /* @@ -845,8 +825,9 @@ __wt_btcur_insert(WT_CURSOR_BTREE *cbt) * Correct to an exact match so we can update whatever we're pointing at. */ cbt->compare = 0; - ret = btree->type == BTREE_ROW ? __cursor_row_modify(cbt, WT_UPDATE_STANDARD) : - __cursor_col_modify(cbt, WT_UPDATE_STANDARD); + ret = btree->type == BTREE_ROW ? + __cursor_row_modify(cbt, &cbt->iface.value, WT_UPDATE_STANDARD) : + __cursor_col_modify(cbt, &cbt->iface.value, WT_UPDATE_STANDARD); if (ret == 0) goto done; @@ -883,7 +864,7 @@ retry: WT_ERR(WT_DUPLICATE_KEY); } - ret = __cursor_row_modify(cbt, WT_UPDATE_STANDARD); + ret = __cursor_row_modify(cbt, &cbt->iface.value, WT_UPDATE_STANDARD); } else if (append_key) { /* * Optionally insert a new record (ignoring the application's record number). The real @@ -892,7 +873,7 @@ retry: cbt->iface.recno = WT_RECNO_OOB; cbt->compare = 1; WT_ERR(__cursor_col_search(cbt, NULL, NULL)); - WT_ERR(__cursor_col_modify(cbt, WT_UPDATE_STANDARD)); + WT_ERR(__cursor_col_modify(cbt, &cbt->iface.value, WT_UPDATE_STANDARD)); cursor->recno = cbt->recno; } else { WT_ERR(__cursor_col_search(cbt, NULL, NULL)); @@ -911,7 +892,7 @@ retry: WT_ERR(WT_DUPLICATE_KEY); } - WT_ERR(__cursor_col_modify(cbt, WT_UPDATE_STANDARD)); + WT_ERR(__cursor_col_modify(cbt, &cbt->iface.value, WT_UPDATE_STANDARD)); } err: @@ -1073,8 +1054,8 @@ __wt_btcur_remove(WT_CURSOR_BTREE *cbt, bool positioned) * Correct to an exact match so we can remove whatever we're pointing at. */ cbt->compare = 0; - ret = btree->type == BTREE_ROW ? __cursor_row_modify(cbt, WT_UPDATE_TOMBSTONE) : - __cursor_col_modify(cbt, WT_UPDATE_TOMBSTONE); + ret = btree->type == BTREE_ROW ? __cursor_row_modify(cbt, NULL, WT_UPDATE_TOMBSTONE) : + __cursor_col_modify(cbt, NULL, WT_UPDATE_TOMBSTONE); if (ret == 0) goto done; goto err; @@ -1108,7 +1089,7 @@ retry: if (!valid) goto search_notfound; - ret = __cursor_row_modify(cbt, WT_UPDATE_TOMBSTONE); + ret = __cursor_row_modify(cbt, NULL, WT_UPDATE_TOMBSTONE); } else { WT_ERR_NOTFOUND_OK(__cursor_col_search(cbt, NULL, NULL), true); if (ret == WT_NOTFOUND) @@ -1137,7 +1118,7 @@ retry: */ cbt->recno = cursor->recno; } else - ret = __cursor_col_modify(cbt, WT_UPDATE_TOMBSTONE); + ret = __cursor_col_modify(cbt, NULL, WT_UPDATE_TOMBSTONE); } err: @@ -1250,8 +1231,8 @@ __btcur_update(WT_CURSOR_BTREE *cbt, WT_ITEM *value, u_int modify_type) * Correct to an exact match so we can update whatever we're pointing at. */ cbt->compare = 0; - ret = btree->type == BTREE_ROW ? __cursor_row_modify_v(cbt, value, modify_type) : - __cursor_col_modify_v(cbt, value, modify_type); + ret = btree->type == BTREE_ROW ? __cursor_row_modify(cbt, value, modify_type) : + __cursor_col_modify(cbt, value, modify_type); if (ret == 0) goto done; @@ -1310,7 +1291,7 @@ update_local: if (!valid) WT_ERR(WT_NOTFOUND); } - ret = __cursor_row_modify_v(cbt, value, modify_type); + ret = __cursor_row_modify(cbt, value, modify_type); } else { /* * If not overwriting, fail if the key doesn't exist. If we find an update for the key, @@ -1326,7 +1307,7 @@ update_local: if ((cbt->compare != 0 || !valid) && !__cursor_fix_implicit(btree, cbt)) WT_ERR(WT_NOTFOUND); } - ret = __cursor_col_modify_v(cbt, value, modify_type); + ret = __cursor_col_modify(cbt, value, modify_type); } err: @@ -1542,7 +1523,7 @@ __wt_btcur_reserve(WT_CURSOR_BTREE *cbt) /* WT_CURSOR.reserve is update-without-overwrite and a special value. */ overwrite = F_ISSET(cursor, WT_CURSTD_OVERWRITE); F_CLR(cursor, WT_CURSTD_OVERWRITE); - ret = __btcur_update(cbt, &cursor->value, WT_UPDATE_RESERVE); + ret = __btcur_update(cbt, NULL, WT_UPDATE_RESERVE); if (overwrite) F_SET(cursor, WT_CURSTD_OVERWRITE); return (ret); @@ -1686,7 +1667,7 @@ __wt_btcur_equals(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *equalp) */ static int __cursor_truncate( - WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop, int (*rmfunc)(WT_CURSOR_BTREE *, u_int)) + WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop, int (*rmfunc)(WT_CURSOR_BTREE *, WT_ITEM *, u_int)) { WT_DECL_RET; WT_SESSION_IMPL *session; @@ -1715,7 +1696,7 @@ retry: WT_ASSERT(session, F_MASK((WT_CURSOR *)start, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT); for (;;) { - WT_ERR(rmfunc(start, WT_UPDATE_TOMBSTONE)); + WT_ERR(rmfunc(start, NULL, WT_UPDATE_TOMBSTONE)); if (stop != NULL && __cursor_equals(start, stop)) return (0); @@ -1741,7 +1722,7 @@ err: */ static int __cursor_truncate_fix( - WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop, int (*rmfunc)(WT_CURSOR_BTREE *, u_int)) + WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop, int (*rmfunc)(WT_CURSOR_BTREE *, WT_ITEM *, u_int)) { WT_DECL_RET; WT_SESSION_IMPL *session; @@ -1774,7 +1755,7 @@ retry: for (;;) { value = (const uint8_t *)start->iface.value.data; if (*value != 0) - WT_ERR(rmfunc(start, WT_UPDATE_TOMBSTONE)); + WT_ERR(rmfunc(start, NULL, WT_UPDATE_TOMBSTONE)); if (stop != NULL && __cursor_equals(start, stop)) return (0); diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index 8e1313b28ef..e3e89620fd5 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -744,13 +744,11 @@ __wt_debug_cursor_hs(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor) WT_TIME_PAIR start, stop; WT_UPDATE *upd; wt_timestamp_t hs_durable_ts; - size_t notused; uint64_t hs_upd_type_full; uint32_t hs_btree_id; uint8_t hs_prep_state, hs_upd_type; ds = &_ds; - notused = 0; WT_ERR(__wt_scr_alloc(session, 0, &hs_key)); WT_ERR(__wt_scr_alloc(session, 0, &hs_value)); @@ -766,7 +764,7 @@ __wt_debug_cursor_hs(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor) hs_upd_type = (uint8_t)hs_upd_type_full; switch (hs_upd_type) { case WT_UPDATE_MODIFY: - WT_ERR(__wt_update_alloc(session, hs_value, &upd, ¬used, hs_upd_type)); + WT_ERR(__wt_upd_alloc(session, hs_value, hs_upd_type, &upd, NULL)); WT_ERR(__debug_modify(ds, upd, "\tM ")); break; case WT_UPDATE_STANDARD: diff --git a/src/third_party/wiredtiger/src/btree/bt_delete.c b/src/third_party/wiredtiger/src/btree/bt_delete.c index e7974765964..94b544f6bc2 100644 --- a/src/third_party/wiredtiger/src/btree/bt_delete.c +++ b/src/third_party/wiredtiger/src/btree/bt_delete.c @@ -263,7 +263,7 @@ __tombstone_update_alloc( { WT_UPDATE *upd; - WT_RET(__wt_update_alloc(session, NULL, &upd, sizep, WT_UPDATE_TOMBSTONE)); + WT_RET(__wt_upd_alloc_tombstone(session, &upd, sizep)); /* * Cleared memory matches the lowest possible transaction ID and timestamp, do nothing. diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index 1537bc52139..a42e11e1d8f 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -1761,10 +1761,8 @@ __slvg_row_build_internal(WT_SESSION_IMPL *session, uint32_t leaf_cnt, WT_STUFF WT_REF *ref, **refp; WT_TRACK *trk; uint32_t i; - u_int decr_cnt; addr = NULL; - decr_cnt = 0; /* Allocate a row-store root (internal) page and fill it in. */ WT_RET(__wt_page_alloc(session, WT_PAGE_ROW_INT, leaf_cnt, true, &page)); @@ -1828,21 +1826,14 @@ __slvg_row_build_internal(WT_SESSION_IMPL *session, uint32_t leaf_cnt, WT_STUFF * the reconciliation of the root page. For now, make sure the eviction threads don't see us * as a threat. */ - if (page->memory_footprint > WT_MEGABYTE) { - ++decr_cnt; + if (page->memory_footprint > WT_MEGABYTE * 2) __wt_cache_page_inmem_decr(session, page, WT_MEGABYTE); - } } - if (decr_cnt != 0) - __wt_cache_page_inmem_incr(session, page, decr_cnt * WT_MEGABYTE); - __wt_root_ref_init(session, &ss->root_ref, page, false); if (0) { err: __wt_free(session, addr); - if (decr_cnt != 0) - __wt_cache_page_inmem_incr(session, page, decr_cnt * WT_MEGABYTE); __wt_page_out(session, &page); } return (ret); diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index 2d80e981da6..a2d85f79db8 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -1528,7 +1528,6 @@ __split_multi_inmem(WT_SESSION_IMPL *session, WT_PAGE *orig, WT_MULTI *multi, WT mod->last_eviction_timestamp = orig->modify->last_eviction_timestamp; mod->rec_max_txn = orig->modify->rec_max_txn; mod->rec_max_timestamp = orig->modify->rec_max_timestamp; - mod->last_stable_timestamp = orig->modify->last_stable_timestamp; /* Add the update/restore flag to any previous state. */ mod->restore_state = orig->modify->restore_state; @@ -1767,6 +1766,7 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref) */ WT_ASSERT(session, __wt_leaf_page_can_split(session, page)); WT_ASSERT(session, __wt_page_is_modified(page)); + WT_ASSERT(session, __wt_page_del_active(session, ref, true) == false); F_SET_ATOMIC(page, WT_PAGE_SPLIT_INSERT); /* Find the last item on the page. */ @@ -1797,9 +1797,6 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref) F_SET(child, WT_REF_FLAG_LEAF); child->state = WT_REF_MEM; - WT_ERR_ASSERT(session, ref->page_del == NULL, WT_PANIC, - "unexpected page-delete structure when splitting a page"); - /* * The address has moved to the replacement WT_REF. Make sure it isn't freed when the original * ref is discarded. diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy.c b/src/third_party/wiredtiger/src/btree/bt_vrfy.c index 170192d0048..c9708e9511b 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy.c @@ -172,7 +172,7 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) uint32_t session_flags; uint8_t root_addr[WT_BTREE_MAX_ADDR_COOKIE]; const char *name; - bool bm_start, is_owner, quit; + bool bm_start, is_owner, quit, skip_hs; btree = S2BT(session); bm = btree->bm; @@ -182,6 +182,13 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) bm_start = false; is_owner = false; /* -Wuninitialized */ + /* + * Skip the history store explicit call if we're performing a metadata verification. The + * metadata file is verified before we verify the history store, and it makes no sense to verify + * the history store against itself. + */ + skip_hs = strcmp(name, WT_METAFILE_URI) == 0 || strcmp(name, WT_HS_URI) == 0; + WT_CLEAR(_vstuff); vs = &_vstuff; WT_ERR(__wt_scr_alloc(session, 0, &vs->max_key)); @@ -209,9 +216,6 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) goto done; } - /* Open a history store cursor. */ - WT_ERR(__wt_hs_cursor(session, &session_flags, &is_owner)); - /* Inform the underlying block manager we're verifying. */ WT_ERR(bm->verify_start(bm, session, ckptbase, cfg)); bm_start = true; @@ -269,6 +273,22 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) session, ret = __verify_tree(session, &btree->root, &addr_unpack, vs)); /* + * The checkpoints are in time-order, so the last one in the list is the most recent. If + * this is the most recent checkpoint, verify the history store against it. + */ + if (ret == 0 && (ckpt + 1)->name == NULL && !skip_hs) { + /* Open a history store cursor. */ + WT_ERR(__wt_hs_cursor(session, &session_flags, &is_owner)); + WT_TRET(__wt_history_store_verify_one(session)); + WT_TRET(__wt_hs_cursor_close(session, session_flags, is_owner)); + /* + * We cannot error out here. If we got an error verifying the history store, we need + * to follow through with reacquiring the exclusive call below. We'll error out + * after that and unloading this checkpoint. + */ + } + + /* * We have an exclusive lock on the handle, but we're swapping root pages in-and-out of * that handle, and there's a race with eviction entering the tree and seeing an invalid * root page. Eviction must work on trees being verified (else we'd have to do our own @@ -302,8 +322,6 @@ err: if (bm_start) WT_TRET(bm->verify_end(bm, session)); - WT_TRET(__wt_hs_cursor_close(session, session_flags, is_owner)); - /* Discard the list of checkpoints. */ if (ckptbase != NULL) __wt_meta_ckptlist_free(session, &ckptbase); diff --git a/src/third_party/wiredtiger/src/btree/col_modify.c b/src/third_party/wiredtiger/src/btree/col_modify.c index b2062354f7c..bfd3ecb9f5c 100644 --- a/src/third_party/wiredtiger/src/btree/col_modify.c +++ b/src/third_party/wiredtiger/src/btree/col_modify.c @@ -39,22 +39,30 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U append = logged = false; /* - * We should have EITHER: - * - A full update list to instantiate with. - * - An update to append the existing update list with. + * We should have one of the following: + * - A full update list to instantiate. + * - An update to append to the existing update list. * - A key/value pair to create an update with and append to the update list. + * - A key with no value to create a reserved or tombstone update to append to the update list. + * + * A "full update list" is distinguished from "an update" by checking whether it has a "next" + * update. */ - WT_ASSERT(session, (value == NULL && upd_arg != NULL) || (value != NULL && upd_arg == NULL)); + WT_ASSERT( + session, ((modify_type == WT_UPDATE_RESERVE || modify_type == WT_UPDATE_TOMBSTONE) && + value == NULL && upd_arg == NULL) || + (!(modify_type == WT_UPDATE_RESERVE || modify_type == WT_UPDATE_TOMBSTONE) && + ((value == NULL && upd_arg != NULL) || (value != NULL && upd_arg == NULL)))); + + /* If we don't yet have a modify structure, we'll need one. */ + WT_RET(__wt_page_modify_init(session, page)); + mod = page->modify; if (upd_arg == NULL) { - if (modify_type == WT_UPDATE_RESERVE || modify_type == WT_UPDATE_TOMBSTONE) { - /* - * Fixed-size column-store doesn't have on-page deleted values, it's a nul byte. - */ - if (modify_type == WT_UPDATE_TOMBSTONE && btree->type == BTREE_COL_FIX) { - modify_type = WT_UPDATE_STANDARD; - value = &col_fix_remove; - } + /* Fixed-size column-store doesn't have on-page deleted values, it's a nul byte. */ + if (modify_type == WT_UPDATE_TOMBSTONE && btree->type == BTREE_COL_FIX) { + modify_type = WT_UPDATE_STANDARD; + value = &col_fix_remove; } /* @@ -74,10 +82,6 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U } } - /* If we don't yet have a modify structure, we'll need one. */ - WT_RET(__wt_page_modify_init(session, page)); - mod = page->modify; - /* * If modifying a record not previously modified, but which is in the same update slot as a * previously modified record, cursor.ins will not be set because there's no list of update @@ -124,7 +128,7 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U WT_ERR(__wt_txn_update_check(session, cbt, old_upd)); /* Allocate a WT_UPDATE structure and transaction ID. */ - WT_ERR(__wt_update_alloc(session, value, &upd, &upd_size, modify_type)); + WT_ERR(__wt_upd_alloc(session, value, modify_type, &upd, &upd_size)); WT_ERR(__wt_txn_modify(session, upd)); logged = true; } else { @@ -179,7 +183,7 @@ __wt_col_modify(WT_CURSOR_BTREE *cbt, uint64_t recno, const WT_ITEM *value, WT_U (recno != WT_RECNO_OOB && mod->mod_col_split_recno > recno)); if (upd_arg == NULL) { - WT_ERR(__wt_update_alloc(session, value, &upd, &upd_size, modify_type)); + WT_ERR(__wt_upd_alloc(session, value, modify_type, &upd, &upd_size)); WT_ERR(__wt_txn_modify(session, upd)); logged = true; diff --git a/src/third_party/wiredtiger/src/btree/row_modify.c b/src/third_party/wiredtiger/src/btree/row_modify.c index d90412eab65..b7b1c5edff8 100644 --- a/src/third_party/wiredtiger/src/btree/row_modify.c +++ b/src/third_party/wiredtiger/src/btree/row_modify.c @@ -62,20 +62,25 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, upd = upd_arg; logged = false; - /* If we don't yet have a modify structure, we'll need one. */ - WT_RET(__wt_page_modify_init(session, page)); - mod = page->modify; - /* - * We should have EITHER: - * - A full update list to instantiate with. - * - An update to append the existing update list with. + * We should have one of the following: + * - A full update list to instantiate. + * - An update to append to the existing update list. * - A key/value pair to create an update with and append to the update list. + * - A key with no value to create a reserved or tombstone update to append to the update list. * - * A full update list is distinguished from an update by checking whether it has any "next" + * A "full update list" is distinguished from "an update" by checking whether it has a "next" * update. */ - WT_ASSERT(session, (value == NULL && upd_arg != NULL) || (value != NULL && upd_arg == NULL)); + WT_ASSERT( + session, ((modify_type == WT_UPDATE_RESERVE || modify_type == WT_UPDATE_TOMBSTONE) && + value == NULL && upd_arg == NULL) || + (!(modify_type == WT_UPDATE_RESERVE || modify_type == WT_UPDATE_TOMBSTONE) && + ((value == NULL && upd_arg != NULL) || (value != NULL && upd_arg == NULL)))); + + /* If we don't yet have a modify structure, we'll need one. */ + WT_RET(__wt_page_modify_init(session, page)); + mod = page->modify; /* * Modify: allocate an update array as necessary, build a WT_UPDATE structure, and call a @@ -99,7 +104,7 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, WT_ERR(__wt_txn_update_check(session, cbt, old_upd = *upd_entry)); /* Allocate a WT_UPDATE structure and transaction ID. */ - WT_ERR(__wt_update_alloc(session, value, &upd, &upd_size, modify_type)); + WT_ERR(__wt_upd_alloc(session, value, modify_type, &upd, &upd_size)); WT_ERR(__wt_txn_modify(session, upd)); logged = true; @@ -159,7 +164,7 @@ __wt_row_modify(WT_CURSOR_BTREE *cbt, const WT_ITEM *key, const WT_ITEM *value, cbt->ins = ins; if (upd_arg == NULL) { - WT_ERR(__wt_update_alloc(session, value, &upd, &upd_size, modify_type)); + WT_ERR(__wt_upd_alloc(session, value, modify_type, &upd, &upd_size)); WT_ERR(__wt_txn_modify(session, upd)); logged = true; @@ -249,40 +254,6 @@ __wt_row_insert_alloc(WT_SESSION_IMPL *session, const WT_ITEM *key, u_int skipde } /* - * __wt_update_alloc -- - * Allocate a WT_UPDATE structure and associated value and fill it in. - */ -int -__wt_update_alloc(WT_SESSION_IMPL *session, const WT_ITEM *value, WT_UPDATE **updp, size_t *sizep, - u_int modify_type) -{ - WT_UPDATE *upd; - - *updp = NULL; - - /* - * The code paths leading here are convoluted: assert we never attempt to allocate an update - * structure if only intending to insert one we already have. - */ - WT_ASSERT(session, modify_type != WT_UPDATE_INVALID); - - if (modify_type == WT_UPDATE_TOMBSTONE || modify_type == WT_UPDATE_RESERVE) - value = NULL; - - /* Allocate the WT_UPDATE structure and room for the value, then copy any value into place. */ - WT_RET(__wt_calloc(session, 1, WT_UPDATE_SIZE + (value == NULL ? 0 : value->size), &upd)); - if (value != NULL && value->size != 0) { - upd->size = WT_STORE_SIZE(value->size); - memcpy(upd->data, value->data, value->size); - } - upd->type = (uint8_t)modify_type; - - *updp = upd; - *sizep = WT_UPDATE_MEMSIZE(upd); - return (0); -} - -/* * __wt_update_obsolete_check -- * Check for obsolete updates. */ diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index b1b2dbf02ca..a7cae12139a 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -357,7 +357,6 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_verify[] = { {"dump_address", "boolean", NULL, NULL, NULL, 0}, {"dump_blocks", "boolean", NULL, NULL, NULL, 0}, {"dump_history", "boolean", NULL, NULL, NULL, 0}, {"dump_layout", "boolean", NULL, NULL, NULL, 0}, {"dump_offsets", "list", NULL, NULL, NULL, 0}, {"dump_pages", "boolean", NULL, NULL, NULL, 0}, - {"history_store", "boolean", NULL, NULL, NULL, 0}, {"stable_timestamp", "boolean", NULL, NULL, NULL, 0}, {"strict", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -594,6 +593,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { "\"thread_group\",\"timestamp\",\"transaction\",\"verify\"," "\"version\",\"write\"]", NULL, 0}, + {"verify_metadata", "boolean", NULL, NULL, NULL, 0}, {"write_through", "list", NULL, "choices=[\"data\",\"log\"]", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -665,7 +665,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { "\"thread_group\",\"timestamp\",\"transaction\",\"verify\"," "\"version\",\"write\"]", NULL, 0}, - {"version", "string", NULL, NULL, NULL, 0}, + {"verify_metadata", "boolean", NULL, NULL, NULL, 0}, {"version", "string", NULL, NULL, NULL, 0}, {"write_through", "list", NULL, "choices=[\"data\",\"log\"]", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -733,7 +733,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { "\"thread_group\",\"timestamp\",\"transaction\",\"verify\"," "\"version\",\"write\"]", NULL, 0}, - {"version", "string", NULL, NULL, NULL, 0}, + {"verify_metadata", "boolean", NULL, NULL, NULL, 0}, {"version", "string", NULL, NULL, NULL, 0}, {"write_through", "list", NULL, "choices=[\"data\",\"log\"]", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -801,6 +801,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { "\"thread_group\",\"timestamp\",\"transaction\",\"verify\"," "\"version\",\"write\"]", NULL, 0}, + {"verify_metadata", "boolean", NULL, NULL, NULL, 0}, {"write_through", "list", NULL, "choices=[\"data\",\"log\"]", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -929,8 +930,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", {"WT_SESSION.verify", "dump_address=false,dump_blocks=false,dump_history=false," "dump_layout=false,dump_offsets=,dump_pages=false," - "history_store=false,stable_timestamp=false,strict=false", - confchk_WT_SESSION_verify, 9}, + "stable_timestamp=false,strict=false", + confchk_WT_SESSION_verify, 8}, {"colgroup.meta", "app_metadata=,collator=,columns=,source=,type=file", confchk_colgroup_meta, 5}, {"file.config", "access_pattern_hint=none,allocation_size=4KB,app_metadata=," @@ -1024,8 +1025,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "timing_stress_for_test=,transaction_sync=(enabled=false," "method=fsync),use_environment=true,use_environment_priv=false," - "verbose=,write_through=", - confchk_wiredtiger_open, 52}, + "verbose=,verify_metadata=false,write_through=", + confchk_wiredtiger_open, 53}, {"wiredtiger_open_all", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_cursors=true," @@ -1056,8 +1057,9 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "timing_stress_for_test=,transaction_sync=(enabled=false," "method=fsync),use_environment=true,use_environment_priv=false," - "verbose=,version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_all, 53}, + "verbose=,verify_metadata=false,version=(major=0,minor=0)," + "write_through=", + confchk_wiredtiger_open_all, 54}, {"wiredtiger_open_basecfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_cursors=true," @@ -1085,8 +1087,9 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "statistics=none,statistics_log=(json=false,on_close=false," "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "timing_stress_for_test=,transaction_sync=(enabled=false," - "method=fsync),verbose=,version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_basecfg, 47}, + "method=fsync),verbose=,verify_metadata=false,version=(major=0," + "minor=0),write_through=", + confchk_wiredtiger_open_basecfg, 48}, {"wiredtiger_open_usercfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" ",builtin_extension_config=,cache_cursors=true," @@ -1114,8 +1117,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator", "statistics=none,statistics_log=(json=false,on_close=false," "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "timing_stress_for_test=,transaction_sync=(enabled=false," - "method=fsync),verbose=,write_through=", - confchk_wiredtiger_open_usercfg, 46}, + "method=fsync),verbose=,verify_metadata=false,write_through=", + confchk_wiredtiger_open_usercfg, 47}, {NULL, NULL, NULL, 0}}; int diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index ffdfe3bc398..82639177844 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -2281,8 +2281,9 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const char *c WT_DECL_ITEM(i3); WT_DECL_RET; const WT_NAME_FLAG *ft; - WT_SESSION_IMPL *session; - bool config_base_set, try_salvage; + WT_SESSION *wt_session; + WT_SESSION_IMPL *session, *verify_session; + bool config_base_set, try_salvage, verify_meta; const char *enc_cfg[] = {NULL, NULL}, *merge_cfg; char version[64]; @@ -2651,20 +2652,31 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const char *c * WE USE TO DECIDE IF WE'RE CREATING OR NOT. */ WT_ERR(__wt_turtle_init(session)); + WT_ERR(__wt_config_gets(session, cfg, "verify_metadata", &cval)); + verify_meta = cval.val; + + /* Only verify the metadata file first. We will verify the history store table later. */ + if (verify_meta) { + wt_session = &session->iface; + ret = wt_session->verify(wt_session, WT_METAFILE_URI, NULL); + WT_ERR(ret); + } /* * If the user wants to salvage, do so before opening the metadata cursor. We do this after the * call to wt_turtle_init because that moves metadata files around from backups and would * overwrite any salvage we did if done before that call. */ - if (F_ISSET(conn, WT_CONN_SALVAGE)) - WT_ERR(__wt_metadata_salvage(session)); + if (F_ISSET(conn, WT_CONN_SALVAGE)) { + wt_session = &session->iface; + WT_ERR(__wt_copy_and_sync(wt_session, WT_METAFILE, WT_METAFILE_SLVG)); + WT_ERR(wt_session->salvage(wt_session, WT_METAFILE_URI, NULL)); + } /* Initialize the connection's base write generation. */ WT_ERR(__wt_metadata_init_base_write_gen(session)); WT_ERR(__wt_metadata_cursor(session, NULL)); - /* * Load any incremental backup information. This reads the metadata so must be done after the * turtle file is initialized. @@ -2675,6 +2687,18 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const char *c WT_ERR(__wt_connection_workers(session, cfg)); /* + * If the user wants to verify WiredTiger metadata, verify the history store now that the + * metadata table may have been salvaged and eviction has been started and recovery run. + */ + if (verify_meta) { + WT_ERR(__wt_open_internal_session(conn, "verify hs", false, 0, &verify_session)); + ret = __wt_history_store_verify(verify_session); + wt_session = &verify_session->iface; + WT_TRET(wt_session->close(wt_session, NULL)); + WT_ERR(ret); + } + + /* * The default session should not open data handles after this point: since it can be shared * between threads, relying on session->dhandle is not safe. */ diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index e6da8446753..3d4edc58167 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -797,8 +797,7 @@ restart: * it's potentially used when discarding other open data handles. Close it before discarding the * underlying metadata handle. */ - if (session->meta_cursor != NULL) - WT_TRET(session->meta_cursor->close(session->meta_cursor)); + WT_TRET(__wt_metadata_cursor_close(session)); /* Close the remaining handles. */ WT_TAILQ_SAFE_REMOVE_BEGIN(dhandle, &conn->dhqh, q, dhandle_tmp) diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c b/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c index a70a288ca6e..301bdf13d1b 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c @@ -144,16 +144,21 @@ __curbackup_incr_next(WT_CURSOR *cursor) */ WT_ERR(__curbackup_incr_blkmod(session, btree, cb)); /* - * If there is no block modification information for this file, there is no information - * to return to the user. + * If there is no block modification information for this file, this is a newly created + * file without any checkpoint information. Return the whole file information. */ - if (cb->bitstring.mem == NULL) - WT_ERR(WT_NOTFOUND); + if (cb->bitstring.mem == NULL) { + WT_ERR(__wt_fs_size(session, cb->incr_file, &size)); + cb->incr_init = true; + __wt_cursor_set_key(cursor, 0, size, WT_BACKUP_FILE); + goto done; + } } __wt_cursor_set_key(cursor, cb->offset + cb->granularity * cb->bit_offset++, cb->granularity, WT_BACKUP_RANGE); } +done: err: F_SET(cursor, raw); __wt_scr_free(session, &buf); @@ -169,7 +174,7 @@ __wt_curbackup_free_incr(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) { __wt_free(session, cb->incr_file); if (cb->incr_cursor != NULL) - __wt_cursor_close(cb->incr_cursor); + cb->incr_cursor->close(cb->incr_cursor); __wt_buf_free(session, &cb->bitstring); } @@ -184,6 +189,7 @@ __wt_curbackup_open_incr(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *o WT_CURSOR_BACKUP *cb, *other_cb; WT_DECL_ITEM(open_uri); WT_DECL_RET; + uint64_t session_cache_flags; cb = (WT_CURSOR_BACKUP *)cursor; other_cb = (WT_CURSOR_BACKUP *)other; @@ -214,16 +220,20 @@ __wt_curbackup_open_incr(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *o if (!F_ISSET(cb, WT_CURBACKUP_FORCE_FULL)) { WT_ERR(__wt_scr_alloc(session, 0, &open_uri)); WT_ERR(__wt_buf_fmt(session, open_uri, "file:%s", cb->incr_file)); - __wt_free(session, cb->incr_file); - WT_ERR(__wt_strdup(session, open_uri->data, &cb->incr_file)); - - WT_ERR(__wt_curfile_open(session, cb->incr_file, NULL, cfg, &cb->incr_cursor)); - WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); - WT_ERR(__wt_strdup(session, cb->incr_cursor->internal_uri, &cb->incr_cursor->internal_uri)); - } else - WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); + /* + * Incremental cursors use file cursors, but in a non-standard way. Turn off cursor caching + * as we open the cursor. + */ + session_cache_flags = F_ISSET(session, WT_SESSION_CACHE_CURSORS); + F_CLR(session, WT_SESSION_CACHE_CURSORS); + WT_ERR(__wt_curfile_open(session, open_uri->data, NULL, cfg, &cb->incr_cursor)); + F_SET(session, session_cache_flags); + } + WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); err: + if (ret != 0) + __wt_curbackup_free_incr(session, cb); __wt_scr_free(session, &open_uri); return (ret); } diff --git a/src/third_party/wiredtiger/src/docs/build-posix.dox b/src/third_party/wiredtiger/src/docs/build-posix.dox index 4d567133e2c..9f612ce95f6 100644 --- a/src/third_party/wiredtiger/src/docs/build-posix.dox +++ b/src/third_party/wiredtiger/src/docs/build-posix.dox @@ -121,14 +121,16 @@ Configure WiredTiger to perform various run-time diagnostic tests. <b>DO NOT</b> configure this option in production environments. @par \c --enable-java -Build the WiredTiger Java API. +Build the WiredTiger Java API; requires <a href="http://swig.org">SWIG</a> +and Java JDK. @par \c --enable-lz4 Configure WiredTiger for <a href="https://github.com/Cyan4973/lz4">LZ4</a> compression; see @ref compression for more information. @par \c --enable-python -Build the WiredTiger <a href="http://www.python.org">Python</a> API. +Build the WiredTiger <a href="http://www.python.org">Python</a> API; +requires <a href="http://swig.org">SWIG</a>. @par \c --enable-snappy Configure WiredTiger for <a href="http://code.google.com/p/snappy/">snappy</a> diff --git a/src/third_party/wiredtiger/src/docs/command-line.dox b/src/third_party/wiredtiger/src/docs/command-line.dox index bdad2a06016..53c36314ce9 100644 --- a/src/third_party/wiredtiger/src/docs/command-line.dox +++ b/src/third_party/wiredtiger/src/docs/command-line.dox @@ -3,7 +3,7 @@ WiredTiger includes a command line utility, \c wt. @section util_global_synopsis SYNOPSIS -<code>wt [-LRrVv] [-C config] [-E secretkey ] [-h directory] command [command-specific arguments]</code> +<code>wt [-BLRrVv] [-C config] [-E secretkey ] [-h directory] command [command-specific arguments]</code> @section util_global_description DESCRIPTION The \c wt tool is a command-line utility that provides access to @@ -12,6 +12,8 @@ various pieces of the WiredTiger functionality. @section util_global_options OPTIONS There are several global options: +@par <code>-B</code> +Maintain release 3.3 log file compatibility. @par <code>-C config</code> Specify configuration strings for the ::wiredtiger_open function. @par <code>-E secretkey</code> @@ -20,6 +22,8 @@ Specify an encryption secret key for the ::wiredtiger_open function. Specify a database home directory. @par <code>-L</code> Forcibly turn off logging subsystem for debugging purposes. +@par <code>-m</code> +Verify the WiredTiger metadata as part of opening the database. @par <code>-R</code> Run recovery if the underlying database is configured to do so. @par <code>-r</code> @@ -126,6 +130,21 @@ The following are command-specific options for the \c create command: Include a configuration string to be passed to WT_SESSION::create. <hr> +@section util_downgrade wt downgrade +Downgrade a database. + +The \c downgrade command downgrades the database to the specified compatibility version. + +@subsection util_downgrade_synopsis Synopsis +<code>wt [-RVv] [-C config] [-E secretkey ] [-h directory] downgrade -V version</code> + +@subsection util_downgrade_options Options +The following are command-specific options for the \c downgrade command: + +@par <code>-V version</code> +The \c -V option is required, and specfies the version to which the database is downgraded. + +<hr> @section util_drop wt drop Drop a table. @@ -155,7 +174,7 @@ The following are command-specific options for the \c dump command: @par <code>-c</code> By default, the \c dump command opens the most recent version of the data -source; the \c -c option changes the \c dump command to open the named +source; the \c -c option changes the \c dump command to dump as of the named checkpoint. @par <code>-f</code> @@ -169,15 +188,15 @@ format. @par <code>-r</code> Dump in reverse order, from largest key to smallest. +@par <code>-t</code> +By default, the \c dump command opens the most recent version of the data +source; the \c -c option changes the \c dump command to dump as of the specified +timestamp. + @par <code>-x</code> Dump all characters in a hexadecimal encoding (the default is to leave printable characters unencoded). -@par <code>-t</code> -By default, the \c dump command reads the most recent timestamp versions of -the data source; the \c -t option changes the \c dump command to read at a -specific timestamp. - <hr> @section util_list wt list List the tables in the database. @@ -418,7 +437,7 @@ The \c verify command verifies the specified table, exiting success if the data source is correct, and failure if the data source is corrupted. @subsection util_verify_synopsis Synopsis -<code>wt [-RrVv] [-C config] [-E secretkey ] [-h directory] verify [-d dump_address | dump_blocks | dump_history | dump_layout | dump_offsets=#,# | dump_pages ] [-s] -a|uri</code> +<code>wt [-RrVv] [-C config] [-E secretkey ] [-h directory] verify [-s] [-d dump_address | dump_blocks | dump_history | dump_layout | dump_offsets=#,# | dump_pages ] [uri]</code> @subsection util_verify_options Options The following are command-specific options for the \c verify command: @@ -427,15 +446,10 @@ The following are command-specific options for the \c verify command: This option allows you to specify values which you want to be displayed when verification is run. See the WT_SESSION::verify configuration options. -<code>-s [config]</code> +<code>-s</code> This option allows you to verify against the stable timestamp, valid only after a rollback-to-stable operation. See the WT_SESSION::verify configuration options. -<code>-a [config]</code> -This options allows you to verify the history store against the current state -of the data store. A uri is not passed along with this option. See the WT_SESSION::verify -configuration options. - <hr> @section util_write wt write Write records to a table. diff --git a/src/third_party/wiredtiger/src/docs/wtperf.dox b/src/third_party/wiredtiger/src/docs/wtperf.dox index 70cfe470c29..b2d7fd16a0e 100644 --- a/src/third_party/wiredtiger/src/docs/wtperf.dox +++ b/src/third_party/wiredtiger/src/docs/wtperf.dox @@ -214,7 +214,7 @@ number of separate tables to be used for scanning. Zero indicates that tables a session configuration string @par session_count_idle (unsigned int, default=0) number of idle sessions to create. Default 0. -@par table_config (string, default="key_format=S,value_format=S,type=lsm,exclusive=true, allocation_size=4kb,internal_page_max=64kb,leaf_page_max=4kb, split_pct=100") +@par table_config (string, default="key_format=S,value_format=S,type=file,exclusive=true, leaf_value_max=64MB,memory_page_max=10m,split_pct=90,checksum=on") table configuration string @par table_count (unsigned int, default=1) number of tables to run operations over. Keys are divided evenly over the tables. Cursors are held open on all tables. Default 1, maximum 99999. diff --git a/src/third_party/wiredtiger/src/history/hs.c b/src/third_party/wiredtiger/src/history/hs.c index fd90d168b6c..2770c48ad53 100644 --- a/src/third_party/wiredtiger/src/history/hs.c +++ b/src/third_party/wiredtiger/src/history/hs.c @@ -205,8 +205,6 @@ __wt_hs_cursor_open(WT_SESSION_IMPL *session) int __wt_hs_cursor(WT_SESSION_IMPL *session, uint32_t *session_flags, bool *is_owner) { - /* We should never reach here if working in context of the default session. */ - WT_ASSERT(session, S2C(session)->default_session != session); /* * We don't want to get tapped for eviction after we start using the history store cursor; save @@ -362,7 +360,6 @@ __hs_insert_record_with_btree_int(WT_SESSION_IMPL *session, WT_CURSOR *cursor, W WT_CURSOR_BTREE *cbt; WT_DECL_RET; WT_UPDATE *hs_upd; - size_t notused; uint32_t session_flags; cbt = (WT_CURSOR_BTREE *)cursor; @@ -380,7 +377,7 @@ __hs_insert_record_with_btree_int(WT_SESSION_IMPL *session, WT_CURSOR *cursor, W * Insert a delete record to represent stop time pair for the actual record to be inserted. Set * the stop time pair as the commit time pair of the history store delete record. */ - WT_ERR(__wt_update_alloc(session, NULL, &hs_upd, ¬used, WT_UPDATE_TOMBSTONE)); + WT_ERR(__wt_upd_alloc_tombstone(session, &hs_upd, NULL)); hs_upd->start_ts = stop_ts_pair.timestamp; hs_upd->durable_ts = stop_ts_pair.timestamp; hs_upd->txnid = stop_ts_pair.txnid; @@ -389,7 +386,7 @@ __hs_insert_record_with_btree_int(WT_SESSION_IMPL *session, WT_CURSOR *cursor, W * Append to the delete record, the actual record to be inserted into the history store. Set the * current update start time pair as the commit time pair to the history store record. */ - WT_ERR(__wt_update_alloc(session, &cursor->value, &hs_upd->next, ¬used, WT_UPDATE_STANDARD)); + WT_ERR(__wt_upd_alloc(session, &cursor->value, WT_UPDATE_STANDARD, &hs_upd->next, NULL)); hs_upd->next->start_ts = upd->start_ts; hs_upd->next->durable_ts = upd->durable_ts; hs_upd->next->txnid = upd->txnid; @@ -880,7 +877,6 @@ __wt_find_hs_upd(WT_SESSION_IMPL *session, WT_ITEM *key, uint64_t recno, WT_UPDA WT_UPDATE *mod_upd, *upd; wt_timestamp_t durable_timestamp, durable_timestamp_tmp, hs_start_ts, hs_start_ts_tmp; wt_timestamp_t hs_stop_ts, hs_stop_ts_tmp, read_timestamp; - size_t notused, size; uint64_t hs_counter, hs_counter_tmp, upd_type_full; uint32_t hs_btree_id, session_flags; uint8_t *p, recno_key_buf[WT_INTPACK64_MAXSIZE], upd_type; @@ -894,7 +890,6 @@ __wt_find_hs_upd(WT_SESSION_IMPL *session, WT_ITEM *key, uint64_t recno, WT_UPDA orig_hs_value_buf = NULL; __wt_modify_vector_init(session, &modifies); txn = session->txn; - notused = size = 0; hs_btree_id = S2BT(session)->id; session_flags = 0; /* [-Werror=maybe-uninitialized] */ WT_NOT_READ(modify, false); @@ -968,8 +963,16 @@ __wt_find_hs_upd(WT_SESSION_IMPL *session, WT_ITEM *key, uint64_t recno, WT_UPDA WT_NOT_READ(modify, true); /* Store this so that we don't have to make a special case for the first modify. */ hs_stop_ts_tmp = hs_stop_ts; + + /* + * Resolving update chains of reverse deltas requires the current transaction to look beyond + * its current snapshot in certain scenarios. This flag allows us to ignore transaction + * visibility checks when reading in order to construct the modify chain, so we can create + * the value we expect. + */ + F_SET(session, WT_SESSION_RESOLVING_MODIFY); while (upd_type == WT_UPDATE_MODIFY) { - WT_ERR(__wt_update_alloc(session, hs_value, &mod_upd, ¬used, upd_type)); + WT_ERR(__wt_upd_alloc(session, hs_value, upd_type, &mod_upd, NULL)); WT_ERR(__wt_modify_vector_push(&modifies, mod_upd)); mod_upd = NULL; @@ -1021,7 +1024,7 @@ __wt_find_hs_upd(WT_SESSION_IMPL *session, WT_ITEM *key, uint64_t recno, WT_UPDA hs_cursor, &hs_stop_ts_tmp, &durable_timestamp_tmp, &upd_type_full, hs_value)); upd_type = (uint8_t)upd_type_full; } - + F_CLR(session, WT_SESSION_RESOLVING_MODIFY); WT_ASSERT(session, upd_type == WT_UPDATE_STANDARD); while (modifies.size > 0) { __wt_modify_vector_pop(&modifies, &mod_upd); @@ -1035,7 +1038,7 @@ __wt_find_hs_upd(WT_SESSION_IMPL *session, WT_ITEM *key, uint64_t recno, WT_UPDA } /* Allocate an update structure for the record found. */ - WT_ERR(__wt_update_alloc(session, hs_value, &upd, &size, upd_type)); + WT_ERR(__wt_upd_alloc(session, hs_value, upd_type, &upd, NULL)); upd->txnid = WT_TXN_NONE; upd->durable_ts = durable_timestamp; upd->start_ts = hs_start_ts; @@ -1050,6 +1053,8 @@ __wt_find_hs_upd(WT_SESSION_IMPL *session, WT_ITEM *key, uint64_t recno, WT_UPDA done: err: + F_CLR(session, WT_SESSION_RESOLVING_MODIFY); + if (orig_hs_value_buf != NULL) __wt_scr_free(session, &orig_hs_value_buf); else @@ -1196,7 +1201,6 @@ __hs_delete_key_from_pos( WT_ITEM hs_key; WT_UPDATE *upd; wt_timestamp_t hs_start_ts; - size_t size; uint64_t hs_counter; uint32_t hs_btree_id; int cmp; @@ -1225,7 +1229,7 @@ __hs_delete_key_from_pos( * Append a globally visible tombstone to the update list. This will effectively make the * value invisible and the key itself will eventually get removed during reconciliation. */ - WT_RET(__wt_update_alloc(session, NULL, &upd, &size, WT_UPDATE_TOMBSTONE)); + WT_RET(__wt_upd_alloc_tombstone(session, &upd, NULL)); upd->txnid = WT_TXN_NONE; upd->start_ts = upd->durable_ts = WT_TS_NONE; WT_ERR(__wt_hs_modify(hs_cbt, upd)); @@ -1240,121 +1244,186 @@ err: } /* - * __wt_verify_history_store_tree -- - * Verify the history store. There can't be an entry in the history store without having the - * latest value for the respective key in the data store. If given a uri, limit the verification - * to the corresponding btree. + * __verify_history_store_id -- + * Verify the history store for a single btree. Given a cursor to the tree, walk all history + * store keys. This function assumes any caller has already opened a cursor to the history + * store. */ -int -__wt_verify_history_store_tree(WT_SESSION_IMPL *session, const char *uri) +static int +__verify_history_store_id(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, uint32_t this_btree_id) { - WT_CURSOR *cursor, *data_cursor; + WT_CURSOR *cursor; WT_DECL_ITEM(hs_key); WT_DECL_ITEM(prev_hs_key); + WT_DECL_ITEM(tmp); WT_DECL_RET; wt_timestamp_t hs_start_ts; uint64_t hs_counter; - uint32_t btree_id, btree_id_given_uri, session_flags, prev_btree_id; - int exact, cmp; - char *uri_itr; - bool is_owner; + uint32_t btree_id; + int cmp; + bool found; - cursor = data_cursor = NULL; - btree_id_given_uri = 0; /* [-Wconditional-uninitialized] */ - session_flags = 0; /* [-Wconditional-uninitialized] */ - prev_btree_id = 0; /* [-Wconditional-uninitialized] */ - uri_itr = NULL; - is_owner = false; /* [-Wconditional-uninitialized] */ + cursor = session->hs_cursor; WT_ERR(__wt_scr_alloc(session, 0, &hs_key)); WT_ERR(__wt_scr_alloc(session, 0, &prev_hs_key)); - WT_ERR(__wt_hs_cursor(session, &session_flags, &is_owner)); - cursor = session->hs_cursor; - /* - * If a uri has been provided, limit verification to the corresponding btree by jumping to the - * first record for that btree in the history store. Otherwise scan the whole history store. + * The caller is responsible for positioning the history store cursor at the first record to + * verify. When we return after moving to a new key the caller is responsible for keeping the + * cursor there or deciding they're done. */ - if (uri != NULL) { - ret = __wt_metadata_uri_to_btree_id(session, uri, &btree_id_given_uri); - if (ret != 0) - WT_ERR_MSG(session, ret, "Unable to locate the URI %s in the metadata file", uri); - - /* - * Position the cursor at the first record of the specified btree, or one after. It is - * possible there are no records in the history store for this btree. - */ - cursor->set_key(cursor, btree_id_given_uri, hs_key, 0, 0, 0, 0); - ret = cursor->search_near(cursor, &exact); - if (ret == 0 && exact < 0) - ret = cursor->next(cursor); - } else - ret = cursor->next(cursor); - - /* We have the history store cursor positioned at the first record that we want to verify. */ for (; ret == 0; ret = cursor->next(cursor)) { WT_ERR(cursor->get_key(cursor, &btree_id, hs_key, &hs_start_ts, &hs_counter)); - /* When limiting our verification to a uri, bail out if the btree-id doesn't match. */ - if (uri != NULL && btree_id != btree_id_given_uri) + /* + * If the btree id does not match the preview one, we're done. It is up to the caller to set + * up for the next tree and call us, if they choose. For a full history store walk, the + * caller sends in WT_BTREE_ID_INVALID and this function will set and use the first btree id + * it finds and will return once it walks off that tree, leaving the cursor set to the first + * key of that new tree. + */ + if (btree_id != this_btree_id) break; /* - * Keep track of the previous comparison. The history store is stored in order, so we can - * avoid redundant comparisons. Previous btree ID isn't set, until data cursor is open. + * If we have already checked against this key, keep going to the next key. We only need to + * check the key once. */ - if (data_cursor == NULL || (prev_btree_id != btree_id)) { - /* - * Check whether this btree-id exists in the metadata. We do that by finding what uri - * this btree belongs to. Using this URI, verify the history store key with the data - * store. - */ - if (data_cursor != NULL) { - WT_ERR(data_cursor->close(data_cursor)); - /* Setting data_cursor to null, to avoid double free */ - data_cursor = NULL; - } - /* - * Using the btree-id find the metadata entry and extract the URI for this btree. Don't - * forget to free the copy of the URI returned. - * - * Re-purpose the previous-key buffer on error, safe because we're about to error out. - */ - __wt_free(session, uri_itr); - if ((ret = __wt_metadata_btree_id_to_uri(session, btree_id, &uri_itr)) != 0) - WT_ERR_MSG(session, ret, - "Unable to find btree-id %" PRIu32 - " in the metadata file for the associated history store key %s", - btree_id, - __wt_buf_set_printable(session, hs_key->data, hs_key->size, prev_hs_key)); - - WT_ERR(__wt_open_cursor(session, uri_itr, NULL, NULL, &data_cursor)); - F_SET(data_cursor, WT_CURSOR_RAW_OK); - } else { - WT_ERR(__wt_compare(session, NULL, hs_key, prev_hs_key, &cmp)); - if (cmp == 0) - continue; - } - WT_ERR(__wt_buf_set(session, prev_hs_key, hs_key->data, hs_key->size)); - prev_btree_id = btree_id; - - /* Re-purpose the previous-key buffer on error, safe because we're about to error out. */ - data_cursor->set_key(data_cursor, hs_key); - if ((ret = data_cursor->search(data_cursor)) == WT_NOTFOUND) - WT_ERR_MSG(session, ret, - "In %s, the associated history store key %s was not found in the data store", uri_itr, - __wt_buf_set_printable(session, hs_key->data, hs_key->size, prev_hs_key)); + WT_ERR(__wt_compare(session, NULL, hs_key, prev_hs_key, &cmp)); + if (cmp == 0) + continue; + WT_WITH_PAGE_INDEX(session, ret = __wt_row_search(cbt, hs_key, false, NULL, false, NULL)); WT_ERR(ret); + +/* FIXME: temporarily disable hs verification. */ +#if 0 + found = cbt->compare == 0; +#else + found = true; +#endif + WT_ERR(__cursor_reset(cbt)); + + if (!found) + WT_ERR_MSG(session, WT_PANIC, + "the associated history store key %s was not found in the data store %s", + __wt_buf_set_printable(session, hs_key->data, hs_key->size, prev_hs_key), + session->dhandle->name); + + /* Swap current/previous buffers. */ + tmp = hs_key; + hs_key = prev_hs_key; + prev_hs_key = tmp; } - WT_ERR_NOTFOUND_OK(ret, false); + WT_ERR_NOTFOUND_OK(ret, true); err: - if (data_cursor != NULL) + __wt_scr_free(session, &hs_key); + __wt_scr_free(session, &prev_hs_key); + return (ret); +} + +/* + * __wt_history_store_verify_one -- + * Verify the history store for the btree that is set up in this session. This must be called + * when we are known to have exclusive access to the btree. + */ +int +__wt_history_store_verify_one(WT_SESSION_IMPL *session) +{ + WT_CURSOR *cursor; + WT_CURSOR_BTREE cbt; + WT_DECL_RET; + WT_ITEM hs_key; + uint32_t btree_id; + int exact; + + cursor = session->hs_cursor; + btree_id = S2BT(session)->id; + + /* + * We are required to position the history store cursor. Set it to the first record of our btree + * in the history store. + */ + memset(&hs_key, 0, sizeof(hs_key)); + cursor->set_key(cursor, btree_id, &hs_key, 0, 0, 0, 0); + ret = cursor->search_near(cursor, &exact); + if (ret == 0 && exact < 0) + ret = cursor->next(cursor); + + /* If we positioned the cursor there is something to verify. */ + if (ret == 0) { + __wt_btcur_init(session, &cbt); + __wt_btcur_open(&cbt); + ret = __verify_history_store_id(session, &cbt, btree_id); + WT_TRET(__wt_btcur_close(&cbt, false)); + } + return (ret == WT_NOTFOUND ? 0 : ret); +} + +/* + * __wt_history_store_verify -- + * Verify the history store. There can't be an entry in the history store without having the + * latest value for the respective key in the data store. + */ +int +__wt_history_store_verify(WT_SESSION_IMPL *session) +{ + WT_CURSOR *cursor, *data_cursor; + WT_DECL_ITEM(buf); + WT_DECL_ITEM(hs_key); + WT_DECL_RET; + wt_timestamp_t hs_start_ts; + uint64_t hs_counter; + uint32_t btree_id, session_flags; + char *uri_data; + bool is_owner, stop; + + /* We should never reach here if working in context of the default session. */ + WT_ASSERT(session, S2C(session)->default_session != session); + + cursor = data_cursor = NULL; + btree_id = WT_BTREE_ID_INVALID; + session_flags = 0; /* [-Wconditional-uninitialized] */ + uri_data = NULL; + is_owner = false; /* [-Wconditional-uninitialized] */ + + WT_ERR(__wt_scr_alloc(session, 0, &buf)); + WT_ERR(__wt_scr_alloc(session, 0, &hs_key)); + WT_ERR(__wt_hs_cursor(session, &session_flags, &is_owner)); + cursor = session->hs_cursor; + ret = cursor->next(cursor); + WT_ERR_NOTFOUND_OK(ret, true); + stop = ret == WT_NOTFOUND ? true : false; + ret = 0; + + /* + * We have the history store cursor positioned at the first record that we want to verify. The + * internal function is expecting a btree cursor, so open and initialize that. + */ + while (!stop) { + /* + * The cursor is positioned either from above or left over from the internal call on the + * first key of a new btree id. + */ + WT_ERR(cursor->get_key(cursor, &btree_id, hs_key, &hs_start_ts, &hs_counter)); + if ((ret = __wt_metadata_btree_id_to_uri(session, btree_id, &uri_data)) != 0) + WT_ERR_MSG(session, WT_PANIC, + "Unable to find btree id %" PRIu32 + " in the metadata file for the associated history store key %s", + btree_id, __wt_buf_set_printable(session, hs_key->data, hs_key->size, buf)); + WT_ERR(__wt_open_cursor(session, uri_data, NULL, NULL, &data_cursor)); + F_SET(data_cursor, WT_CURSOR_RAW_OK); + ret = __verify_history_store_id(session, (WT_CURSOR_BTREE *)data_cursor, btree_id); + if (ret == WT_NOTFOUND) + stop = true; WT_TRET(data_cursor->close(data_cursor)); + WT_ERR_NOTFOUND_OK(ret, false); + } +err: WT_TRET(__wt_hs_cursor_close(session, session_flags, is_owner)); + __wt_scr_free(session, &buf); __wt_scr_free(session, &hs_key); - __wt_scr_free(session, &prev_hs_key); - __wt_free(session, uri_itr); + __wt_free(session, uri_data); return (ret); } diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index 6985cce0508..513e0106e53 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -279,9 +279,6 @@ struct __wt_page_modify { uint64_t rec_max_txn; wt_timestamp_t rec_max_timestamp; - /* Stable timestamp at last reconciliation. */ - wt_timestamp_t last_stable_timestamp; - /* The largest update transaction ID (approximate). */ uint64_t update_txn; @@ -1290,8 +1287,9 @@ struct __wt_insert_head { for ((i) = 0, (v) = (i) < (dsk)->u.entries ? \ __bit_getv(WT_PAGE_HEADER_BYTE(btree, dsk), 0, (btree)->bitcnt) : \ 0; \ - (i) < (dsk)->u.entries; \ - ++(i), (v) = __bit_getv(WT_PAGE_HEADER_BYTE(btree, dsk), i, (btree)->bitcnt)) + (i) < (dsk)->u.entries; ++(i), (v) = (i) < (dsk)->u.entries ? \ + __bit_getv(WT_PAGE_HEADER_BYTE(btree, dsk), i, (btree)->bitcnt) : \ + 0) /* * Manage split generation numbers. Splits walk the list of sessions to check when it is safe to diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h index 53bd608efb0..abf19855a3f 100644 --- a/src/third_party/wiredtiger/src/include/btree.h +++ b/src/third_party/wiredtiger/src/include/btree.h @@ -60,6 +60,11 @@ #define WT_BTREE_MIN_SPLIT_PCT 50 /* + * An invalid btree file ID value. ID 0 is reserved for the metadata file. + */ +#define WT_BTREE_ID_INVALID UINT32_MAX + +/* * WT_BTREE -- * A btree handle. */ diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index 7cbfddbd381..cd1952b00d1 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -515,6 +515,8 @@ __wt_page_only_modify_set(WT_SESSION_IMPL *session, WT_PAGE *page) { uint64_t last_running; + WT_ASSERT(session, !F_ISSET(session->dhandle, WT_DHANDLE_DEAD)); + last_running = 0; if (page->modify->page_state == WT_PAGE_CLEAN) last_running = S2C(session)->txn_global.last_running; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 0888eeee453..a3e71435037 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -742,6 +742,10 @@ extern int __wt_hex2byte(const u_char *from, u_char *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_hex_to_raw(WT_SESSION_IMPL *session, const char *from, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_history_store_verify(WT_SESSION_IMPL *session) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_history_store_verify_one(WT_SESSION_IMPL *session) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_hs_config(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_hs_create(WT_SESSION_IMPL *session, const char **cfg) @@ -1060,6 +1064,8 @@ extern int __wt_metadata_btree_id_to_uri(WT_SESSION_IMPL *session, uint32_t btre WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_metadata_cursor(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_metadata_cursor_close(WT_SESSION_IMPL *session) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_metadata_cursor_open(WT_SESSION_IMPL *session, const char *config, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_metadata_cursor_release(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) @@ -1073,8 +1079,6 @@ extern int __wt_metadata_insert(WT_SESSION_IMPL *session, const char *key, const WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_metadata_remove(WT_SESSION_IMPL *session, const char *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_metadata_salvage(WT_SESSION_IMPL *session) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_metadata_search(WT_SESSION_IMPL *session, const char *key, char **valuep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_metadata_turtle_rewrite(WT_SESSION_IMPL *session) @@ -1083,8 +1087,6 @@ extern int __wt_metadata_update(WT_SESSION_IMPL *session, const char *key, const WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_metadata_update_base_write_gen(WT_SESSION_IMPL *session, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_metadata_uri_to_btree_id(WT_SESSION_IMPL *session, const char *uri, - uint32_t *btree_id) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_modify_apply(WT_CURSOR *cursor, const void *modify) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_modify_apply_api(WT_CURSOR *cursor, WT_MODIFY *entries, int nentries) @@ -1509,8 +1511,6 @@ extern int __wt_txn_update_pinned_timestamp(WT_SESSION_IMPL *session, bool force extern int __wt_unexpected_object_type( WT_SESSION_IMPL *session, const char *uri, const char *expect) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_update_alloc(WT_SESSION_IMPL *session, const WT_ITEM *value, WT_UPDATE **updp, - size_t *sizep, u_int modify_type) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_upgrade(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_value_return(WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) @@ -1546,8 +1546,6 @@ extern int __wt_verify_dsk(WT_SESSION_IMPL *session, const char *tag, WT_ITEM *b extern int __wt_verify_dsk_image(WT_SESSION_IMPL *session, const char *tag, const WT_PAGE_HEADER *dsk, size_t size, WT_ADDR *addr, bool empty_page_ok) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_verify_history_store_tree(WT_SESSION_IMPL *session, const char *uri) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int64_t __wt_log_slot_release(WT_MYSLOT *myslot, int64_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern size_t __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode) @@ -2018,8 +2016,10 @@ static inline int __wt_txn_search_check(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_txn_update_check(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline int __wt_upd_alloc_tombstone(WT_SESSION_IMPL *session, WT_UPDATE **updp) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline int __wt_upd_alloc(WT_SESSION_IMPL *session, const WT_ITEM *value, u_int modify_type, + WT_UPDATE **updp, size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline int __wt_upd_alloc_tombstone(WT_SESSION_IMPL *session, WT_UPDATE **updp, + size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_update_serial(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_PAGE *page, WT_UPDATE **srch_upd, WT_UPDATE **updp, size_t upd_size, bool exclusive) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index bd877622ca1..25447813d55 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -188,10 +188,11 @@ struct __wt_session_impl { #define WT_SESSION_NO_SCHEMA_LOCK 0x01000000u #define WT_SESSION_QUIET_CORRUPT_FILE 0x02000000u #define WT_SESSION_READ_WONT_NEED 0x04000000u -#define WT_SESSION_RESOLVING_TXN 0x08000000u -#define WT_SESSION_ROLLBACK_TO_STABLE 0x10000000u -#define WT_SESSION_SCHEMA_TXN 0x20000000u -#define WT_SESSION_SERVER_ASYNC 0x40000000u +#define WT_SESSION_RESOLVING_MODIFY 0x08000000u +#define WT_SESSION_RESOLVING_TXN 0x10000000u +#define WT_SESSION_ROLLBACK_TO_STABLE 0x20000000u +#define WT_SESSION_SCHEMA_TXN 0x40000000u +#define WT_SESSION_SERVER_ASYNC 0x80000000u /* AUTOMATIC FLAG VALUE GENERATION STOP */ uint32_t flags; diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index fd54e279171..483fd429496 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -46,7 +46,9 @@ typedef enum { #define WT_TXNID_LT(t1, t2) ((t1) < (t2)) -#define WT_SESSION_TXN_SHARED(s) (&S2C(s)->txn_global.txn_shared_list[(s)->id]) +#define WT_SESSION_TXN_SHARED(s) \ + (S2C(s)->txn_global.txn_shared_list == NULL ? NULL : \ + &S2C(s)->txn_global.txn_shared_list[(s)->id]) #define WT_SESSION_IS_CHECKPOINT(s) ((s)->id != 0 && (s)->id == S2C(s)->txn_global.checkpoint_id) diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index 574eece2e5f..42d9233676c 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -710,24 +710,58 @@ __wt_txn_upd_visible(WT_SESSION_IMPL *session, WT_UPDATE *upd) } /* - * __wt_upd_alloc_tombstone -- - * Allocate a tombstone update with default values. + * __wt_upd_alloc -- + * Allocate a WT_UPDATE structure and associated value and fill it in. */ static inline int -__wt_upd_alloc_tombstone(WT_SESSION_IMPL *session, WT_UPDATE **updp) +__wt_upd_alloc(WT_SESSION_IMPL *session, const WT_ITEM *value, u_int modify_type, WT_UPDATE **updp, + size_t *sizep) { - size_t notused; + WT_UPDATE *upd; + + *updp = NULL; /* - * The underlying allocation code clears memory, which is the equivalent of setting: - * + * The code paths leading here are convoluted: assert we never attempt to allocate an update + * structure if only intending to insert one we already have, or pass in a value with a type + * that doesn't support values. + */ + WT_ASSERT(session, modify_type != WT_UPDATE_INVALID); + WT_ASSERT(session, + (value == NULL && (modify_type == WT_UPDATE_RESERVE || modify_type == WT_UPDATE_TOMBSTONE)) || + (value != NULL && + !(modify_type == WT_UPDATE_RESERVE || modify_type == WT_UPDATE_TOMBSTONE))); + + /* + * Allocate the WT_UPDATE structure and room for the value, then copy any value into place. + * Memory is cleared, which is the equivalent of setting: * WT_UPDATE.txnid = WT_TXN_NONE; * WT_UPDATE.durable_ts = WT_TS_NONE; * WT_UPDATE.start_ts = WT_TS_NONE; * WT_UPDATE.prepare_state = WT_PREPARE_INIT; * WT_UPDATE.flags = 0; */ - return (__wt_update_alloc(session, NULL, updp, ¬used, WT_UPDATE_TOMBSTONE)); + WT_RET(__wt_calloc(session, 1, WT_UPDATE_SIZE + (value == NULL ? 0 : value->size), &upd)); + if (value != NULL && value->size != 0) { + upd->size = WT_STORE_SIZE(value->size); + memcpy(upd->data, value->data, value->size); + } + upd->type = (uint8_t)modify_type; + + *updp = upd; + if (sizep != NULL) + *sizep = WT_UPDATE_MEMSIZE(upd); + return (0); +} + +/* + * __wt_upd_alloc_tombstone -- + * Allocate a tombstone update. + */ +static inline int +__wt_upd_alloc_tombstone(WT_SESSION_IMPL *session, WT_UPDATE **updp, size_t *sizep) +{ + return (__wt_upd_alloc(session, NULL, WT_UPDATE_TOMBSTONE, updp, sizep)); } /* @@ -780,7 +814,6 @@ __wt_txn_read(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, uint WT_DECL_RET; WT_ITEM buf; WT_TIME_PAIR start, stop; - size_t size; *updp = NULL; WT_RET(__wt_txn_read_upd_list(session, upd, updp)); @@ -789,7 +822,7 @@ __wt_txn_read(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, uint /* If there is no ondisk value, there can't be anything in the history store either. */ if (cbt->ref->page->dsk == NULL || cbt->slot == UINT32_MAX) - return (__wt_upd_alloc_tombstone(session, updp)); + return (__wt_upd_alloc_tombstone(session, updp, NULL)); buf.data = NULL; buf.size = 0; @@ -822,7 +855,7 @@ __wt_txn_read(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, uint (!WT_IS_HS(S2BT(session)) || !F_ISSET(session, WT_SESSION_IGNORE_HS_TOMBSTONE)) && __wt_txn_visible(session, stop.txnid, stop.timestamp)) { __wt_buf_free(session, &buf); - WT_RET(__wt_upd_alloc_tombstone(session, updp)); + WT_RET(__wt_upd_alloc_tombstone(session, updp, NULL)); (*updp)->txnid = stop.txnid; /* FIXME: Reevaluate this as part of PM-1524. */ (*updp)->durable_ts = (*updp)->start_ts = stop.timestamp; @@ -837,8 +870,15 @@ __wt_txn_read(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, uint * an update. This allocation is expensive and doesn't serve a purpose other than to work within * the current system. */ - if (__wt_txn_visible(session, start.txnid, start.timestamp)) { - ret = __wt_update_alloc(session, &buf, updp, &size, WT_UPDATE_STANDARD); + if (__wt_txn_visible(session, start.txnid, start.timestamp) || + F_ISSET(session, WT_SESSION_RESOLVING_MODIFY)) { + + /* If we are resolving a modify then the btree must be the history store. */ + WT_ASSERT( + session, (F_ISSET(session, WT_SESSION_RESOLVING_MODIFY) && WT_IS_HS(S2BT(session))) || + !F_ISSET(session, WT_SESSION_RESOLVING_MODIFY)); + + ret = __wt_upd_alloc(session, &buf, WT_UPDATE_STANDARD, updp, NULL); __wt_buf_free(session, &buf); WT_RET(ret); (*updp)->txnid = start.txnid; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index f939a421dac..1a9bc7b7519 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1722,7 +1722,6 @@ struct __wt_session { * @config{dump_pages, Display the contents of in-memory pages as they are verified\, using * the application's message handler\, intended for debugging., a boolean flag; default \c * false.} - * @config{history_store, Verify the history store., a boolean flag; default \c false.} * @config{stable_timestamp, Ensure that no data has a start timestamp after the stable * timestamp\, to be run after rollback_to_stable., a boolean flag; default \c false.} * @config{strict, Treat any verification problem as an error; by default\, verify will @@ -3072,6 +3071,9 @@ struct __wt_connection { * "recovery_progress"\, \c "rts"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, \c "temporary"\, * \c "thread_group"\, \c "timestamp"\, \c "transaction"\, \c "verify"\, \c "version"\, \c "write"; * default empty.} + * @config{verify_metadata, open connection and verify any WiredTiger metadata. This API allows + * verification and detection of corruption in WiredTiger metadata., a boolean flag; default \c + * false.} * @config{write_through, Use \c FILE_FLAG_WRITE_THROUGH on Windows to write to files. Ignored on * non-Windows systems. Options are given as a list\, such as <code>"write_through=[data]"</code>. * Configuring \c write_through requires care\, see @ref tuning_system_buffer_cache_direct_io for diff --git a/src/third_party/wiredtiger/src/meta/meta_table.c b/src/third_party/wiredtiger/src/meta/meta_table.c index 2737e4ed6df..9a2bd723099 100644 --- a/src/third_party/wiredtiger/src/meta/meta_table.c +++ b/src/third_party/wiredtiger/src/meta/meta_table.c @@ -134,6 +134,21 @@ __wt_metadata_cursor(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) } /* + * __wt_metadata_cursor_close -- + * Close a metadata cursor. + */ +int +__wt_metadata_cursor_close(WT_SESSION_IMPL *session) +{ + WT_DECL_RET; + + if (session->meta_cursor != NULL) + ret = session->meta_cursor->close(session->meta_cursor); + session->meta_cursor = NULL; + return (ret); +} + +/* * __wt_metadata_cursor_release -- * Release a metadata cursor. */ @@ -322,54 +337,6 @@ err: } /* - * __wt_metadata_salvage -- - * Salvage the metadata file. This is a destructive operation. Save a copy of the original - * metadata. - */ -int -__wt_metadata_salvage(WT_SESSION_IMPL *session) -{ - WT_SESSION *wt_session; - - wt_session = &session->iface; - /* - * Copy the original metadata. - */ - WT_RET(__wt_copy_and_sync(wt_session, WT_METAFILE, WT_METAFILE_SLVG)); - - /* - * Now salvage the metadata. We know we're in wiredtiger_open and single threaded. - */ - WT_RET(wt_session->salvage(wt_session, WT_METAFILE_URI, NULL)); - return (0); -} - -/* - * __wt_metadata_uri_to_btree_id -- - * Given a uri, find the btree id from the metadata. WT_NOTFOUND is returned for a non-file uri. - */ -int -__wt_metadata_uri_to_btree_id(WT_SESSION_IMPL *session, const char *uri, uint32_t *btree_id) -{ - WT_CONFIG_ITEM id; - WT_DECL_RET; - char *value; - - value = NULL; - - if (!WT_PREFIX_MATCH(uri, "file:")) - return (WT_NOTFOUND); - - WT_ERR(__wt_metadata_search(session, uri, &value)); - WT_ERR(__wt_config_getones(session, value, "id", &id)); - *btree_id = (uint32_t)id.val; - -err: - __wt_free(session, value); - return (ret); -} - -/* * __wt_metadata_btree_id_to_uri -- * Given a btree id, find the matching entry in the metadata and return a copy of the uri. The * caller has to free the returned uri. diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c index 9a237065ba8..9bedba65310 100644 --- a/src/third_party/wiredtiger/src/meta/meta_turtle.c +++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c @@ -135,6 +135,12 @@ __metadata_load_bulk(WT_SESSION_IMPL *session) err: WT_TRET(__wt_metadata_cursor_release(session, &cursor)); + /* + * We want to explicitly close, not just release the metadata cursor here. We know we are in + * initialization and this open cursor holds a lock on the metadata and we may need to verify + * the metadata. + */ + WT_TRET(__wt_metadata_cursor_close(session)); return (ret); } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c index e25b02a3104..36e2de9ccc5 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c @@ -58,12 +58,12 @@ __rec_append_orig_value( { WT_DECL_ITEM(tmp); WT_DECL_RET; - WT_UPDATE *append, *tombstone; + WT_UPDATE *append, *oldest_upd, *tombstone; size_t size, total_size; WT_ASSERT(session, upd != NULL && unpack != NULL && unpack->type != WT_CELL_DEL); - append = tombstone = NULL; + append = oldest_upd = tombstone = NULL; total_size = 0; /* Review the current update list, checking conditions that mean no work is needed. */ @@ -75,8 +75,13 @@ __rec_append_orig_value( if (F_ISSET(upd, WT_UPDATE_RESTORED_FOR_ROLLBACK)) return (0); - /* Done if the on page value already appears on the update list. */ - if (unpack->start_ts == upd->start_ts && unpack->start_txn == upd->txnid) + /* + * Done if the on page value already appears on the update list. We can't do the same check + * for stop time pair because we may still need to append the onpage value if only the + * tombstone is on the update chain. + */ + if (unpack->start_ts == upd->start_ts && unpack->start_txn == upd->txnid && + upd->type != WT_UPDATE_TOMBSTONE) return (0); /* @@ -90,6 +95,9 @@ __rec_append_orig_value( if (WT_UPDATE_DATA_VALUE(upd) && __wt_txn_upd_visible_all(session, upd)) return (0); + if (upd->txnid != WT_TXN_ABORTED) + oldest_upd = upd; + /* Leave reference pointing to the last item in the update list. */ if (upd->next == NULL) break; @@ -103,7 +111,7 @@ __rec_append_orig_value( /* We need the original on-page value for some reader: get a copy. */ WT_ERR(__wt_scr_alloc(session, 0, &tmp)); WT_ERR(__wt_page_cell_data_ref(session, page, unpack, tmp)); - WT_ERR(__wt_update_alloc(session, tmp, &append, &size, WT_UPDATE_STANDARD)); + WT_ERR(__wt_upd_alloc(session, tmp, WT_UPDATE_STANDARD, &append, &size)); total_size += size; append->txnid = unpack->start_txn; append->start_ts = unpack->start_ts; @@ -116,14 +124,19 @@ __rec_append_orig_value( * the tombstone to tell us there is no value between 10 and 20. */ if (unpack->stop_ts != WT_TS_MAX || unpack->stop_txn != WT_TXN_MAX) { - WT_ERR(__wt_update_alloc(session, NULL, &tombstone, &size, WT_UPDATE_TOMBSTONE)); - total_size += size; - tombstone->txnid = unpack->stop_txn; - tombstone->start_ts = unpack->stop_ts; - tombstone->durable_ts = unpack->durable_stop_ts; - - tombstone->next = append; - append = tombstone; + /* No need to append the tombstone if it is already in the update chain. */ + if (oldest_upd->type != WT_UPDATE_TOMBSTONE) { + WT_ERR(__wt_upd_alloc_tombstone(session, &tombstone, &size)); + total_size += size; + tombstone->txnid = unpack->stop_txn; + tombstone->start_ts = unpack->stop_ts; + tombstone->durable_ts = unpack->durable_stop_ts; + + tombstone->next = append; + append = tombstone; + } else + WT_ASSERT(session, + unpack->stop_ts == oldest_upd->start_ts && unpack->stop_txn == oldest_upd->txnid); } /* Append the new entry into the update list. */ @@ -178,9 +191,9 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v WT_DECL_ITEM(tmp); WT_DECL_RET; WT_PAGE *page; - WT_UPDATE *first_txn_upd, *first_upd, *upd, *last_upd; + WT_UPDATE *first_txn_upd, *first_upd, *upd, *last_upd, *tombstone; wt_timestamp_t max_ts; - size_t size, upd_memsize; + size_t upd_memsize; uint64_t max_txn, txnid; bool has_newer_updates, is_hs_page, supd_restore, upd_saved; @@ -198,7 +211,7 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v upd_select->prepare = false; page = r->page; - first_txn_upd = upd = last_upd = NULL; + first_txn_upd = upd = last_upd = tombstone = NULL; upd_memsize = 0; max_ts = WT_TS_NONE; max_txn = WT_TXN_NONE; @@ -239,11 +252,21 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v */ if (!is_hs_page && (F_ISSET(r, WT_REC_VISIBLE_ALL) ? WT_TXNID_LE(r->last_running, txnid) : !__txn_visible_id(session, txnid))) { + /* + * Rare case: when applications run at low isolation levels, eviction may see a + * committed update followed by uncommitted updates. Give up in that case because we + * can't move uncommitted updates to the history store. + */ + if (upd_select->upd != NULL) + return (__wt_set_return(session, EBUSY)); + has_newer_updates = true; continue; } + if (upd->prepare_state == WT_PREPARE_LOCKED || upd->prepare_state == WT_PREPARE_INPROGRESS) { + WT_ASSERT(session, upd_select->upd == NULL); has_newer_updates = true; if (upd->start_ts > max_ts) max_ts = upd->start_ts; @@ -265,19 +288,9 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v if (upd_select->upd == NULL) upd_select->upd = upd; - if (!__rec_update_stable(session, r, upd)) { - if (F_ISSET(r, WT_REC_EVICT)) - ++r->updates_unstable; - - /* - * Rare case: when applications run at low isolation levels, update/restore eviction may - * see a stable update followed by an uncommitted update. Give up in that case: we need - * to discard updates from the stable update and older for correctness and we can't - * discard an uncommitted update. - */ - if (upd_select->upd != NULL && has_newer_updates) - return (__wt_set_return(session, EBUSY)); - } else if (!F_ISSET(r, WT_REC_EVICT)) + if (F_ISSET(r, WT_REC_EVICT) && !__rec_update_stable(session, r, upd)) + ++r->updates_unstable; + else if (!F_ISSET(r, WT_REC_EVICT)) break; } @@ -339,6 +352,7 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v upd_select->stop_ts = upd->start_ts; upd_select->stop_txn = upd->txnid; upd_select->stop_durable_ts = upd->durable_ts; + tombstone = upd; /* Find the update this tombstone applies to. */ if (!__wt_txn_visible_all(session, upd->txnid, upd->start_ts)) { @@ -357,7 +371,7 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v upd_select->start_txn = upd->txnid; } else if (upd_select->stop_ts != WT_TS_NONE || upd_select->stop_txn != WT_TXN_NONE) { /* If we only have a tombstone in the update list, we must have an ondisk value. */ - WT_ASSERT(session, vpack != NULL); + WT_ASSERT(session, vpack != NULL && tombstone != NULL); /* * It's possible to have a tombstone as the only update in the update list. If we * reconciled before with only a single update and then read the page back into cache, @@ -368,25 +382,15 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v * keep the same on-disk value but set the stop time pair to indicate that the validity * window ends when this tombstone started. */ - upd_select->start_ts = vpack->start_ts; - upd_select->start_durable_ts = vpack->durable_start_ts; - upd_select->start_txn = vpack->start_txn; - - /* - * Leaving the update unset means that we can skip reconciling. If we've set the stop - * time pair because of a tombstone after the on-disk value, we still have work to do so - * that is NOT ok. Let's append the on-disk value to the chain. - */ - WT_ERR(__wt_scr_alloc(session, 0, &tmp)); - WT_ERR(__wt_page_cell_data_ref(session, page, vpack, tmp)); - WT_ERR(__wt_update_alloc(session, tmp, &upd, &size, WT_UPDATE_STANDARD)); - upd->durable_ts = vpack->durable_start_ts; - upd->start_ts = vpack->start_ts; - upd->txnid = vpack->start_txn; - WT_PUBLISH(last_upd->next, upd); - /* This is going in our update list so it should be accounted for in cache usage. */ - __wt_cache_page_inmem_incr(session, page, size); - upd_select->upd = upd; + WT_ERR(__rec_append_orig_value(session, page, tombstone, vpack)); + WT_ASSERT(session, last_upd->next != NULL && + last_upd->next->txnid == vpack->start_txn && + last_upd->next->start_ts == vpack->start_ts && + last_upd->next->type == WT_UPDATE_STANDARD && last_upd->next->next == NULL); + upd_select->upd = last_upd->next; + upd_select->start_ts = last_upd->next->start_ts; + upd_select->start_durable_ts = last_upd->next->durable_ts; + upd_select->start_txn = last_upd->next->txnid; } } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index ff4fe361abe..365e5841347 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -141,12 +141,10 @@ __reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage, u WT_BTREE *btree; WT_DECL_RET; WT_PAGE *page; - WT_PAGE_MODIFY *mod; WT_RECONCILE *r; btree = S2BT(session); page = ref->page; - mod = page->modify; /* Save the eviction state. */ __reconcile_save_evict_state(session, ref, flags); @@ -193,12 +191,6 @@ __reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage, u else WT_TRET(__rec_write_wrapup_err(session, r, page)); - /* - * If reconciliation completes successfully, save the stable timestamp. - */ - if (ret == 0 && S2C(session)->txn_global.has_stable_timestamp) - mod->last_stable_timestamp = S2C(session)->txn_global.stable_timestamp; - /* Release the reconciliation lock. */ *page_lockedp = false; WT_PAGE_UNLOCK(session, page); diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 24acc8da2d9..c843d592160 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -1563,14 +1563,11 @@ err: static int __session_verify(WT_SESSION *wt_session, const char *uri, const char *config) { - WT_CONFIG_ITEM cval; WT_DECL_RET; WT_SESSION_IMPL *session; session = (WT_SESSION_IMPL *)wt_session; - SESSION_API_CALL(session, verify, config, cfg); - WT_ERR(__wt_inmem_unsupported_op(session, NULL)); /* @@ -1581,24 +1578,10 @@ __session_verify(WT_SESSION *wt_session, const char *uri, const char *config) F_SET(session, WT_SESSION_IGNORE_HS_TOMBSTONE); /* Block out checkpoints to avoid spurious EBUSY errors. */ - WT_ERR(__wt_config_gets(session, cfg, "history_store", &cval)); - if (cval.val == true) { - /* Can't give a URI with history store verification. */ - if (uri != NULL) - WT_ERR_MSG(session, EINVAL, "URI not applicable when verifying the history store"); - - WT_WITH_CHECKPOINT_LOCK(session, - WT_WITH_SCHEMA_LOCK(session, ret = __wt_verify_history_store_tree(session, NULL))); - } else { - WT_WITH_CHECKPOINT_LOCK(session, - WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_verify, NULL, - cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_VERIFY))); - WT_ERR(ret); - /* TODO: WT-5643 Add history store verification for non file URI */ - if (WT_PREFIX_MATCH(uri, "file:")) - WT_WITH_CHECKPOINT_LOCK(session, - WT_WITH_SCHEMA_LOCK(session, ret = __wt_verify_history_store_tree(session, uri))); - } + WT_WITH_CHECKPOINT_LOCK( + session, WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_verify, + NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_VERIFY))); + WT_ERR(ret); err: F_CLR(session, WT_SESSION_IGNORE_HS_TOMBSTONE); if (ret != 0) diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index b238d2dfd3a..b27342c3f93 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -151,8 +151,10 @@ __rollback_row_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW WT_DECL_RET; WT_ITEM full_value; WT_UPDATE *hs_upd, *upd; - wt_timestamp_t durable_ts, hs_start_ts, hs_stop_ts, newer_hs_ts; - size_t size; + wt_timestamp_t durable_ts, hs_start_ts, hs_stop_ts; +#ifdef HAVE_DIAGNOSTIC + wt_timestamp_t newer_hs_ts; +#endif uint64_t hs_counter, type_full; uint32_t hs_btree_id, session_flags; uint8_t type; @@ -162,7 +164,10 @@ __rollback_row_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW hs_cursor = NULL; hs_upd = upd = NULL; - durable_ts = hs_start_ts = newer_hs_ts = WT_TS_NONE; + durable_ts = hs_start_ts = WT_TS_NONE; +#ifdef HAVE_DIAGNOSTIC + newer_hs_ts = WT_TS_NONE; +#endif hs_btree_id = S2BT(session)->id; session_flags = 0; is_owner = valid_update_found = false; @@ -271,12 +276,14 @@ __rollback_row_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW __wt_timestamp_to_string(hs_stop_ts, ts_string[2]), __wt_timestamp_to_string(rollback_timestamp, ts_string[3])); +#ifdef HAVE_DIAGNOSTIC /* * Durable timestamp of the current record is used as stop timestamp of previous record. * Save it to verify against previous record. */ newer_hs_ts = durable_ts; - WT_ERR(__wt_upd_alloc_tombstone(session, &hs_upd)); +#endif + WT_ERR(__wt_upd_alloc_tombstone(session, &hs_upd, NULL)); WT_ERR(__wt_hs_modify(cbt, hs_upd)); WT_STAT_CONN_INCR(session, txn_rts_hs_removed); hs_upd = NULL; @@ -288,7 +295,7 @@ __rollback_row_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW * list. Otherwise remove the key by adding a tombstone. */ if (valid_update_found) { - WT_ERR(__wt_update_alloc(session, &full_value, &upd, &size, WT_UPDATE_STANDARD)); + WT_ERR(__wt_upd_alloc(session, &full_value, WT_UPDATE_STANDARD, &upd, NULL)); upd->txnid = WT_TXN_NONE; upd->durable_ts = durable_ts; @@ -304,7 +311,7 @@ __rollback_row_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW */ F_SET(upd, WT_UPDATE_RESTORED_FOR_ROLLBACK); } else { - WT_ERR(__wt_upd_alloc_tombstone(session, &upd)); + WT_ERR(__wt_upd_alloc_tombstone(session, &upd, NULL)); WT_STAT_CONN_INCR(session, txn_rts_keys_removed); __wt_verbose(session, WT_VERB_RTS, "%p: key removed", (void *)key); } @@ -315,7 +322,7 @@ __rollback_row_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW /* Finally remove that update from history store. */ if (valid_update_found) { - WT_ERR(__wt_upd_alloc_tombstone(session, &hs_upd)); + WT_ERR(__wt_upd_alloc_tombstone(session, &hs_upd, NULL)); WT_ERR(__wt_hs_modify(cbt, hs_upd)); WT_STAT_CONN_INCR(session, txn_rts_hs_removed); hs_upd = NULL; @@ -345,7 +352,6 @@ __rollback_abort_row_ondisk_kv( WT_DECL_RET; WT_ITEM buf; WT_UPDATE *upd; - size_t size; char ts_string[3][WT_TS_INT_STRING_SIZE]; vpack = &_vpack; @@ -365,7 +371,7 @@ __rollback_abort_row_ondisk_kv( * In-memory database don't have a history store to provide a stable update, so remove * the key. */ - WT_RET(__wt_upd_alloc_tombstone(session, &upd)); + WT_RET(__wt_upd_alloc_tombstone(session, &upd, NULL)); WT_STAT_CONN_INCR(session, txn_rts_keys_removed); } } else if (vpack->durable_stop_ts != WT_TS_NONE && @@ -384,7 +390,7 @@ __rollback_abort_row_ondisk_kv( /* Take the value from the original page cell. */ WT_RET(__wt_page_cell_data_ref(session, page, vpack, &buf)); - WT_RET(__wt_update_alloc(session, &buf, &upd, &size, WT_UPDATE_STANDARD)); + WT_RET(__wt_upd_alloc(session, &buf, WT_UPDATE_STANDARD, &upd, NULL)); upd->txnid = vpack->start_txn; upd->durable_ts = vpack->durable_start_ts; upd->start_ts = vpack->start_ts; @@ -957,7 +963,7 @@ __rollback_to_stable_btree_hs_truncate(WT_SESSION_IMPL *session, uint32_t btree_ "rollback to stable history store cleanup of update with start timestamp: %s", __wt_timestamp_to_string(hs_start_ts, ts_string)); - WT_ERR(__wt_upd_alloc_tombstone(session, &hs_upd)); + WT_ERR(__wt_upd_alloc_tombstone(session, &hs_upd, NULL)); WT_ERR(__wt_hs_modify(cbt, hs_upd)); WT_STAT_CONN_INCR(session, txn_rts_hs_removed); hs_upd = NULL; diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c index a000e86cc87..f81e7b54acc 100644 --- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c +++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c @@ -817,13 +817,13 @@ __wt_txn_set_prepare_timestamp(WT_SESSION_IMPL *session, wt_timestamp_t prepare_ { WT_TXN *txn; WT_TXN_GLOBAL *txn_global; - WT_TXN_SHARED *prev_shared, *txn_shared; + WT_TXN_SHARED *prev_shared; wt_timestamp_t oldest_ts; char ts_string[2][WT_TS_INT_STRING_SIZE]; txn = session->txn; txn_global = &S2C(session)->txn_global; - prev_shared = txn_shared = WT_SESSION_TXN_SHARED(session); + prev_shared = WT_SESSION_TXN_SHARED(session); WT_RET(__wt_txn_context_prepare_check(session)); diff --git a/src/third_party/wiredtiger/src/utilities/util.h b/src/third_party/wiredtiger/src/utilities/util.h index d0078edc45f..fc3e3b93128 100644 --- a/src/third_party/wiredtiger/src/utilities/util.h +++ b/src/third_party/wiredtiger/src/utilities/util.h @@ -53,5 +53,6 @@ int util_str2num(WT_SESSION *, const char *, bool, uint64_t *); int util_truncate(WT_SESSION *, int, char *[]); int util_upgrade(WT_SESSION *, int, char *[]); char *util_uri(WT_SESSION *, const char *, const char *); +void util_usage(const char *, const char *, const char *[]); int util_verify(WT_SESSION *, int, char *[]); int util_write(WT_SESSION *, int, char *[]); diff --git a/src/third_party/wiredtiger/src/utilities/util_alter.c b/src/third_party/wiredtiger/src/utilities/util_alter.c index be7524b4b5c..6d5a9520a90 100644 --- a/src/third_party/wiredtiger/src/utilities/util_alter.c +++ b/src/third_party/wiredtiger/src/utilities/util_alter.c @@ -8,7 +8,12 @@ #include "util.h" -static int usage(void); +static int +usage(void) +{ + util_usage("alter uri configuration ...", NULL, NULL); + return (1); +} int util_alter(WT_SESSION *session, int argc, char *argv[]) @@ -38,13 +43,3 @@ util_alter(WT_SESSION *session, int argc, char *argv[]) } return (0); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "alter uri configuration ...\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_backup.c b/src/third_party/wiredtiger/src/utilities/util_backup.c index a7a0e8d0a38..f4b066955c3 100644 --- a/src/third_party/wiredtiger/src/utilities/util_backup.c +++ b/src/third_party/wiredtiger/src/utilities/util_backup.c @@ -9,7 +9,16 @@ #include "util.h" static int copy(WT_SESSION *, const char *, const char *); -static int usage(void); + +static int +usage(void) +{ + static const char *options[] = {"-t uri", + "backup the named data sources (by default the entire database is backed up)", NULL, NULL}; + + util_usage("backup [-t uri] directory", "options:", options); + return (1); +} int util_backup(WT_SESSION *session, int argc, char *argv[]) @@ -114,13 +123,3 @@ err: free(to); return (ret); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "backup [-t uri] directory\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_compact.c b/src/third_party/wiredtiger/src/utilities/util_compact.c index 298f4b234e0..e4f23b3feef 100644 --- a/src/third_party/wiredtiger/src/utilities/util_compact.c +++ b/src/third_party/wiredtiger/src/utilities/util_compact.c @@ -8,7 +8,12 @@ #include "util.h" -static int usage(void); +static int +usage(void) +{ + util_usage("compact uri", NULL, NULL); + return (1); +} int util_compact(WT_SESSION *session, int argc, char *argv[]) @@ -39,13 +44,3 @@ util_compact(WT_SESSION *session, int argc, char *argv[]) free(uri); return (ret); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "compact uri\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_create.c b/src/third_party/wiredtiger/src/utilities/util_create.c index 0b6bb4fb4ad..12abf2b2eda 100644 --- a/src/third_party/wiredtiger/src/utilities/util_create.c +++ b/src/third_party/wiredtiger/src/utilities/util_create.c @@ -8,7 +8,15 @@ #include "util.h" -static int usage(void); +static int +usage(void) +{ + static const char *options[] = { + "-c config", "a configuration string to be passed to WT_SESSION.create", NULL, NULL}; + + util_usage("create [-c configuration] uri", "options:", options); + return (1); +} int util_create(WT_SESSION *session, int argc, char *argv[]) @@ -44,13 +52,3 @@ util_create(WT_SESSION *session, int argc, char *argv[]) free(uri); return (ret); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "create [-c configuration] uri\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_downgrade.c b/src/third_party/wiredtiger/src/utilities/util_downgrade.c index c091a4cc9ef..f942611432d 100644 --- a/src/third_party/wiredtiger/src/utilities/util_downgrade.c +++ b/src/third_party/wiredtiger/src/utilities/util_downgrade.c @@ -8,7 +8,15 @@ #include "util.h" -static int usage(void); +static int +usage(void) +{ + static const char *options[] = { + "-V", "a required option, the version to which the database is downgraded", NULL, NULL}; + + util_usage("downgrade -V release", "options:", options); + return (1); +} int util_downgrade(WT_SESSION *session, int argc, char *argv[]) @@ -45,13 +53,3 @@ util_downgrade(WT_SESSION *session, int argc, char *argv[]) return (0); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "downgrade -V release\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_drop.c b/src/third_party/wiredtiger/src/utilities/util_drop.c index 594e5629e1e..6784735b340 100644 --- a/src/third_party/wiredtiger/src/utilities/util_drop.c +++ b/src/third_party/wiredtiger/src/utilities/util_drop.c @@ -8,7 +8,12 @@ #include "util.h" -static int usage(void); +static int +usage(void) +{ + util_usage("drop uri", NULL, NULL); + return (1); +} int util_drop(WT_SESSION *session, int argc, char *argv[]) @@ -40,13 +45,3 @@ util_drop(WT_SESSION *session, int argc, char *argv[]) free(uri); return (ret); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "drop uri\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_dump.c b/src/third_party/wiredtiger/src/utilities/util_dump.c index e1aacbd3408..47b6f99bbe9 100755 --- a/src/third_party/wiredtiger/src/utilities/util_dump.c +++ b/src/third_party/wiredtiger/src/utilities/util_dump.c @@ -26,7 +26,24 @@ static int dump_table_parts_config(WT_SESSION *, WT_CURSOR *, const char *, cons static int dup_json_string(const char *, char **); static int print_config(WT_SESSION *, const char *, const char *, bool, bool); static int time_pair_to_timestamp(WT_SESSION_IMPL *, char *, WT_ITEM *); -static int usage(void); + +static int +usage(void) +{ + static const char *options[] = {"-c checkpoint", + "dump as of the named checkpoint (the default is the most recent version of the data)", + "-f output", "dump to the specified file (the default is stdout)", "-j", + "dump in JSON format", "-r", "dump in reverse order", "-t timestamp", + "dump as of the specified timestamp (the default is the most recent version of the data)", + "-x", + "dump all characters in a hexadecimal encoding (by default printable characters are not " + "encoded)", + NULL, NULL}; + + util_usage( + "dump [-jrx] [-c checkpoint] [-f output-file] [-t timestamp] uri", "options:", options); + return (1); +} static FILE *fp; @@ -54,15 +71,15 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) case 'f': ofile = __wt_optarg; break; - case 't': - timestamp = __wt_optarg; - break; case 'j': json = true; break; case 'r': reverse = true; break; + case 't': + timestamp = __wt_optarg; + break; case 'x': hex = true; break; @@ -664,13 +681,3 @@ print_config(WT_SESSION *session, const char *key, const char *cfg, bool json, b return (util_err(session, EIO, NULL)); return (0); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "dump [-jrx] [-c checkpoint] [-f output-file] [-t timestamp] uri\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_import.c b/src/third_party/wiredtiger/src/utilities/util_import.c index 9420ff35b20..8f3ec92f492 100644 --- a/src/third_party/wiredtiger/src/utilities/util_import.c +++ b/src/third_party/wiredtiger/src/utilities/util_import.c @@ -11,10 +11,7 @@ static int usage(void) { - (void)fprintf(stderr, - "usage: %s %s " - "import uri\n", - progname, usage_prefix); + util_usage("import uri", NULL, NULL); return (1); } diff --git a/src/third_party/wiredtiger/src/utilities/util_list.c b/src/third_party/wiredtiger/src/utilities/util_list.c index 1463ba23862..ab2e6ad1299 100644 --- a/src/third_party/wiredtiger/src/utilities/util_list.c +++ b/src/third_party/wiredtiger/src/utilities/util_list.c @@ -11,7 +11,18 @@ static int list_get_allocsize(WT_SESSION *, const char *, size_t *); static int list_print(WT_SESSION *, const char *, bool, bool); static int list_print_checkpoint(WT_SESSION *, const char *); -static int usage(void); + +static int +usage(void) +{ + static const char *options[] = {"-c", + "display checkpoints in human-readable format (by default checkpoints are not displayed)", + "-v", "display the complete schema table (by default only a subset is displayed)", NULL, + NULL}; + + util_usage("list [-cv] [uri]", "options:", options); + return (1); +} int util_list(WT_SESSION *session, int argc, char *argv[]) @@ -276,13 +287,3 @@ list_print_checkpoint(WT_SESSION *session, const char *key) __wt_metadata_free_ckptlist(session, ckptbase); return (0); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "list [-cv] [uri]\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_load.c b/src/third_party/wiredtiger/src/utilities/util_load.c index de1931835bb..482074d85fb 100644 --- a/src/third_party/wiredtiger/src/utilities/util_load.c +++ b/src/third_party/wiredtiger/src/utilities/util_load.c @@ -14,7 +14,6 @@ static int config_rename(WT_SESSION *, char **, const char *); static int format(WT_SESSION *); static int insert(WT_CURSOR *, const char *); static int load_dump(WT_SESSION *); -static int usage(void); static bool append = false; /* -a append (ignore number keys) */ static char *cmdname; /* -r rename */ @@ -22,6 +21,20 @@ static char **cmdconfig; /* configuration pairs */ static bool json = false; /* -j input is JSON format */ static bool no_overwrite = false; /* -n don't overwrite existing data */ +static int +usage(void) +{ + static const char *options[] = {"-a", + "ignore record number keys in the input and assign new record number keys", "-f input", + "read from the specified file (by default records are read from stdin)", "-j", + "read in JSON format", "-n", "fail at any attempt to overwrite existing data", "-r name", + "use the argument as the table name, ignoring any name in the source", NULL, NULL}; + + util_usage( + "load [-as] [-f input-file] [-r name] [object configuration ...]", "options:", options); + return (1); +} + int util_load(WT_SESSION *session, int argc, char *argv[]) { @@ -561,13 +574,3 @@ err: return (ret); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "load [-as] [-f input-file] [-r name] [object configuration ...]\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_loadtext.c b/src/third_party/wiredtiger/src/utilities/util_loadtext.c index afd845c95c6..2ae1374a241 100644 --- a/src/third_party/wiredtiger/src/utilities/util_loadtext.c +++ b/src/third_party/wiredtiger/src/utilities/util_loadtext.c @@ -10,7 +10,16 @@ static int insert(WT_CURSOR *, const char *, bool); static int text(WT_SESSION *, const char *); -static int usage(void); + +static int +usage(void) +{ + static const char *options[] = { + "-f", "read from the specified file (by default rows are read from stdin)", NULL, NULL}; + + util_usage("loadtext [-f input-file] uri", "options:", options); + return (1); +} int util_loadtext(WT_SESSION *session, int argc, char *argv[]) @@ -151,13 +160,3 @@ insert(WT_CURSOR *cursor, const char *name, bool readkey) return (0); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "loadtext [-f input-file] uri\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_main.c b/src/third_party/wiredtiger/src/utilities/util_main.c index 6966971ec20..3859c87cde7 100644 --- a/src/third_party/wiredtiger/src/utilities/util_main.c +++ b/src/third_party/wiredtiger/src/utilities/util_main.c @@ -11,7 +11,7 @@ const char *home = "."; /* Home directory */ const char *progname; /* Program name */ /* Global arguments */ -const char *usage_prefix = "[-LRrSVv] [-C config] [-E secretkey] [-h home]"; +const char *usage_prefix = "[-LmRrSVv] [-C config] [-E secretkey] [-h home]"; bool verbose = false; /* Verbose flag */ static const char *command; /* Command name */ @@ -25,85 +25,32 @@ static const char *command; /* Command name */ static void usage(void) { - fprintf(stderr, "WiredTiger Data Engine (version %d.%d)\n", WIREDTIGER_VERSION_MAJOR, - WIREDTIGER_VERSION_MINOR); - fprintf(stderr, - "global options:\n" - "\t" - "-C\t" - "wiredtiger_open configuration\n" - "\t" - "-E\t" - "secret encryption key\n" - "\t" - "-h\t" - "database directory\n" - "\t" - "-L\t" - "turn logging off for debug-mode\n" - "\t" - "-R\t" - "run recovery (if recovery configured)\n" - "\t" - "-r\t" - "access the database via a readonly connection\n" - "\t" - "-S\t" - "run salvage recovery (if recovery configured)\n" - "\t" - "-V\t" - "display library version and exit\n" - "\t" - "-v\t" - "verbose\n"); - fprintf(stderr, - "commands:\n" - "\t" - "alter\t alter an object\n" - "\t" - "backup\t database backup\n" - "\t" - "compact\t compact an object\n" - "\t" - "copyright copyright information\n" - "\t" - "create\t create an object\n" - "\t" - "downgrade downgrade a database\n" - "\t" - "drop\t drop an object\n" - "\t" - "dump\t dump an object\n" + static const char *options[] = {"-B", "maintain release 3.3 log file compatibility", + "-C config", "wiredtiger_open configuration", "-E key", "secret encryption key", "-h home", + "database directory", "-L", "turn logging off for debug-mode", "-m", "run verify on metadata", + "-R", "run recovery (if recovery configured)", "-r", + "access the database via a readonly connection", "-S", + "run salvage recovery (if recovery configured)", "-V", "display library version and exit", + "-v", "verbose", NULL, NULL}; + static const char *commands[] = {"alter", "alter an object", "backup", "database backup", + "compact", "compact an object", "copyright", "display copyright information", "create", + "create an object", "downgrade", "downgrade a database", "drop", "drop an object", "dump", + "dump an object", /* * Import is not documented. - * "\t" "import\t import an object\n" + * "import", "import an object" */ - "\t" - "list\t list database objects\n" - "\t" - "load\t load an object\n" - "\t" - "loadtext load an object from a text file\n" - "\t" - "printlog display the database log\n" - "\t" - "read\t read values from an object\n" - "\t" - "rebalance rebalance an object\n" - "\t" - "rename\t rename an object\n" - "\t" - "salvage\t salvage a file\n" - "\t" - "stat\t display statistics for an object\n" - "\t" - "truncate truncate an object, removing all content\n" - "\t" - "upgrade\t upgrade an object\n" - "\t" - "verify\t verify an object\n" - "\t" - "write\t write values to an object\n"); + "list", "list database objects", "load", "load an object", "loadtext", + "load an object from a text file", "printlog", "display the database log", "read", + "read values from an object", "rebalance", "rebalance an object", "rename", + "rename an object", "salvage", "salvage a file", "stat", "display statistics for an object", + "truncate", "truncate an object, removing all content", "upgrade", "upgrade an object", + "verify", "verify an object", "write", "write values to an object", NULL, NULL}; + + fprintf(stderr, "WiredTiger Data Engine (version %d.%d)\n", WIREDTIGER_VERSION_MAJOR, + WIREDTIGER_VERSION_MINOR); + util_usage(NULL, "global_options:", options); + util_usage(NULL, "commands:", commands); } int @@ -116,7 +63,7 @@ main(int argc, char *argv[]) int ch, major_v, minor_v, tret, (*func)(WT_SESSION *, int, char *[]); char *p, *secretkey; const char *cmd_config, *config, *p1, *p2, *p3, *readonly_config, *rec_config; - bool logoff, readonly, recover, salvage; + bool backward_compatible, logoff, meta_verify, readonly, recover, salvage; conn = NULL; p = NULL; @@ -145,10 +92,13 @@ main(int argc, char *argv[]) * needed, the user can specify -R to run recovery. */ rec_config = REC_ERROR; - logoff = readonly = recover = salvage = false; + backward_compatible = logoff = meta_verify = readonly = recover = salvage = false; /* Check for standard options. */ - while ((ch = __wt_getopt(progname, argc, argv, "C:E:h:LRrSVv")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "BC:E:h:LmRrSVv")) != EOF) switch (ch) { + case 'B': /* backward compatibility */ + backward_compatible = true; + break; case 'C': /* wiredtiger_open config */ cmd_config = __wt_optarg; break; @@ -167,6 +117,10 @@ main(int argc, char *argv[]) rec_config = REC_LOGOFF; logoff = true; break; + case 'm': /* verify metadata on connection open */ + cmd_config = "verify_metadata=true"; + meta_verify = true; + break; case 'R': /* recovery */ rec_config = REC_RECOVER; recover = true; @@ -201,8 +155,11 @@ main(int argc, char *argv[]) argc -= __wt_optind; argv += __wt_optind; + func = NULL; /* The next argument is the command name. */ if (argc < 1) { + if (meta_verify) + goto open; usage(); goto err; } @@ -210,7 +167,6 @@ main(int argc, char *argv[]) /* Reset getopt. */ __wt_optreset = __wt_optind = 1; - func = NULL; switch (command[0]) { case 'a': if (strcmp(command, "alter") == 0) @@ -300,6 +256,7 @@ main(int argc, char *argv[]) goto err; } +open: /* Build the configuration string. */ len = 10; /* some slop */ p1 = p2 = p3 = ""; @@ -329,17 +286,21 @@ main(int argc, char *argv[]) } config = p; - /* Open the database and a session. */ if ((ret = wiredtiger_open(home, verbose ? verbose_handler : NULL, config, &conn)) != 0) { (void)util_err(NULL, ret, NULL); goto err; } + + /* If we only want to verify the metadata, that is done in wiredtiger_open. We're done. */ + if (func == NULL && meta_verify) + goto done; + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) { (void)util_err(NULL, ret, NULL); goto err; } - /* Call the function. */ + /* Call the function after opening the database and session. */ ret = func(session, argc, argv); if (0) { @@ -348,9 +309,16 @@ err: } done: - /* Close the database. */ - if (conn != NULL && (tret = conn->close(conn, NULL)) != 0 && ret == 0) - ret = tret; + if (conn != NULL) { + /* Maintain backward compatibility. */ + if (backward_compatible && + (tret = conn->reconfigure(conn, "compatibility=(release=3.3)")) != 0 && ret == 0) + ret = tret; + + /* Close the database. */ + if ((tret = conn->close(conn, NULL)) != 0 && ret == 0) + ret = tret; + } free(p); free(secretkey); diff --git a/src/third_party/wiredtiger/src/utilities/util_misc.c b/src/third_party/wiredtiger/src/utilities/util_misc.c index a999f334e3f..eb8eb5782fb 100644 --- a/src/third_party/wiredtiger/src/utilities/util_misc.c +++ b/src/third_party/wiredtiger/src/utilities/util_misc.c @@ -152,3 +152,21 @@ util_flush(WT_SESSION *session, const char *uri) (void)util_err(session, ret, "%s: session.drop", uri); return (1); } + +/* + * util_usage -- + * Display a usage statement. + */ +void +util_usage(const char *usage, const char *tag, const char *list[]) +{ + const char **p; + + if (usage != NULL) + fprintf(stderr, "usage: %s %s %s\n", progname, usage_prefix, usage); + if (tag != NULL) + fprintf(stderr, "%s\n", tag); + if (list != NULL) + for (p = list; *p != NULL; p += 2) + fprintf(stderr, " %s%s%s\n", p[0], strlen(p[0]) > 2 ? "\n " : " ", p[1]); +} diff --git a/src/third_party/wiredtiger/src/utilities/util_printlog.c b/src/third_party/wiredtiger/src/utilities/util_printlog.c index fa96ae2065c..ee46e98159f 100644 --- a/src/third_party/wiredtiger/src/utilities/util_printlog.c +++ b/src/third_party/wiredtiger/src/utilities/util_printlog.c @@ -8,7 +8,15 @@ #include "util.h" -static int usage(void); +static int +usage(void) +{ + static const char *options[] = {"-f", "output to the specified file", "-x", + "display key and value items in hexadecimal format", NULL, NULL}; + + util_usage("printlog [-x] [-f output-file]", "options:", options); + return (1); +} int util_printlog(WT_SESSION *session, int argc, char *argv[]) @@ -43,13 +51,3 @@ util_printlog(WT_SESSION *session, int argc, char *argv[]) return (ret); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "printlog [-x] [-f output-file]\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_read.c b/src/third_party/wiredtiger/src/utilities/util_read.c index 9266bf5c026..7c4d94cc124 100644 --- a/src/third_party/wiredtiger/src/utilities/util_read.c +++ b/src/third_party/wiredtiger/src/utilities/util_read.c @@ -8,7 +8,12 @@ #include "util.h" -static int usage(void); +static int +usage(void) +{ + util_usage("read uri key ...", NULL, NULL); + return (1); +} int util_read(WT_SESSION *session, int argc, char *argv[]) @@ -94,13 +99,3 @@ util_read(WT_SESSION *session, int argc, char *argv[]) return (rval ? 1 : 0); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "read uri key ...\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_rebalance.c b/src/third_party/wiredtiger/src/utilities/util_rebalance.c index 279c0d46958..82d719b2aa9 100644 --- a/src/third_party/wiredtiger/src/utilities/util_rebalance.c +++ b/src/third_party/wiredtiger/src/utilities/util_rebalance.c @@ -8,7 +8,12 @@ #include "util.h" -static int usage(void); +static int +usage(void) +{ + util_usage("rebalance uri", NULL, NULL); + return (1); +} int util_rebalance(WT_SESSION *session, int argc, char *argv[]) @@ -46,13 +51,3 @@ util_rebalance(WT_SESSION *session, int argc, char *argv[]) free(uri); return (ret); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "rebalance uri\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_rename.c b/src/third_party/wiredtiger/src/utilities/util_rename.c index 4dc03e83243..0c865881ed7 100644 --- a/src/third_party/wiredtiger/src/utilities/util_rename.c +++ b/src/third_party/wiredtiger/src/utilities/util_rename.c @@ -8,7 +8,12 @@ #include "util.h" -static int usage(void); +static int +usage(void) +{ + util_usage("rename uri newuri", NULL, NULL); + return (1); +} int util_rename(WT_SESSION *session, int argc, char *argv[]) @@ -40,13 +45,3 @@ util_rename(WT_SESSION *session, int argc, char *argv[]) free(uri); return (ret); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "rename uri newuri\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_salvage.c b/src/third_party/wiredtiger/src/utilities/util_salvage.c index f6ff1a1fbb2..d26e6a3a049 100644 --- a/src/third_party/wiredtiger/src/utilities/util_salvage.c +++ b/src/third_party/wiredtiger/src/utilities/util_salvage.c @@ -8,7 +8,16 @@ #include "util.h" -static int usage(void); +static int +usage(void) +{ + static const char *options[] = {"-F", + "force salvage (by default salvage will refuse to salvage tables that fail basic tests)", + NULL, NULL}; + + util_usage("salvage [-F] uri", "options:", options); + return (1); +} int util_salvage(WT_SESSION *session, int argc, char *argv[]) @@ -51,13 +60,3 @@ util_salvage(WT_SESSION *session, int argc, char *argv[]) free(uri); return (ret); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "salvage [-F] uri\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_stat.c b/src/third_party/wiredtiger/src/utilities/util_stat.c index 4ffed2b5376..af9072e8e37 100644 --- a/src/third_party/wiredtiger/src/utilities/util_stat.c +++ b/src/third_party/wiredtiger/src/utilities/util_stat.c @@ -8,7 +8,15 @@ #include "util.h" -static int usage(void); +static int +usage(void) +{ + static const char *options[] = { + "-f", "include only \"fast\" statistics in the output", NULL, NULL}; + + util_usage("stat [-f] [uri]", "options:", options); + return (1); +} int util_stat(WT_SESSION *session, int argc, char *argv[]) @@ -103,13 +111,3 @@ err: return (ret); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "stat [-f] [uri]\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_truncate.c b/src/third_party/wiredtiger/src/utilities/util_truncate.c index d6904a18e51..2b47783fb25 100644 --- a/src/third_party/wiredtiger/src/utilities/util_truncate.c +++ b/src/third_party/wiredtiger/src/utilities/util_truncate.c @@ -8,7 +8,12 @@ #include "util.h" -static int usage(void); +static int +usage(void) +{ + util_usage("truncate uri", NULL, NULL); + return (1); +} int util_truncate(WT_SESSION *session, int argc, char *argv[]) @@ -40,13 +45,3 @@ util_truncate(WT_SESSION *session, int argc, char *argv[]) free(uri); return (ret); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "truncate uri\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_upgrade.c b/src/third_party/wiredtiger/src/utilities/util_upgrade.c index 2aad1d1e8d3..b85b88b325c 100644 --- a/src/third_party/wiredtiger/src/utilities/util_upgrade.c +++ b/src/third_party/wiredtiger/src/utilities/util_upgrade.c @@ -8,7 +8,12 @@ #include "util.h" -static int usage(void); +static int +usage(void) +{ + util_usage("upgrade uri", NULL, NULL); + return (1); +} int util_upgrade(WT_SESSION *session, int argc, char *argv[]) @@ -46,13 +51,3 @@ util_upgrade(WT_SESSION *session, int argc, char *argv[]) free(uri); return (ret); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "upgrade uri\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_verify.c b/src/third_party/wiredtiger/src/utilities/util_verify.c index daf80e4bb1f..dcdab59e142 100644 --- a/src/third_party/wiredtiger/src/utilities/util_verify.c +++ b/src/third_party/wiredtiger/src/utilities/util_verify.c @@ -8,7 +8,20 @@ #include "util.h" -static int usage(void); +static int +usage(void) +{ + static const char *options[] = {"-a", "verify the history store", "-d config", + "display underlying information during verification", "-s", + "verify against the specified timestamp", NULL, NULL}; + + util_usage( + "verify [-as] [-d dump_address | dump_blocks | dump_history | dump_layout | dump_offsets=#,# " + "| dump_pages] [uri]", + "options:", options); + + return (1); +} int util_verify(WT_SESSION *session, int argc, char *argv[]) @@ -17,17 +30,12 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) size_t size; int ch; char *config, *dump_offsets, *uri; - bool dump_address, dump_blocks, dump_layout, dump_pages, dump_history, hs_verify, - stable_timestamp; + bool dump_address, dump_blocks, dump_layout, dump_pages, dump_history, stable_timestamp; - dump_address = dump_blocks = dump_history = dump_layout = dump_pages = hs_verify = - stable_timestamp = false; + dump_address = dump_blocks = dump_history = dump_layout = dump_pages = stable_timestamp = false; config = dump_offsets = uri = NULL; - while ((ch = __wt_getopt(progname, argc, argv, "ad:s")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "d:s")) != EOF) switch (ch) { - case 'a': - hs_verify = true; - break; case 'd': if (strcmp(__wt_optarg, "dump_address") == 0) dump_address = true; @@ -65,22 +73,13 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) * The remaining argument is the table name. If we are verifying the history store we do not * accept a URI. Otherwise, we need a URI top operate on. */ - if (hs_verify && argc != 0) - (void)util_err(session, 0, "-a can't be used along with a uri"); - if (!hs_verify) { - if (argc != 1) - return (usage()); - if ((uri = util_uri(session, *argv, "table")) == NULL) - return (1); - } - - if (hs_verify && (dump_address || dump_blocks || dump_layout || dump_offsets != NULL || - dump_pages || stable_timestamp)) { - (void)util_err(session, 0, "-a and -d are not supported together"); - } + if (argc != 1) + return (usage()); + if ((uri = util_uri(session, *argv, "table")) == NULL) + return (1); if (dump_address || dump_blocks || dump_history || dump_layout || dump_offsets != NULL || - dump_pages || hs_verify || stable_timestamp) { + dump_pages || stable_timestamp) { size = strlen("dump_address,") + strlen("dump_blocks,") + strlen("dump_history") + strlen("dump_layout,") + strlen("dump_pages,") + strlen("dump_offsets[],") + (dump_offsets == NULL ? 0 : strlen(dump_offsets)) + strlen("history_store") + @@ -89,13 +88,13 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) ret = util_err(session, errno, NULL); goto err; } - if ((ret = __wt_snprintf(config, size, "%s%s%s%s%s%s%s%s%s%s", + if ((ret = __wt_snprintf(config, size, "%s%s%s%s%s%s%s%s%s", dump_address ? "dump_address," : "", dump_blocks ? "dump_blocks," : "", dump_history ? "dump_history," : "", dump_layout ? "dump_layout," : "", dump_offsets != NULL ? "dump_offsets=[" : "", dump_offsets != NULL ? dump_offsets : "", dump_offsets != NULL ? "]," : "", - dump_pages ? "dump_pages," : "", hs_verify ? "history_store" : "", - stable_timestamp ? "stable_timestamp," : "")) != 0) { + dump_pages ? "dump_pages," : "", stable_timestamp ? "stable_timestamp," : "")) != + 0) { (void)util_err(session, ret, NULL); goto err; } @@ -115,15 +114,3 @@ err: free(uri); return (ret); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "verify %s\n", - progname, usage_prefix, - "[-d dump_address | dump_blocks | dump_history | dump_layout | " - "dump_offsets=#,# | dump_pages] [-s] -a|uri"); - return (1); -} diff --git a/src/third_party/wiredtiger/src/utilities/util_write.c b/src/third_party/wiredtiger/src/utilities/util_write.c index fa5b56b0d44..fdb5f5a716c 100644 --- a/src/third_party/wiredtiger/src/utilities/util_write.c +++ b/src/third_party/wiredtiger/src/utilities/util_write.c @@ -8,7 +8,15 @@ #include "util.h" -static int usage(void); +static int +usage(void) +{ + static const char *options[] = {"-a", "append each value as a new record in the data source", + "-o", "allow overwrite of previously existing records", NULL, NULL}; + + util_usage("write [-ao] uri key ...", "options:", options); + return (1); +} int util_write(WT_SESSION *session, int argc, char *argv[]) @@ -102,13 +110,3 @@ util_write(WT_SESSION *session, int argc, char *argv[]) return (0); } - -static int -usage(void) -{ - (void)fprintf(stderr, - "usage: %s %s " - "write [-ao] uri key ...\n", - progname, usage_prefix); - return (1); -} diff --git a/src/third_party/wiredtiger/test/csuite/Makefile.am b/src/third_party/wiredtiger/test/csuite/Makefile.am index 89c02c290ad..8eaf1a83756 100644 --- a/src/third_party/wiredtiger/test/csuite/Makefile.am +++ b/src/third_party/wiredtiger/test/csuite/Makefile.am @@ -53,7 +53,8 @@ all_TESTS += test_wt2403_lsm_workload test_wt2246_col_append_SOURCES = wt2246_col_append/main.c noinst_PROGRAMS += test_wt2246_col_append -all_TESTS += test_wt2246_col_append +# Temporarily disabled (WT-5790) +# all_TESTS += test_wt2246_col_append test_wt2323_join_visibility_SOURCES = wt2323_join_visibility/main.c noinst_PROGRAMS += test_wt2323_join_visibility diff --git a/src/third_party/wiredtiger/test/csuite/incr_backup/main.c b/src/third_party/wiredtiger/test/csuite/incr_backup/main.c index 9892eaa373e..fe0e0677996 100644 --- a/src/third_party/wiredtiger/test/csuite/incr_backup/main.c +++ b/src/third_party/wiredtiger/test/csuite/incr_backup/main.c @@ -58,9 +58,8 @@ static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); */ static bool slow_incremental = false; -/* TODO: rename and drop are not currently working, they give resource busy. */ -static bool do_rename = false; -static bool do_drop = false; +static bool do_drop = true; +static bool do_rename = true; #define VERBOSE(level, fmt, ...) \ do { \ diff --git a/src/third_party/wiredtiger/test/csuite/random_abort/main.c b/src/third_party/wiredtiger/test/csuite/random_abort/main.c index 9d75cb4e463..906492c6c20 100644 --- a/src/third_party/wiredtiger/test/csuite/random_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/random_abort/main.c @@ -156,9 +156,15 @@ thread_run(void *arg) testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); - /* Make sure that alternative threads operate on column-store table */ +#if 0 + /* + * Make sure that alternative threads operate on column-store table + * + * FIXME: temporarily turn off column store test. + */ if (td->id % 2 != 0) columnar_table = true; +#endif if (columnar_table) testutil_check(session->open_cursor(session, col_uri, NULL, NULL, &cursor)); @@ -174,6 +180,9 @@ thread_run(void *arg) if (i == 0) i++; + /* FIXME: temporarily turn off tests for lower isolation levels. */ + testutil_check(session->begin_transaction(session, "isolation=snapshot")); + /* * The value is the insert- with key appended. */ @@ -198,6 +207,10 @@ thread_run(void *arg) } cursor->set_value(cursor, &data); testutil_check(cursor->insert(cursor)); + + /* FIXME: temporarily turn off tests for lower isolation levels. */ + testutil_check(session->commit_transaction(session, NULL)); + /* * Save the key separately for checking later. */ @@ -208,6 +221,9 @@ thread_run(void *arg) * Decide what kind of operation can be performed on the already inserted data. */ if (i % MAX_NUM_OPS == OP_TYPE_DELETE) { + /* FIXME: temporarily turn off tests for lower isolation levels. */ + testutil_check(session->begin_transaction(session, "isolation=snapshot")); + if (columnar_table) cursor->set_key(cursor, i); else @@ -215,12 +231,13 @@ thread_run(void *arg) testutil_check(cursor->remove(cursor)); + /* FIXME: temporarily turn off tests for lower isolation levels. */ + testutil_check(session->commit_transaction(session, NULL)); + /* Save the key separately for checking later.*/ if (fprintf(fp[DELETE_RECORD_FILE_ID], "%" PRIu64 "\n", i) == -1) testutil_die(errno, "fprintf"); - } else if (i % MAX_NUM_OPS == OP_TYPE_INSERT) - continue; - else if (i % MAX_NUM_OPS == OP_TYPE_MODIFY) { + } else if (i % MAX_NUM_OPS == OP_TYPE_MODIFY) { testutil_check(__wt_snprintf(new_buf, sizeof(new_buf), "modify-%" PRIu64, i)); new_buf_size = (data.size < MAX_VAL - 1 ? data.size : MAX_VAL - 1); @@ -259,7 +276,7 @@ thread_run(void *arg) */ if (fprintf(fp[MODIFY_RECORD_FILE_ID], "%s %" PRIu64 "\n", new_buf, i) == -1) testutil_die(errno, "fprintf"); - } else + } else if (i % MAX_NUM_OPS != OP_TYPE_INSERT) /* Dead code. To catch any op type misses */ testutil_die(0, "Unsupported operation type."); } @@ -360,6 +377,8 @@ recover_and_verify(uint32_t nthreads) absent = count = 0; fatal = false; for (i = 0; i < nthreads; ++i) { + +#if 0 /* * Every alternative thread is operated on column-store table. Make sure that proper cursor * is used for verification of recovered records. @@ -371,6 +390,11 @@ recover_and_verify(uint32_t nthreads) columnar_table = false; cursor = row_cursor; } +#else + /* FIXME: temporarily turn off column store test. */ + columnar_table = false; + cursor = row_cursor; +#endif middle = 0; testutil_check(__wt_snprintf(fname[DELETE_RECORD_FILE_ID], diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index d5eaa4bd8e0..2649f1ee703 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -1098,20 +1098,21 @@ tasks: ${test_env_vars|} $(pwd)/test/csuite/test_rwlock 2>&1 - - name: csuite-wt2246-col-append-test - tags: ["pull_request"] - depends_on: - - name: compile - commands: - - func: "fetch artifacts" - - command: shell.exec - params: - working_dir: "wiredtiger/build_posix" - script: | - set -o errexit - set -o verbose + # Temporarily disabled (WT-5790) + # - name: csuite-wt2246-col-append-test + # tags: ["pull_request"] + # depends_on: + # - name: compile + # commands: + # - func: "fetch artifacts" + # - command: shell.exec + # params: + # working_dir: "wiredtiger/build_posix" + # script: | + # set -o errexit + # set -o verbose - ${test_env_vars|} $(pwd)/test/csuite/test_wt2246_col_append 2>&1 + # ${test_env_vars|} $(pwd)/test/csuite/test_wt2246_col_append 2>&1 # Temporarily disabled # - name: csuite-wt2323-join-visibility-test @@ -1543,7 +1544,7 @@ tasks: ulimit -c unlimited largescale/run-million-collection-test.sh . - - name: compatibility-test-for-mongodb-releases + - name: compatibility-test-for-releases commands: - func: "get project" - command: shell.exec @@ -1552,7 +1553,7 @@ tasks: script: | set -o errexit set -o verbose - test/evergreen/compatibility_test_for_mongodb_releases.sh + test/evergreen/compatibility_test_for_releases.sh # Temporarily disabled # - name: generate-datafile-little-endian @@ -2209,14 +2210,6 @@ tasks: exit 1 fi - - name: compatibility-test-for-wiredtiger-releases - commands: - - func: "get project" - - command: subprocess.exec - params: - working_dir: "wiredtiger" - command: bash test/evergreen/compatibility_test_for_wiredtiger_releases.sh - - name: format-stress-sanitizer-ppc-test # Set 2.5 hours timeout (60 * 60 * 2.5) exec_timeout_secs: 9000 @@ -2410,8 +2403,7 @@ buildvariants: run_on: - ubuntu1804-test tasks: - - name: compatibility-test-for-mongodb-releases - - name: compatibility-test-for-wiredtiger-releases + - name: compatibility-test-for-releases - name: windows-64 display_name: Windows 64-bit diff --git a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_mongodb_releases.sh b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_mongodb_releases.sh deleted file mode 100755 index cf442ab4ff5..00000000000 --- a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_mongodb_releases.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env bash -############################################################################################## -# Check releases to ensure backward compatibility. -############################################################################################## - -set -e - -############################################################# -# build_release: -# arg1: release -############################################################# -build_release() -{ - echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - echo "Building release: \"$1\"" - echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - - git clone --quiet https://github.com/wiredtiger/wiredtiger.git "$1" - cd "$1" - git checkout --quiet "$1" - - config="" - config+="--enable-diagnostic " - config+="--enable-snappy " - (sh build_posix/reconf && - ./configure $config && make -j $(grep -c ^processor /proc/cpuinfo)) > /dev/null -} - -############################################################# -# run_format: -# arg1: release -# arg2: access methods list -# arg3: -B for compatibility testing -############################################################# -run_format() -{ - echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - echo "Running format in release: \"$1\"" - echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - - cd "$1/test/format" - - args="" - args+="cache=80 " # Medium cache so there's eviction - args+="checkpoints=1 " # Force periodic writes - args+="compression=snappy " # We only built with snappy, force the choice - args+="data_source=table " - args+="in_memory=0 " # Interested in the on-disk format - args+="leak_memory=1 " # Faster runs - args+="logging=1 " # Test log compatibility - args+="logging_compression=snappy " # We only built with snappy, force the choice - args+="rebalance=0 " # Faster runs - args+="rows=1000000 " - args+="salvage=0 " # Faster runs - args+="timer=4 " - args+="verify=0 " # Faster runs - - for am in $2; do - dir="RUNDIR.$am" - echo "./t running $am access method..." - ./t -1q $3 -h $dir "file_type=$am" $args - - # Remove the version string from the base configuration file. (MongoDB does not create - # a base configuration file, but format does, so we need to remove its version string - # to allow backward compatibility testing.) - (echo '/^version=/d' - echo w) | ed -s $dir/WiredTiger.basecfg > /dev/null - done -} - -EXT="extensions=[" -EXT+="ext/compressors/snappy/.libs/libwiredtiger_snappy.so," -EXT+="ext/collators/reverse/.libs/libwiredtiger_reverse_collator.so, " -EXT+="ext/encryptors/rotn/.libs/libwiredtiger_rotn.so, " -EXT+="]" - -############################################################# -# verify_release: -# arg1: release #1 -# arg2: release #2 -# arg3: access methods list -############################################################# -verify_release() -{ - echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - echo "Release \"$1\" verifying \"$2\"" - echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" - - cd "$1" - for am in $3; do - dir="$2/test/format/RUNDIR.$am" - echo "$1/wt verifying $2 access method $am..." - - WIREDTIGER_CONFIG="$EXT" ./wt -h "../$dir" verify table:wt - done -} - -# Create a directory in which to do the work. -top="test-compatibility-run" -rm -rf "$top" && mkdir "$top" -cd "$top" - -# Build the releases. -(build_release mongodb-3.4) -(build_release mongodb-3.6) -(build_release mongodb-4.0) -(build_release mongodb-4.2) -#(build_release mongodb-4.4) -(build_release "develop") - -# Run format in each release for supported access methods. -(run_format mongodb-3.4 "fix row var") -(run_format mongodb-3.6 "fix row var") -(run_format mongodb-4.0 "fix row var") -(run_format mongodb-4.2 "fix row var") -#(run_format mongodb-4.4 "row") -(run_format "develop" "row" "-B") - -# Verify backward compatibility for supported access methods. -(verify_release mongodb-3.6 mongodb-3.4 "fix row var") -(verify_release mongodb-4.0 mongodb-3.6 "fix row var") -(verify_release mongodb-4.2 mongodb-4.0 "fix row var") -#(verify_release mongodb-4.4 mongodb-4.2 "fix row var") -#(verify_release develop mongodb-4.4 "row") -(verify_release develop mongodb-4.2 "fix row var") - -# Verify forward compatibility for supported access methods. -(verify_release mongodb-4.2 develop "row") - -exit 0 diff --git a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh new file mode 100755 index 00000000000..a030f80c712 --- /dev/null +++ b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_releases.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash +############################################################################################## +# Check branches to ensure forward/backward compatibility, including some upgrade/downgrade testing. +############################################################################################## + +set -e + +############################################################# +# format_b_flag: +# arg1: branch name +############################################################# +format_b_flag() +{ + # Return if the branch's format command takes the -B flag for backward compatibility. + test "$1" = "develop" && echo "-B " + test "$1" = "mongodb-4.4" && echo "-B " + return 0 +} + +############################################################# +# get_prev_version: +# arg1: branch name +############################################################# +get_prev_version() +{ + # Sort the list of WiredTiger tags numerically, then pick out the argument number of releases + # from the end of the list. That is, get a list of releases in numeric order, then pick out + # the last release (argument "1"), the next-to-last release (argument "2") and so on. Assumes + # WiredTiger releases are tagged with just numbers and decimal points. + echo "$(git tag | egrep '^[0-9][0-9.]*$' | sort -g | tail -$1 | head -1)" +} + +############################################################# +# build_branch: +# arg1: branch name +############################################################# +build_branch() +{ + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + echo "Building branch: \"$1\"" + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + + git clone --quiet https://github.com/wiredtiger/wiredtiger.git "$1" + cd "$1" + git checkout --quiet "$1" + + config="" + config+="--enable-snappy " + (sh build_posix/reconf && + ./configure $config && make -j $(grep -c ^processor /proc/cpuinfo)) > /dev/null +} + +############################################################# +# run_format: +# arg1: branch name +# arg2: access methods list +############################################################# +run_format() +{ + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + echo "Running format in branch: \"$1\"" + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + + cd "$1/test/format" + + flags="-1q $(format_b_flag $1)" + + args="" + args+="cache=80 " # Medium cache so there's eviction + args+="checkpoints=1 " # Force periodic writes + args+="compression=snappy " # We only built with snappy, force the choice + args+="data_source=table " + args+="in_memory=0 " # Interested in the on-disk format + args+="leak_memory=1 " # Faster runs + args+="logging=1 " # Test log compatibility + args+="logging_compression=snappy " # We only built with snappy, force the choice + args+="rebalance=0 " # Faster runs + args+="rows=1000000 " + args+="salvage=0 " # Faster runs + args+="timer=4 " + args+="verify=0 " # Faster runs + + for am in $2; do + dir="RUNDIR.$am" + echo "./t running $am access method..." + ./t $flags -h $dir "file_type=$am" $args + + # Remove the version string from the base configuration file. (MongoDB does not create + # a base configuration file, but format does, so we need to remove its version string + # to allow backward compatibility testing.) + (echo '/^version=/d' + echo w) | ed -s $dir/WiredTiger.basecfg > /dev/null + done +} + +EXT="extensions=[" +EXT+="ext/compressors/snappy/.libs/libwiredtiger_snappy.so," +EXT+="ext/collators/reverse/.libs/libwiredtiger_reverse_collator.so, " +EXT+="ext/encryptors/rotn/.libs/libwiredtiger_rotn.so, " +EXT+="]" + +############################################################# +# verify_branches: +# arg1: branch name #1 +# arg2: branch name #2 +# arg3: access methods list +############################################################# +verify_branches() +{ + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + echo "Release \"$1\" verifying \"$2\"" + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + + cd "$1" + for am in $3; do + echo "$1/wt verifying $2 access method $am..." + dir="$2/test/format/RUNDIR.$am" + WIREDTIGER_CONFIG="$EXT" ./wt -h "../$dir" verify table:wt + done +} + +############################################################# +# upgrade_downgrade: +# arg1: branch name #1 +# arg2: branch name #2 +# arg3: access methods list +############################################################# +upgrade_downgrade() +{ + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + echo "Upgrade/downgrade testing with \"$1\" and \"$2\"" + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + + # Alternate running each branch format test program on the second branch's build. + # Loop twice, that is, run format twice using each branch. + top="$PWD" + for am in $3; do + for reps in {1..2}; do + echo "$1 format running on $2 access method $am..." + cd "$top/$1/test/format" + flags="-1qR $(format_b_flag $1)" + ./t $flags -h "$top/$2/test/format/RUNDIR.$am" timer=2 + + echo "$2 format running on $2 access method $am..." + cd "$top/$2/test/format" + flags="-1qR $(format_b_flag $2)" + ./t $flags -h "RUNDIR.$am" timer=2 + done + done +} + +# Create a directory in which to do the work. +top="test-compatibility-run" +rm -rf "$top" && mkdir "$top" +cd "$top" + +# Build the branches. +(build_branch mongodb-3.4) +(build_branch mongodb-3.6) +(build_branch mongodb-4.0) +(build_branch mongodb-4.2) +(build_branch mongodb-4.4) +(build_branch develop) + +# Get the names of the last two WiredTiger releases, wt1 is the most recent release, wt2 is the +# release before that. Minor trickiness, we depend on the "develop" directory already existing +# so we have a source in which to do git commands. +cd develop; wt1=$(get_prev_version 1); cd .. +(build_branch "$wt1") +cd develop; wt2=$(get_prev_version 2); cd .. +(build_branch "$wt2") + +# Run format in each branch for supported access methods. +(run_format mongodb-3.4 "fix row var") +(run_format mongodb-3.6 "fix row var") +(run_format mongodb-4.0 "fix row var") +(run_format mongodb-4.2 "fix row var") +(run_format mongodb-4.4 "row") +(run_format develop "row") +(run_format "$wt1" "fix row var") +(run_format "$wt2" "fix row var") + +# Verify backward compatibility for supported access methods. +(verify_branches mongodb-3.6 mongodb-3.4 "fix row var") +(verify_branches mongodb-4.0 mongodb-3.6 "fix row var") +(verify_branches mongodb-4.2 mongodb-4.0 "fix row var") +### (verify_branches mongodb-4.4 mongodb-4.2 "fix row var") +### (verify_branches develop mongodb-4.4 "row") +(verify_branches develop mongodb-4.2 "row") +(verify_branches "$wt1" "$wt2" "row") +(verify_branches develop "$wt1" "row") + +# Verify forward compatibility for supported access methods. +### (verify_branches mongodb-4.2 mongodb-4.4 "row") +(verify_branches mongodb-4.2 develop "row") +### (verify_branches mongodb-4.4 develop "row") + +# Upgrade/downgrade testing for supported access methods. +### (upgrade_downgrade mongodb-4.2 mongodb-4.4 "row") +(upgrade_downgrade mongodb-4.2 develop "row") +### (upgrade_downgrade mongodb-4.4 develop "row") + +exit 0 diff --git a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_wiredtiger_releases.sh b/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_wiredtiger_releases.sh deleted file mode 100755 index 9a571e733b4..00000000000 --- a/src/third_party/wiredtiger/test/evergreen/compatibility_test_for_wiredtiger_releases.sh +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env bash -############################################################################################## -# Run WiredTiger test/format check against below 3 releases to ensure backward compatibility: -# - current release (develop branch) -# - previous release (WTx.y.z tag) -# - previous previous release (WTa.b.c tag) -############################################################################################## - -set -e - -BUILD_DIR="build_posix" # The relative directory of WiredTiger repo tree - -########################################################################### -# This function will -# - retrieve the previous release tag number based on the count provided -# -# arg1: count of previous release -########################################################################### -get_release() -{ - prev_cnt=$1 - rel=$(git tag | grep -v release | egrep "^[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2}" | tail -$prev_cnt | head -1) - echo "$rel" -} - -############################################################# -# This function will -# - checkout git tree of the desired release (via arg1) -# - make a build -# - use the generated binary 't' to test the configuration -# -# arg1: release indicator (mapped to release branch/tag) -############################################################# -build_test_db() -{ - rel_ind="$1" - echo "Starting to build and test format for \"$rel_ind\" branch/release ..." - - # Parse the release indicator into release number. - # Checkout the release, and in the case of a branch, refresh it. - case "$rel_ind" in - "develop") # current release - rel="develop" - git checkout $rel - git pull --rebase - ;; - "r1") # previous release - rel="$(get_release 1)" - git checkout $rel - ;; - "r2") # previous previous release - rel="$(get_release 2)" - git checkout $rel - ;; - *) - echo "Unexpected branch/release number: \"$rel_ind\"" - exit 1 - ;; - esac - - echo "Building release: \"$rel\"" - - # Ensure read only testing hasn't left things difficult to cleanup. - chmod -R u+w ${BUILD_DIR} - - # Configure and build - cd ${BUILD_DIR} - sh reconf - ../configure --disable-strict --disable-shared --enable-diagnostic - make -j $(grep -c ^processor /proc/cpuinfo) - - # Test the configuration - cd test/format - cat > CONFIG <<- EOF - reverse=0 - runs=1 - rows=10000 - ops=1000 - threads=2 - evict_max=0 - compression=none - logging_compression=none - encryption=none - EOF - ./t - cd ../.. - - rm -f db - - # Go back to WiredTiger repo directory - cd .. - - # Archive the whole build_posix directory - echo "Archiving ${BUILD_DIR} directory to ${BUILD_DIR}.${rel_ind} ..." - rm -rf ${BUILD_DIR}.${rel_ind} # if dir exists, the next cp command would write it as a sub-dir - cp -R ${BUILD_DIR} ${BUILD_DIR}.${rel_ind} -} - -############################################################# -# This function will -# - go into the archived build directory for the release -# - use the 'wt' binary to cross-check the wt URI -# -# arg1: release indicator (mapped to release branch/tag) -############################################################# -verify_uri() -{ - rel_ind="$1" - echo "Starting to verify URI for \"$rel_ind\" branch/release ..." - - # Go into the archived build directory of the release - cd ${BUILD_DIR}.${rel_ind} - - dir=test/format/RUNDIR - - # Along the way, we changed the running configuration from - # RUNDIR/run to RUNDIR/CONFIG. Check for both. Once that - # change propogates back, we can just use $dir/CONFIG. - if test -e $dir/CONFIG; then - cfile=$dir/CONFIG - else - cfile=$dir/run - fi - isfile=`grep data_source $cfile | grep -c file || exit 0` - if test "$isfile" -ne 0; then - uri="file:wt" - else - uri="table:wt" - fi - - # Cross-check the wt URI against the build directory - case "$rel_ind" in - "develop") # Test directory of current release using wt binaries from "all 3 releases" - echo "Verifying $uri using $devwt binary" - # Remove basecfg as the statistic logging block - rm -f $dir/WiredTiger.basecfg - $devwt -h $dir verify $uri - # We don't guarantee that the wiredtiger_open config is backwards - # and forwards compatible. Just delete it for now. - echo "Verifying $uri using $r1wt binary" - $r1wt -h $dir verify $uri - echo "Verifying $uri using $r2wt binary" - $r2wt -h $dir verify $uri - ;; - "r1") # Test directory of previous release using wt binaries from both "current" and "previous" releases - rm -f $dir/WiredTiger.basecfg - echo "Verifying $uri using $r1wt binary" - $r1wt -h $dir verify $uri - echo "Verifying $uri using $devwt binary" - $devwt -h $dir verify $uri - ;; - "r2") # Test directory of previous previous release using wt binaries from both "current" and "previous previous" releases - rm -f $dir/WiredTiger.basecfg - echo "Verifying $uri using $r2wt binary" - $r2wt -h $dir verify $uri - echo "Verifying $uri using $devwt binary" - $devwt -h $dir verify $uri - ;; - *) - echo "Unexpected argument value" - ;; - esac - - # Go back to WiredTiger repo directory - cd .. -} - -# Firstly, do some cleanup for the previous run. -git reset --hard && git clean -fdqx -e '*.tgz' - -# Build and test format, then set the archived wt binary for each release -build_test_db "develop" -devwt=../${BUILD_DIR}.develop/wt -build_test_db "r1" -r1wt=../${BUILD_DIR}.r1/wt -build_test_db "r2" -r2wt=../${BUILD_DIR}.r2/wt - -# Cross-check the URI for the 3 releases -verify_uri "develop" -verify_uri "r1" -verify_uri "r2" - diff --git a/src/third_party/wiredtiger/test/format/t.c b/src/third_party/wiredtiger/test/format/t.c index a2fcf405cf9..b596124087b 100644 --- a/src/third_party/wiredtiger/test/format/t.c +++ b/src/third_party/wiredtiger/test/format/t.c @@ -378,7 +378,7 @@ usage(void) progname); fprintf(stderr, "%s", "\t-1 run once then quit\n" - "\t-B create backward compatible configurations\n" + "\t-B maintain 3.3 release log and configuration option compatibility\n" "\t-C specify wiredtiger_open configuration arguments\n" "\t-c read test program configuration from a file (default 'CONFIG')\n" "\t-h home directory (default 'RUNDIR')\n" diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index 5691c2877ca..ea72c4c7f0f 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -281,6 +281,9 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) CONFIG_APPEND(p, ",split_8"); CONFIG_APPEND(p, "]"); + if (g.c_verify) + CONFIG_APPEND(p, ",verify_metadata=true"); + /* Extensions. */ CONFIG_APPEND(p, ",extensions=[" diff --git a/src/third_party/wiredtiger/test/suite/test_backup08.py b/src/third_party/wiredtiger/test/suite/test_backup08.py index f238a865514..6c4b04edd2a 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup08.py +++ b/src/third_party/wiredtiger/test/suite/test_backup08.py @@ -31,7 +31,7 @@ # import os, shutil -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wtscenario import make_scenarios def timestamp_str(t): diff --git a/src/third_party/wiredtiger/test/suite/test_bug008.py b/src/third_party/wiredtiger/test/suite/test_bug008.py index 6af51e1eaba..cb2987c234a 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug008.py +++ b/src/third_party/wiredtiger/test/suite/test_bug008.py @@ -29,7 +29,7 @@ # test_bug008.py # Regression tests. -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wtdataset import SimpleDataSet from wtscenario import make_scenarios diff --git a/src/third_party/wiredtiger/test/suite/test_compact01.py b/src/third_party/wiredtiger/test/suite/test_compact01.py index 0486429b972..9e5d0e19c5e 100644 --- a/src/third_party/wiredtiger/test/suite/test_compact01.py +++ b/src/third_party/wiredtiger/test/suite/test_compact01.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import unittest, wiredtiger, wttest +import wiredtiger, wttest from suite_subprocess import suite_subprocess from wtdataset import SimpleDataSet, ComplexDataSet from wiredtiger import stat diff --git a/src/third_party/wiredtiger/test/suite/test_compact02.py b/src/third_party/wiredtiger/test/suite/test_compact02.py index ccbec469433..c15fb5bc78b 100644 --- a/src/third_party/wiredtiger/test/suite/test_compact02.py +++ b/src/third_party/wiredtiger/test/suite/test_compact02.py @@ -30,7 +30,7 @@ # Test that compact reduces the file size. # -import time, unittest, wiredtiger, wttest +import time, wiredtiger, wttest from wiredtiger import stat from wtscenario import make_scenarios @@ -109,7 +109,6 @@ class test_compact02(wttest.WiredTigerTestCase): self.session = self.conn.open_session(None) # Create a table, add keys with both big and small values. - @unittest.skip("Temporarily disabled") def test_compact02(self): self.ConnectionOpen(self.cacheSize) diff --git a/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py b/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py index 7e434a53597..e7eb35f515d 100644 --- a/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. from helper import copy_wiredtiger_home -import wiredtiger, wttest, unittest +import wiredtiger, wttest from suite_subprocess import suite_subprocess from wtdataset import SimpleDataSet from wtscenario import make_scenarios diff --git a/src/third_party/wiredtiger/test/suite/test_gc01.py b/src/third_party/wiredtiger/test/suite/test_gc01.py index 8f84c61b245..ff6a0d6a85c 100755 --- a/src/third_party/wiredtiger/test/suite/test_gc01.py +++ b/src/third_party/wiredtiger/test/suite/test_gc01.py @@ -28,7 +28,7 @@ import time from helper import copy_wiredtiger_home -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wtdataset import SimpleDataSet from wiredtiger import stat diff --git a/src/third_party/wiredtiger/test/suite/test_hs01.py b/src/third_party/wiredtiger/test/suite/test_hs01.py index 3559acb0645..b7ed671e4fb 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs01.py +++ b/src/third_party/wiredtiger/test/suite/test_hs01.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. from helper import copy_wiredtiger_home -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wtdataset import SimpleDataSet def timestamp_str(t): diff --git a/src/third_party/wiredtiger/test/suite/test_hs03.py b/src/third_party/wiredtiger/test/suite/test_hs03.py index 2172284ea02..ea98944c3e1 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs03.py +++ b/src/third_party/wiredtiger/test/suite/test_hs03.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. from helper import copy_wiredtiger_home -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wiredtiger import stat from wtdataset import SimpleDataSet diff --git a/src/third_party/wiredtiger/test/suite/test_hs07.py b/src/third_party/wiredtiger/test/suite/test_hs07.py index e2242cd4a7e..98bca0ab7ba 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs07.py +++ b/src/third_party/wiredtiger/test/suite/test_hs07.py @@ -28,7 +28,7 @@ import time from helper import copy_wiredtiger_home -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wtdataset import SimpleDataSet def timestamp_str(t): diff --git a/src/third_party/wiredtiger/test/suite/test_hs08.py b/src/third_party/wiredtiger/test/suite/test_hs08.py index 48afefef941..b0d0d497dd0 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs08.py +++ b/src/third_party/wiredtiger/test/suite/test_hs08.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import unittest, wiredtiger, wttest, time +import wiredtiger, wttest, time from wiredtiger import stat from wtscenario import make_scenarios diff --git a/src/third_party/wiredtiger/test/suite/test_hs09.py b/src/third_party/wiredtiger/test/suite/test_hs09.py index a0d6790a87a..43ced8ad589 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs09.py +++ b/src/third_party/wiredtiger/test/suite/test_hs09.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wiredtiger import stat from wtscenario import make_scenarios diff --git a/src/third_party/wiredtiger/test/suite/test_hs10.py b/src/third_party/wiredtiger/test/suite/test_hs10.py index 4ba3b25c4f0..4a33ced8125 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs10.py +++ b/src/third_party/wiredtiger/test/suite/test_hs10.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import unittest, wiredtiger, wttest, time +import wiredtiger, wttest, time from wiredtiger import stat from wtscenario import make_scenarios diff --git a/src/third_party/wiredtiger/test/suite/test_hs13.py b/src/third_party/wiredtiger/test/suite/test_hs13.py new file mode 100644 index 00000000000..7ada5d7a0b6 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hs13.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python
+#
+# Public Domain 2014-2020 MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+import wiredtiger, wttest
+def timestamp_str(t):
+ return '%x' % t
+
+# test_hs13.py
+# Verify reverse modify traversal after eviction.
+class test_hs13(wttest.WiredTigerTestCase):
+ conn_config = 'cache_size=2MB,statistics=(all),eviction=(threads_max=1)'
+ session_config = 'isolation=snapshot'
+
+ def test_reverse_modifies_constructed_after_eviction(self):
+ uri = "table:test_hs13"
+ create_params = 'value_format=S,key_format=i'
+ value1 = 'a' * 10000
+ value2 = 'b' * 10000
+ value3 = 'e' * 10000
+
+ self.session.create(uri, create_params)
+ cursor = self.session.open_cursor(uri)
+ session2 = self.setUpSessionOpen(self.conn)
+ cursor2 = session2.open_cursor(uri)
+
+ # Insert a full value.
+ self.session.begin_transaction()
+ cursor[1] = value1
+ self.session.commit_transaction()
+
+ # Insert a modify.
+ self.session.begin_transaction()
+ cursor.set_key(1)
+ cursor.modify([wiredtiger.Modify('A', 0, 0)])
+ self.session.commit_transaction()
+
+ # Validate that we do see the correct value.
+ session2.begin_transaction()
+ cursor2.set_key(1)
+ cursor2.search()
+ self.assertEquals(cursor2.get_value(), 'A' + value1)
+ session2.commit_transaction()
+
+ # Reset the cursor.
+ cursor2.reset()
+
+ # Begin session2's transaction so it gets a snapshot prior to the full value being
+ # inserted below.
+ session2.begin_transaction()
+
+ # Insert a second modify.
+ self.session.begin_transaction()
+ cursor.set_key(1)
+ cursor.modify([wiredtiger.Modify('B', 1, 0)])
+ self.session.commit_transaction()
+
+ # Insert one more value.
+ self.session.begin_transaction()
+ cursor.set_key(1)
+ cursor[1] = value2
+ self.session.commit_transaction()
+
+ # Insert a whole bunch of data into the table to force wiredtiger to evict data
+ # from the previous table.
+ self.session.begin_transaction()
+ for i in range(2, 10000):
+ cursor[i] = value3
+ self.session.commit_transaction()
+
+ # Try to find the value we saw earlier.
+ cursor2.set_key(1)
+ cursor2.search()
+ self.assertEquals(cursor2.get_value(), 'A' + value1)
+
+if __name__ == '__main__':
+ wttest.run()
diff --git a/src/third_party/wiredtiger/test/suite/test_inmem01.py b/src/third_party/wiredtiger/test/suite/test_inmem01.py index 2bc7df0d403..879872b9c07 100644 --- a/src/third_party/wiredtiger/test/suite/test_inmem01.py +++ b/src/third_party/wiredtiger/test/suite/test_inmem01.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import unittest, wiredtiger, wttest +import wiredtiger, wttest from time import sleep from wtdataset import SimpleDataSet from wtscenario import make_scenarios diff --git a/src/third_party/wiredtiger/test/suite/test_intpack.py b/src/third_party/wiredtiger/test/suite/test_intpack.py index 5a90f309b0b..1a1c5725792 100644 --- a/src/third_party/wiredtiger/test/suite/test_intpack.py +++ b/src/third_party/wiredtiger/test/suite/test_intpack.py @@ -30,7 +30,7 @@ # Tests integer packing using public methods # -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wtscenario import make_scenarios class PackTester: diff --git a/src/third_party/wiredtiger/test/suite/test_jsondump01.py b/src/third_party/wiredtiger/test/suite/test_jsondump01.py index e5bc170c45c..97af4764622 100644 --- a/src/third_party/wiredtiger/test/suite/test_jsondump01.py +++ b/src/third_party/wiredtiger/test/suite/test_jsondump01.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import os, json, unittest +import os, json import wiredtiger, wttest from wtdataset import SimpleDataSet, SimpleLSMDataSet, SimpleIndexDataSet, \ ComplexDataSet, ComplexLSMDataSet diff --git a/src/third_party/wiredtiger/test/suite/test_jsondump02.py b/src/third_party/wiredtiger/test/suite/test_jsondump02.py index 0fbe4da25db..eec0e5e97fd 100755 --- a/src/third_party/wiredtiger/test/suite/test_jsondump02.py +++ b/src/third_party/wiredtiger/test/suite/test_jsondump02.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import os, sys, unittest +import os, sys import wiredtiger, wttest from suite_subprocess import suite_subprocess diff --git a/src/third_party/wiredtiger/test/suite/test_prepare07.py b/src/third_party/wiredtiger/test/suite/test_prepare07.py index 441cef2b5bf..da42843e26d 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare07.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare07.py @@ -28,7 +28,7 @@ import fnmatch, os, shutil, time from helper import copy_wiredtiger_home -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wtdataset import SimpleDataSet def timestamp_str(t): diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py index 15d9e9f29da..7281cdaf99b 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py @@ -28,7 +28,7 @@ import time from helper import copy_wiredtiger_home -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wtdataset import SimpleDataSet from wiredtiger import stat from wtscenario import make_scenarios diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py index 32dbcfea39c..771e3f36b9d 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py @@ -28,7 +28,7 @@ import time from helper import copy_wiredtiger_home -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wtdataset import SimpleDataSet from wiredtiger import stat from wtscenario import make_scenarios diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py index 53672178c7b..37b9958091d 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py @@ -28,7 +28,7 @@ import time from helper import copy_wiredtiger_home -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wtdataset import SimpleDataSet from wiredtiger import stat from wtscenario import make_scenarios diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py index eddff33aaf1..36daf16573f 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py @@ -28,7 +28,7 @@ import time from helper import copy_wiredtiger_home -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wtdataset import SimpleDataSet from wiredtiger import stat from wtscenario import make_scenarios diff --git a/src/third_party/wiredtiger/test/suite/test_schema08.py b/src/third_party/wiredtiger/test/suite/test_schema08.py index 9a20b0bee6d..91925ffc975 100644 --- a/src/third_party/wiredtiger/test/suite/test_schema08.py +++ b/src/third_party/wiredtiger/test/suite/test_schema08.py @@ -28,7 +28,7 @@ import fnmatch, os, shutil, sys from suite_subprocess import suite_subprocess -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wtscenario import make_scenarios # test_schema08.py diff --git a/src/third_party/wiredtiger/test/suite/test_stat04.py b/src/third_party/wiredtiger/test/suite/test_stat04.py index b86e0187dd8..a3d0d0eae65 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat04.py +++ b/src/third_party/wiredtiger/test/suite/test_stat04.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import os, struct, unittest +import os, struct from suite_subprocess import suite_subprocess from wtscenario import make_scenarios import wiredtiger, wttest diff --git a/src/third_party/wiredtiger/test/suite/test_sweep02.py b/src/third_party/wiredtiger/test/suite/test_sweep02.py index 690c433285b..b7463e7a8ed 100644 --- a/src/third_party/wiredtiger/test/suite/test_sweep02.py +++ b/src/third_party/wiredtiger/test/suite/test_sweep02.py @@ -31,7 +31,6 @@ # import wiredtiger, wttest -from wttest import unittest class test_sweep02(wttest.WiredTigerTestCase): base_config = 'create,' diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp03.py b/src/third_party/wiredtiger/test/suite/test_timestamp03.py index 767bfcc888f..2d32bfefa9b 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp03.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp03.py @@ -33,7 +33,7 @@ from helper import copy_wiredtiger_home import random from suite_subprocess import suite_subprocess -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wtscenario import make_scenarios def timestamp_str(t): diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp04.py b/src/third_party/wiredtiger/test/suite/test_timestamp04.py index d5dffd6a841..fd7d24050d7 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp04.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp04.py @@ -31,7 +31,7 @@ # from suite_subprocess import suite_subprocess -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wiredtiger import stat from wtscenario import make_scenarios diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp06.py b/src/third_party/wiredtiger/test/suite/test_timestamp06.py index 9129d2ddf24..26bbda36ad2 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp06.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp06.py @@ -33,7 +33,7 @@ from helper import copy_wiredtiger_home import random from suite_subprocess import suite_subprocess -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wtscenario import make_scenarios def timestamp_str(t): diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp07.py b/src/third_party/wiredtiger/test/suite/test_timestamp07.py index 2b56e85b3ea..6a5db08c73a 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp07.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp07.py @@ -33,7 +33,7 @@ from helper import copy_wiredtiger_home import random from suite_subprocess import suite_subprocess -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wtscenario import make_scenarios def timestamp_str(t): diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp10.py b/src/third_party/wiredtiger/test/suite/test_timestamp10.py index 46d42003885..fb24992d840 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp10.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp10.py @@ -31,7 +31,7 @@ # from suite_subprocess import suite_subprocess -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wtscenario import make_scenarios def timestamp_str(t): diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp11.py b/src/third_party/wiredtiger/test/suite/test_timestamp11.py index 0b31abbccb9..3c56b66bf66 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp11.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp11.py @@ -31,7 +31,7 @@ # from suite_subprocess import suite_subprocess -import wiredtiger, wttest, unittest +import wiredtiger, wttest def timestamp_str(t): return '%x' % t diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp12.py b/src/third_party/wiredtiger/test/suite/test_timestamp12.py index cf258814a87..fcbaf5d01ef 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp12.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp12.py @@ -30,7 +30,7 @@ # Timestamps: Test the use_timestamp setting when closing the connection. # -import shutil, os, unittest, wiredtiger, wttest +import shutil, os, wiredtiger, wttest from wtscenario import make_scenarios def timestamp_str(t): diff --git a/src/third_party/wiredtiger/test/suite/test_truncate02.py b/src/third_party/wiredtiger/test/suite/test_truncate02.py index f4e41e1aef2..c60698b2a15 100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate02.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate02.py @@ -30,7 +30,7 @@ # session level operations on tables # -import unittest, wiredtiger, wttest +import wiredtiger, wttest from wtdataset import SimpleDataSet from wtscenario import make_scenarios diff --git a/src/third_party/wiredtiger/test/suite/test_txn02.py b/src/third_party/wiredtiger/test/suite/test_txn02.py index 648e31b5374..ac6626a9b28 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn02.py +++ b/src/third_party/wiredtiger/test/suite/test_txn02.py @@ -30,7 +30,7 @@ # Transactions: commits and rollbacks # -import fnmatch, os, shutil, time, unittest +import fnmatch, os, shutil, time from suite_subprocess import suite_subprocess from wtscenario import make_scenarios import wttest diff --git a/src/third_party/wiredtiger/test/suite/test_txn04.py b/src/third_party/wiredtiger/test/suite/test_txn04.py index 10c0f4ab3c5..98a0aca1240 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn04.py +++ b/src/third_party/wiredtiger/test/suite/test_txn04.py @@ -33,7 +33,7 @@ import shutil, os from suite_subprocess import suite_subprocess from wtscenario import make_scenarios -import unittest, wttest +import wttest class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): logmax = "100K" diff --git a/src/third_party/wiredtiger/test/suite/test_txn05.py b/src/third_party/wiredtiger/test/suite/test_txn05.py index 53fc6496bae..4ef8e8c9b83 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn05.py +++ b/src/third_party/wiredtiger/test/suite/test_txn05.py @@ -30,7 +30,7 @@ # Transactions: commits and rollbacks # -import fnmatch, os, shutil, time, unittest +import fnmatch, os, shutil, time from suite_subprocess import suite_subprocess from wtscenario import make_scenarios import wttest diff --git a/src/third_party/wiredtiger/test/suite/test_txn07.py b/src/third_party/wiredtiger/test/suite/test_txn07.py index 0fe78cbb7ab..011d10c0df4 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn07.py +++ b/src/third_party/wiredtiger/test/suite/test_txn07.py @@ -34,7 +34,7 @@ import fnmatch, os, shutil, run, time from suite_subprocess import suite_subprocess from wiredtiger import stat from wtscenario import make_scenarios -import unittest, wttest +import wttest class test_txn07(wttest.WiredTigerTestCase, suite_subprocess): logmax = "100K" diff --git a/src/third_party/wiredtiger/test/suite/test_txn09.py b/src/third_party/wiredtiger/test/suite/test_txn09.py index a6ca3a64082..98c7dbd9c48 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn09.py +++ b/src/third_party/wiredtiger/test/suite/test_txn09.py @@ -30,7 +30,7 @@ # Transactions: recovery toggling logging # -import fnmatch, os, shutil, time, unittest +import fnmatch, os, shutil, time from suite_subprocess import suite_subprocess from wtscenario import make_scenarios import wttest diff --git a/src/third_party/wiredtiger/test/suite/test_txn16.py b/src/third_party/wiredtiger/test/suite/test_txn16.py index 34a363a83e4..9e2829246e5 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn16.py +++ b/src/third_party/wiredtiger/test/suite/test_txn16.py @@ -33,7 +33,7 @@ import fnmatch, os, shutil, time from suite_subprocess import suite_subprocess -import unittest, wttest +import wttest class test_txn16(wttest.WiredTigerTestCase, suite_subprocess): t1 = 'table:test_txn16_1' diff --git a/src/third_party/wiredtiger/test/suite/test_txn19.py b/src/third_party/wiredtiger/test/suite/test_txn19.py index 702515612d8..cd68c0c9931 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn19.py +++ b/src/third_party/wiredtiger/test/suite/test_txn19.py @@ -535,8 +535,7 @@ class test_txn19_meta(wttest.WiredTigerTestCase, suite_subprocess): # But, immediately after the corruption, if we run # wiredtiger_open with salvage, it will fail. # This anomoly should be fixed or explained. - if salvagedir == newdir and self.kind == 'removal' and \ - self.filename == 'WiredTiger.turtle': + if self.kind == 'removal' and self.filename == 'WiredTiger.turtle': continue if self.is_salvageable(): diff --git a/src/third_party/wiredtiger/test/suite/test_util01.py b/src/third_party/wiredtiger/test/suite/test_util01.py index ba0e71caba0..30a53453647 100755 --- a/src/third_party/wiredtiger/test/suite/test_util01.py +++ b/src/third_party/wiredtiger/test/suite/test_util01.py @@ -28,7 +28,7 @@ import string, os, sys, random from suite_subprocess import suite_subprocess -import wiredtiger, wttest, unittest +import wiredtiger, wttest _python3 = (sys.version_info >= (3, 0, 0)) @@ -232,7 +232,6 @@ class test_util01(wttest.WiredTigerTestCase, suite_subprocess): def test_dump_api_hex(self): self.dump(True, True, None, None) - @unittest.skip("Temporarily Disabled") def test_dump_process_timestamp_old(self): self.dump(False, False, 5, 5) diff --git a/src/third_party/wiredtiger/test/suite/test_util04.py b/src/third_party/wiredtiger/test/suite/test_util04.py index 69a76471741..518933e226c 100644 --- a/src/third_party/wiredtiger/test/suite/test_util04.py +++ b/src/third_party/wiredtiger/test/suite/test_util04.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import os, unittest +import os from suite_subprocess import suite_subprocess import wiredtiger, wttest diff --git a/src/third_party/wiredtiger/test/suite/test_util11.py b/src/third_party/wiredtiger/test/suite/test_util11.py index aec700e51d2..92f3dbc5c75 100644 --- a/src/third_party/wiredtiger/test/suite/test_util11.py +++ b/src/third_party/wiredtiger/test/suite/test_util11.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import os, struct, unittest +import os, struct from suite_subprocess import suite_subprocess import wiredtiger, wttest diff --git a/src/third_party/wiredtiger/test/suite/test_util16.py b/src/third_party/wiredtiger/test/suite/test_util16.py index 2d2db88e361..f2e50665c85 100644 --- a/src/third_party/wiredtiger/test/suite/test_util16.py +++ b/src/third_party/wiredtiger/test/suite/test_util16.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import os, unittest +import os from suite_subprocess import suite_subprocess import wiredtiger, wttest |