diff options
112 files changed, 3278 insertions, 1100 deletions
diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.cxx b/src/third_party/wiredtiger/bench/workgen/workgen.cxx index 2f7c0c4f5c4..b1753981540 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen.cxx +++ b/src/third_party/wiredtiger/bench/workgen/workgen.cxx @@ -26,8 +26,16 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#define __STDC_LIMIT_MACROS // needed to get UINT64_MAX in C++ -#define __STDC_FORMAT_MACROS // needed to get PRIuXX macros in C++ +/* Needed to get UINT64_MAX in C++. */ +#ifndef __STDC_LIMIT_MACROS +#define __STDC_LIMIT_MACROS +#endif + +/* Needed to get PRIuXX macros in C++. */ +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif + #include <iomanip> #include <iostream> #include <fstream> diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/perf_stat.py b/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/perf_stat.py new file mode 100644 index 00000000000..de62d328373 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/perf_stat.py @@ -0,0 +1,45 @@ +class PerfStat: + def __init__(self, + short_label: str, + pattern: str, + input_offset: int, + output_label: str, + output_precision: int = 0, + conversion_function=int): + self.short_label: str = short_label + self.pattern: str = pattern + self.input_offset: int = input_offset + self.output_label: str = output_label + self.output_precision: int = output_precision + self.conversion_function = conversion_function + self.values = [] + + def add_value(self, value): + converted_value = self.conversion_function(value) + self.values.append(converted_value) + + def get_num_values(self): + return len(self.values) + + def get_average(self): + num_values = len(self.values) + total = sum(self.values) + average = self.conversion_function(total / num_values) + return average + + def get_skipminmax_average(self): + num_values = len(self.values) + assert num_values >= 3 + minimum = min(self.values) + maximum = max(self.values) + total = sum(self.values) + total_skipminmax = total - maximum - minimum + num_values_skipminmax = num_values - 2 + skipminmax_average = self.conversion_function(total_skipminmax / num_values_skipminmax) + return skipminmax_average + + def get_core_average(self): + if len(self.values) >= 3: + return self.get_skipminmax_average() + else: + return self.get_average() diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/perf_stat_collection.py b/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/perf_stat_collection.py new file mode 100644 index 00000000000..d2784901311 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/perf_stat_collection.py @@ -0,0 +1,37 @@ +import re +from perf_stat import PerfStat + +def find_stat(test_stat_path: str, pattern: str, position_of_value: int): + for line in open(test_stat_path): + match = re.match(pattern, line) + if match: + return float(line.split()[position_of_value]) + return 0 + +class PerfStatCollection: + def __init__(self): + self.perf_stats = {} + + def add_stat(self, perf_stat: PerfStat): + self.perf_stats[perf_stat.short_label] = perf_stat + + def find_stats(self, test_stat_path: str): + for stat in self.perf_stats.values(): + value = find_stat(test_stat_path=test_stat_path, + pattern=stat.pattern, + position_of_value=stat.input_offset) + stat.add_value(value=value) + + def to_value_list(self): + as_list = [] + for stat in self.perf_stats.values(): + as_list.append({ + 'name': stat.output_label, + 'value': stat.get_core_average(), + 'values': stat.values + }) + return as_list + + def to_dict(self): + as_dict = {'metrics': self.to_value_list()} + return as_dict diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/wtperf_config.py b/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/wtperf_config.py new file mode 100644 index 00000000000..d2b70dff2dc --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/wtperf_config.py @@ -0,0 +1,23 @@ +class WTPerfConfig: + def __init__(self, + wtperf_path: str, + home_dir: str, + test: str, + environment: str = None, + run_max: int = 1, + verbose: bool = False): + self.wtperf_path: str = wtperf_path + self.home_dir: str = home_dir + self.test: str = test + self.environment: str = environment + self.run_max: int = run_max + self.verbose: bool = verbose + + def to_value_dict(self): + as_dict = {'wt_perf_path': self.wtperf_path, + 'test': self.test, + 'home_dir': self.home_dir, + 'environment': self.environment, + 'run_max': self.run_max, + 'verbose': self.verbose} + return as_dict diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/wtperf_run.py b/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/wtperf_run.py new file mode 100644 index 00000000000..fcd9e8d0cb8 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_run_py/wtperf_run.py @@ -0,0 +1,167 @@ +import argparse +import json +import os.path +import re +import subprocess +import sys +import platform +import psutil + +from wtperf_config import WTPerfConfig +from perf_stat import PerfStat +from perf_stat_collection import PerfStatCollection + +# example parameters: -p /Users/jeremy.thorp/Git/wiredtiger/build/bench/wtperf/wtperf -t ../runners/small-lsm.wtperf -v -ho WT_TEST -m 3 + +# the 'test.stat' file is where wt-perf.c writes out it's statistics +# (within the directory specified by the 'home' parameter) +test_stats_file = 'test.stat' + +def create_test_home_path(home: str, test_run: int): + return '{}_{}'.format(home, test_run) + +def create_test_stat_path(test_home_path: str): + return os.path.join(test_home_path, test_stats_file) + +def find_stat(test_stat_path: str, pattern: str, position_of_value: int): + for line in open(test_stat_path): + match = re.match(pattern, line) + if match: + return line.split()[position_of_value] + return 0 + +def construct_wtperf_command_line(wtperf: str, env: str, test: str, home: str): + command_line = [] + if env is not None: + command_line.append(env) + command_line.append(wtperf) + if test is not None: + command_line.append('-O') + command_line.append(test) + if home is not None: + command_line.append('-h') + command_line.append(home) + return command_line + +def run_test(config: WTPerfConfig, test_run: int): + test_home = create_test_home_path(home=config.home_dir, test_run=test_run) + command_line = construct_wtperf_command_line( + wtperf=config.wtperf_path, + env=config.environment, + test=config.test, + home=test_home) + # print('Command Line for test: {}'.format(command_line)) + subprocess.run(command_line) + +def process_results(config: WTPerfConfig, perf_stats: PerfStatCollection): + for test_run in range(config.run_max): + test_home = create_test_home_path(home=config.home_dir, test_run=test_run) + test_stats_path = create_test_stat_path(test_home) + if config.verbose: + print('Reading test stats file: {}'.format(test_stats_path)) + perf_stats.find_stats(test_stat_path=test_stats_path) + + total_memory_gb = psutil.virtual_memory().total / (1024 * 1024 * 1024) + as_dict = {'config': config.to_value_dict(), + 'metrics': perf_stats.to_value_list(), + 'system': { + 'cpu_physical_cores': psutil.cpu_count(logical=False), + 'cpu_logical_cores': psutil.cpu_count(), + 'total_physical_memory_gb': total_memory_gb, + 'platform': platform.platform()} + } + return as_dict + +def setup_perf_stats(): + perf_stats = PerfStatCollection() + perf_stats.add_stat(PerfStat(short_label="load", + pattern='Load time:', + input_offset=2, + output_label='Load time:', + output_precision=2, + conversion_function=float)) + perf_stats.add_stat(PerfStat(short_label="insert", + pattern='Executed \d+ insert operations', + input_offset=1, + output_label='Insert count:')) + perf_stats.add_stat(PerfStat(short_label="modify", + pattern='Executed \d+ modify operations', + input_offset=1, + output_label='Modify count:')) + perf_stats.add_stat(PerfStat(short_label="read", + pattern='Executed \d+ read operations', + input_offset=1, + output_label='Read count:')) + perf_stats.add_stat(PerfStat(short_label="truncate", + pattern='Executed \d+ truncate operations', + input_offset=1, + output_label='Truncate count:')) + perf_stats.add_stat(PerfStat(short_label="update", + pattern='Executed \d+ update operations', + input_offset=1, + output_label='Update count:')) + return perf_stats + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', '--wtperf', help='path of the wtperf executable') + parser.add_argument('-e', '--env', help='any environment variables that need to be set for running wtperf') + parser.add_argument('-t', '--test', help='path of the wtperf test to execute') + parser.add_argument('-o', '--outfile', help='path of the file to write test output to') + parser.add_argument('-m', '--runmax', type=int, default=1, help='maximum number of times to run the test') + parser.add_argument('-ho', '--home', help='path of the "home" directory that wtperf will use') + parser.add_argument('-re', + '--reuse', + action="store_true", + help='reuse and reanalyse results from previous tests rather than running tests again') + parser.add_argument('-v', '--verbose', action="store_true", help='be verbose') + args = parser.parse_args() + + if args.verbose: + print('WTPerfPy') + print('========') + print("Configuration:") + print(" WtPerf path: {}".format(args.wtperf)) + print(" Environment: {}".format(args.env)) + print(" Test path: {}".format(args.test)) + print(" Home base: {}".format(args.home)) + print(" Outfile: {}".format(args.outfile)) + print(" Runmax: {}".format(args.runmax)) + print(" Reuse results: {}".format(args.reuse)) + + if args.wtperf is None: + sys.exit('The path to the wtperf executable is required') + if args.test is None: + sys.exit('The path to the test file is required') + if args.home is None: + sys.exit('The path to the "home" directory is required') + + config = WTPerfConfig(wtperf_path=args.wtperf, + home_dir=args.home, + test=args.test, + environment=args.env, + run_max=args.runmax, + verbose=args.verbose) + + perf_stats: PerfStatCollection = setup_perf_stats() + + # Run tests (if we're not reusing results) + if not args.reuse: + for test_run in range(args.runmax): + print("Starting test {}".format(test_run)) + run_test(config=config, test_run=test_run) + print("Completed test {}".format(test_run)) + + # Process results + perf_dict = process_results(config, perf_stats) + perf_json = json.dumps(perf_dict, indent=4, sort_keys=True) + + if args.verbose: + print("JSON: {}".format(perf_json)) + + if args.outfile: + with open(args.outfile, 'w') as outfile: + json.dump(perf_dict, outfile, indent=4, sort_keys=True) + +if __name__ == '__main__': + main() diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 70307efdd2e..3d202b5f500 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -805,7 +805,7 @@ connection_runtime_config = [ 'failpoint_history_store_delete_key_from_ts', 'failpoint_history_store_insert_1', 'failpoint_history_store_insert_2', 'history_store_checkpoint_delay', 'history_store_search', 'history_store_sweep_race', 'prepare_checkpoint_delay', 'split_1', - 'split_2', 'split_3', 'split_4', 'split_5', 'split_6', 'split_7', 'split_8']), + 'split_2', 'split_3', 'split_4', 'split_5', 'split_6', 'split_7']), Config('verbose', '[]', r''' enable messages for various events. Options are given as a list, such as <code>"verbose=[evictserver,read]"</code>''', diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index c660bde80c1..6c0cc16592c 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -1136,6 +1136,7 @@ numSymbols numbare objs offpage +ofh ok oldv ondisk diff --git a/src/third_party/wiredtiger/dist/s_void b/src/third_party/wiredtiger/dist/s_void index da6e7b77710..249c89fbbab 100755 --- a/src/third_party/wiredtiger/dist/s_void +++ b/src/third_party/wiredtiger/dist/s_void @@ -60,6 +60,7 @@ func_ok() -e '/int __wt_block_manager_size$/d' \ -e '/int __wt_block_tiered_load$/d' \ -e '/int __wt_block_write_size$/d' \ + -e '/int __wt_btcur_skip_page$/d' \ -e '/int __wt_buf_catfmt$/d' \ -e '/int __wt_buf_fmt$/d' \ -e '/int __wt_count_birthmarks$/d' \ diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 6af0af26729..362712d5de7 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -505,8 +505,9 @@ conn_stats = [ ########################################## # Tiered storage statistics ########################################## - StorageStat('flush_state_races', 'flush state races'), StorageStat('flush_tier', 'flush_tier operation calls'), + StorageStat('local_objects_inuse', 'attempts to remove a local object and the object is in use'), + StorageStat('local_objects_removed', 'local objects removed'), ########################################## # Thread Count statistics @@ -819,13 +820,11 @@ conn_dsrc_stats = [ CursorStat('cursor_next_hs_tombstone', 'cursor next calls that skip due to a globally visible history store tombstone'), CursorStat('cursor_next_skip_ge_100', 'cursor next calls that skip greater than or equal to 100 entries'), CursorStat('cursor_next_skip_lt_100', 'cursor next calls that skip less than 100 entries'), - CursorStat('cursor_next_skip_page_count', 'Total number of pages skipped without reading by cursor next calls'), CursorStat('cursor_next_skip_total', 'Total number of entries skipped by cursor next calls'), CursorStat('cursor_open_count', 'open cursor count', 'no_clear,no_scale'), CursorStat('cursor_prev_hs_tombstone', 'cursor prev calls that skip due to a globally visible history store tombstone'), CursorStat('cursor_prev_skip_ge_100', 'cursor prev calls that skip greater than or equal to 100 entries'), CursorStat('cursor_prev_skip_lt_100', 'cursor prev calls that skip less than 100 entries'), - CursorStat('cursor_prev_skip_page_count', 'Total number of pages skipped without reading by cursor prev calls'), CursorStat('cursor_prev_skip_total', 'Total number of entries skipped by cursor prev calls'), CursorStat('cursor_search_near_prefix_fast_paths', 'Total number of times a search near has exited due to prefix config'), CursorStat('cursor_skip_hs_cur_position', 'Total number of entries skipped to position the history store cursor'), diff --git a/src/third_party/wiredtiger/dist/test_data.py b/src/third_party/wiredtiger/dist/test_data.py index c074877056a..f35d400be4e 100644 --- a/src/third_party/wiredtiger/dist/test_data.py +++ b/src/third_party/wiredtiger/dist/test_data.py @@ -107,7 +107,7 @@ component_config = throttle_config transaction_config = [ Config('ops_per_transaction', '', r''' Defines how many operations a transaction can perform, the range is defined with a minimum - and a maximum and a random number is chosen between the two using a linear distrubtion.''', + and a maximum and a random number is chosen between the two using a linear distribution.''', type='category',subconfig=range_config), ] @@ -116,7 +116,7 @@ thread_count = [ Specifies the number of threads that will be used to perform a certain function.''', min=0) ] -read_thread_config = thread_count + throttle_config + transaction_config +read_thread_config = thread_count + throttle_config + transaction_config + record_config update_insert_thread_config = thread_count + transaction_config + throttle_config + record_config # @@ -220,4 +220,6 @@ methods = { How long the insertions will occur for.''')]), 'example_test' : Method(test_config), 'hs_cleanup' : Method(test_config), + 'search_near_01' : Method(test_config), + 'search_near_02' : Method(test_config), } diff --git a/src/third_party/wiredtiger/examples/c/ex_cursor.c b/src/third_party/wiredtiger/examples/c/ex_cursor.c index ac4fc1d22bc..14b8e866e8f 100644 --- a/src/third_party/wiredtiger/examples/c/ex_cursor.c +++ b/src/third_party/wiredtiger/examples/c/ex_cursor.c @@ -38,6 +38,7 @@ int cursor_search_near(WT_CURSOR *cursor); int cursor_insert(WT_CURSOR *cursor); int cursor_update(WT_CURSOR *cursor); int cursor_remove(WT_CURSOR *cursor); +int cursor_largest_key(WT_CURSOR *cursor); static const char *home; @@ -155,6 +156,14 @@ cursor_remove(WT_CURSOR *cursor) } /*! [cursor remove] */ +/*! [cursor largest key] */ +int +cursor_largest_key(WT_CURSOR *cursor) +{ + return (cursor->largest_key(cursor)); +} +/*! [cursor largest key] */ + int main(int argc, char *argv[]) { @@ -197,6 +206,7 @@ main(int argc, char *argv[]) error_check(cursor_search_near(cursor)); error_check(cursor_update(cursor)); error_check(cursor_remove(cursor)); + error_check(cursor_largest_key(cursor)); error_check(cursor->close(cursor)); /* Note: closing the connection implicitly closes open session(s). */ diff --git a/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c b/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c index 8bcb6911d06..ee5720a1844 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c +++ b/src/third_party/wiredtiger/ext/storage_sources/local_store/local_store.c @@ -637,6 +637,8 @@ local_file_copy(LOCAL_STORAGE *local, WT_SESSION *session, const char *src_path, goto err; } } + if (ret == 0 && (ret = chmod(tmp_path, 0444)) < 0) + ret = local_err(local, session, errno, "%s: file_copy chmod failed", tmp_path); if ((ret = rename(tmp_path, dest_path)) != 0) { ret = local_err(local, session, errno, "%s: cannot rename from %s", dest_path, tmp_path); goto err; @@ -693,7 +695,7 @@ err: /* * local_flush_finish -- - * Move a file from the default file system to the cache in the new file system. + * Cache a file in the new file system. */ static int local_flush_finish(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, @@ -715,9 +717,13 @@ local_flush_finish(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, goto err; local->op_count++; - if ((ret = rename(source, dest_path)) != 0) { + /* + * Link the object with the original local object. The could be replaced by a file copy if + * portability is an issue. + */ + if ((ret = link(source, dest_path)) != 0) { ret = local_err( - local, session, errno, "ss_flush_finish rename %s to %s failed", source, dest_path); + local, session, errno, "ss_flush_finish link %s to %s failed", source, dest_path); goto err; } /* Set the file to readonly in the cache. */ diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index ffc75973557..34b33ccf645 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "a9bd69a7ca5dd14ae36ab76574ac7eb1c0207758" + "commit": "e8bf2e5fcb33197b73c7bb550742926f59e22218" } diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger.i b/src/third_party/wiredtiger/lang/python/wiredtiger.i index 0105a5a70d6..5d221c6863b 100644 --- a/src/third_party/wiredtiger/lang/python/wiredtiger.i +++ b/src/third_party/wiredtiger/lang/python/wiredtiger.i @@ -582,6 +582,7 @@ NOTFOUND_OK(__wt_cursor::remove) NOTFOUND_OK(__wt_cursor::search) NOTFOUND_OK(__wt_cursor::update) NOTFOUND_OK(__wt_cursor::_modify) +NOTFOUND_OK(__wt_cursor::largest_key) ANY_OK(__wt_modify::__wt_modify) ANY_OK(__wt_modify::~__wt_modify) diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index 8506083e006..6953eb3dc02 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -343,10 +343,12 @@ __cursor_row_next( WT_PAGE *page; WT_ROW *rip; WT_SESSION_IMPL *session; + bool prefix_search; - session = CUR2S(cbt); - page = cbt->ref->page; key = &cbt->iface.key; + page = cbt->ref->page; + session = CUR2S(cbt); + prefix_search = prefix != NULL && F_ISSET(&cbt->iface, WT_CURSTD_PREFIX_SEARCH); *skippedp = 0; /* If restarting after a prepare conflict, jump to the right spot. */ @@ -394,6 +396,14 @@ restart_read_insert: if ((ins = cbt->ins) != NULL) { key->data = WT_INSERT_KEY(ins); key->size = WT_INSERT_KEY_SIZE(ins); + /* + * If the cursor has prefix search configured we can early exit here if the key that we + * are visiting is after our prefix. + */ + if (prefix_search && __wt_prefix_match(prefix, key) < 0) { + WT_STAT_CONN_DATA_INCR(session, cursor_search_near_prefix_fast_paths); + return (WT_NOTFOUND); + } WT_RET(__wt_txn_read_upd_list(session, cbt, ins->upd)); if (cbt->upd_value->type == WT_UPDATE_INVALID) { ++*skippedp; @@ -435,10 +445,7 @@ restart_read_page: * If the cursor has prefix search configured we can early exit here if the key that we are * visiting is after our prefix. */ - if (F_ISSET(&cbt->iface, WT_CURSTD_PREFIX_SEARCH) && prefix != NULL && - __wt_prefix_match(prefix, &cbt->iface.key) < 0) { - /* It is not okay for the user to have a custom collator. */ - WT_ASSERT(session, CUR2BT(cbt)->collator == NULL); + if (prefix_search && __wt_prefix_match(prefix, &cbt->iface.key) < 0) { WT_STAT_CONN_DATA_INCR(session, cursor_search_near_prefix_fast_paths); return (WT_NOTFOUND); } @@ -681,13 +688,12 @@ __wt_btcur_next_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) WT_DECL_RET; WT_PAGE *page; WT_SESSION_IMPL *session; - size_t pages_skipped_count, total_skipped, skipped; + size_t total_skipped, skipped; uint32_t flags; bool newpage, restart; cursor = &cbt->iface; session = CUR2S(cbt); - pages_skipped_count = 0; total_skipped = 0; WT_STAT_CONN_DATA_INCR(session, cursor_next); @@ -707,7 +713,7 @@ __wt_btcur_next_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) __wt_btcur_iterate_setup(cbt); /* - * Walk any page we're holding until the underlying call returns not- found. Then, move to the + * Walk any page we're holding until the underlying call returns not-found. Then, move to the * next page, until we reach the end of the file. */ restart = F_ISSET(cbt, WT_CBT_ITERATE_RETRY_NEXT); @@ -715,16 +721,6 @@ __wt_btcur_next_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) for (newpage = false;; newpage = true, restart = false) { page = cbt->ref == NULL ? NULL : cbt->ref->page; - /* - * Determine if all records on the page have been deleted and all the tombstones are visible - * to our transaction. If so, we can avoid reading the records on the page and move to the - * next page. - */ - if (__wt_btcur_skip_page(cbt)) { - pages_skipped_count++; - goto skip_page; - } - if (F_ISSET(cbt, WT_CBT_ITERATE_APPEND)) { /* The page cannot be NULL if the above flag is set. */ WT_ASSERT(session, page != NULL); @@ -796,16 +792,26 @@ __wt_btcur_next_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) WT_STAT_CONN_INCR(session, cache_eviction_force_delete); } cbt->page_deleted_count = 0; -skip_page: + if (F_ISSET(cbt, WT_CBT_READ_ONCE)) LF_SET(WT_READ_WONT_NEED); - WT_ERR(__wt_tree_walk(session, &cbt->ref, flags)); + + /* + * If we are running with snapshot isolation, and not interested in returning tombstones, we + * could potentially skip pages. The skip function looks at the aggregated timestamp + * information to determine if something is visible on the page. If nothing is, the page is + * skipped. + */ + if (session->txn->isolation == WT_ISO_SNAPSHOT && + !F_ISSET(&cbt->iface, WT_CURSTD_IGNORE_TOMBSTONE)) + WT_ERR( + __wt_tree_walk_custom_skip(session, &cbt->ref, __wt_btcur_skip_page, NULL, flags)); + else + WT_ERR(__wt_tree_walk(session, &cbt->ref, flags)); WT_ERR_TEST(cbt->ref == NULL, WT_NOTFOUND, false); } err: - WT_STAT_CONN_DATA_INCRV(session, cursor_next_skip_page_count, pages_skipped_count); - if (total_skipped < 100) WT_STAT_CONN_DATA_INCR(session, cursor_next_skip_lt_100); else diff --git a/src/third_party/wiredtiger/src/btree/bt_curprev.c b/src/third_party/wiredtiger/src/btree/bt_curprev.c index bcf9f7e6d5a..3941ada539c 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curprev.c +++ b/src/third_party/wiredtiger/src/btree/bt_curprev.c @@ -484,10 +484,12 @@ __cursor_row_prev( WT_PAGE *page; WT_ROW *rip; WT_SESSION_IMPL *session; + bool prefix_search; - session = CUR2S(cbt); - page = cbt->ref->page; key = &cbt->iface.key; + page = cbt->ref->page; + session = CUR2S(cbt); + prefix_search = prefix != NULL && F_ISSET(&cbt->iface, WT_CURSTD_PREFIX_SEARCH); *skippedp = 0; /* If restarting after a prepare conflict, jump to the right spot. */ @@ -541,6 +543,14 @@ restart_read_insert: if ((ins = cbt->ins) != NULL) { key->data = WT_INSERT_KEY(ins); key->size = WT_INSERT_KEY_SIZE(ins); + /* + * If the cursor has prefix search configured we can early exit here if the key we are + * visiting is before our prefix. + */ + if (prefix_search && __wt_prefix_match(prefix, key) > 0) { + WT_STAT_CONN_DATA_INCR(session, cursor_search_near_prefix_fast_paths); + return (WT_NOTFOUND); + } WT_RET(__wt_txn_read_upd_list(session, cbt, ins->upd)); if (cbt->upd_value->type == WT_UPDATE_INVALID) { ++*skippedp; @@ -584,10 +594,7 @@ restart_read_page: * If the cursor has prefix search configured we can early exit here if the key we are * visiting is before our prefix. */ - if (F_ISSET(&cbt->iface, WT_CURSTD_PREFIX_SEARCH) && prefix != NULL && - __wt_prefix_match(prefix, &cbt->iface.key) > 0) { - /* It is not okay for the user to have a custom collator. */ - WT_ASSERT(session, CUR2BT(cbt)->collator == NULL); + if (prefix_search && __wt_prefix_match(prefix, &cbt->iface.key) > 0) { WT_STAT_CONN_DATA_INCR(session, cursor_search_near_prefix_fast_paths); return (WT_NOTFOUND); } @@ -620,13 +627,12 @@ __wt_btcur_prev_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) WT_DECL_RET; WT_PAGE *page; WT_SESSION_IMPL *session; - size_t pages_skipped_count, total_skipped, skipped; + size_t total_skipped, skipped; uint32_t flags; bool newpage, restart; cursor = &cbt->iface; session = CUR2S(cbt); - pages_skipped_count = 0; total_skipped = 0; WT_STAT_CONN_DATA_INCR(session, cursor_prev); @@ -647,7 +653,7 @@ __wt_btcur_prev_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) __wt_btcur_iterate_setup(cbt); /* - * Walk any page we're holding until the underlying call returns not- found. Then, move to the + * Walk any page we're holding until the underlying call returns not-found. Then, move to the * previous page, until we reach the start of the file. */ restart = F_ISSET(cbt, WT_CBT_ITERATE_RETRY_PREV); @@ -656,16 +662,6 @@ __wt_btcur_prev_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) page = cbt->ref == NULL ? NULL : cbt->ref->page; /* - * Determine if all records on the page have been deleted and all the tombstones are visible - * to our transaction. If so, we can avoid reading the records on the page and move to the - * next page. - */ - if (__wt_btcur_skip_page(cbt)) { - pages_skipped_count++; - goto skip_page; - } - - /* * Column-store pages may have appended entries. Handle it separately from the usual cursor * code, it's in a simple format. */ @@ -737,16 +733,26 @@ __wt_btcur_prev_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) WT_STAT_CONN_INCR(session, cache_eviction_force_delete); } cbt->page_deleted_count = 0; -skip_page: + if (F_ISSET(cbt, WT_CBT_READ_ONCE)) LF_SET(WT_READ_WONT_NEED); - WT_ERR(__wt_tree_walk(session, &cbt->ref, flags)); + + /* + * If we are running with snapshot isolation, and not interested in returning tombstones, we + * could potentially skip pages. The skip function looks at the aggregated timestamp + * information to determine if something is visible on the page. If nothing is, the page is + * skipped. + */ + if (session->txn->isolation == WT_ISO_SNAPSHOT && + !F_ISSET(&cbt->iface, WT_CURSTD_IGNORE_TOMBSTONE)) + WT_ERR( + __wt_tree_walk_custom_skip(session, &cbt->ref, __wt_btcur_skip_page, NULL, flags)); + else + WT_ERR(__wt_tree_walk(session, &cbt->ref, flags)); WT_ERR_TEST(cbt->ref == NULL, WT_NOTFOUND, false); } err: - WT_STAT_CONN_DATA_INCRV(session, cursor_prev_skip_page_count, pages_skipped_count); - if (total_skipped < 100) WT_STAT_CONN_DATA_INCR(session, cursor_prev_skip_lt_100); else diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index b3d879fae8f..6cd2b3c3fc7 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -684,15 +684,8 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) } /* - * If a valid key has been found and we are doing a prefix search near, we want to return the - * key only if it is a prefix match. - */ - if (valid && F_ISSET(cursor, WT_CURSTD_PREFIX_SEARCH) && - __wt_prefix_match(&state.key, cbt->tmp) != 0) - valid = false; - - /* - * If we find a valid key, return it. + * If we find a valid key, check if we are performing a prefix search near. If we are, return + * the record only if it is a prefix match. If not, return the record. * * Else, creating a record past the end of the tree in a fixed-length column-store implicitly * fills the gap with empty records. In this case, we instantiate the empty record, it's an @@ -708,8 +701,22 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) */ if (valid) { exact = cbt->compare; - ret = __cursor_kv_return(cbt, cbt->upd_value); - } else if (__cursor_fix_implicit(btree, cbt)) { + /* + * Set the cursor key before the prefix search near check. If the prefix doesn't match, + * restore the cursor state, and continue to search for a valid key. Otherwise set the + * cursor value and return the valid record. + */ + WT_ERR(__wt_key_return(cbt)); + if (F_ISSET(cursor, WT_CURSTD_PREFIX_SEARCH) && + __wt_prefix_match(&state.key, &cursor->key) != 0) + __cursor_state_restore(cursor, &state); + else { + WT_ERR(__wt_value_return(cbt, cbt->upd_value)); + goto done; + } + } + + if (__cursor_fix_implicit(btree, cbt)) { cbt->recno = cursor->recno; cbt->v = 0; cursor->value.data = &cbt->v; diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index 687c21ec086..3d6a8e82215 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -258,6 +258,7 @@ read: /* We just read a page, don't evict it before we have a chance to use it. */ evict_skip = true; + F_CLR(session->dhandle, WT_DHANDLE_EVICTED); /* * If configured to not trash the cache, leave the page generation unset, we'll set it diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index cc8c8d47803..699270b3ffa 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -2140,7 +2140,12 @@ __slvg_ovfl_reconcile(WT_SESSION_IMPL *session, WT_STUFF *ss) WT_ERR(__wt_calloc_def(session, trk->trk_ovfl_cnt, &slot)); for (j = 0; j < trk->trk_ovfl_cnt; ++j) { addr = &trk->trk_ovfl_addr[j]; - searchp = + /* + * It is possible that salvage found a leaf page that points to an overflow item, but + * there were no overflow items at all. + */ + searchp = ss->ovfl == NULL ? + NULL : bsearch(addr, ss->ovfl, ss->ovfl_next, sizeof(WT_TRACK *), __slvg_ovfl_compare); /* diff --git a/src/third_party/wiredtiger/src/btree/bt_walk.c b/src/third_party/wiredtiger/src/btree/bt_walk.c index 69ece30ac04..233cc2a2514 100644 --- a/src/third_party/wiredtiger/src/btree/bt_walk.c +++ b/src/third_party/wiredtiger/src/btree/bt_walk.c @@ -375,9 +375,6 @@ restart: empty_internal = false; } - /* Encourage races. */ - __wt_timing_stress(session, WT_TIMING_STRESS_SPLIT_8); - /* Optionally return internal pages. */ if (LF_ISSET(WT_READ_SKIP_INTL)) continue; diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 9b95fe6762d..6ea091cd3f6 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -145,7 +145,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { "\"history_store_checkpoint_delay\",\"history_store_search\"," "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\"]", + "\"split_6\",\"split_7\"]", NULL, 0}, {"verbose", "list", NULL, "choices=[\"api\",\"backup\",\"block\",\"checkpoint\"," @@ -877,7 +877,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { "\"history_store_checkpoint_delay\",\"history_store_search\"," "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\"]", + "\"split_6\",\"split_7\"]", NULL, 0}, {"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2}, @@ -959,7 +959,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { "\"history_store_checkpoint_delay\",\"history_store_search\"," "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\"]", + "\"split_6\",\"split_7\"]", NULL, 0}, {"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2}, @@ -1038,7 +1038,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { "\"history_store_checkpoint_delay\",\"history_store_search\"," "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\"]", + "\"split_6\",\"split_7\"]", NULL, 0}, {"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2}, @@ -1115,7 +1115,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { "\"history_store_checkpoint_delay\",\"history_store_search\"," "\"history_store_sweep_race\",\"prepare_checkpoint_delay\"," "\"split_1\",\"split_2\",\"split_3\",\"split_4\",\"split_5\"," - "\"split_6\",\"split_7\",\"split_8\"]", + "\"split_6\",\"split_7\"]", NULL, 0}, {"transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2}, diff --git a/src/third_party/wiredtiger/src/config/test_config.c b/src/third_party/wiredtiger/src/config/test_config.c index 30bf6d8a3c3..a9f954ace43 100644 --- a/src/third_party/wiredtiger/src/config/test_config.c +++ b/src/third_party/wiredtiger/src/config/test_config.c @@ -49,9 +49,10 @@ static const WT_CONFIG_CHECK confchk_populate_config_subconfigs[] = { {"value_size", "int", NULL, "min=0,max=1000000000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_read_config_subconfigs[] = { - {"op_rate", "string", NULL, NULL, NULL, 0}, + {"key_size", "int", NULL, "min=0,max=10000", NULL, 0}, {"op_rate", "string", NULL, NULL, NULL, 0}, {"ops_per_transaction", "category", NULL, NULL, confchk_ops_per_transaction_subconfigs, 2}, - {"thread_count", "int", NULL, "min=0", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; + {"thread_count", "int", NULL, "min=0", NULL, 0}, + {"value_size", "int", NULL, "min=0,max=1000000000", NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; static const WT_CONFIG_CHECK confchk_update_config_subconfigs[] = { {"key_size", "int", NULL, "min=0,max=10000", NULL, 0}, {"op_rate", "string", NULL, NULL, NULL, 0}, @@ -64,7 +65,7 @@ static const WT_CONFIG_CHECK confchk_workload_generator_subconfigs[] = { {"insert_config", "category", NULL, NULL, confchk_insert_config_subconfigs, 5}, {"op_rate", "string", NULL, NULL, NULL, 0}, {"populate_config", "category", NULL, NULL, confchk_populate_config_subconfigs, 5}, - {"read_config", "category", NULL, NULL, confchk_read_config_subconfigs, 3}, + {"read_config", "category", NULL, NULL, confchk_read_config_subconfigs, 5}, {"update_config", "category", NULL, NULL, confchk_update_config_subconfigs, 5}, {NULL, NULL, NULL, NULL, NULL, 0}}; @@ -125,6 +126,32 @@ static const WT_CONFIG_CHECK confchk_hs_cleanup[] = { {"workload_tracking", "category", NULL, NULL, confchk_workload_tracking_subconfigs, 2}, {NULL, NULL, NULL, NULL, NULL, 0}}; +static const WT_CONFIG_CHECK confchk_search_near_01[] = { + {"cache_size_mb", "int", NULL, "min=0,max=100000000000", NULL, 0}, + {"checkpoint_manager", "category", NULL, NULL, confchk_checkpoint_manager_subconfigs, 2}, + {"compression_enabled", "boolean", NULL, NULL, NULL, 0}, + {"duration_seconds", "int", NULL, "min=0,max=1000000", NULL, 0}, + {"enable_logging", "boolean", NULL, NULL, NULL, 0}, + {"runtime_monitor", "category", NULL, NULL, confchk_runtime_monitor_subconfigs, 5}, + {"statistics_config", "category", NULL, NULL, confchk_statistics_config_subconfigs, 2}, + {"timestamp_manager", "category", NULL, NULL, confchk_timestamp_manager_subconfigs, 4}, + {"workload_generator", "category", NULL, NULL, confchk_workload_generator_subconfigs, 6}, + {"workload_tracking", "category", NULL, NULL, confchk_workload_tracking_subconfigs, 2}, + {NULL, NULL, NULL, NULL, NULL, 0}}; + +static const WT_CONFIG_CHECK confchk_search_near_02[] = { + {"cache_size_mb", "int", NULL, "min=0,max=100000000000", NULL, 0}, + {"checkpoint_manager", "category", NULL, NULL, confchk_checkpoint_manager_subconfigs, 2}, + {"compression_enabled", "boolean", NULL, NULL, NULL, 0}, + {"duration_seconds", "int", NULL, "min=0,max=1000000", NULL, 0}, + {"enable_logging", "boolean", NULL, NULL, NULL, 0}, + {"runtime_monitor", "category", NULL, NULL, confchk_runtime_monitor_subconfigs, 5}, + {"statistics_config", "category", NULL, NULL, confchk_statistics_config_subconfigs, 2}, + {"timestamp_manager", "category", NULL, NULL, confchk_timestamp_manager_subconfigs, 4}, + {"workload_generator", "category", NULL, NULL, confchk_workload_generator_subconfigs, 6}, + {"workload_tracking", "category", NULL, NULL, confchk_workload_tracking_subconfigs, 2}, + {NULL, NULL, NULL, NULL, NULL, 0}}; + static const WT_CONFIG_ENTRY config_entries[] = { {"base_test", "cache_size_mb=0,checkpoint_manager=(enabled=false,op_rate=1s)," @@ -138,10 +165,10 @@ static const WT_CONFIG_ENTRY config_entries[] = { "insert_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," "min=0),thread_count=0,value_size=5),op_rate=1s," "populate_config=(collection_count=1,key_count_per_collection=0," - "key_size=5,thread_count=1,value_size=5),read_config=(op_rate=1s," - "ops_per_transaction=(max=1,min=0),thread_count=0)," - "update_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," - "min=0),thread_count=0,value_size=5))," + "key_size=5,thread_count=1,value_size=5),read_config=(key_size=5," + "op_rate=1s,ops_per_transaction=(max=1,min=0),thread_count=0," + "value_size=5),update_config=(key_size=5,op_rate=1s," + "ops_per_transaction=(max=1,min=0),thread_count=0,value_size=5))," "workload_tracking=(enabled=true,op_rate=1s)", confchk_base_test, 10}, {"burst_inserts", @@ -157,10 +184,10 @@ static const WT_CONFIG_ENTRY config_entries[] = { "insert_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," "min=0),thread_count=0,value_size=5),op_rate=1s," "populate_config=(collection_count=1,key_count_per_collection=0," - "key_size=5,thread_count=1,value_size=5),read_config=(op_rate=1s," - "ops_per_transaction=(max=1,min=0),thread_count=0)," - "update_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," - "min=0),thread_count=0,value_size=5))," + "key_size=5,thread_count=1,value_size=5),read_config=(key_size=5," + "op_rate=1s,ops_per_transaction=(max=1,min=0),thread_count=0," + "value_size=5),update_config=(key_size=5,op_rate=1s," + "ops_per_transaction=(max=1,min=0),thread_count=0,value_size=5))," "workload_tracking=(enabled=true,op_rate=1s)", confchk_burst_inserts, 11}, {"example_test", @@ -175,10 +202,10 @@ static const WT_CONFIG_ENTRY config_entries[] = { "insert_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," "min=0),thread_count=0,value_size=5),op_rate=1s," "populate_config=(collection_count=1,key_count_per_collection=0," - "key_size=5,thread_count=1,value_size=5),read_config=(op_rate=1s," - "ops_per_transaction=(max=1,min=0),thread_count=0)," - "update_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," - "min=0),thread_count=0,value_size=5))," + "key_size=5,thread_count=1,value_size=5),read_config=(key_size=5," + "op_rate=1s,ops_per_transaction=(max=1,min=0),thread_count=0," + "value_size=5),update_config=(key_size=5,op_rate=1s," + "ops_per_transaction=(max=1,min=0),thread_count=0,value_size=5))," "workload_tracking=(enabled=true,op_rate=1s)", confchk_example_test, 10}, {"hs_cleanup", @@ -193,12 +220,48 @@ static const WT_CONFIG_ENTRY config_entries[] = { "insert_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," "min=0),thread_count=0,value_size=5),op_rate=1s," "populate_config=(collection_count=1,key_count_per_collection=0," - "key_size=5,thread_count=1,value_size=5),read_config=(op_rate=1s," - "ops_per_transaction=(max=1,min=0),thread_count=0)," - "update_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," - "min=0),thread_count=0,value_size=5))," + "key_size=5,thread_count=1,value_size=5),read_config=(key_size=5," + "op_rate=1s,ops_per_transaction=(max=1,min=0),thread_count=0," + "value_size=5),update_config=(key_size=5,op_rate=1s," + "ops_per_transaction=(max=1,min=0),thread_count=0,value_size=5))," "workload_tracking=(enabled=true,op_rate=1s)", confchk_hs_cleanup, 10}, + {"search_near_01", + "cache_size_mb=0,checkpoint_manager=(enabled=false,op_rate=1s)," + "compression_enabled=false,duration_seconds=0," + "enable_logging=false,runtime_monitor=(enabled=true,op_rate=1s," + "postrun_statistics=[],stat_cache_size=(enabled=false,limit=0)," + "stat_db_size=(enabled=false,limit=0))," + "statistics_config=(enable_logging=true,type=all)," + "timestamp_manager=(enabled=true,oldest_lag=1,op_rate=1s," + "stable_lag=1),workload_generator=(enabled=true," + "insert_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," + "min=0),thread_count=0,value_size=5),op_rate=1s," + "populate_config=(collection_count=1,key_count_per_collection=0," + "key_size=5,thread_count=1,value_size=5),read_config=(key_size=5," + "op_rate=1s,ops_per_transaction=(max=1,min=0),thread_count=0," + "value_size=5),update_config=(key_size=5,op_rate=1s," + "ops_per_transaction=(max=1,min=0),thread_count=0,value_size=5))," + "workload_tracking=(enabled=true,op_rate=1s)", + confchk_search_near_01, 10}, + {"search_near_02", + "cache_size_mb=0,checkpoint_manager=(enabled=false,op_rate=1s)," + "compression_enabled=false,duration_seconds=0," + "enable_logging=false,runtime_monitor=(enabled=true,op_rate=1s," + "postrun_statistics=[],stat_cache_size=(enabled=false,limit=0)," + "stat_db_size=(enabled=false,limit=0))," + "statistics_config=(enable_logging=true,type=all)," + "timestamp_manager=(enabled=true,oldest_lag=1,op_rate=1s," + "stable_lag=1),workload_generator=(enabled=true," + "insert_config=(key_size=5,op_rate=1s,ops_per_transaction=(max=1," + "min=0),thread_count=0,value_size=5),op_rate=1s," + "populate_config=(collection_count=1,key_count_per_collection=0," + "key_size=5,thread_count=1,value_size=5),read_config=(key_size=5," + "op_rate=1s,ops_per_transaction=(max=1,min=0),thread_count=0," + "value_size=5),update_config=(key_size=5,op_rate=1s," + "ops_per_transaction=(max=1,min=0),thread_count=0,value_size=5))," + "workload_tracking=(enabled=true,op_rate=1s)", + confchk_search_near_02, 10}, {NULL, NULL, NULL, 0}}; /* diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index c06ab898882..2e55aedcd63 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -2148,7 +2148,7 @@ __wt_timing_stress_config(WT_SESSION_IMPL *session, const char *cfg[]) {"split_1", WT_TIMING_STRESS_SPLIT_1}, {"split_2", WT_TIMING_STRESS_SPLIT_2}, {"split_3", WT_TIMING_STRESS_SPLIT_3}, {"split_4", WT_TIMING_STRESS_SPLIT_4}, {"split_5", WT_TIMING_STRESS_SPLIT_5}, {"split_6", WT_TIMING_STRESS_SPLIT_6}, - {"split_7", WT_TIMING_STRESS_SPLIT_7}, {"split_8", WT_TIMING_STRESS_SPLIT_8}, {NULL, 0}}; + {"split_7", WT_TIMING_STRESS_SPLIT_7}, {NULL, 0}}; WT_CONFIG_ITEM cval, sval; WT_CONNECTION_IMPL *conn; WT_DECL_RET; diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index decf70dd7e4..215eb5a47f0 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -319,6 +319,9 @@ __wt_conn_dhandle_close(WT_SESSION_IMPL *session, bool final, bool mark_dead) /* Reset the tree's eviction priority (if any). */ __wt_evict_priority_clear(session); + + /* Mark the advisory bit that the tree has been evicted. */ + F_SET(dhandle, WT_DHANDLE_EVICTED); } /* diff --git a/src/third_party/wiredtiger/src/conn/conn_tiered.c b/src/third_party/wiredtiger/src/conn/conn_tiered.c index 58b2026c5b8..6f34064ca7e 100644 --- a/src/third_party/wiredtiger/src/conn/conn_tiered.c +++ b/src/third_party/wiredtiger/src/conn/conn_tiered.c @@ -20,6 +20,16 @@ #endif /* + * __tiered_server_run_chk -- + * Check to decide if the tiered storage server should continue running. + */ +static bool +__tiered_server_run_chk(WT_SESSION_IMPL *session) +{ + return (FLD_ISSET(S2C(session)->server_flags, WT_CONN_SERVER_TIERED)); +} + +/* * __flush_tier_wait -- * Wait for all previous work units queued to be processed. */ @@ -121,55 +131,63 @@ err: /* * __tier_storage_remove_local -- - * Perform one iteration of tiered storage local tier removal. + * Perform one iteration of tiered storage local object removal. */ static int -__tier_storage_remove_local(WT_SESSION_IMPL *session, const char *uri, bool force) +__tier_storage_remove_local(WT_SESSION_IMPL *session) { - WT_CONFIG_ITEM cval; WT_DECL_RET; - size_t len; + WT_TIERED_WORK_UNIT *entry; uint64_t now; - char *config, *newfile; - const char *cfg[2], *filename; - - config = newfile = NULL; - if (uri == NULL) - return (0); - __wt_verbose(session, WT_VERB_TIERED, "Removing tree %s", uri); - filename = uri; - WT_PREFIX_SKIP_REQUIRED(session, filename, "tiered:"); - len = strlen("file:") + strlen(filename) + 1; - WT_ERR(__wt_calloc_def(session, len, &newfile)); - WT_ERR(__wt_snprintf(newfile, len, "file:%s", filename)); + const char *object; - /* - * If the file:URI of the tiered object does not exist, there is nothing to do. - */ - ret = __wt_metadata_search(session, newfile, &config); - if (ret == WT_NOTFOUND) { - ret = 0; - goto err; - } - WT_ERR(ret); + entry = NULL; + for (;;) { + /* Check if we're quitting or being reconfigured. */ + if (!__tiered_server_run_chk(session)) + break; - /* - * We have a local version of this tiered data. Check its metadata for when it expires and - * remove if necessary. - */ - cfg[0] = config; - cfg[1] = NULL; - WT_ERR(__wt_config_gets(session, cfg, "local_retention", &cval)); - __wt_seconds(session, &now); - if (force || (uint64_t)cval.val + S2C(session)->bstorage->retain_secs >= now) + __wt_seconds(session, &now); + __wt_tiered_get_drop_local(session, now, &entry); + if (entry == NULL) + break; + WT_ERR(__wt_tiered_name( + session, &entry->tiered->iface, entry->id, WT_TIERED_NAME_OBJECT, &object)); + __wt_verbose(session, WT_VERB_TIERED, "REMOVE_LOCAL: %s at %" PRIu64, object, now); + WT_PREFIX_SKIP_REQUIRED(session, object, "object:"); /* - * We want to remove the entry and the file. Probably do a schema_drop on the file:uri. + * If the handle is still open, it could still be in use for reading. In that case put the + * work unit back on the work queue and keep trying. */ - __wt_verbose(session, WT_VERB_TIERED, "Would remove %s. Local retention expired", newfile); - + if (__wt_handle_is_open(session, object)) { + __wt_verbose(session, WT_VERB_TIERED, "REMOVE_LOCAL: %s in USE, queue again", object); + WT_STAT_CONN_INCR(session, local_objects_inuse); + /* + * FIXME-WT-7470: If the object we want to remove is in use this is the place to call + * object sweep to clean up block->ofh file handles. Another alternative would be to try + * to sweep and then try the remove call below rather than pushing it back on the work + * queue. NOTE: Remove 'ofh' from s_string.ok when removing this comment. + * + * Update the time on the entry before pushing it back on the queue so that we don't get + * into an infinite loop trying to drop an open file that may be in use a while. + */ + WT_ASSERT(session, entry->tiered != NULL && entry->tiered->bstorage != NULL); + entry->op_val = now + entry->tiered->bstorage->retain_secs; + __wt_tiered_push_work(session, entry); + } else { + __wt_verbose(session, WT_VERB_TIERED, "REMOVE_LOCAL: actually remove %s", object); + WT_STAT_CONN_INCR(session, local_objects_removed); + WT_ERR(__wt_fs_remove(session, object, false)); + /* + * We are responsible for freeing the work unit when we're done with it. + */ + __wt_tiered_work_free(session, entry); + } + entry = NULL; + } err: - __wt_free(session, config); - __wt_free(session, newfile); + if (entry != NULL) + __wt_tiered_work_free(session, entry); return (ret); } @@ -230,8 +248,8 @@ err: * Perform one iteration of copying newly flushed objects to the shared storage. */ int -__wt_tier_do_flush( - WT_SESSION_IMPL *session, WT_TIERED *tiered, const char *local_uri, const char *obj_uri) +__wt_tier_do_flush(WT_SESSION_IMPL *session, WT_TIERED *tiered, uint32_t id, const char *local_uri, + const char *obj_uri) { WT_DECL_RET; WT_FILE_SYSTEM *bucket_fs; @@ -260,6 +278,11 @@ __wt_tier_do_flush( */ WT_RET(storage_source->ss_flush_finish( storage_source, &session->iface, bucket_fs, local_name, obj_name, NULL)); + /* + * After successful flushing, push a work unit to drop the local object in the future. The + * object will be removed locally after the local retention period expires. + */ + WT_RET(__wt_tiered_put_drop_local(session, tiered, id)); return (0); } @@ -276,7 +299,7 @@ __wt_tier_flush(WT_SESSION_IMPL *session, WT_TIERED *tiered, uint32_t id) local_uri = obj_uri = NULL; WT_ERR(__wt_tiered_name(session, &tiered->iface, id, WT_TIERED_NAME_LOCAL, &local_uri)); WT_ERR(__wt_tiered_name(session, &tiered->iface, id, WT_TIERED_NAME_OBJECT, &obj_uri)); - WT_ERR(__wt_tier_do_flush(session, tiered, local_uri, obj_uri)); + WT_ERR(__wt_tier_do_flush(session, tiered, id, local_uri, obj_uri)); err: __wt_free(session, local_uri); @@ -296,6 +319,10 @@ __tier_storage_copy(WT_SESSION_IMPL *session) entry = NULL; for (;;) { + /* Check if we're quitting or being reconfigured. */ + if (!__tiered_server_run_chk(session)) + break; + /* * We probably need some kind of flush generation so that we don't process flush items for * tables that are added during an in-progress flush_tier. This thread could run due to a @@ -334,7 +361,7 @@ __tier_storage_remove(WT_SESSION_IMPL *session, bool force) * We want to walk the metadata perhaps and for each tiered URI, call remove on its file:URI * version. */ - WT_RET(__tier_storage_remove_local(session, NULL, force)); + WT_RET(__tier_storage_remove_local(session)); return (0); } @@ -448,16 +475,6 @@ __tiered_manager_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) } /* - * __tiered_server_run_chk -- - * Check to decide if the tiered storage server should continue running. - */ -static bool -__tiered_server_run_chk(WT_SESSION_IMPL *session) -{ - return (FLD_ISSET(S2C(session)->server_flags, WT_CONN_SERVER_TIERED)); -} - -/* * __tiered_server -- * The tiered storage server thread. */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index fd416d3c452..820ea49aca0 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -268,6 +268,7 @@ __wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, __wt_cursor_notsup, /* remove */ __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ + __wt_cursor_notsup, /* largest_key */ __wt_cursor_notsup, /* cache */ __wt_cursor_reopen_notsup, /* reopen */ __curbackup_close); /* close */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_config.c b/src/third_party/wiredtiger/src/cursor/cur_config.c index ade673bba41..0fbf1066b27 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_config.c +++ b/src/third_party/wiredtiger/src/cursor/cur_config.c @@ -51,6 +51,7 @@ __wt_curconfig_open( __wt_cursor_notsup, /* remove */ __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ + __wt_cursor_notsup, /* largest_key */ __wt_cursor_notsup, /* cache */ __wt_cursor_reopen_notsup, /* reopen */ __curconfig_close); diff --git a/src/third_party/wiredtiger/src/cursor/cur_ds.c b/src/third_party/wiredtiger/src/cursor/cur_ds.c index 7139f1a8b08..c3e28dd5f59 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_ds.c +++ b/src/third_party/wiredtiger/src/cursor/cur_ds.c @@ -425,6 +425,7 @@ __wt_curds_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, con __curds_remove, /* remove */ __curds_reserve, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ + __wt_cursor_notsup, /* largest_key */ __wt_cursor_notsup, /* cache */ __wt_cursor_reopen_notsup, /* reopen */ __curds_close); /* close */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_dump.c b/src/third_party/wiredtiger/src/cursor/cur_dump.c index be4c1c7ef34..64e273f801b 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_dump.c +++ b/src/third_party/wiredtiger/src/cursor/cur_dump.c @@ -396,6 +396,7 @@ __wt_curdump_create(WT_CURSOR *child, WT_CURSOR *owner, WT_CURSOR **cursorp) __curdump_remove, /* remove */ __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ + __wt_cursor_notsup, /* largest_key */ __wt_cursor_notsup, /* cache */ __wt_cursor_reopen_notsup, /* reopen */ __curdump_close); /* close */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_file.c b/src/third_party/wiredtiger/src/cursor/cur_file.c index ce54fd23abf..23c0a37776c 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_file.c +++ b/src/third_party/wiredtiger/src/cursor/cur_file.c @@ -671,6 +671,7 @@ __curfile_create(WT_SESSION_IMPL *session, WT_CURSOR *owner, const char *cfg[], __curfile_remove, /* remove */ __curfile_reserve, /* reserve */ __wt_cursor_reconfigure, /* reconfigure */ + __wt_cursor_largest_key, /* largest_key */ __curfile_cache, /* cache */ __curfile_reopen, /* reopen */ __curfile_close); /* close */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_hs.c b/src/third_party/wiredtiger/src/cursor/cur_hs.c index 2a2657b828b..00d419915ac 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_hs.c +++ b/src/third_party/wiredtiger/src/cursor/cur_hs.c @@ -1097,6 +1097,7 @@ __wt_curhs_open(WT_SESSION_IMPL *session, WT_CURSOR *owner, WT_CURSOR **cursorp) __curhs_remove, /* remove */ __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ + __wt_cursor_notsup, /* largest_key */ __wt_cursor_notsup, /* cache */ __wt_cursor_reopen_notsup, /* reopen */ __curhs_close); /* close */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_index.c b/src/third_party/wiredtiger/src/cursor/cur_index.c index ea57c060196..c5c17e92453 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_index.c +++ b/src/third_party/wiredtiger/src/cursor/cur_index.c @@ -453,6 +453,7 @@ __wt_curindex_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, __wt_cursor_notsup, /* remove */ __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ + __wt_cursor_notsup, /* largest_key */ __wt_cursor_notsup, /* cache */ __wt_cursor_reopen_notsup, /* reopen */ __curindex_close); /* close */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_join.c b/src/third_party/wiredtiger/src/cursor/cur_join.c index 7f00ea9bc3d..b721fe9d0ee 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_join.c +++ b/src/third_party/wiredtiger/src/cursor/cur_join.c @@ -576,6 +576,7 @@ __curjoin_entry_member( __wt_cursor_notsup, /* remove */ __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ + __wt_cursor_notsup, /* largest_key */ __wt_cursor_notsup, /* cache */ __wt_cursor_reopen_notsup, /* reopen */ __wt_cursor_notsup); /* close */ @@ -1222,6 +1223,7 @@ __wt_curjoin_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, c __wt_cursor_notsup, /* remove */ __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ + __wt_cursor_notsup, /* largest_key */ __wt_cursor_notsup, /* cache */ __wt_cursor_reopen_notsup, /* reopen */ __curjoin_close); /* close */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_log.c b/src/third_party/wiredtiger/src/cursor/cur_log.c index f40ef6b7d1a..281050785d4 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_log.c +++ b/src/third_party/wiredtiger/src/cursor/cur_log.c @@ -340,6 +340,7 @@ __wt_curlog_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], W __wt_cursor_notsup, /* remove */ __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ + __wt_cursor_notsup, /* largest_key */ __wt_cursor_notsup, /* cache */ __wt_cursor_reopen_notsup, /* reopen */ __curlog_close); /* close */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_metadata.c b/src/third_party/wiredtiger/src/cursor/cur_metadata.c index cfd0191a782..f0b04128704 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_metadata.c +++ b/src/third_party/wiredtiger/src/cursor/cur_metadata.c @@ -575,6 +575,7 @@ __wt_curmetadata_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owne __curmetadata_remove, /* remove */ __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ + __wt_cursor_notsup, /* largest_key */ __wt_cursor_notsup, /* cache */ __wt_cursor_reopen_notsup, /* reopen */ __curmetadata_close); /* close */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_stat.c b/src/third_party/wiredtiger/src/cursor/cur_stat.c index cafa0c49b35..ca78b4bff52 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_stat.c +++ b/src/third_party/wiredtiger/src/cursor/cur_stat.c @@ -608,6 +608,7 @@ __wt_curstat_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, c __wt_cursor_notsup, /* remove */ __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ + __wt_cursor_notsup, /* largest_key */ __wt_cursor_notsup, /* cache */ __wt_cursor_reopen_notsup, /* reopen */ __curstat_close); /* close */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 776ad980d3f..de5115845c7 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -856,7 +856,7 @@ __wt_cursor_cache_get(WT_SESSION_IMPL *session, const char *uri, uint64_t hash_v * than flag values, so fix them up according to the given configuration. */ F_CLR(cursor, - WT_CURSTD_APPEND | WT_CURSTD_PREFIX_SEARCH | WT_CURSTD_RAW | WT_CURSTD_OVERWRITE); + WT_CURSTD_APPEND | WT_CURSTD_OVERWRITE | WT_CURSTD_PREFIX_SEARCH | WT_CURSTD_RAW); F_SET(cursor, overwrite_flag); /* * If this is a btree cursor, clear its read_once flag. @@ -1084,7 +1084,7 @@ __wt_cursor_reconfigure(WT_CURSOR *cursor, const char *config) WT_ERR_NOTFOUND_OK(ret, false); /* Set the prefix search near flag. */ - if ((ret = __wt_config_getones(session, config, "prefix_key", &cval)) == 0) { + if ((ret = __wt_config_getones(session, config, "prefix_search", &cval)) == 0) { if (cval.val) { /* Prefix search near configuration can only be used for row-store. */ if (WT_CURSOR_RECNO(cursor)) @@ -1114,6 +1114,52 @@ err: } /* + * __wt_cursor_largest_key -- + * WT_CURSOR->largest_key default implementation.. + */ +int +__wt_cursor_largest_key(WT_CURSOR *cursor) +{ + WT_DECL_ITEM(key); + WT_DECL_RET; + WT_SESSION_IMPL *session; + bool ignore_tombstone; + + ignore_tombstone = F_ISSET(cursor, WT_CURSTD_IGNORE_TOMBSTONE); + CURSOR_API_CALL(cursor, session, largest_key, NULL); + + if (F_ISSET(session->txn, WT_TXN_SHARED_TS_READ)) + WT_ERR_MSG(session, EINVAL, "largest key cannot be called with a read timestamp"); + + WT_ERR(__wt_scr_alloc(session, 0, &key)); + + /* Reset the cursor to give up the cursor position. */ + WT_ERR(cursor->reset(cursor)); + + /* Ignore deletion */ + F_SET(cursor, WT_CURSTD_IGNORE_TOMBSTONE); + + /* Call cursor prev with read uncommitted isolation level. */ + WT_WITH_TXN_ISOLATION(session, WT_ISO_READ_UNCOMMITTED, ret = cursor->prev(cursor)); + WT_ERR(ret); + + /* Copy the key as we will reset the cursor after that. */ + WT_ERR(__wt_buf_set(session, key, cursor->key.data, cursor->key.size)); + WT_ERR(cursor->reset(cursor)); + WT_ERR(__wt_buf_set(session, &cursor->key, key->data, key->size)); + /* Set the key as external. */ + F_SET(cursor, WT_CURSTD_KEY_EXT); + +err: + if (!ignore_tombstone) + F_CLR(cursor, WT_CURSTD_IGNORE_TOMBSTONE); + __wt_scr_free(session, &key); + if (ret != 0) + WT_TRET(cursor->reset(cursor)); + API_END_RET(session, ret); +} + +/* * __wt_cursor_dup_position -- * Set a cursor to another cursor's position. */ diff --git a/src/third_party/wiredtiger/src/cursor/cur_table.c b/src/third_party/wiredtiger/src/cursor/cur_table.c index e4505544544..68776b18ba3 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_table.c +++ b/src/third_party/wiredtiger/src/cursor/cur_table.c @@ -102,6 +102,7 @@ __wt_apply_single_idx(WT_SESSION_IMPL *session, WT_INDEX *idx, WT_CURSOR *cur, __wt_cursor_notsup, /* remove */ __wt_cursor_notsup, /* reserve */ __wt_cursor_reconfigure_notsup, /* reconfigure */ + __wt_cursor_notsup, /* largest_key */ __wt_cursor_notsup, /* cache */ __wt_cursor_reopen_notsup, /* reopen */ __wt_cursor_notsup); /* close */ @@ -789,6 +790,31 @@ err: } /* + * __curtable_largest_key -- + * WT_CURSOR->largest_key method for the table cursor type. + */ +static int +__curtable_largest_key(WT_CURSOR *cursor) +{ + WT_CURSOR *primary; + WT_CURSOR_TABLE *ctable; + WT_DECL_RET; + WT_SESSION_IMPL *session; + + ctable = (WT_CURSOR_TABLE *)cursor; + JOINABLE_CURSOR_API_CALL(cursor, session, largest_key, NULL); + + WT_ERR(cursor->reset(cursor)); + primary = *ctable->cg_cursors; + WT_ERR(primary->largest_key(primary)); + +err: + if (ret != 0) + WT_TRET(cursor->reset(cursor)); + API_END_RET(session, ret); +} + +/* * __curtable_close -- * WT_CURSOR->close method for the table cursor type. */ @@ -968,6 +994,7 @@ __wt_curtable_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, __curtable_remove, /* remove */ __curtable_reserve, /* reserve */ __wt_cursor_reconfigure, /* reconfigure */ + __curtable_largest_key, /* largest_key */ __wt_cursor_notsup, /* cache */ __wt_cursor_reopen_notsup, /* reopen */ __curtable_close); /* close */ diff --git a/src/third_party/wiredtiger/src/docs/cursor-ops.dox b/src/third_party/wiredtiger/src/docs/cursor-ops.dox index c9d75295ddc..82b763b563f 100644 --- a/src/third_party/wiredtiger/src/docs/cursor-ops.dox +++ b/src/third_party/wiredtiger/src/docs/cursor-ops.dox @@ -103,6 +103,16 @@ fail with ::WT_DUPLICATE_KEY if the record previously exists, and WT_CURSOR::update and WT_CURSOR::remove will fail with ::WT_NOTFOUND if the record does not previously exist. +@snippet ex_cursor.c cursor largest key + +The WT_SESSION::largest_key \c gets the largest key in a table regardless of +visibility. + +It can only be called without a read timestamp, otherwise it returns an invalid +argument error. Any following prev or next calls will behave as if they were +invoked on an unpositioned cursor no matter the largest key call is successful +or not. + @section cursor_error Cursor position after error After any cursor handle method failure, the cursor's position is diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index f34eace96d9..6a5b7f8dd5d 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -658,6 +658,7 @@ unescaped unicode uninstall unittest +unpositioned untyped uri useconds diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 0488f2346f6..0bfaf1c9f33 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -285,7 +285,7 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) * busy and then opens a different file (in this case, the HS file), it can deadlock with a * thread waiting for the first file to drain from the eviction queue. See WT-5946 for details. */ - WT_RET(__wt_curhs_cache(session)); + WT_ERR(__wt_curhs_cache(session)); if (conn->evict_server_running && __wt_spin_trylock(session, &cache->evict_pass_lock) == 0) { /* * Cannot use WT_WITH_PASS_LOCK because this is a try lock. Fix when that is supported. We @@ -349,10 +349,11 @@ __wt_evict_thread_stop(WT_SESSION_IMPL *session, WT_THREAD *thread) WT_WITH_PASS_LOCK(session, ret = __evict_clear_all_walks(session)); WT_ERR(ret); /* - * The only two cases when the eviction server is expected to stop are when recovery is finished - * or when the connection is closing. + * The only cases when the eviction server is expected to stop are when recovery is finished, + * when the connection is closing or when an error has occurred and connection panic flag is + * set. */ - WT_ASSERT(session, F_ISSET(conn, WT_CONN_CLOSING | WT_CONN_RECOVERING)); + WT_ASSERT(session, F_ISSET(conn, WT_CONN_CLOSING | WT_CONN_PANIC | WT_CONN_RECOVERING)); /* Clear the eviction thread session flag. */ F_CLR(session, WT_SESSION_EVICTION); @@ -2046,9 +2047,14 @@ fast: */ if (pages_queued < target_pages / 2 && !urgent_queued) btree->evict_walk_period = WT_MIN(WT_MAX(1, 2 * btree->evict_walk_period), 100); - else if (pages_queued == target_pages) + else if (pages_queued == target_pages) { btree->evict_walk_period = 0; - else if (btree->evict_walk_period > 0) + /* + * If there's a chance the Btree was fully evicted, update the evicted flag in the handle. + */ + if (__wt_btree_bytes_evictable(session) == 0) + F_SET(session->dhandle, WT_DHANDLE_EVICTED); + } else if (btree->evict_walk_period > 0) btree->evict_walk_period /= 2; /* diff --git a/src/third_party/wiredtiger/src/include/btree_inline.h b/src/third_party/wiredtiger/src/include/btree_inline.h index adcf81b83bb..7b9a2b74a5f 100644 --- a/src/third_party/wiredtiger/src/include/btree_inline.h +++ b/src/third_party/wiredtiger/src/include/btree_inline.h @@ -2003,50 +2003,44 @@ __wt_page_swap_func(WT_SESSION_IMPL *session, WT_REF *held, WT_REF *want, uint32 * Return if the cursor is pointing to a page with deleted records and can be skipped for cursor * traversal. */ -static inline bool -__wt_btcur_skip_page(WT_CURSOR_BTREE *cbt) +static inline int +__wt_btcur_skip_page(WT_SESSION_IMPL *session, WT_REF *ref, void *context, bool *skipp) { WT_ADDR_COPY addr; - WT_PAGE *page; - WT_REF *ref; - WT_SESSION_IMPL *session; uint8_t previous_state; - bool can_skip; - session = CUR2S(cbt); - ref = cbt->ref; - page = cbt->ref == NULL ? NULL : cbt->ref->page; + WT_UNUSED(context); - if (page == NULL) - return false; - - previous_state = ref->state; - can_skip = false; + *skipp = false; /* Default to reading */ /* * Determine if all records on the page have been deleted and all the tombstones are visible to * our transaction. If so, we can avoid reading the records on the page and move to the next * page. We base this decision on the aggregate stop point added to the page during the last - * reconciliation. We can skip this test if the page has been modified since it was reconciled - * or the underlying cursor is configured to ignore tombstones. + * reconciliation. We can skip this test if the page has been modified since it was reconciled. + * We also skip this test on an internal page, as we rely on reconciliation to mark the internal + * page dirty. There could be a period of time when the internal page is marked clean but the + * leaf page is dirty and has newer data than let on by the internal page's aggregated + * information. * * We are making these decisions while holding a lock for the page as checkpoint or eviction can - * make changes to the data structures (i.e., aggregate timestamps) we are reading. + * make changes to the data structures (i.e., aggregate timestamps) we are reading. It is okay + * if the page is not in memory, or gets evicted before we lock it. In such a case, we can forgo + * checking if the page has been modified. So, only do a page modified check if the page was in + * memory before locking. */ - if (session->txn->isolation == WT_ISO_SNAPSHOT && !__wt_page_is_modified(page) && - !F_ISSET(&cbt->iface, WT_CURSTD_IGNORE_TOMBSTONE) && previous_state == WT_REF_MEM) { - - /* We only try to lock the page once. */ - if (!WT_REF_CAS_STATE(session, ref, previous_state, WT_REF_LOCKED)) - return false; + if (F_ISSET(ref, WT_REF_FLAG_INTERNAL)) + return (0); - if (__wt_ref_addr_copy(session, ref, &addr) && - __wt_txn_visible(session, addr.ta.newest_stop_txn, addr.ta.newest_stop_ts) && - __wt_txn_visible(session, addr.ta.newest_stop_txn, addr.ta.newest_stop_durable_ts)) - can_skip = true; + WT_REF_LOCK(session, ref, &previous_state); + if ((previous_state == WT_REF_DISK || previous_state == WT_REF_DELETED || + (previous_state == WT_REF_MEM && !__wt_page_is_modified(ref->page))) && + __wt_ref_addr_copy(session, ref, &addr) && addr.ta.newest_stop_txn != WT_TXN_MAX && + addr.ta.newest_stop_ts != WT_TS_MAX && + __wt_txn_visible(session, addr.ta.newest_stop_txn, addr.ta.newest_stop_ts)) + *skipp = true; - WT_REF_SET_STATE(ref, previous_state); - } + WT_REF_UNLOCK(ref, previous_state); - return (can_skip); + return (0); } diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index c94dd274fbc..5b8f7e9f8e4 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -614,7 +614,6 @@ struct __wt_connection_impl { #define WT_TIMING_STRESS_SPLIT_5 0x08000u #define WT_TIMING_STRESS_SPLIT_6 0x10000u #define WT_TIMING_STRESS_SPLIT_7 0x20000u -#define WT_TIMING_STRESS_SPLIT_8 0x40000u /* AUTOMATIC FLAG VALUE GENERATION STOP 64 */ uint64_t timing_stress_flags; diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index 569da6aabaa..85797573e23 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -12,27 +12,27 @@ /* * Initialize a static WT_CURSOR structure. */ -#define WT_CURSOR_STATIC_INIT(n, get_key, get_value, set_key, set_value, compare, equals, next, \ - prev, reset, search, search_near, insert, modify, update, remove, reserve, reconfigure, cache, \ - reopen, close) \ - static const WT_CURSOR n = { \ - NULL, /* session */ \ - NULL, /* uri */ \ - NULL, /* key_format */ \ - NULL, /* value_format */ \ - get_key, get_value, set_key, set_value, compare, equals, next, prev, reset, search, \ - search_near, insert, modify, update, remove, reserve, close, reconfigure, cache, reopen, \ - 0, /* uri_hash */ \ - {NULL, NULL}, /* TAILQ_ENTRY q */ \ - 0, /* recno key */ \ - {0}, /* recno raw buffer */ \ - NULL, /* json_private */ \ - NULL, /* lang_private */ \ - {NULL, 0, NULL, 0, 0}, /* WT_ITEM key */ \ - {NULL, 0, NULL, 0, 0}, /* WT_ITEM value */ \ - 0, /* int saved_err */ \ - NULL, /* internal_uri */ \ - 0 /* uint32_t flags */ \ +#define WT_CURSOR_STATIC_INIT(n, get_key, get_value, set_key, set_value, compare, equals, next, \ + prev, reset, search, search_near, insert, modify, update, remove, reserve, reconfigure, \ + largest_key, cache, reopen, close) \ + static const WT_CURSOR n = { \ + NULL, /* session */ \ + NULL, /* uri */ \ + NULL, /* key_format */ \ + NULL, /* value_format */ \ + get_key, get_value, set_key, set_value, compare, equals, next, prev, reset, search, \ + search_near, insert, modify, update, remove, reserve, close, reconfigure, largest_key, \ + cache, reopen, 0, /* uri_hash */ \ + {NULL, NULL}, /* TAILQ_ENTRY q */ \ + 0, /* recno key */ \ + {0}, /* recno raw buffer */ \ + NULL, /* json_private */ \ + NULL, /* lang_private */ \ + {NULL, 0, NULL, 0, 0}, /* WT_ITEM key */ \ + {NULL, 0, NULL, 0, 0}, /* WT_ITEM value */ \ + 0, /* int saved_err */ \ + NULL, /* internal_uri */ \ + 0 /* uint32_t flags */ \ } struct __wt_cursor_backup { diff --git a/src/third_party/wiredtiger/src/include/dhandle.h b/src/third_party/wiredtiger/src/include/dhandle.h index 133ab688544..d4e23e52c8f 100644 --- a/src/third_party/wiredtiger/src/include/dhandle.h +++ b/src/third_party/wiredtiger/src/include/dhandle.h @@ -119,11 +119,12 @@ struct __wt_data_handle { #define WT_DHANDLE_DISCARD 0x002u /* Close on release */ #define WT_DHANDLE_DISCARD_KILL 0x004u /* Mark dead on release */ #define WT_DHANDLE_DROPPED 0x008u /* Handle is dropped */ -#define WT_DHANDLE_EXCLUSIVE 0x010u /* Exclusive access */ -#define WT_DHANDLE_HS 0x020u /* History store table */ -#define WT_DHANDLE_IS_METADATA 0x040u /* Metadata handle */ -#define WT_DHANDLE_LOCK_ONLY 0x080u /* Handle only used as a lock */ -#define WT_DHANDLE_OPEN 0x100u /* Handle is open */ +#define WT_DHANDLE_EVICTED 0x010u /* Btree is evicted (advisory) */ +#define WT_DHANDLE_EXCLUSIVE 0x020u /* Exclusive access */ +#define WT_DHANDLE_HS 0x040u /* History store table */ +#define WT_DHANDLE_IS_METADATA 0x080u /* Metadata handle */ +#define WT_DHANDLE_LOCK_ONLY 0x100u /* Handle only used as a lock */ +#define WT_DHANDLE_OPEN 0x200u /* Handle is open */ /* AUTOMATIC FLAG VALUE GENERATION STOP 12 */ uint32_t flags; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 3468dbfa24b..0f4f3d84a67 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -577,6 +577,8 @@ extern int __wt_cursor_key_order_init(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_kv_not_set(WT_CURSOR *cursor, bool key) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_largest_key(WT_CURSOR *cursor) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_modify_notsup(WT_CURSOR *cursor, WT_MODIFY *entries, int nentries) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_modify_value_format_notsup(WT_CURSOR *cursor, WT_MODIFY *entries, @@ -1363,8 +1365,6 @@ extern int __wt_scr_alloc_func(WT_SESSION_IMPL *session, size_t size, WT_ITEM ** WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_search_insert(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_INSERT_HEAD *ins_head, WT_ITEM *srch_key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_session_blocking_checkpoint(WT_SESSION_IMPL *session, bool force, uint64_t seconds) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_breakpoint(WT_SESSION *wt_session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_close_internal(WT_SESSION_IMPL *session) @@ -1474,8 +1474,8 @@ extern int __wt_thread_group_destroy(WT_SESSION_IMPL *session, WT_THREAD_GROUP * extern int __wt_thread_group_resize(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, uint32_t new_min, uint32_t new_max, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_tier_do_flush(WT_SESSION_IMPL *session, WT_TIERED *tiered, const char *local_uri, - const char *obj_uri) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tier_do_flush(WT_SESSION_IMPL *session, WT_TIERED *tiered, uint32_t id, + const char *local_uri, const char *obj_uri) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_tier_flush(WT_SESSION_IMPL *session, WT_TIERED *tiered, uint32_t id) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_tiered_bucket_config(WT_SESSION_IMPL *session, const char *cfg[], @@ -1895,8 +1895,6 @@ static inline WT_IKEY *__wt_ref_key_instantiated(WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline WT_VISIBLE_TYPE __wt_txn_upd_visible_type(WT_SESSION_IMPL *session, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline bool __wt_btcur_skip_page(WT_CURSOR_BTREE *cbt) - WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline bool __wt_btree_can_evict_dirty(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline bool __wt_btree_dominating_cache(WT_SESSION_IMPL *session, WT_BTREE *btree) @@ -1982,6 +1980,8 @@ static inline bool __wt_txn_visible_id_snapshot(uint64_t id, uint64_t snap_min, uint64_t *snapshot, uint32_t snapshot_count) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline double __wt_eviction_dirty_target(WT_CACHE *cache) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline int __wt_btcur_skip_page(WT_SESSION_IMPL *session, WT_REF *ref, void *context, + bool *skipp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_btree_block_free(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_buf_extend(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) diff --git a/src/third_party/wiredtiger/src/include/os_fs_inline.h b/src/third_party/wiredtiger/src/include/os_fs_inline.h index 2276f096312..2010032a5c2 100644 --- a/src/third_party/wiredtiger/src/include/os_fs_inline.h +++ b/src/third_party/wiredtiger/src/include/os_fs_inline.h @@ -141,7 +141,7 @@ __wt_fs_remove(WT_SESSION_IMPL *session, const char *name, bool durable) * It is a layering violation to retrieve a WT_FH here, but it is a useful diagnostic to ensure * WiredTiger doesn't have the handle open. */ - if (__wt_handle_is_open(session, name)) + if (__wt_handle_is_open(session, name) && !F_ISSET(session, WT_SESSION_QUIET_TIERED)) WT_RET_MSG(session, EINVAL, "%s: file-remove: file has open handles", name); #endif diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 465f641954b..5206910f139 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -203,10 +203,11 @@ struct __wt_session_impl { #define WT_SESSION_NO_LOGGING 0x00800u #define WT_SESSION_NO_RECONCILE 0x01000u #define WT_SESSION_QUIET_CORRUPT_FILE 0x02000u -#define WT_SESSION_READ_WONT_NEED 0x04000u -#define WT_SESSION_RESOLVING_TXN 0x08000u -#define WT_SESSION_ROLLBACK_TO_STABLE 0x10000u -#define WT_SESSION_SCHEMA_TXN 0x20000u +#define WT_SESSION_QUIET_TIERED 0x04000u +#define WT_SESSION_READ_WONT_NEED 0x08000u +#define WT_SESSION_RESOLVING_TXN 0x10000u +#define WT_SESSION_ROLLBACK_TO_STABLE 0x20000u +#define WT_SESSION_SCHEMA_TXN 0x40000u /* AUTOMATIC FLAG VALUE GENERATION STOP 32 */ uint32_t flags; diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 2eb26c749dc..df1cd95de67 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -508,8 +508,6 @@ struct __wt_connection_stats { int64_t cursor_next_skip_total; int64_t cursor_prev_skip_total; int64_t cursor_skip_hs_cur_position; - int64_t cursor_next_skip_page_count; - int64_t cursor_prev_skip_page_count; int64_t cursor_search_near_prefix_fast_paths; int64_t cursor_cached_count; int64_t cursor_insert_bulk; @@ -688,8 +686,9 @@ struct __wt_connection_stats { int64_t rec_time_window_stop_txn; int64_t rec_split_stashed_bytes; int64_t rec_split_stashed_objects; - int64_t flush_state_races; + int64_t local_objects_inuse; int64_t flush_tier; + int64_t local_objects_removed; int64_t session_open; int64_t session_query_ts; int64_t session_table_alter_fail; @@ -946,8 +945,6 @@ struct __wt_dsrc_stats { int64_t cursor_next_skip_total; int64_t cursor_prev_skip_total; int64_t cursor_skip_hs_cur_position; - int64_t cursor_next_skip_page_count; - int64_t cursor_prev_skip_page_count; int64_t cursor_search_near_prefix_fast_paths; int64_t cursor_insert_bulk; int64_t cursor_reopen; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index daba823f589..fc8e542c33a 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -661,6 +661,19 @@ struct __wt_cursor { */ int __F(reconfigure)(WT_CURSOR *cursor, const char *config); + /*! + * Get the largest key of the cursor regardless of visibility. + * The cursor is not positioned after calling this api. Only supported by data cursors. + * + * @snippet ex_all.c Reset the cursor + * + * @param cursor the cursor handle + * @errors + * If \c read_timestamp is set, EINVAL is returned. + * Also, calling \c WT_CURSOR::get_key after this api call returns EINVAL. + */ + int __F(largest_key)(WT_CURSOR *cursor); + /* * Protected fields, only to be used by cursor implementations. */ @@ -5468,728 +5481,720 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); */ #define WT_STAT_CONN_CURSOR_SKIP_HS_CUR_POSITION 1194 /*! - * cursor: Total number of pages skipped without reading by cursor next - * calls - */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_PAGE_COUNT 1195 -/*! - * cursor: Total number of pages skipped without reading by cursor prev - * calls - */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_PAGE_COUNT 1196 -/*! * cursor: Total number of times a search near has exited due to prefix * config */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR_PREFIX_FAST_PATHS 1197 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR_PREFIX_FAST_PATHS 1195 /*! cursor: cached cursor count */ -#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1198 +#define WT_STAT_CONN_CURSOR_CACHED_COUNT 1196 /*! cursor: cursor bulk loaded cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT_BULK 1199 +#define WT_STAT_CONN_CURSOR_INSERT_BULK 1197 /*! cursor: cursor close calls that result in cache */ -#define WT_STAT_CONN_CURSOR_CACHE 1200 +#define WT_STAT_CONN_CURSOR_CACHE 1198 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1201 +#define WT_STAT_CONN_CURSOR_CREATE 1199 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1202 +#define WT_STAT_CONN_CURSOR_INSERT 1200 /*! cursor: cursor insert key and value bytes */ -#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1203 +#define WT_STAT_CONN_CURSOR_INSERT_BYTES 1201 /*! cursor: cursor modify calls */ -#define WT_STAT_CONN_CURSOR_MODIFY 1204 +#define WT_STAT_CONN_CURSOR_MODIFY 1202 /*! cursor: cursor modify key and value bytes affected */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1205 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES 1203 /*! cursor: cursor modify value bytes modified */ -#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1206 +#define WT_STAT_CONN_CURSOR_MODIFY_BYTES_TOUCH 1204 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1207 +#define WT_STAT_CONN_CURSOR_NEXT 1205 /*! * cursor: cursor next calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE 1208 +#define WT_STAT_CONN_CURSOR_NEXT_HS_TOMBSTONE 1206 /*! * cursor: cursor next calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_GE_100 1209 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_GE_100 1207 /*! cursor: cursor next calls that skip less than 100 entries */ -#define WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100 1210 +#define WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100 1208 /*! cursor: cursor operation restarted */ -#define WT_STAT_CONN_CURSOR_RESTART 1211 +#define WT_STAT_CONN_CURSOR_RESTART 1209 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1212 +#define WT_STAT_CONN_CURSOR_PREV 1210 /*! * cursor: cursor prev calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE 1213 +#define WT_STAT_CONN_CURSOR_PREV_HS_TOMBSTONE 1211 /*! * cursor: cursor prev calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_GE_100 1214 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_GE_100 1212 /*! cursor: cursor prev calls that skip less than 100 entries */ -#define WT_STAT_CONN_CURSOR_PREV_SKIP_LT_100 1215 +#define WT_STAT_CONN_CURSOR_PREV_SKIP_LT_100 1213 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1216 +#define WT_STAT_CONN_CURSOR_REMOVE 1214 /*! cursor: cursor remove key bytes removed */ -#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1217 +#define WT_STAT_CONN_CURSOR_REMOVE_BYTES 1215 /*! cursor: cursor reserve calls */ -#define WT_STAT_CONN_CURSOR_RESERVE 1218 +#define WT_STAT_CONN_CURSOR_RESERVE 1216 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1219 +#define WT_STAT_CONN_CURSOR_RESET 1217 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1220 +#define WT_STAT_CONN_CURSOR_SEARCH 1218 /*! cursor: cursor search history store calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_HS 1221 +#define WT_STAT_CONN_CURSOR_SEARCH_HS 1219 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1222 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1220 /*! cursor: cursor sweep buckets */ -#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1223 +#define WT_STAT_CONN_CURSOR_SWEEP_BUCKETS 1221 /*! cursor: cursor sweep cursors closed */ -#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1224 +#define WT_STAT_CONN_CURSOR_SWEEP_CLOSED 1222 /*! cursor: cursor sweep cursors examined */ -#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1225 +#define WT_STAT_CONN_CURSOR_SWEEP_EXAMINED 1223 /*! cursor: cursor sweeps */ -#define WT_STAT_CONN_CURSOR_SWEEP 1226 +#define WT_STAT_CONN_CURSOR_SWEEP 1224 /*! cursor: cursor truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1227 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1225 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1228 +#define WT_STAT_CONN_CURSOR_UPDATE 1226 /*! cursor: cursor update key and value bytes */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1229 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES 1227 /*! cursor: cursor update value size change */ -#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1230 +#define WT_STAT_CONN_CURSOR_UPDATE_BYTES_CHANGED 1228 /*! cursor: cursors reused from cache */ -#define WT_STAT_CONN_CURSOR_REOPEN 1231 +#define WT_STAT_CONN_CURSOR_REOPEN 1229 /*! cursor: open cursor count */ -#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1232 +#define WT_STAT_CONN_CURSOR_OPEN_COUNT 1230 /*! data-handle: connection data handle size */ -#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1233 +#define WT_STAT_CONN_DH_CONN_HANDLE_SIZE 1231 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1234 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1232 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1235 +#define WT_STAT_CONN_DH_SWEEP_REF 1233 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1236 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1234 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1237 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1235 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1238 +#define WT_STAT_CONN_DH_SWEEP_TOD 1236 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1239 +#define WT_STAT_CONN_DH_SWEEPS 1237 /*! * data-handle: connection sweeps skipped due to checkpoint gathering * handles */ -#define WT_STAT_CONN_DH_SWEEP_SKIP_CKPT 1240 +#define WT_STAT_CONN_DH_SWEEP_SKIP_CKPT 1238 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1241 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1239 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1242 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1240 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1243 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1241 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1244 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1242 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1245 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1243 /*! lock: dhandle lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1246 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_APPLICATION 1244 /*! lock: dhandle lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1247 +#define WT_STAT_CONN_LOCK_DHANDLE_WAIT_INTERNAL 1245 /*! lock: dhandle read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1248 +#define WT_STAT_CONN_LOCK_DHANDLE_READ_COUNT 1246 /*! lock: dhandle write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1249 +#define WT_STAT_CONN_LOCK_DHANDLE_WRITE_COUNT 1247 /*! * lock: durable timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1250 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_APPLICATION 1248 /*! * lock: durable timestamp queue lock internal thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1251 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WAIT_INTERNAL 1249 /*! lock: durable timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1252 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_READ_COUNT 1250 /*! lock: durable timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1253 +#define WT_STAT_CONN_LOCK_DURABLE_TIMESTAMP_WRITE_COUNT 1251 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1254 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1252 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1255 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1253 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1256 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1254 /*! * lock: read timestamp queue lock application thread time waiting * (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1257 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_APPLICATION 1255 /*! lock: read timestamp queue lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1258 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WAIT_INTERNAL 1256 /*! lock: read timestamp queue read lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1259 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_READ_COUNT 1257 /*! lock: read timestamp queue write lock acquisitions */ -#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1260 +#define WT_STAT_CONN_LOCK_READ_TIMESTAMP_WRITE_COUNT 1258 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1261 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1259 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1262 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1260 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1263 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1261 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1264 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1262 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1265 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1263 /*! lock: table read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1266 +#define WT_STAT_CONN_LOCK_TABLE_READ_COUNT 1264 /*! lock: table write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1267 +#define WT_STAT_CONN_LOCK_TABLE_WRITE_COUNT 1265 /*! lock: txn global lock application thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1268 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_APPLICATION 1266 /*! lock: txn global lock internal thread time waiting (usecs) */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1269 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WAIT_INTERNAL 1267 /*! lock: txn global read lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1270 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_READ_COUNT 1268 /*! lock: txn global write lock acquisitions */ -#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1271 +#define WT_STAT_CONN_LOCK_TXN_GLOBAL_WRITE_COUNT 1269 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1272 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1270 /*! log: force archive time sleeping (usecs) */ -#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1273 +#define WT_STAT_CONN_LOG_FORCE_ARCHIVE_SLEEP 1271 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1274 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1272 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1275 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1273 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1276 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1274 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1277 +#define WT_STAT_CONN_LOG_FLUSH 1275 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1278 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1276 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1279 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1277 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1280 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1278 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1281 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1279 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1282 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1280 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1283 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1281 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1284 +#define WT_STAT_CONN_LOG_SCANS 1282 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1285 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1283 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1286 +#define WT_STAT_CONN_LOG_WRITE_LSN 1284 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1287 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1285 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1288 +#define WT_STAT_CONN_LOG_SYNC 1286 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1289 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1287 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1290 +#define WT_STAT_CONN_LOG_SYNC_DIR 1288 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1291 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1289 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1292 +#define WT_STAT_CONN_LOG_WRITES 1290 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1293 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1291 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1294 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1292 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1295 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1293 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1296 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1294 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1297 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1295 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1298 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1296 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1299 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1297 /*! log: slot close lost race */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1300 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_RACE 1298 /*! log: slot close unbuffered waits */ -#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1301 +#define WT_STAT_CONN_LOG_SLOT_CLOSE_UNBUF 1299 /*! log: slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1302 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1300 /*! log: slot join atomic update races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1303 +#define WT_STAT_CONN_LOG_SLOT_RACES 1301 /*! log: slot join calls atomic updates raced */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1304 +#define WT_STAT_CONN_LOG_SLOT_YIELD_RACE 1302 /*! log: slot join calls did not yield */ -#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1305 +#define WT_STAT_CONN_LOG_SLOT_IMMEDIATE 1303 /*! log: slot join calls found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1306 +#define WT_STAT_CONN_LOG_SLOT_YIELD_CLOSE 1304 /*! log: slot join calls slept */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1307 +#define WT_STAT_CONN_LOG_SLOT_YIELD_SLEEP 1305 /*! log: slot join calls yielded */ -#define WT_STAT_CONN_LOG_SLOT_YIELD 1308 +#define WT_STAT_CONN_LOG_SLOT_YIELD 1306 /*! log: slot join found active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1309 +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1307 /*! log: slot joins yield time (usecs) */ -#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1310 +#define WT_STAT_CONN_LOG_SLOT_YIELD_DURATION 1308 /*! log: slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1311 +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1309 /*! log: slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1312 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1310 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1313 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1311 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1314 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1312 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1315 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1313 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1316 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1314 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1317 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1315 /*! perf: file system read latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1318 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT50 1316 /*! perf: file system read latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1319 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT100 1317 /*! perf: file system read latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1320 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT250 1318 /*! perf: file system read latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1321 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT500 1319 /*! perf: file system read latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1322 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_LT1000 1320 /*! perf: file system read latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1323 +#define WT_STAT_CONN_PERF_HIST_FSREAD_LATENCY_GT1000 1321 /*! perf: file system write latency histogram (bucket 1) - 10-49ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1324 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT50 1322 /*! perf: file system write latency histogram (bucket 2) - 50-99ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1325 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT100 1323 /*! perf: file system write latency histogram (bucket 3) - 100-249ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1326 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT250 1324 /*! perf: file system write latency histogram (bucket 4) - 250-499ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1327 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT500 1325 /*! perf: file system write latency histogram (bucket 5) - 500-999ms */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1328 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_LT1000 1326 /*! perf: file system write latency histogram (bucket 6) - 1000ms+ */ -#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1329 +#define WT_STAT_CONN_PERF_HIST_FSWRITE_LATENCY_GT1000 1327 /*! perf: operation read latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1330 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT250 1328 /*! perf: operation read latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1331 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT500 1329 /*! perf: operation read latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1332 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT1000 1330 /*! perf: operation read latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1333 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_LT10000 1331 /*! perf: operation read latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1334 +#define WT_STAT_CONN_PERF_HIST_OPREAD_LATENCY_GT10000 1332 /*! perf: operation write latency histogram (bucket 1) - 100-249us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1335 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT250 1333 /*! perf: operation write latency histogram (bucket 2) - 250-499us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1336 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT500 1334 /*! perf: operation write latency histogram (bucket 3) - 500-999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1337 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT1000 1335 /*! perf: operation write latency histogram (bucket 4) - 1000-9999us */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1338 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_LT10000 1336 /*! perf: operation write latency histogram (bucket 5) - 10000us+ */ -#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1339 +#define WT_STAT_CONN_PERF_HIST_OPWRITE_LATENCY_GT10000 1337 /*! reconciliation: approximate byte size of timestamps in pages written */ -#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1340 +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TS 1338 /*! * reconciliation: approximate byte size of transaction IDs in pages * written */ -#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1341 +#define WT_STAT_CONN_REC_TIME_WINDOW_BYTES_TXN 1339 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1342 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1340 /*! reconciliation: internal-page overflow keys */ -#define WT_STAT_CONN_REC_OVERFLOW_KEY_INTERNAL 1343 +#define WT_STAT_CONN_REC_OVERFLOW_KEY_INTERNAL 1341 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_CONN_REC_OVERFLOW_KEY_LEAF 1344 +#define WT_STAT_CONN_REC_OVERFLOW_KEY_LEAF 1342 /*! reconciliation: maximum seconds spent in a reconciliation call */ -#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1345 +#define WT_STAT_CONN_REC_MAXIMUM_SECONDS 1343 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1346 +#define WT_STAT_CONN_REC_PAGES 1344 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1347 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1345 /*! * reconciliation: page reconciliation calls that resulted in values with * prepared transaction metadata */ -#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1348 +#define WT_STAT_CONN_REC_PAGES_WITH_PREPARE 1346 /*! * reconciliation: page reconciliation calls that resulted in values with * timestamps */ -#define WT_STAT_CONN_REC_PAGES_WITH_TS 1349 +#define WT_STAT_CONN_REC_PAGES_WITH_TS 1347 /*! * reconciliation: page reconciliation calls that resulted in values with * transaction ids */ -#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1350 +#define WT_STAT_CONN_REC_PAGES_WITH_TXN 1348 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1351 +#define WT_STAT_CONN_REC_PAGE_DELETE 1349 /*! * reconciliation: pages written including an aggregated newest start * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1352 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 1350 /*! * reconciliation: pages written including an aggregated newest stop * durable timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1353 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 1351 /*! * reconciliation: pages written including an aggregated newest stop * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1354 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TS 1352 /*! * reconciliation: pages written including an aggregated newest stop * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1355 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_STOP_TXN 1353 /*! * reconciliation: pages written including an aggregated newest * transaction ID */ -#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_TXN 1356 +#define WT_STAT_CONN_REC_TIME_AGGR_NEWEST_TXN 1354 /*! * reconciliation: pages written including an aggregated oldest start * timestamp */ -#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1357 +#define WT_STAT_CONN_REC_TIME_AGGR_OLDEST_START_TS 1355 /*! reconciliation: pages written including an aggregated prepare */ -#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1358 +#define WT_STAT_CONN_REC_TIME_AGGR_PREPARED 1356 /*! reconciliation: pages written including at least one prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1359 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_PREPARED 1357 /*! * reconciliation: pages written including at least one start durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1360 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 1358 /*! reconciliation: pages written including at least one start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1361 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TS 1359 /*! * reconciliation: pages written including at least one start transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1362 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_START_TXN 1360 /*! * reconciliation: pages written including at least one stop durable * timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1363 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 1361 /*! reconciliation: pages written including at least one stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1364 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TS 1362 /*! * reconciliation: pages written including at least one stop transaction * ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1365 +#define WT_STAT_CONN_REC_TIME_WINDOW_PAGES_STOP_TXN 1363 /*! reconciliation: records written including a prepare state */ -#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1366 +#define WT_STAT_CONN_REC_TIME_WINDOW_PREPARED 1364 /*! reconciliation: records written including a start durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1367 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_START_TS 1365 /*! reconciliation: records written including a start timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1368 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TS 1366 /*! reconciliation: records written including a start transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1369 +#define WT_STAT_CONN_REC_TIME_WINDOW_START_TXN 1367 /*! reconciliation: records written including a stop durable timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1370 +#define WT_STAT_CONN_REC_TIME_WINDOW_DURABLE_STOP_TS 1368 /*! reconciliation: records written including a stop timestamp */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1371 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TS 1369 /*! reconciliation: records written including a stop transaction ID */ -#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1372 +#define WT_STAT_CONN_REC_TIME_WINDOW_STOP_TXN 1370 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1373 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1371 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1374 -/*! session: flush state races */ -#define WT_STAT_CONN_FLUSH_STATE_RACES 1375 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1372 +/*! session: attempts to remove a local object and the object is in use */ +#define WT_STAT_CONN_LOCAL_OBJECTS_INUSE 1373 /*! session: flush_tier operation calls */ -#define WT_STAT_CONN_FLUSH_TIER 1376 +#define WT_STAT_CONN_FLUSH_TIER 1374 +/*! session: local objects removed */ +#define WT_STAT_CONN_LOCAL_OBJECTS_REMOVED 1375 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1377 +#define WT_STAT_CONN_SESSION_OPEN 1376 /*! session: session query timestamp calls */ -#define WT_STAT_CONN_SESSION_QUERY_TS 1378 +#define WT_STAT_CONN_SESSION_QUERY_TS 1377 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1379 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1378 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1380 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1379 /*! session: table alter triggering checkpoint calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_TRIGGER_CHECKPOINT 1381 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_TRIGGER_CHECKPOINT 1380 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1382 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1381 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1383 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1382 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1384 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1383 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1385 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1384 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1386 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1385 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1387 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1386 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1388 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1387 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1389 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1388 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1390 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1389 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1391 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1390 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1392 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1391 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1393 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1392 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1394 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1393 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1395 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1394 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1396 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1395 /*! session: tiered operations dequeued and processed */ -#define WT_STAT_CONN_TIERED_WORK_UNITS_DEQUEUED 1397 +#define WT_STAT_CONN_TIERED_WORK_UNITS_DEQUEUED 1396 /*! session: tiered operations scheduled */ -#define WT_STAT_CONN_TIERED_WORK_UNITS_CREATED 1398 +#define WT_STAT_CONN_TIERED_WORK_UNITS_CREATED 1397 /*! session: tiered storage local retention time (secs) */ -#define WT_STAT_CONN_TIERED_RETENTION 1399 +#define WT_STAT_CONN_TIERED_RETENTION 1398 /*! session: tiered storage object size */ -#define WT_STAT_CONN_TIERED_OBJECT_SIZE 1400 +#define WT_STAT_CONN_TIERED_OBJECT_SIZE 1399 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1401 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1400 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1402 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1401 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1403 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1402 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1404 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1403 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1405 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1404 /*! * thread-yield: connection close blocked waiting for transaction state * stabilization */ -#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1406 +#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1405 /*! thread-yield: connection close yielded for lsm manager shutdown */ -#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1407 +#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1406 /*! thread-yield: data handle lock yielded */ -#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1408 +#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1407 /*! * thread-yield: get reference for page index and slot time sleeping * (usecs) */ -#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1409 +#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1408 /*! thread-yield: page access yielded due to prepare state change */ -#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1410 +#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1409 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1411 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1410 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1412 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1411 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1413 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1412 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1414 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1413 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1415 +#define WT_STAT_CONN_PAGE_SLEEP 1414 /*! * thread-yield: page delete rollback time sleeping for state change * (usecs) */ -#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1416 +#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1415 /*! thread-yield: page reconciliation yielded due to child modification */ -#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1417 +#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1416 /*! transaction: Number of prepared updates */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES 1418 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES 1417 /*! transaction: Number of prepared updates committed */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COMMITTED 1419 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COMMITTED 1418 /*! transaction: Number of prepared updates repeated on the same key */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_KEY_REPEATED 1420 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_KEY_REPEATED 1419 /*! transaction: Number of prepared updates rolled back */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_ROLLEDBACK 1421 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_ROLLEDBACK 1420 /*! transaction: prepared transactions */ -#define WT_STAT_CONN_TXN_PREPARE 1422 +#define WT_STAT_CONN_TXN_PREPARE 1421 /*! transaction: prepared transactions committed */ -#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1423 +#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1422 /*! transaction: prepared transactions currently active */ -#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1424 +#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1423 /*! transaction: prepared transactions rolled back */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1425 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1424 /*! * transaction: prepared transactions rolled back and do not remove the * history store entry */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK_DO_NOT_REMOVE_HS_UPDATE 1426 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK_DO_NOT_REMOVE_HS_UPDATE 1425 /*! * transaction: prepared transactions rolled back and fix the history * store entry with checkpoint reserved transaction id */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK_FIX_HS_UPDATE_WITH_CKPT_RESERVED_TXNID 1427 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK_FIX_HS_UPDATE_WITH_CKPT_RESERVED_TXNID 1426 /*! transaction: query timestamp calls */ -#define WT_STAT_CONN_TXN_QUERY_TS 1428 +#define WT_STAT_CONN_TXN_QUERY_TS 1427 /*! transaction: race to read prepared update retry */ -#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1429 +#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1428 /*! transaction: rollback to stable calls */ -#define WT_STAT_CONN_TXN_RTS 1430 +#define WT_STAT_CONN_TXN_RTS 1429 /*! * transaction: rollback to stable history store records with stop * timestamps older than newer records */ -#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1431 +#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1430 /*! transaction: rollback to stable inconsistent checkpoint */ -#define WT_STAT_CONN_TXN_RTS_INCONSISTENT_CKPT 1432 +#define WT_STAT_CONN_TXN_RTS_INCONSISTENT_CKPT 1431 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1433 +#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1432 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1434 +#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1433 /*! transaction: rollback to stable pages visited */ -#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1435 +#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1434 /*! transaction: rollback to stable restored tombstones from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1436 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1435 /*! transaction: rollback to stable restored updates from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES 1437 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES 1436 /*! transaction: rollback to stable skipping delete rle */ -#define WT_STAT_CONN_TXN_RTS_DELETE_RLE_SKIPPED 1438 +#define WT_STAT_CONN_TXN_RTS_DELETE_RLE_SKIPPED 1437 /*! transaction: rollback to stable skipping stable rle */ -#define WT_STAT_CONN_TXN_RTS_STABLE_RLE_SKIPPED 1439 +#define WT_STAT_CONN_TXN_RTS_STABLE_RLE_SKIPPED 1438 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1440 +#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1439 /*! transaction: rollback to stable tree walk skipping pages */ -#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1441 +#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1440 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1442 +#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1441 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1443 +#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1442 /*! transaction: sessions scanned in each walk of concurrent sessions */ -#define WT_STAT_CONN_TXN_SESSIONS_WALKED 1444 +#define WT_STAT_CONN_TXN_SESSIONS_WALKED 1443 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1445 +#define WT_STAT_CONN_TXN_SET_TS 1444 /*! transaction: set timestamp durable calls */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1446 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1445 /*! transaction: set timestamp durable updates */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1447 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1446 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1448 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1447 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1449 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1448 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1450 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1449 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1451 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1450 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1452 +#define WT_STAT_CONN_TXN_BEGIN 1451 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1453 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1452 /*! * transaction: transaction checkpoint currently running for history * store file */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING_HS 1454 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING_HS 1453 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1455 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1454 /*! * transaction: transaction checkpoint history store file duration * (usecs) */ -#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1456 +#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1455 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1457 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1456 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1458 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1457 /*! * transaction: transaction checkpoint most recent duration for gathering * all handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1459 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1458 /*! * transaction: transaction checkpoint most recent duration for gathering * applied handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1460 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1459 /*! * transaction: transaction checkpoint most recent duration for gathering * skipped handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1461 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1460 /*! transaction: transaction checkpoint most recent handles applied */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1462 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1461 /*! transaction: transaction checkpoint most recent handles skipped */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1463 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1462 /*! transaction: transaction checkpoint most recent handles walked */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1464 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1463 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1465 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1464 /*! transaction: transaction checkpoint prepare currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1466 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1465 /*! transaction: transaction checkpoint prepare max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1467 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1466 /*! transaction: transaction checkpoint prepare min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1468 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1467 /*! transaction: transaction checkpoint prepare most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1469 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1468 /*! transaction: transaction checkpoint prepare total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1470 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1469 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1471 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1470 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1472 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1471 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1473 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1472 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1474 +#define WT_STAT_CONN_TXN_CHECKPOINT 1473 /*! transaction: transaction checkpoints due to obsolete pages */ -#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1475 +#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1474 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1476 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1475 /*! transaction: transaction failures due to history store */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1477 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1476 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1478 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1477 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1479 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1478 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1480 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1479 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1481 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1480 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1482 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1481 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1483 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1482 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1484 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1483 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1485 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1484 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1486 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1485 /*! transaction: transaction rollback to stable currently running */ -#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1487 +#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1486 /*! transaction: transaction walk of concurrent sessions */ -#define WT_STAT_CONN_TXN_WALK_SESSIONS 1488 +#define WT_STAT_CONN_TXN_WALK_SESSIONS 1487 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1489 +#define WT_STAT_CONN_TXN_COMMIT 1488 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1490 +#define WT_STAT_CONN_TXN_ROLLBACK 1489 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1491 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1490 /*! * @} @@ -6617,241 +6622,231 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); */ #define WT_STAT_DSRC_CURSOR_SKIP_HS_CUR_POSITION 2133 /*! - * cursor: Total number of pages skipped without reading by cursor next - * calls - */ -#define WT_STAT_DSRC_CURSOR_NEXT_SKIP_PAGE_COUNT 2134 -/*! - * cursor: Total number of pages skipped without reading by cursor prev - * calls - */ -#define WT_STAT_DSRC_CURSOR_PREV_SKIP_PAGE_COUNT 2135 -/*! * cursor: Total number of times a search near has exited due to prefix * config */ -#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR_PREFIX_FAST_PATHS 2136 +#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR_PREFIX_FAST_PATHS 2134 /*! cursor: bulk loaded cursor insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2137 +#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2135 /*! cursor: cache cursors reuse count */ -#define WT_STAT_DSRC_CURSOR_REOPEN 2138 +#define WT_STAT_DSRC_CURSOR_REOPEN 2136 /*! cursor: close calls that result in cache */ -#define WT_STAT_DSRC_CURSOR_CACHE 2139 +#define WT_STAT_DSRC_CURSOR_CACHE 2137 /*! cursor: create calls */ -#define WT_STAT_DSRC_CURSOR_CREATE 2140 +#define WT_STAT_DSRC_CURSOR_CREATE 2138 /*! * cursor: cursor next calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_DSRC_CURSOR_NEXT_HS_TOMBSTONE 2141 +#define WT_STAT_DSRC_CURSOR_NEXT_HS_TOMBSTONE 2139 /*! * cursor: cursor next calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_DSRC_CURSOR_NEXT_SKIP_GE_100 2142 +#define WT_STAT_DSRC_CURSOR_NEXT_SKIP_GE_100 2140 /*! cursor: cursor next calls that skip less than 100 entries */ -#define WT_STAT_DSRC_CURSOR_NEXT_SKIP_LT_100 2143 +#define WT_STAT_DSRC_CURSOR_NEXT_SKIP_LT_100 2141 /*! * cursor: cursor prev calls that skip due to a globally visible history * store tombstone */ -#define WT_STAT_DSRC_CURSOR_PREV_HS_TOMBSTONE 2144 +#define WT_STAT_DSRC_CURSOR_PREV_HS_TOMBSTONE 2142 /*! * cursor: cursor prev calls that skip greater than or equal to 100 * entries */ -#define WT_STAT_DSRC_CURSOR_PREV_SKIP_GE_100 2145 +#define WT_STAT_DSRC_CURSOR_PREV_SKIP_GE_100 2143 /*! cursor: cursor prev calls that skip less than 100 entries */ -#define WT_STAT_DSRC_CURSOR_PREV_SKIP_LT_100 2146 +#define WT_STAT_DSRC_CURSOR_PREV_SKIP_LT_100 2144 /*! cursor: insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT 2147 +#define WT_STAT_DSRC_CURSOR_INSERT 2145 /*! cursor: insert key and value bytes */ -#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2148 +#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2146 /*! cursor: modify */ -#define WT_STAT_DSRC_CURSOR_MODIFY 2149 +#define WT_STAT_DSRC_CURSOR_MODIFY 2147 /*! cursor: modify key and value bytes affected */ -#define WT_STAT_DSRC_CURSOR_MODIFY_BYTES 2150 +#define WT_STAT_DSRC_CURSOR_MODIFY_BYTES 2148 /*! cursor: modify value bytes modified */ -#define WT_STAT_DSRC_CURSOR_MODIFY_BYTES_TOUCH 2151 +#define WT_STAT_DSRC_CURSOR_MODIFY_BYTES_TOUCH 2149 /*! cursor: next calls */ -#define WT_STAT_DSRC_CURSOR_NEXT 2152 +#define WT_STAT_DSRC_CURSOR_NEXT 2150 /*! cursor: open cursor count */ -#define WT_STAT_DSRC_CURSOR_OPEN_COUNT 2153 +#define WT_STAT_DSRC_CURSOR_OPEN_COUNT 2151 /*! cursor: operation restarted */ -#define WT_STAT_DSRC_CURSOR_RESTART 2154 +#define WT_STAT_DSRC_CURSOR_RESTART 2152 /*! cursor: prev calls */ -#define WT_STAT_DSRC_CURSOR_PREV 2155 +#define WT_STAT_DSRC_CURSOR_PREV 2153 /*! cursor: remove calls */ -#define WT_STAT_DSRC_CURSOR_REMOVE 2156 +#define WT_STAT_DSRC_CURSOR_REMOVE 2154 /*! cursor: remove key bytes removed */ -#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2157 +#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2155 /*! cursor: reserve calls */ -#define WT_STAT_DSRC_CURSOR_RESERVE 2158 +#define WT_STAT_DSRC_CURSOR_RESERVE 2156 /*! cursor: reset calls */ -#define WT_STAT_DSRC_CURSOR_RESET 2159 +#define WT_STAT_DSRC_CURSOR_RESET 2157 /*! cursor: search calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH 2160 +#define WT_STAT_DSRC_CURSOR_SEARCH 2158 /*! cursor: search history store calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH_HS 2161 +#define WT_STAT_DSRC_CURSOR_SEARCH_HS 2159 /*! cursor: search near calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2162 +#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2160 /*! cursor: truncate calls */ -#define WT_STAT_DSRC_CURSOR_TRUNCATE 2163 +#define WT_STAT_DSRC_CURSOR_TRUNCATE 2161 /*! cursor: update calls */ -#define WT_STAT_DSRC_CURSOR_UPDATE 2164 +#define WT_STAT_DSRC_CURSOR_UPDATE 2162 /*! cursor: update key and value bytes */ -#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2165 +#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2163 /*! cursor: update value size change */ -#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES_CHANGED 2166 +#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES_CHANGED 2164 /*! reconciliation: approximate byte size of timestamps in pages written */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_BYTES_TS 2167 +#define WT_STAT_DSRC_REC_TIME_WINDOW_BYTES_TS 2165 /*! * reconciliation: approximate byte size of transaction IDs in pages * written */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_BYTES_TXN 2168 +#define WT_STAT_DSRC_REC_TIME_WINDOW_BYTES_TXN 2166 /*! reconciliation: dictionary matches */ -#define WT_STAT_DSRC_REC_DICTIONARY 2169 +#define WT_STAT_DSRC_REC_DICTIONARY 2167 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2170 +#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2168 /*! * reconciliation: internal page key bytes discarded using suffix * compression */ -#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2171 +#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2169 /*! reconciliation: internal page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2172 +#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2170 /*! reconciliation: internal-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2173 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2171 /*! reconciliation: leaf page key bytes discarded using prefix compression */ -#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2174 +#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2172 /*! reconciliation: leaf page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2175 +#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2173 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2176 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2174 /*! reconciliation: maximum blocks required for a page */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2177 +#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2175 /*! reconciliation: overflow values written */ -#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2178 +#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2176 /*! reconciliation: page checksum matches */ -#define WT_STAT_DSRC_REC_PAGE_MATCH 2179 +#define WT_STAT_DSRC_REC_PAGE_MATCH 2177 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_DSRC_REC_PAGES 2180 +#define WT_STAT_DSRC_REC_PAGES 2178 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_DSRC_REC_PAGES_EVICTION 2181 +#define WT_STAT_DSRC_REC_PAGES_EVICTION 2179 /*! reconciliation: pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE 2182 +#define WT_STAT_DSRC_REC_PAGE_DELETE 2180 /*! * reconciliation: pages written including an aggregated newest start * durable timestamp */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 2183 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_START_DURABLE_TS 2181 /*! * reconciliation: pages written including an aggregated newest stop * durable timestamp */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 2184 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_DURABLE_TS 2182 /*! * reconciliation: pages written including an aggregated newest stop * timestamp */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_TS 2185 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_TS 2183 /*! * reconciliation: pages written including an aggregated newest stop * transaction ID */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_TXN 2186 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_STOP_TXN 2184 /*! * reconciliation: pages written including an aggregated newest * transaction ID */ -#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_TXN 2187 +#define WT_STAT_DSRC_REC_TIME_AGGR_NEWEST_TXN 2185 /*! * reconciliation: pages written including an aggregated oldest start * timestamp */ -#define WT_STAT_DSRC_REC_TIME_AGGR_OLDEST_START_TS 2188 +#define WT_STAT_DSRC_REC_TIME_AGGR_OLDEST_START_TS 2186 /*! reconciliation: pages written including an aggregated prepare */ -#define WT_STAT_DSRC_REC_TIME_AGGR_PREPARED 2189 +#define WT_STAT_DSRC_REC_TIME_AGGR_PREPARED 2187 /*! reconciliation: pages written including at least one prepare */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_PREPARED 2190 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_PREPARED 2188 /*! * reconciliation: pages written including at least one start durable * timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 2191 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_START_TS 2189 /*! reconciliation: pages written including at least one start timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_START_TS 2192 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_START_TS 2190 /*! * reconciliation: pages written including at least one start transaction * ID */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_START_TXN 2193 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_START_TXN 2191 /*! * reconciliation: pages written including at least one stop durable * timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 2194 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_DURABLE_STOP_TS 2192 /*! reconciliation: pages written including at least one stop timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TS 2195 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TS 2193 /*! * reconciliation: pages written including at least one stop transaction * ID */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TXN 2196 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PAGES_STOP_TXN 2194 /*! reconciliation: records written including a prepare */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_PREPARED 2197 +#define WT_STAT_DSRC_REC_TIME_WINDOW_PREPARED 2195 /*! reconciliation: records written including a start durable timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_DURABLE_START_TS 2198 +#define WT_STAT_DSRC_REC_TIME_WINDOW_DURABLE_START_TS 2196 /*! reconciliation: records written including a start timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_START_TS 2199 +#define WT_STAT_DSRC_REC_TIME_WINDOW_START_TS 2197 /*! reconciliation: records written including a start transaction ID */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_START_TXN 2200 +#define WT_STAT_DSRC_REC_TIME_WINDOW_START_TXN 2198 /*! reconciliation: records written including a stop durable timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_DURABLE_STOP_TS 2201 +#define WT_STAT_DSRC_REC_TIME_WINDOW_DURABLE_STOP_TS 2199 /*! reconciliation: records written including a stop timestamp */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TS 2202 +#define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TS 2200 /*! reconciliation: records written including a stop transaction ID */ -#define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TXN 2203 +#define WT_STAT_DSRC_REC_TIME_WINDOW_STOP_TXN 2201 /*! session: object compaction */ -#define WT_STAT_DSRC_SESSION_COMPACT 2204 +#define WT_STAT_DSRC_SESSION_COMPACT 2202 /*! session: tiered operations dequeued and processed */ -#define WT_STAT_DSRC_TIERED_WORK_UNITS_DEQUEUED 2205 +#define WT_STAT_DSRC_TIERED_WORK_UNITS_DEQUEUED 2203 /*! session: tiered operations scheduled */ -#define WT_STAT_DSRC_TIERED_WORK_UNITS_CREATED 2206 +#define WT_STAT_DSRC_TIERED_WORK_UNITS_CREATED 2204 /*! session: tiered storage local retention time (secs) */ -#define WT_STAT_DSRC_TIERED_RETENTION 2207 +#define WT_STAT_DSRC_TIERED_RETENTION 2205 /*! session: tiered storage object size */ -#define WT_STAT_DSRC_TIERED_OBJECT_SIZE 2208 +#define WT_STAT_DSRC_TIERED_OBJECT_SIZE 2206 /*! transaction: race to read prepared update retry */ -#define WT_STAT_DSRC_TXN_READ_RACE_PREPARE_UPDATE 2209 +#define WT_STAT_DSRC_TXN_READ_RACE_PREPARE_UPDATE 2207 /*! * transaction: rollback to stable history store records with stop * timestamps older than newer records */ -#define WT_STAT_DSRC_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 2210 +#define WT_STAT_DSRC_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 2208 /*! transaction: rollback to stable inconsistent checkpoint */ -#define WT_STAT_DSRC_TXN_RTS_INCONSISTENT_CKPT 2211 +#define WT_STAT_DSRC_TXN_RTS_INCONSISTENT_CKPT 2209 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_DSRC_TXN_RTS_KEYS_REMOVED 2212 +#define WT_STAT_DSRC_TXN_RTS_KEYS_REMOVED 2210 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_DSRC_TXN_RTS_KEYS_RESTORED 2213 +#define WT_STAT_DSRC_TXN_RTS_KEYS_RESTORED 2211 /*! transaction: rollback to stable restored tombstones from history store */ -#define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_TOMBSTONES 2214 +#define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_TOMBSTONES 2212 /*! transaction: rollback to stable restored updates from history store */ -#define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_UPDATES 2215 +#define WT_STAT_DSRC_TXN_RTS_HS_RESTORE_UPDATES 2213 /*! transaction: rollback to stable skipping delete rle */ -#define WT_STAT_DSRC_TXN_RTS_DELETE_RLE_SKIPPED 2216 +#define WT_STAT_DSRC_TXN_RTS_DELETE_RLE_SKIPPED 2214 /*! transaction: rollback to stable skipping stable rle */ -#define WT_STAT_DSRC_TXN_RTS_STABLE_RLE_SKIPPED 2217 +#define WT_STAT_DSRC_TXN_RTS_STABLE_RLE_SKIPPED 2215 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_DSRC_TXN_RTS_SWEEP_HS_KEYS 2218 +#define WT_STAT_DSRC_TXN_RTS_SWEEP_HS_KEYS 2216 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_DSRC_TXN_RTS_HS_REMOVED 2219 +#define WT_STAT_DSRC_TXN_RTS_HS_REMOVED 2217 /*! transaction: transaction checkpoints due to obsolete pages */ -#define WT_STAT_DSRC_TXN_CHECKPOINT_OBSOLETE_APPLIED 2220 +#define WT_STAT_DSRC_TXN_CHECKPOINT_OBSOLETE_APPLIED 2218 /*! transaction: update conflicts */ -#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2221 +#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2219 /*! * @} diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index 6b136c6cc84..0222230a473 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -462,14 +462,13 @@ __wt_log_get_backup_files( * log file will be removed from the list of files returned. New writes will not be included in * the backup. */ - if (active_only) - F_SET(log, WT_LOG_FORCE_NEWFILE); + F_SET(log, WT_LOG_FORCE_NEWFILE); WT_RET(__wt_log_force_write(session, 1, NULL)); WT_RET(__log_get_files(session, WT_LOG_FILENAME, &files, &count)); for (max = 0, i = 0; i < count;) { WT_ERR(__wt_log_extract_lognum(session, files[i], &id)); - if (active_only && (id < min_file || id > max_file)) { + if ((active_only && id < min_file) || id > max_file) { /* * Any files not being returned are individually freed and the array adjusted. */ diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c index 4c3aa2a3204..f1503d9bbf3 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c @@ -1685,6 +1685,7 @@ __wt_clsm_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, cons __clsm_remove, /* remove */ __clsm_reserve, /* reserve */ __wt_cursor_reconfigure, /* reconfigure */ + __wt_cursor_notsup, /* largest_key */ __wt_cursor_notsup, /* cache */ __wt_cursor_reopen_notsup, /* reopen */ __wt_clsm_close); /* close */ diff --git a/src/third_party/wiredtiger/src/os_common/os_fhandle.c b/src/third_party/wiredtiger/src/os_common/os_fhandle.c index f39fbd599e7..18bdb756176 100644 --- a/src/third_party/wiredtiger/src/os_common/os_fhandle.c +++ b/src/third_party/wiredtiger/src/os_common/os_fhandle.c @@ -41,7 +41,6 @@ __fhandle_method_finalize(WT_SESSION_IMPL *session, WT_FILE_HANDLE *handle, bool return (0); } -#ifdef HAVE_DIAGNOSTIC /* * __wt_handle_is_open -- * Return if there's an open handle matching a name. @@ -72,7 +71,6 @@ __wt_handle_is_open(WT_SESSION_IMPL *session, const char *name) return (found); } -#endif /* * __handle_search -- @@ -288,9 +286,8 @@ __handle_close(WT_SESSION_IMPL *session, WT_FH *fh, bool locked) conn = S2C(session); - if (fh->ref != 0) { + if (fh->ref != 0) __wt_errx(session, "Closing a file handle with open references: %s", fh->name); - } /* Remove from the list. */ bucket = fh->name_hash & (conn->hash_size - 1); diff --git a/src/third_party/wiredtiger/src/os_posix/os_fs.c b/src/third_party/wiredtiger/src/os_posix/os_fs.c index 1ae6259e5d8..3898eb74343 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_fs.c +++ b/src/third_party/wiredtiger/src/os_posix/os_fs.c @@ -797,12 +797,16 @@ __posix_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const cha /* Create/Open the file. */ WT_SYSCALL_RETRY(((pfh->fd = open(name, f, mode)) == -1 ? -1 : 0), ret); - if (ret != 0) + if (ret != 0) { + /* If we don't want error messages, just return the error value. */ + if (F_ISSET(session, WT_SESSION_QUIET_TIERED) && ret == ENOENT) + goto err; WT_ERR_MSG(session, ret, pfh->direct_io ? "%s: handle-open: open: failed with direct I/O configured, some " "filesystem types do not support direct I/O" : "%s: handle-open: open", name); + } #ifdef __linux__ /* diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 5a4320b493e..8353a6e9d5a 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -672,22 +672,18 @@ err: } /* - * __wt_session_blocking_checkpoint -- + * __session_blocking_checkpoint -- * Perform a checkpoint or wait if it is already running to resolve an EBUSY error. */ -int -__wt_session_blocking_checkpoint(WT_SESSION_IMPL *session, bool force, uint64_t seconds) +static int +__session_blocking_checkpoint(WT_SESSION_IMPL *session) { WT_DECL_RET; WT_TXN_GLOBAL *txn_global; uint64_t txn_gen; - const char *cfg[3] = {NULL, NULL, NULL}; + const char *checkpoint_cfg[] = {WT_CONFIG_BASE(session, WT_SESSION_checkpoint), NULL}; - cfg[0] = WT_CONFIG_BASE(session, WT_SESSION_checkpoint); - if (force) - cfg[1] = "force=1"; - - if ((ret = __wt_txn_checkpoint(session, cfg, false)) == 0) + if ((ret = __wt_txn_checkpoint(session, checkpoint_cfg, false)) == 0) return (0); WT_RET_BUSY_OK(ret); @@ -704,13 +700,6 @@ __wt_session_blocking_checkpoint(WT_SESSION_IMPL *session, bool force, uint64_t */ if (!txn_global->checkpoint_running || txn_gen != __wt_gen(session, WT_GEN_CHECKPOINT)) break; - - /* If there's a timeout, give up. */ - if (seconds == 0) - continue; - if (seconds <= WT_CKPT_WAIT) - return (EBUSY); - seconds -= WT_CKPT_WAIT; } return (0); @@ -735,7 +724,7 @@ __session_alter(WT_SESSION *wt_session, const char *uri, const char *config) */ ret = __session_alter_internal(session, uri, config); if (ret == EBUSY) { - WT_RET(__wt_session_blocking_checkpoint(session, false, 0)); + WT_RET(__session_blocking_checkpoint(session)); WT_STAT_CONN_INCR(session, session_table_alter_trigger_checkpoint); ret = __session_alter_internal(session, uri, config); } diff --git a/src/third_party/wiredtiger/src/session/session_compact.c b/src/third_party/wiredtiger/src/session/session_compact.c index 1f83d6e4c83..9e0ac6ce549 100644 --- a/src/third_party/wiredtiger/src/session/session_compact.c +++ b/src/third_party/wiredtiger/src/session/session_compact.c @@ -193,6 +193,26 @@ __wt_session_compact_check_timeout(WT_SESSION_IMPL *session) } /* + * __compact_checkpoint -- + * This function does wait and force checkpoint. + */ +static int +__compact_checkpoint(WT_SESSION_IMPL *session) +{ + /* + * Force compaction checkpoints: we don't want to skip it because the work we need to have done + * is done in the underlying block manager. + */ + const char *checkpoint_cfg[] = { + WT_CONFIG_BASE(session, WT_SESSION_checkpoint), "force=1", NULL}; + + /* Checkpoints take a lot of time, check if we've run out. */ + WT_RET(__wt_session_compact_check_timeout(session)); + + return (__wt_txn_checkpoint(session, checkpoint_cfg, true)); +} + +/* * __compact_worker -- * Function to alternate between checkpoints and compaction calls. */ @@ -213,7 +233,7 @@ __compact_worker(WT_SESSION_IMPL *session) /* * Perform an initial checkpoint (see this file's leading comment for details). */ - WT_ERR(__wt_session_blocking_checkpoint(session, true, session->compact->max_time)); + WT_ERR(__compact_checkpoint(session)); /* * We compact 10% of a file on each pass (but the overall size of the file is decreasing each @@ -263,8 +283,8 @@ __compact_worker(WT_SESSION_IMPL *session) /* * Perform two checkpoints (see this file's leading comment for details). */ - WT_ERR(__wt_session_blocking_checkpoint(session, true, session->compact->max_time)); - WT_ERR(__wt_session_blocking_checkpoint(session, true, session->compact->max_time)); + WT_ERR(__compact_checkpoint(session)); + WT_ERR(__compact_checkpoint(session)); } err: diff --git a/src/third_party/wiredtiger/src/session/session_dhandle.c b/src/third_party/wiredtiger/src/session/session_dhandle.c index 8172fc8841d..685bc6e0c46 100644 --- a/src/third_party/wiredtiger/src/session/session_dhandle.c +++ b/src/third_party/wiredtiger/src/session/session_dhandle.c @@ -362,7 +362,6 @@ __session_dhandle_sweep(WT_SESSION_IMPL *session) WT_DATA_HANDLE *dhandle; WT_DATA_HANDLE_CACHE *dhandle_cache, *dhandle_cache_tmp; uint64_t now; - bool empty_btree; conn = S2C(session); @@ -379,15 +378,16 @@ __session_dhandle_sweep(WT_SESSION_IMPL *session) TAILQ_FOREACH_SAFE(dhandle_cache, &session->dhandles, q, dhandle_cache_tmp) { dhandle = dhandle_cache->dhandle; - empty_btree = false; - if (WT_DHANDLE_BTREE(dhandle)) - WT_WITH_DHANDLE( - session, dhandle, empty_btree = (__wt_btree_bytes_evictable(session) == 0)); + /* + * Only discard handles that are dead or dying and, in the case of btrees, have been + * evicted. These checks are not done with any locks in place, other than the data handle + * reference, so we cannot peer past what is in the dhandle directly. + */ if (dhandle != session->dhandle && dhandle->session_inuse == 0 && (WT_DHANDLE_INACTIVE(dhandle) || - (dhandle->timeofdeath != 0 && now - dhandle->timeofdeath > conn->sweep_idle_time) || - empty_btree)) { + (dhandle->timeofdeath != 0 && now - dhandle->timeofdeath > conn->sweep_idle_time)) && + (!WT_DHANDLE_BTREE(dhandle) || F_ISSET(dhandle, WT_DHANDLE_EVICTED))) { WT_STAT_CONN_INCR(session, dh_session_handles); WT_ASSERT(session, !WT_IS_METADATA(dhandle)); __session_discard_dhandle(session, dhandle_cache); diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 0376a34f998..f5ae97c8cea 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -146,8 +146,6 @@ static const char *const __stats_dsrc_desc[] = { "cursor: Total number of entries skipped by cursor next calls", "cursor: Total number of entries skipped by cursor prev calls", "cursor: Total number of entries skipped to position the history store cursor", - "cursor: Total number of pages skipped without reading by cursor next calls", - "cursor: Total number of pages skipped without reading by cursor prev calls", "cursor: Total number of times a search near has exited due to prefix config", "cursor: bulk loaded cursor insert calls", "cursor: cache cursors reuse count", @@ -409,8 +407,6 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->cursor_next_skip_total = 0; stats->cursor_prev_skip_total = 0; stats->cursor_skip_hs_cur_position = 0; - stats->cursor_next_skip_page_count = 0; - stats->cursor_prev_skip_page_count = 0; stats->cursor_search_near_prefix_fast_paths = 0; stats->cursor_insert_bulk = 0; stats->cursor_reopen = 0; @@ -661,8 +657,6 @@ __wt_stat_dsrc_aggregate_single(WT_DSRC_STATS *from, WT_DSRC_STATS *to) to->cursor_next_skip_total += from->cursor_next_skip_total; to->cursor_prev_skip_total += from->cursor_prev_skip_total; to->cursor_skip_hs_cur_position += from->cursor_skip_hs_cur_position; - to->cursor_next_skip_page_count += from->cursor_next_skip_page_count; - to->cursor_prev_skip_page_count += from->cursor_prev_skip_page_count; to->cursor_search_near_prefix_fast_paths += from->cursor_search_near_prefix_fast_paths; to->cursor_insert_bulk += from->cursor_insert_bulk; to->cursor_reopen += from->cursor_reopen; @@ -916,8 +910,6 @@ __wt_stat_dsrc_aggregate(WT_DSRC_STATS **from, WT_DSRC_STATS *to) to->cursor_next_skip_total += WT_STAT_READ(from, cursor_next_skip_total); to->cursor_prev_skip_total += WT_STAT_READ(from, cursor_prev_skip_total); to->cursor_skip_hs_cur_position += WT_STAT_READ(from, cursor_skip_hs_cur_position); - to->cursor_next_skip_page_count += WT_STAT_READ(from, cursor_next_skip_page_count); - to->cursor_prev_skip_page_count += WT_STAT_READ(from, cursor_prev_skip_page_count); to->cursor_search_near_prefix_fast_paths += WT_STAT_READ(from, cursor_search_near_prefix_fast_paths); to->cursor_insert_bulk += WT_STAT_READ(from, cursor_insert_bulk); @@ -1222,8 +1214,6 @@ static const char *const __stats_connection_desc[] = { "cursor: Total number of entries skipped by cursor next calls", "cursor: Total number of entries skipped by cursor prev calls", "cursor: Total number of entries skipped to position the history store cursor", - "cursor: Total number of pages skipped without reading by cursor next calls", - "cursor: Total number of pages skipped without reading by cursor prev calls", "cursor: Total number of times a search near has exited due to prefix config", "cursor: cached cursor count", "cursor: cursor bulk loaded cursor insert calls", @@ -1403,8 +1393,9 @@ static const char *const __stats_connection_desc[] = { "reconciliation: records written including a stop transaction ID", "reconciliation: split bytes currently awaiting free", "reconciliation: split objects currently awaiting free", - "session: flush state races", + "session: attempts to remove a local object and the object is in use", "session: flush_tier operation calls", + "session: local objects removed", "session: open session count", "session: session query timestamp calls", "session: table alter failed calls", @@ -1758,8 +1749,6 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cursor_next_skip_total = 0; stats->cursor_prev_skip_total = 0; stats->cursor_skip_hs_cur_position = 0; - stats->cursor_next_skip_page_count = 0; - stats->cursor_prev_skip_page_count = 0; stats->cursor_search_near_prefix_fast_paths = 0; /* not clearing cursor_cached_count */ stats->cursor_insert_bulk = 0; @@ -1938,8 +1927,9 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->rec_time_window_stop_txn = 0; /* not clearing rec_split_stashed_bytes */ /* not clearing rec_split_stashed_objects */ - stats->flush_state_races = 0; + stats->local_objects_inuse = 0; stats->flush_tier = 0; + stats->local_objects_removed = 0; /* not clearing session_open */ stats->session_query_ts = 0; /* not clearing session_table_alter_fail */ @@ -2296,8 +2286,6 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->cursor_next_skip_total += WT_STAT_READ(from, cursor_next_skip_total); to->cursor_prev_skip_total += WT_STAT_READ(from, cursor_prev_skip_total); to->cursor_skip_hs_cur_position += WT_STAT_READ(from, cursor_skip_hs_cur_position); - to->cursor_next_skip_page_count += WT_STAT_READ(from, cursor_next_skip_page_count); - to->cursor_prev_skip_page_count += WT_STAT_READ(from, cursor_prev_skip_page_count); to->cursor_search_near_prefix_fast_paths += WT_STAT_READ(from, cursor_search_near_prefix_fast_paths); to->cursor_cached_count += WT_STAT_READ(from, cursor_cached_count); @@ -2485,8 +2473,9 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->rec_time_window_stop_txn += WT_STAT_READ(from, rec_time_window_stop_txn); to->rec_split_stashed_bytes += WT_STAT_READ(from, rec_split_stashed_bytes); to->rec_split_stashed_objects += WT_STAT_READ(from, rec_split_stashed_objects); - to->flush_state_races += WT_STAT_READ(from, flush_state_races); + to->local_objects_inuse += WT_STAT_READ(from, local_objects_inuse); to->flush_tier += WT_STAT_READ(from, flush_tier); + to->local_objects_removed += WT_STAT_READ(from, local_objects_removed); to->session_open += WT_STAT_READ(from, session_open); to->session_query_ts += WT_STAT_READ(from, session_query_ts); to->session_table_alter_fail += WT_STAT_READ(from, session_table_alter_fail); diff --git a/src/third_party/wiredtiger/src/support/thread_group.c b/src/third_party/wiredtiger/src/support/thread_group.c index 6bac6d5450d..0f3e3084b54 100644 --- a/src/third_party/wiredtiger/src/support/thread_group.c +++ b/src/third_party/wiredtiger/src/support/thread_group.c @@ -42,14 +42,15 @@ err: WT_IGNORE_RET(__wt_panic(session, ret, "Unrecoverable utility thread error")); /* - * The three cases when threads are expected to stop are: + * The cases when threads are expected to stop are: * 1. When recovery is done. * 2. When the connection is closing. * 3. When a shutdown has been requested via clearing the run flag. + * 4. When an error has occurred and the connection panic flag is set. */ WT_ASSERT(session, !F_ISSET(thread, WT_THREAD_RUN) || - F_ISSET(S2C(session), WT_CONN_CLOSING | WT_CONN_RECOVERING)); + F_ISSET(S2C(session), WT_CONN_CLOSING | WT_CONN_PANIC | WT_CONN_RECOVERING)); return (WT_THREAD_RET_VALUE); } diff --git a/src/third_party/wiredtiger/src/tiered/tiered_handle.c b/src/third_party/wiredtiger/src/tiered/tiered_handle.c index 5cfd583b1c9..5be43f315c6 100644 --- a/src/third_party/wiredtiger/src/tiered/tiered_handle.c +++ b/src/third_party/wiredtiger/src/tiered/tiered_handle.c @@ -441,9 +441,7 @@ __wt_tiered_switch(WT_SESSION_IMPL *session, const char *config) /* * __wt_tiered_name -- - * Given a dhandle structure and object number generate the URI name of the given type. XXX - * Currently this is only used in this file but I anticipate it may be of use outside. If not, - * make this static and tiered_name instead. + * Given a dhandle structure and object number generate the URI name of the given type. */ int __wt_tiered_name( @@ -567,9 +565,7 @@ __tiered_open(WT_SESSION_IMPL *session, const char *cfg[]) /* Temp code to keep s_all happy. */ FLD_SET(unused, WT_TIERED_OBJ_LOCAL | WT_TIERED_TREE_UNUSED); FLD_SET(unused, WT_TIERED_WORK_FORCE | WT_TIERED_WORK_FREE); - WT_ERR(__wt_tiered_put_drop_local(session, tiered, tiered->current_id)); WT_ERR(__wt_tiered_put_drop_shared(session, tiered, tiered->current_id)); - __wt_tiered_get_drop_local(session, 0, &entry); __wt_tiered_get_drop_shared(session, &entry); } #endif diff --git a/src/third_party/wiredtiger/src/tiered/tiered_opener.c b/src/third_party/wiredtiger/src/tiered/tiered_opener.c index 5ca027510fc..b6f3df90fa7 100644 --- a/src/third_party/wiredtiger/src/tiered/tiered_opener.c +++ b/src/third_party/wiredtiger/src/tiered/tiered_opener.c @@ -20,44 +20,44 @@ __tiered_opener_open(WT_BLOCK_FILE_OPENER *opener, WT_SESSION_IMPL *session, uin WT_DECL_RET; WT_TIERED *tiered; const char *object_name, *object_uri; + bool local_only; tiered = opener->cookie; object_uri = NULL; + local_only = false; WT_ASSERT(session, (object_id > 0 && object_id <= tiered->current_id) || object_id == WT_TIERED_CURRENT_ID); /* - * FIXME-WT-7590 we will need some kind of locking while we're looking at the tiered structure. - * This can be called at any time, because we are opening the objects lazily. + * First look for the local file. This will be the fastest access and we retain recent objects + * in the local database for a while. */ if (object_id == tiered->current_id || object_id == WT_TIERED_CURRENT_ID) { bstorage = NULL; object_name = tiered->tiers[WT_TIERED_INDEX_LOCAL].name; - if (!WT_PREFIX_SKIP(object_name, "file:")) - WT_RET_MSG(session, EINVAL, "expected a 'file:' URI"); - WT_ERR(__wt_open(session, object_name, type, flags, fhp)); + WT_PREFIX_SKIP_REQUIRED(session, object_name, "file:"); + local_only = true; } else { WT_ERR( __wt_tiered_name(session, &tiered->iface, object_id, WT_TIERED_NAME_OBJECT, &object_uri)); object_name = object_uri; WT_PREFIX_SKIP_REQUIRED(session, object_name, "object:"); + LF_SET(WT_FS_OPEN_READONLY); + WT_ASSERT(session, !FLD_ISSET(flags, WT_FS_OPEN_CREATE)); + F_SET(session, WT_SESSION_QUIET_TIERED); + } + ret = __wt_open(session, object_name, type, flags, fhp); + F_CLR(session, WT_SESSION_QUIET_TIERED); + + /* + * FIXME-WT-7590 we will need some kind of locking while we're looking at the tiered structure. + * This can be called at any time, because we are opening the objects lazily. + */ + if (!local_only && ret != 0) { bstorage = tiered->bstorage; - flags |= WT_FS_OPEN_READONLY; + LF_SET(WT_FS_OPEN_READONLY); WT_WITH_BUCKET_STORAGE( bstorage, session, { ret = __wt_open(session, object_name, type, flags, fhp); }); - if (ret == ENOENT) { - /* - * There is a window where the object may not be copied yet to the bucket. If it isn't - * found try the local system. If it isn't found there then try the bucket one more - * time. - */ - ret = __wt_open(session, object_name, type, flags, fhp); - __wt_errx(session, "OPENER: local %s ret %d", object_name, ret); - if (ret == ENOENT) - WT_WITH_BUCKET_STORAGE( - bstorage, session, { ret = __wt_open(session, object_name, type, flags, fhp); }); - WT_ERR(ret); - } } err: __wt_free(session, object_uri); diff --git a/src/third_party/wiredtiger/src/tiered/tiered_work.c b/src/third_party/wiredtiger/src/tiered/tiered_work.c index b3fd1b7f9b5..efc80ea86bd 100644 --- a/src/third_party/wiredtiger/src/tiered/tiered_work.c +++ b/src/third_party/wiredtiger/src/tiered/tiered_work.c @@ -9,6 +9,24 @@ #include "wt_internal.h" /* + * __tiered_flush_state -- + * Account for flush work units so threads can know when shared storage flushing is complete. + */ +static void +__tiered_flush_state(WT_SESSION_IMPL *session, uint32_t type, bool incr) +{ + WT_CONNECTION_IMPL *conn; + + if (type != WT_TIERED_WORK_FLUSH) + return; + conn = S2C(session); + if (incr) + (void)__wt_atomic_addv32(&conn->flush_state, 1); + else + (void)__wt_atomic_subv32(&conn->flush_state, 1); +} + +/* * __wt_tiered_work_free -- * Free a work unit and account for it in the flush state. */ @@ -16,18 +34,9 @@ void __wt_tiered_work_free(WT_SESSION_IMPL *session, WT_TIERED_WORK_UNIT *entry) { WT_CONNECTION_IMPL *conn; - uint32_t new_state, old_state; conn = S2C(session); - for (;;) { - WT_BARRIER(); - old_state = conn->flush_state; - new_state = old_state - 1; - if (__wt_atomic_casv32(&conn->flush_state, old_state, new_state)) - break; - WT_STAT_CONN_INCR(session, flush_state_races); - __wt_yield(); - } + __tiered_flush_state(session, entry->type, false); /* If all work is done signal any waiting thread waiting for sync. */ if (WT_FLUSH_STATE_DONE(conn->flush_state)) __wt_cond_signal(session, conn->flush_cond); @@ -42,23 +51,13 @@ void __wt_tiered_push_work(WT_SESSION_IMPL *session, WT_TIERED_WORK_UNIT *entry) { WT_CONNECTION_IMPL *conn; - uint32_t new_state, old_state; conn = S2C(session); - __wt_spin_lock(session, &conn->tiered_lock); TAILQ_INSERT_TAIL(&conn->tieredqh, entry, q); WT_STAT_CONN_INCR(session, tiered_work_units_created); __wt_spin_unlock(session, &conn->tiered_lock); - for (;;) { - WT_BARRIER(); - old_state = conn->flush_state; - new_state = old_state + 1; - if (__wt_atomic_casv32(&conn->flush_state, old_state, new_state)) - break; - WT_STAT_CONN_INCR(session, flush_state_races); - __wt_yield(); - } + __tiered_flush_state(session, entry->type, true); __wt_cond_signal(session, conn->tiered_cond); return; } @@ -87,10 +86,10 @@ __wt_tiered_pop_work( if (FLD_ISSET(type, entry->type) && (maxval == 0 || entry->op_val < maxval)) { TAILQ_REMOVE(&conn->tieredqh, entry, q); WT_STAT_CONN_INCR(session, tiered_work_units_dequeued); + *entryp = entry; break; } } - *entryp = entry; __wt_spin_unlock(session, &conn->tiered_lock); return; } diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index 04df57cf66f..6aee858e94b 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -155,17 +155,19 @@ __rollback_abort_update(WT_SESSION_IMPL *session, WT_ITEM *key, WT_UPDATE *first /* * __rollback_abort_insert_list -- - * Apply the update abort check to each entry in an insert skip list. + * Apply the update abort check to each entry in an insert skip list. Return how many entries + * had stable updates. */ static int __rollback_abort_insert_list(WT_SESSION_IMPL *session, WT_PAGE *page, WT_INSERT_HEAD *head, - wt_timestamp_t rollback_timestamp, bool *stable_update_found) + wt_timestamp_t rollback_timestamp, uint32_t *stable_updates_count) { WT_DECL_ITEM(key); WT_DECL_RET; WT_INSERT *ins; uint64_t recno; uint8_t *memp; + bool stable_update_found; WT_ERR( __wt_scr_alloc(session, page->type == WT_PAGE_ROW_LEAF ? 0 : WT_INTPACK64_MAXSIZE, &key)); @@ -182,7 +184,9 @@ __rollback_abort_insert_list(WT_SESSION_IMPL *session, WT_PAGE *page, WT_INSERT_ key->size = WT_PTRDIFF(memp, key->data); } WT_ERR(__rollback_abort_update( - session, key, ins->upd, rollback_timestamp, stable_update_found)); + session, key, ins->upd, rollback_timestamp, &stable_update_found)); + if (stable_update_found && stable_updates_count != NULL) + (*stable_updates_count)++; } err: @@ -191,6 +195,19 @@ err: } /* + * __rollback_has_stable_update -- + * Check if an update chain has a stable update on it. Assume the update chain has already been + * processed so all we need to do is look for a valid, non-aborted entry. + */ +static bool +__rollback_has_stable_update(WT_UPDATE *upd) +{ + while (upd != NULL && (upd->type == WT_UPDATE_INVALID || upd->txnid == WT_TXN_ABORTED)) + upd = upd->next; + return upd != NULL; +} + +/* * __rollback_col_modify -- * Add the provided update to the head of the update list. */ @@ -225,57 +242,27 @@ err: * Add the provided update to the head of the update list. */ static inline int -__rollback_row_modify(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW *rip, WT_UPDATE *upd) +__rollback_row_modify(WT_SESSION_IMPL *session, WT_REF *ref, WT_UPDATE *upd, WT_ITEM *key) { + WT_CURSOR_BTREE cbt; WT_DECL_RET; - WT_PAGE_MODIFY *mod; - WT_UPDATE *last_upd, *old_upd, **upd_entry; - size_t upd_size; - - last_upd = NULL; - /* If we don't yet have a modify structure, we'll need one. */ - WT_RET(__wt_page_modify_init(session, page)); - mod = page->modify; - - /* Allocate an update array as necessary. */ - WT_PAGE_ALLOC_AND_SWAP(session, page, mod->mod_row_update, upd_entry, page->entries); - - /* Set the WT_UPDATE array reference. */ - upd_entry = &mod->mod_row_update[WT_ROW_SLOT(page, rip)]; - upd_size = __wt_update_list_memsize(upd); - /* If there are existing updates, append them after the new updates. */ - for (last_upd = upd; last_upd->next != NULL; last_upd = last_upd->next) - ; - last_upd->next = *upd_entry; - - /* - * We can either put a tombstone plus an update or a single update on the update chain. - * - * Set the "old" entry to the second update in the list so that the serialization function - * succeeds in swapping the first update into place. - */ - if (upd->next != NULL) - *upd_entry = upd->next; - old_upd = *upd_entry; + __wt_btcur_init(session, &cbt); + __wt_btcur_open(&cbt); - /* - * Point the new WT_UPDATE item to the next element in the list. The serialization function acts - * as our memory barrier to flush this write. - */ - upd->next = old_upd; + /* Search the page. */ + WT_ERR(__wt_row_search(&cbt, key, true, ref, true, NULL)); - /* - * Serialize the update. Rollback to stable doesn't need to check the visibility of the on page - * value to detect conflict. - */ - WT_ERR(__wt_update_serial(session, NULL, page, upd_entry, &upd, upd_size, true)); + /* Apply the modification. */ +#ifdef HAVE_DIAGNOSTIC + WT_ERR(__wt_row_modify(&cbt, key, NULL, upd, WT_UPDATE_INVALID, true, false)); +#else + WT_ERR(__wt_row_modify(&cbt, key, NULL, upd, WT_UPDATE_INVALID, true)); +#endif - if (0) { err: - if (last_upd != NULL) - last_upd->next = NULL; - } + /* Free any resources that may have been cached in the cursor. */ + WT_TRET(__wt_btcur_close(&cbt, true)); return (ret); } @@ -606,7 +593,7 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, } if (rip != NULL) - WT_ERR(__rollback_row_modify(session, page, rip, upd)); + WT_ERR(__rollback_row_modify(session, ref, upd, key)); else WT_ERR(__rollback_col_modify(session, ref, upd, recno)); @@ -644,6 +631,7 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, u WT_ITEM *row_key, WT_CELL_UNPACK_KV *vpack, wt_timestamp_t rollback_timestamp, bool *is_ondisk_stable) { + WT_DECL_ITEM(key); WT_DECL_ITEM(tmp); WT_DECL_RET; WT_PAGE *page; @@ -766,14 +754,24 @@ __rollback_abort_ondisk_kv(WT_SESSION_IMPL *session, WT_REF *ref, WT_ROW *rip, u return (0); } - if (rip != NULL) - WT_ERR(__rollback_row_modify(session, page, rip, upd)); - else + if (rip != NULL) { + if (row_key != NULL) + key = row_key; + else { + /* Unpack a row key. */ + WT_ERR(__wt_scr_alloc(session, 0, &key)); + WT_ERR(__wt_row_leaf_key(session, page, rip, key, false)); + } + WT_ERR(__rollback_row_modify(session, ref, upd, key)); + } else WT_ERR(__rollback_col_modify(session, ref, upd, recno)); - return (0); + if (0) { err: - __wt_free(session, upd); + __wt_free(session, upd); + } + if (rip != NULL && row_key == NULL) + __wt_scr_free(session, &key); return (ret); } @@ -788,11 +786,12 @@ __rollback_abort_col_var(WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t r WT_CELL *kcell; WT_CELL_UNPACK_KV unpack; WT_COL *cip; - WT_INSERT_HEAD *ins; + WT_INSERT *ins; + WT_INSERT_HEAD *inshead; WT_PAGE *page; - uint64_t recno, rle; - uint32_t i, j; - bool is_ondisk_stable, stable_update_found; + uint64_t ins_recno, recno, rle; + uint32_t i, j, stable_updates_count; + bool is_ondisk_stable; page = ref->page; /* @@ -805,11 +804,11 @@ __rollback_abort_col_var(WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t r /* Review the changes to the original on-page data items. */ WT_COL_FOREACH (page, cip, i) { - stable_update_found = false; + stable_updates_count = 0; - if ((ins = WT_COL_UPDATE(page, cip)) != NULL) + if ((inshead = WT_COL_UPDATE(page, cip)) != NULL) WT_RET(__rollback_abort_insert_list( - session, page, ins, rollback_timestamp, &stable_update_found)); + session, page, inshead, rollback_timestamp, &stable_updates_count)); if (page->dsk != NULL) { /* Unpack the cell. We need its RLE count whether or not we're going to iterate it. */ @@ -818,44 +817,76 @@ __rollback_abort_col_var(WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t r rle = __wt_cell_rle(&unpack); /* - * If we found a stable update on the insert list, this key needs no further attention. - * Any other keys in this cell with stable updates also do not require attention. But - * beyond that, the on-disk value must be older than the update we found. That means it - * too is stable(*), so any keys in the cell that _don't_ have stable updates on the - * update list don't need further attention either. (And any unstable updates were just - * handled above.) Thus we can skip iterating over the cell. + * Each key whose on-disk value is not stable and has no stable update on the update + * list must be processed downstream. + * + * If we can determine that the cell's on-disk value is stable, we can skip iterating + * over the cell; likewise, if we can determine that every key in the cell has a stable + * update on the update list, we can skip the iteration. Otherwise we have to try each + * key. + * + * If the on-disk cell is deleted, it is stable, because cells only appear as deleted + * when there is no older value that might need to be restored. * - * Furthermore, if the cell is deleted it must be - * itself stable, because cells only appear as deleted if there is no older value that - * might need to be restored. We can skip iterating over the cell. + * Note that in a purely timestamped world, the presence of any stable update for any + * key in the cell means the on-disk value must be stable, because the update must be + * newer than the on-disk value. However, this is no longer true if the stable update + * has no timestamp. It may also not be true if the on-disk value is prepared, or other + * corner cases. Therefore, we must iterate the cell unless _every_ key has a stable + * update. * - * (*) Either that, or the update is not timestamped, in which case the on-disk value - * might not be stable but the non-timestamp update will hide it until the next - * reconciliation and then overwrite it. + * We can, however, stop iterating as soon as the downstream code reports back that the + * on-disk value is actually stable. */ - if (stable_update_found) - WT_STAT_CONN_DATA_INCR(session, txn_rts_stable_rle_skipped); - else if (unpack.type == WT_CELL_DEL) + if (unpack.type == WT_CELL_DEL) WT_STAT_CONN_DATA_INCR(session, txn_rts_delete_rle_skipped); + else if (stable_updates_count == rle) + WT_STAT_CONN_DATA_INCR(session, txn_rts_stable_rle_skipped); else { - for (j = 0; j < rle; j++) { + j = 0; + if (inshead != NULL) { + WT_SKIP_FOREACH (ins, inshead) { + /* If the update list goes past the end of the cell, something's wrong. */ + WT_ASSERT(session, j < rle); + ins_recno = WT_INSERT_RECNO(ins); + /* Process all the keys before this update. */ + while (recno + j < ins_recno) { + WT_RET(__rollback_abort_ondisk_kv(session, ref, NULL, recno + j, NULL, + &unpack, rollback_timestamp, &is_ondisk_stable)); + /* We can stop right away if the on-disk version is stable. */ + if (is_ondisk_stable) { + if (rle > 1) + WT_STAT_CONN_DATA_INCR(session, txn_rts_stable_rle_skipped); + goto stop; + } + j++; + } + /* If this key has a stable update, skip over it. */ + if (recno + j == ins_recno && __rollback_has_stable_update(ins->upd)) + j++; + } + } + /* Process the rest of the keys. */ + while (j < rle) { WT_RET(__rollback_abort_ondisk_kv(session, ref, NULL, recno + j, NULL, &unpack, rollback_timestamp, &is_ondisk_stable)); /* We can stop right away if the on-disk version is stable. */ if (is_ondisk_stable) { if (rle > 1) WT_STAT_CONN_DATA_INCR(session, txn_rts_stable_rle_skipped); - break; + goto stop; } + j++; } } +stop: recno += rle; } } /* Review the append list */ - if ((ins = WT_COL_APPEND(page)) != NULL) - WT_RET(__rollback_abort_insert_list(session, page, ins, rollback_timestamp, NULL)); + if ((inshead = WT_COL_APPEND(page)) != NULL) + WT_RET(__rollback_abort_insert_list(session, page, inshead, rollback_timestamp, NULL)); /* Mark the page as dirty to reconcile the page. */ if (page->modify) diff --git a/src/third_party/wiredtiger/test/checkpoint/workers.c b/src/third_party/wiredtiger/test/checkpoint/workers.c index de2798413ee..3c9313c4c99 100644 --- a/src/third_party/wiredtiger/test/checkpoint/workers.c +++ b/src/third_party/wiredtiger/test/checkpoint/workers.c @@ -332,7 +332,7 @@ real_worker(void) /* If we have specified to run with mix mode deletes we need to do it in it's own txn. */ if (g.use_timestamps && g.mixed_mode_deletes && new_txn && __wt_random(&rnd) % 72 == 0) { new_txn = false; - for (j = 0; ret == 0 && j < g.ntables; j++) { + for (j = 0; j < g.ntables; j++) { ret = worker_mm_delete(cursors[j], keyno); if (ret == WT_ROLLBACK || ret == WT_PREPARE_CONFLICT) break; diff --git a/src/third_party/wiredtiger/test/cppsuite/Makefile.am b/src/third_party/wiredtiger/test/cppsuite/Makefile.am index 82c30442482..5e896ce2e04 100644 --- a/src/third_party/wiredtiger/test/cppsuite/Makefile.am +++ b/src/third_party/wiredtiger/test/cppsuite/Makefile.am @@ -11,7 +11,7 @@ all: all_TESTS= noinst_PROGRAMS= -run_SOURCES = test_harness/core/component.cxx \ +test_harness = test_harness/core/component.cxx \ test_harness/core/configuration.cxx \ test_harness/core/throttle.cxx \ test_harness/util/logger.cxx \ @@ -28,9 +28,15 @@ run_SOURCES = test_harness/core/component.cxx \ test_harness/test.cxx \ test_harness/thread_manager.cxx \ test_harness/timestamp_manager.cxx \ - test_harness/workload_generator.cxx \ - tests/run.cxx + test_harness/workload_generator.cxx +# If you prefer to not use the run binary you can add a test via this +# mechanism but it is generally frowned upon. +csuite_style_example_test_SOURCES = $(test_harness) tests/csuite_style_example_test.cxx +noinst_PROGRAMS += csuite_style_example_test +all_TESTS += csuite_style_example_test + +run_SOURCES = $(test_harness) tests/run.cxx noinst_PROGRAMS += run all_TESTS += run diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/search_near_01_default.txt b/src/third_party/wiredtiger/test/cppsuite/configs/search_near_01_default.txt new file mode 100644 index 00000000000..b3b3d6c668a --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/configs/search_near_01_default.txt @@ -0,0 +1,22 @@ +# Configuration for search_near_01. +# The test will generate key_count_per_collection number of keys for each prefix in aaa -> zzz. +# This config will have a 3 minute duration, with 3 tables of an entry for each prefix. +duration_seconds=180, +cache_size_mb=1000, +timestamp_manager= +( + enabled=false, +), +workload_generator= +( + populate_config= + ( + collection_count=3, + key_count_per_collection=1, + key_size=5, + ), + read_config= + ( + thread_count=10 + ) +),
\ No newline at end of file diff --git a/src/third_party/wiredtiger/test/cppsuite/configs/search_near_02_default.txt b/src/third_party/wiredtiger/test/cppsuite/configs/search_near_02_default.txt new file mode 100644 index 00000000000..34ab79e1980 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/configs/search_near_02_default.txt @@ -0,0 +1,31 @@ +# Configuration for search_near_02. +# The configuration creates: +# - threads that continuously insert random keys and values. +# - threads that continuously perform prefix search_near calls on random keys. +duration_seconds=180, +cache_size_mb=500, +timestamp_manager= +( + # This will let us randomly pick a read timestamp in a bigger range to trigger visibility + # checks. + oldest_lag=10, +), +workload_generator= +( + populate_config= + ( + collection_count=10, + ), + insert_config= + ( + key_size=5, + op_rate=100ms, + thread_count=10 + ), + read_config= + ( + key_size=5, + op_rate=250ms, + thread_count=10 + ) +) diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.cxx index 6ff134c7c96..8a8b75b7b8f 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.cxx @@ -80,6 +80,12 @@ connection_manager::create_session() return (session); } +WT_CONNECTION * +connection_manager::get_connection() +{ + return (_conn); +} + /* * set_timestamp calls into the connection API in a thread safe manner to set global timestamps. */ diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.h index c6245160df1..a5d44903717 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/connection_manager.h @@ -30,8 +30,12 @@ #define CONN_API_H /* Following definitions are required in order to use printing format specifiers in C++. */ +#ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS +#endif +#ifndef __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS +#endif #include <mutex> @@ -60,6 +64,8 @@ class connection_manager { void create(const std::string &config, const std::string &home = DEFAULT_DIR); scoped_session create_session(); + WT_CONNECTION *get_connection(); + /* * set_timestamp calls into the connection API in a thread safe manner to set global timestamps. */ diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.cxx index ebb6520b465..0e454a4f4f0 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.cxx @@ -36,16 +36,6 @@ namespace test_harness { /* Static methods implementation. */ -static void -get_stat(scoped_cursor &cursor, int stat_field, int64_t *valuep) -{ - const char *desc, *pvalue; - cursor->set_key(cursor.get(), stat_field); - testutil_check(cursor->search(cursor.get())); - testutil_check(cursor->get_value(cursor.get(), &desc, &pvalue, valuep)); - testutil_check(cursor->reset(cursor.get())); -} - static std::string collection_name_to_file_name(const std::string &collection_name) { @@ -100,9 +90,9 @@ cache_limit_statistic::check(scoped_cursor &cursor) int64_t cache_bytes_image, cache_bytes_other, cache_bytes_max; double use_percent; /* Three statistics are required to compute cache use percentage. */ - get_stat(cursor, WT_STAT_CONN_CACHE_BYTES_IMAGE, &cache_bytes_image); - get_stat(cursor, WT_STAT_CONN_CACHE_BYTES_OTHER, &cache_bytes_other); - get_stat(cursor, WT_STAT_CONN_CACHE_BYTES_MAX, &cache_bytes_max); + runtime_monitor::get_stat(cursor, WT_STAT_CONN_CACHE_BYTES_IMAGE, &cache_bytes_image); + runtime_monitor::get_stat(cursor, WT_STAT_CONN_CACHE_BYTES_OTHER, &cache_bytes_other); + runtime_monitor::get_stat(cursor, WT_STAT_CONN_CACHE_BYTES_MAX, &cache_bytes_max); /* * Assert that we never exceed our configured limit for cache usage. Add 0.0 to avoid floating * point conversion errors. @@ -222,7 +212,7 @@ postrun_statistic_check::check_stat(scoped_cursor &cursor, const postrun_statist int64_t stat_value; testutil_assert(cursor.get() != nullptr); - get_stat(cursor, stat.field, &stat_value); + runtime_monitor::get_stat(cursor, stat.field, &stat_value); if (stat_value < stat.min_limit || stat_value > stat.max_limit) { const std::string error_string = "runtime_monitor: Postrun stat \"" + stat.name + "\" was outside of the specified limits. Min=" + std::to_string(stat.min_limit) + @@ -236,6 +226,16 @@ postrun_statistic_check::check_stat(scoped_cursor &cursor, const postrun_statist } /* runtime_monitor class implementation */ +void +runtime_monitor::get_stat(scoped_cursor &cursor, int stat_field, int64_t *valuep) +{ + const char *desc, *pvalue; + cursor->set_key(cursor.get(), stat_field); + testutil_check(cursor->search(cursor.get())); + testutil_check(cursor->get_value(cursor.get(), &desc, &pvalue, valuep)); + testutil_check(cursor->reset(cursor.get())); +} + runtime_monitor::runtime_monitor(configuration *config, database &database) : component("runtime_monitor", config), _postrun_stats(config), _database(database) { diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h index e7e69302d1d..0f63585290d 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/runtime_monitor.h @@ -116,6 +116,9 @@ class postrun_statistic_check { */ class runtime_monitor : public component { public: + static void get_stat(scoped_cursor &, int, int64_t *); + + public: runtime_monitor(configuration *config, database &database); ~runtime_monitor(); diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.h index 8d67bfd7e27..2c35ef0d162 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/logger.h @@ -30,8 +30,12 @@ #define DEBUG_UTILS_H /* Following definitions are required in order to use printing format specifiers in C++. */ +#ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS +#endif +#ifndef __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS +#endif #include <chrono> #include <iostream> diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.h index 47b8592ede0..edb38e3e22c 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/util/scoped_types.h @@ -30,8 +30,12 @@ #define SCOPED_TYPES_H /* Following definitions are required in order to use printing format specifiers in C++. */ +#ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS +#endif +#ifndef __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS +#endif extern "C" { #include "test_util.h" diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.cxx index c539fbc34fc..3973af7242c 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.cxx @@ -44,10 +44,11 @@ random_generator::instance() std::string random_generator::generate_random_string(std::size_t length, characters_type type) { + const std::string characters = get_characters(type); std::string str; while (str.size() < length) - str += get_characters(type); + str += characters; std::shuffle(str.begin(), str.end(), _generator); return (str.substr(0, length)); diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.h index 31f44bbe98e..967d5566ce1 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/random_generator.h @@ -30,8 +30,12 @@ #define RANDOM_GENERATOR_H /* Following definitions are required in order to use printing format specifiers in C++. */ +#ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS +#endif +#ifndef __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS +#endif #include <random> #include <string> diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h index 61e1b99f28a..28c1ee4265b 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/thread_context.h @@ -159,6 +159,7 @@ class thread_context { public: scoped_session session; scoped_cursor op_track_cursor; + scoped_cursor stat_cursor; transaction_context transaction; timestamp_manager *tsm; workload_tracking *tracking; diff --git a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cxx b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cxx index 1211749ec28..e0e7738590d 100644 --- a/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/test_harness/workload/workload_tracking.cxx @@ -99,7 +99,8 @@ workload_tracking::do_work() /* Take a copy of the oldest so that we sweep with a consistent timestamp. */ oldest_ts = _tsm.get_oldest_ts(); - while ((ret = _sweep_cursor->prev(_sweep_cursor.get())) == 0) { + /* We need to check if the component is still running to avoid unecessary iterations. */ + while (_running && (ret = _sweep_cursor->prev(_sweep_cursor.get())) == 0) { testutil_check(_sweep_cursor->get_key(_sweep_cursor.get(), &collection_id, &key, &ts)); testutil_check(_sweep_cursor->get_value(_sweep_cursor.get(), &op_type, &value)); /* @@ -137,7 +138,13 @@ workload_tracking::do_work() free(sweep_key); - if (ret != WT_NOTFOUND) + /* + * If we get here and the test is still running, it means we must have reached the end of the + * table. We can also get here because the test is no longer running. In this case, the cursor + * can either be at the end of the table or still on a valid entry since we interrupted the + * work. + */ + if (ret != 0 && ret != WT_NOTFOUND) testutil_die(LOG_ERROR, "Tracking table sweep failed: cursor->next() returned an unexpected error %d.", ret); diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/csuite_style_example_test.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/csuite_style_example_test.cxx new file mode 100644 index 00000000000..d0059880446 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/tests/csuite_style_example_test.cxx @@ -0,0 +1,169 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * This file provides an example of how to create a test in C++ using a few features from the + * framework if any. This file can be used as a template for quick testing and/or when stress + * testing is not required. For any stress testing, it is encouraged to use the framework, see + * example_test.cxx and create_script.sh. + */ + +#include "test_harness/connection_manager.h" +#include "test_harness/thread_manager.h" +#include "test_harness/util/api_const.h" +#include "test_harness/util/logger.h" +#include "test_harness/workload/random_generator.h" + +extern "C" { +#include "wiredtiger.h" +} + +using namespace test_harness; + +bool do_inserts = false; +bool do_reads = false; + +void +insert_op(WT_CURSOR *cursor, int key_size, int value_size) +{ + logger::log_msg(LOG_INFO, "called insert_op"); + + /* Insert random data. */ + std::string key, value; + while (do_inserts) { + key = random_generator::instance().generate_random_string(key_size); + value = random_generator::instance().generate_random_string(value_size); + cursor->set_key(cursor, key.c_str()); + cursor->set_value(cursor, value.c_str()); + testutil_check(cursor->insert(cursor)); + } +} + +void +read_op(WT_CURSOR *cursor, int key_size) +{ + logger::log_msg(LOG_INFO, "called read_op"); + + /* Read random data. */ + std::string key; + while (do_reads) { + key = random_generator::instance().generate_random_string(key_size); + cursor->set_key(cursor, key.c_str()); + cursor->search(cursor); + } +} + +int +main(int argc, char *argv[]) +{ + /* Set the program name for error messages. */ + const std::string progname = testutil_set_progname(argv); + + /* Set the tracing level for the logger component. */ + logger::trace_level = LOG_INFO; + + /* Printing some messages. */ + logger::log_msg(LOG_INFO, "Starting " + progname); + logger::log_msg(LOG_ERROR, "This could be an error."); + + /* Create a connection, set the cache size and specify the home directory. */ + const std::string conn_config = std::string(CONNECTION_CREATE) + ",cache_size=500MB"; + const std::string home_dir = std::string(DEFAULT_DIR) + '_' + progname; + connection_manager::instance().create(conn_config, home_dir); + WT_CONNECTION *conn = connection_manager::instance().get_connection(); + + /* Open different sessions. */ + WT_SESSION *insert_session, *read_session; + testutil_check(conn->open_session(conn, nullptr, nullptr, &insert_session)); + testutil_check(conn->open_session(conn, nullptr, nullptr, &read_session)); + + /* Create a collection. */ + const std::string collection_name = "table:my_collection"; + testutil_check( + insert_session->create(insert_session, collection_name.c_str(), DEFAULT_FRAMEWORK_SCHEMA)); + + /* Open different cursors. */ + WT_CURSOR *insert_cursor, *read_cursor; + const std::string cursor_config = ""; + testutil_check(insert_session->open_cursor( + insert_session, collection_name.c_str(), nullptr, cursor_config.c_str(), &insert_cursor)); + testutil_check(read_session->open_cursor( + read_session, collection_name.c_str(), nullptr, cursor_config.c_str(), &read_cursor)); + + /* Store cursors. */ + std::vector<WT_CURSOR *> cursors; + cursors.push_back(insert_cursor); + cursors.push_back(read_cursor); + + /* Insert some data. */ + std::string key = "a"; + const std::string value = "b"; + insert_cursor->set_key(insert_cursor, key.c_str()); + insert_cursor->set_value(insert_cursor, value.c_str()); + testutil_check(insert_cursor->insert(insert_cursor)); + + /* Read some data. */ + key = "b"; + read_cursor->set_key(read_cursor, key.c_str()); + testutil_assert(read_cursor->search(read_cursor) == WT_NOTFOUND); + + key = "a"; + read_cursor->set_key(read_cursor, key.c_str()); + testutil_check(read_cursor->search(read_cursor)); + + /* Create a thread manager and spawn some threads that will work. */ + thread_manager t; + int key_size = 1, value_size = 2; + + do_inserts = true; + t.add_thread(insert_op, insert_cursor, key_size, value_size); + + do_reads = true; + t.add_thread(read_op, read_cursor, key_size); + + /* Sleep for the test duration. */ + int test_duration_s = 5; + std::this_thread::sleep_for(std::chrono::seconds(test_duration_s)); + + /* Stop the threads. */ + do_reads = false; + do_inserts = false; + t.join(); + + /* Close cursors. */ + for (auto c : cursors) + testutil_check(c->close(c)); + + /* Close the connection. */ + connection_manager::instance().close(); + + /* Another message. */ + logger::log_msg(LOG_INFO, "End of test."); + + return (0); +} diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx index 5a1996b45a1..4b49ad2b148 100644 --- a/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/example_test.cxx @@ -37,6 +37,13 @@ class example_test : public test_harness::test { example_test(const test_harness::test_args &args) : test(args) {} void + run() + { + /* You can remove the call to the base class to fully customized your test. */ + test::run(); + } + + void populate(test_harness::database &, test_harness::timestamp_manager *, test_harness::configuration *, test_harness::workload_tracking *) override final { diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx index 0c231e6568d..627921a9aa4 100755 --- a/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx +++ b/src/third_party/wiredtiger/test/cppsuite/tests/run.cxx @@ -34,9 +34,11 @@ #include "test_harness/test.h" #include "base_test.cxx" -#include "example_test.cxx" #include "burst_inserts.cxx" +#include "example_test.cxx" #include "hs_cleanup.cxx" +#include "search_near_01.cxx" +#include "search_near_02.cxx" std::string parse_configuration_from_file(const std::string &filename) @@ -72,7 +74,7 @@ print_help() std::cout << "\trun -C [WIREDTIGER_OPEN_CONFIGURATION]" << std::endl; std::cout << "\trun -c [TEST_FRAMEWORK_CONFIGURATION]" << std::endl; std::cout << "\trun -f [FILE]" << std::endl; - std::cout << "\trun -l [TRACEL_LEVEL]" << std::endl; + std::cout << "\trun -l [TRACE_LEVEL]" << std::endl; std::cout << "\trun -t [TEST_NAME]" << std::endl; std::cout << std::endl; std::cout << "DESCRIPTION" << std::endl; @@ -113,10 +115,14 @@ run_test(const std::string &test_name, const std::string &config, const std::str base_test(test_harness::test_args{config, test_name, wt_open_config}).run(); else if (test_name == "example_test") example_test(test_harness::test_args{config, test_name, wt_open_config}).run(); + else if (test_name == "search_near_02") + search_near_02(test_harness::test_args{config, test_name, wt_open_config}).run(); else if (test_name == "hs_cleanup") hs_cleanup(test_harness::test_args{config, test_name, wt_open_config}).run(); else if (test_name == "burst_inserts") burst_inserts(test_harness::test_args{config, test_name, wt_open_config}).run(); + else if (test_name == "search_near_01") + search_near_01(test_harness::test_args{config, test_name, wt_open_config}).run(); else { test_harness::logger::log_msg(LOG_ERROR, "Test not found: " + test_name); error_code = -1; @@ -139,8 +145,8 @@ main(int argc, char *argv[]) { std::string cfg, config_filename, current_cfg, current_test_name, test_name, wt_open_config; int64_t error_code = 0; - const std::vector<std::string> all_tests = { - "example_test", "burst_inserts", "hs_cleanup", "base_test"}; + const std::vector<std::string> all_tests = {"base_test", "burst_inserts", "example_test", + "hs_cleanup", "search_near_01", "search_near_02"}; /* Set the program name for error messages. */ (void)testutil_set_progname(argv); @@ -219,12 +225,20 @@ main(int argc, char *argv[]) } } else { current_test_name = test_name; - /* Configuration parsing. */ - if (!config_filename.empty()) - cfg = parse_configuration_from_file(config_filename); - else if (cfg.empty()) - cfg = parse_configuration_from_file(get_default_config_path(current_test_name)); - error_code = run_test(current_test_name, cfg, wt_open_config); + /* Check the test exists. */ + if (std::find(all_tests.begin(), all_tests.end(), current_test_name) == + all_tests.end()) { + test_harness::logger::log_msg( + LOG_ERROR, "The test " + current_test_name + " was not found."); + error_code = -1; + } else { + /* Configuration parsing. */ + if (!config_filename.empty()) + cfg = parse_configuration_from_file(config_filename); + else if (cfg.empty()) + cfg = parse_configuration_from_file(get_default_config_path(current_test_name)); + error_code = run_test(current_test_name, cfg, wt_open_config); + } } if (error_code != 0) diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/search_near_01.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_01.cxx new file mode 100644 index 00000000000..abacf3dc196 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_01.cxx @@ -0,0 +1,262 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "test_harness/util/api_const.h" +#include "test_harness/workload/random_generator.h" +#include "test_harness/workload/thread_context.h" +#include "test_harness/test.h" +#include "test_harness/thread_manager.h" + +using namespace test_harness; +/* + * In this test, we want to verify that search_near with prefix enabled only traverses the portion + * of the tree that follows the prefix portion of the search key. The test is composed of a populate + * phase followed by a read phase. The populate phase will insert a set of random generated keys + * with a prefix of aaa -> zzz. The read phase will continuously perform prefix search near calls, + * and validate that the number of entries traversed is within bounds of the search key. + */ +class search_near_01 : public test_harness::test { + uint64_t keys_per_prefix = 0; + uint64_t srchkey_len = 0; + const std::string ALPHABET{"abcdefghijklmnopqrstuvwxyz"}; + const uint64_t PREFIX_KEY_LEN = 3; + + static void + populate_worker(thread_context *tc, const std::string &ALPHABET, uint64_t PREFIX_KEY_LEN) + { + logger::log_msg(LOG_INFO, "Populate with thread id: " + std::to_string(tc->id)); + + std::string prefix_key; + uint64_t collections_per_thread = tc->collection_count; + const uint64_t MAX_ROLLBACKS = 100; + uint32_t rollback_retries = 0; + int cmpp; + + /* + * Generate a table of data with prefix keys aaa -> zzz. We have 26 threads from ids + * starting from 0 to 26. Each populate thread will insert separate prefix keys based on the + * id. + */ + for (int64_t i = 0; i < collections_per_thread; ++i) { + collection &coll = tc->db.get_collection(i); + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); + for (uint64_t j = 0; j < ALPHABET.size(); ++j) { + for (uint64_t k = 0; k < ALPHABET.size(); ++k) { + for (uint64_t count = 0; count < tc->key_count; ++count) { + tc->transaction.begin(); + /* + * Generate the prefix key, and append a random generated key string based + * on the key size configuration. + */ + prefix_key = {ALPHABET.at(tc->id), ALPHABET.at(j), ALPHABET.at(k)}; + prefix_key += random_generator::instance().generate_random_string( + tc->key_size - PREFIX_KEY_LEN); + if (!tc->insert(cursor, coll.id, prefix_key)) { + testutil_assert(rollback_retries < MAX_ROLLBACKS); + /* We failed to insert, rollback our transaction and retry. */ + tc->transaction.rollback(); + ++rollback_retries; + --count; + } else { + /* Commit txn at commit timestamp 100. */ + tc->transaction.commit( + "commit_timestamp=" + tc->tsm->decimal_to_hex(100)); + rollback_retries = 0; + } + } + } + } + } + } + + public: + search_near_01(const test_harness::test_args &args) : test(args) {} + + void + populate(test_harness::database &database, test_harness::timestamp_manager *tsm, + test_harness::configuration *config, test_harness::workload_tracking *tracking) override final + { + uint64_t collection_count, key_size; + std::vector<thread_context *> workers; + thread_manager tm; + + /* Validate our config. */ + collection_count = config->get_int(COLLECTION_COUNT); + keys_per_prefix = config->get_int(KEY_COUNT_PER_COLLECTION); + key_size = config->get_int(KEY_SIZE); + testutil_assert(collection_count > 0); + testutil_assert(keys_per_prefix > 0); + /* Check the prefix length is not greater than the key size. */ + testutil_assert(key_size >= PREFIX_KEY_LEN); + + logger::log_msg(LOG_INFO, + "Populate configuration with key size: " + std::to_string(key_size) + + " key count: " + std::to_string(keys_per_prefix) + + " number of collections: " + std::to_string(collection_count)); + + /* Create n collections as per the configuration. */ + for (uint64_t i = 0; i < collection_count; ++i) + /* + * The database model will call into the API and create the collection, with its own + * session. + */ + database.add_collection(); + + /* Spawn 26 threads to populate the database. */ + for (uint64_t i = 0; i < ALPHABET.size(); ++i) { + thread_context *tc = new thread_context(i, thread_type::INSERT, config, + connection_manager::instance().create_session(), tsm, tracking, database); + workers.push_back(tc); + tm.add_thread(populate_worker, tc, ALPHABET, PREFIX_KEY_LEN); + } + + /* Wait for our populate threads to finish and then join them. */ + logger::log_msg(LOG_INFO, "Populate: waiting for threads to complete."); + tm.join(); + + /* Cleanup our workers. */ + for (auto &it : workers) { + delete it; + it = nullptr; + } + + /* Force evict all the populated keys in all of the collections. */ + int cmpp; + scoped_session session = connection_manager::instance().create_session(); + for (uint64_t count = 0; count < collection_count; ++count) { + collection &coll = database.get_collection(count); + scoped_cursor evict_cursor = + session.open_scoped_cursor(coll.name.c_str(), "debug=(release_evict=true)"); + + for (uint64_t i = 0; i < ALPHABET.size(); ++i) { + for (uint64_t j = 0; j < ALPHABET.size(); ++j) { + for (uint64_t k = 0; k < ALPHABET.size(); ++k) { + std::string key = {ALPHABET.at(i), ALPHABET.at(j), ALPHABET.at(k)}; + evict_cursor->set_key(evict_cursor.get(), key.c_str()); + evict_cursor->search_near(evict_cursor.get(), &cmpp); + testutil_check(evict_cursor->reset(evict_cursor.get())); + } + } + } + } + srchkey_len = + random_generator::instance().generate_integer(static_cast<uint64_t>(1), PREFIX_KEY_LEN); + logger::log_msg(LOG_INFO, "Populate: finished."); + } + + void + read_operation(test_harness::thread_context *tc) override final + { + /* Make sure that thread statistics cursor is null before we open it. */ + testutil_assert(tc->stat_cursor.get() == nullptr); + logger::log_msg( + LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); + std::map<uint64_t, scoped_cursor> cursors; + tc->stat_cursor = tc->session.open_scoped_cursor(STATISTICS_URI); + std::string srch_key; + int64_t entries_stat, prefix_stat, prev_entries_stat, prev_prefix_stat, expected_entries; + int cmpp; + + cmpp = 0; + prev_entries_stat = 0; + prev_prefix_stat = 0; + + /* + * The number of expected entries is calculated to account for the maximum allowed entries + * per search near function call. The key we search near can be different in length, which + * will increase the number of entries search by a factor of 26. + */ + expected_entries = tc->thread_count * keys_per_prefix * 2 * + pow(ALPHABET.size(), PREFIX_KEY_LEN - srchkey_len); + + /* + * Read at timestamp 10, so that no keys are visible to this transaction. This allows prefix + * search near to early exit out of it's prefix range when it's trying to search for a + * visible key in the tree. + */ + tc->transaction.begin("read_timestamp=" + tc->tsm->decimal_to_hex(10)); + while (tc->running()) { + + /* Get a collection and find a cached cursor. */ + collection &coll = tc->db.get_random_collection(); + if (cursors.find(coll.id) == cursors.end()) { + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); + cursor->reconfigure(cursor.get(), "prefix_search=true"); + cursors.emplace(coll.id, std::move(cursor)); + } + + /* Generate search prefix key of random length between a -> zzz. */ + srch_key = random_generator::instance().generate_random_string( + srchkey_len, characters_type::ALPHABET); + logger::log_msg(LOG_INFO, + "Read thread {" + std::to_string(tc->id) + + "} performing prefix search near with key: " + srch_key); + + /* Do a second lookup now that we know it exists. */ + auto &cursor = cursors[coll.id]; + if (tc->transaction.active()) { + runtime_monitor::get_stat( + tc->stat_cursor, WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100, &prev_entries_stat); + runtime_monitor::get_stat(tc->stat_cursor, + WT_STAT_CONN_CURSOR_SEARCH_NEAR_PREFIX_FAST_PATHS, &prev_prefix_stat); + + cursor->set_key(cursor.get(), srch_key.c_str()); + testutil_assert(cursor->search_near(cursor.get(), &cmpp) == WT_NOTFOUND); + + runtime_monitor::get_stat( + tc->stat_cursor, WT_STAT_CONN_CURSOR_NEXT_SKIP_LT_100, &entries_stat); + runtime_monitor::get_stat( + tc->stat_cursor, WT_STAT_CONN_CURSOR_SEARCH_NEAR_PREFIX_FAST_PATHS, &prefix_stat); + logger::log_msg(LOG_INFO, + "Read thread {" + std::to_string(tc->id) + + "} skipped entries: " + std::to_string(entries_stat - prev_entries_stat) + + " prefix fash path: " + std::to_string(prefix_stat - prev_prefix_stat)); + + /* + * It is possible that WiredTiger increments the entries skipped stat irrelevant to + * prefix search near. This is dependent on how many read threads are present in the + * test. Account for this by creating a small buffer using thread count. Assert that + * the number of expected entries is the upper limit which the prefix search near + * can traverse and the prefix fast path is incremented. + */ + testutil_assert( + (expected_entries + (2 * tc->thread_count)) >= entries_stat - prev_entries_stat); + testutil_assert(prefix_stat > prev_prefix_stat); + + tc->transaction.add_op(); + tc->sleep(); + } + /* Reset our cursor to avoid pinning content. */ + testutil_check(cursor->reset(cursor.get())); + } + tc->transaction.commit(); + /* Make sure the last transaction is rolled back now the work is finished. */ + if (tc->transaction.active()) + tc->transaction.rollback(); + } +}; diff --git a/src/third_party/wiredtiger/test/cppsuite/tests/search_near_02.cxx b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_02.cxx new file mode 100644 index 00000000000..1df75a83bc0 --- /dev/null +++ b/src/third_party/wiredtiger/test/cppsuite/tests/search_near_02.cxx @@ -0,0 +1,425 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "test_harness/test.h" +#include "test_harness/util/api_const.h" +#include "test_harness/workload/random_generator.h" + +using namespace test_harness; + +/* + * In this test, we want to verify search_near with prefix enabled returns the correct key. + * During the test duration: + * - N threads will keep inserting new random keys + * - M threads will execute search_near calls with prefix enabled using random prefixes as well. + * Each search_near call with prefix enabled is verified using the default search_near. + */ +class search_near_02 : public test_harness::test { + public: + search_near_02(const test_harness::test_args &args) : test(args) {} + + void + populate(test_harness::database &database, test_harness::timestamp_manager *, + test_harness::configuration *config, test_harness::workload_tracking *) override final + { + /* + * The populate phase only creates empty collections. The number of collections is defined + * in the configuration. + */ + int64_t collection_count = config->get_int(COLLECTION_COUNT); + + logger::log_msg( + LOG_INFO, "Populate: " + std::to_string(collection_count) + " creating collections."); + + for (uint64_t i = 0; i < collection_count; ++i) + database.add_collection(); + + logger::log_msg(LOG_INFO, "Populate: finished."); + } + + void + insert_operation(test_harness::thread_context *tc) override final + { + /* Each insert operation will insert new keys in the collections. */ + logger::log_msg( + LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); + + /* Helper struct which stores a pointer to a collection and a cursor associated with it. */ + struct collection_cursor { + collection_cursor(collection &coll, scoped_cursor &&cursor) + : coll(coll), cursor(std::move(cursor)) + { + } + collection &coll; + scoped_cursor cursor; + }; + + /* Collection cursor vector. */ + std::vector<collection_cursor> ccv; + int64_t collection_count = tc->db.get_collection_count(); + int64_t collections_per_thread = collection_count / tc->thread_count; + + /* Must have unique collections for each thread. */ + testutil_assert(collection_count % tc->thread_count == 0); + const uint64_t thread_offset = tc->id * collections_per_thread; + for (uint64_t i = thread_offset; + i < thread_offset + collections_per_thread && tc->running(); ++i) { + collection &coll = tc->db.get_collection(i); + scoped_cursor cursor = tc->session.open_scoped_cursor(coll.name.c_str()); + ccv.push_back({coll, std::move(cursor)}); + } + + std::string key; + uint64_t counter = 0; + + while (tc->running()) { + + auto &cc = ccv[counter]; + tc->transaction.begin(); + + while (tc->transaction.active() && tc->running()) { + + /* Generate a random key. */ + key = random_generator::instance().generate_random_string(tc->key_size); + + /* Insert a key value pair. */ + if (tc->insert(cc.cursor, cc.coll.id, key)) { + if (tc->transaction.can_commit()) + /* We are not checking the result of commit as it is not necessary. */ + tc->transaction.commit(); + } else { + tc->transaction.rollback(); + } + + /* Sleep the duration defined by the configuration. */ + tc->sleep(); + } + + /* Rollback any transaction that could not commit before the end of the test. */ + if (tc->transaction.active()) + tc->transaction.rollback(); + + /* Reset our cursor to avoid pinning content. */ + testutil_check(cc.cursor->reset(cc.cursor.get())); + if (++counter == ccv.size()) + counter = 0; + testutil_assert(counter < collections_per_thread); + } + } + + void + read_operation(test_harness::thread_context *tc) override final + { + /* + * Each read operation performs search_near calls with and without prefix enabled on random + * collections. Each prefix is randomly generated. The result of the seach_near call with + * prefix enabled is then validated using the search_near call without prefix enabled. + */ + logger::log_msg( + LOG_INFO, type_string(tc->type) + " thread {" + std::to_string(tc->id) + "} commencing."); + + const char *key_prefix; + int exact_prefix, ret; + int64_t prefix_size; + std::map<uint64_t, scoped_cursor> cursors; + std::string generated_prefix, key_prefix_str; + + while (tc->running()) { + /* Get a random collection to work on. */ + collection &coll = tc->db.get_random_collection(); + + /* Find a cached cursor or create one if none exists. */ + if (cursors.find(coll.id) == cursors.end()) { + cursors.emplace( + coll.id, std::move(tc->session.open_scoped_cursor(coll.name.c_str()))); + auto &cursor_prefix = cursors[coll.id]; + /* The cached cursors have the prefix configuration enabled. */ + testutil_check( + cursor_prefix.get()->reconfigure(cursor_prefix.get(), "prefix_search=true")); + } + + auto &cursor_prefix = cursors[coll.id]; + + /* + * Pick a random timestamp between the oldest and now. Get rid of the last 32 bits as + * they represent an increment for uniqueness. + */ + wt_timestamp_t ts = random_generator::instance().generate_integer( + (tc->tsm->get_oldest_ts() >> 32), (tc->tsm->get_next_ts() >> 32)); + /* Put back the timestamp in the correct format. */ + ts <<= 32; + + /* + * The oldest timestamp might move ahead and the reading timestamp might become invalid. + * To tackle this issue, we round the timestamp to the oldest timestamp value. + */ + tc->transaction.begin( + "roundup_timestamps=(read=true),read_timestamp=" + tc->tsm->decimal_to_hex(ts)); + + while (tc->transaction.active() && tc->running()) { + /* + * Generate a random prefix. For this, we start by generating a random size and then + * its value. + */ + prefix_size = random_generator::instance().generate_integer( + static_cast<int64_t>(1), tc->key_size); + generated_prefix = random_generator::instance().generate_random_string( + prefix_size, characters_type::ALPHABET); + + /* Call search near with the prefix cursor. */ + cursor_prefix->set_key(cursor_prefix.get(), generated_prefix.c_str()); + ret = cursor_prefix->search_near(cursor_prefix.get(), &exact_prefix); + testutil_assert(ret == 0 || ret == WT_NOTFOUND); + if (ret == 0) { + testutil_check(cursor_prefix->get_key(cursor_prefix.get(), &key_prefix)); + key_prefix_str = key_prefix; + } else { + key_prefix_str = ""; + } + + /* Open a cursor with the default configuration on the selected collection. */ + scoped_cursor cursor_default(tc->session.open_scoped_cursor(coll.name.c_str())); + + /* Verify the prefix search_near output using the default cursor. */ + validate_prefix_search_near( + ret, exact_prefix, key_prefix_str, cursor_default, generated_prefix); + + tc->transaction.add_op(); + tc->transaction.try_rollback(); + tc->sleep(); + } + testutil_check(cursor_prefix->reset(cursor_prefix.get())); + } + /* Roll back the last transaction if still active now the work is finished. */ + if (tc->transaction.active()) + tc->transaction.rollback(); + } + + private: + /* Validate prefix search_near call outputs using a cursor without prefix key enabled. */ + void + validate_prefix_search_near(int ret_prefix, int exact_prefix, const std::string &key_prefix, + scoped_cursor &cursor_default, const std::string &prefix) + { + /* Call search near with the default cursor using the given prefix. */ + int exact_default; + cursor_default->set_key(cursor_default.get(), prefix.c_str()); + int ret_default = cursor_default->search_near(cursor_default.get(), &exact_default); + + /* + * It is not possible to have a prefix search near call successful and the default search + * near call unsuccessful. + */ + testutil_assert( + ret_default == ret_prefix || (ret_default == 0 && ret_prefix == WT_NOTFOUND)); + + /* We only have to perform validation when the default search near call is successful. */ + if (ret_default == 0) { + /* Both calls are successful. */ + if (ret_prefix == 0) + validate_successful_calls( + ret_prefix, exact_prefix, key_prefix, cursor_default, exact_default, prefix); + /* The prefix search near call failed. */ + else + validate_unsuccessful_prefix_call(cursor_default, prefix, exact_default); + } + } + + /* + * Validate a successful prefix enabled search near call using a successful default search near + * call. + * The exact value set by the prefix search near call has to be either 0 or 1. Indeed, it cannot + * be -1 as the key needs to contain the prefix. + * - If it is 0, both search near calls should return the same outputs and both cursors should + * be positioned on the prefix we are looking for. + * - If it is 1, it will depend on the exact value set by the default search near call which can + * be -1 or 1. If it is -1, calling next on the default cursor should get us ti the key found by + * the prefix search near call. If it is 1, it means both search near calls have found the same + * key that is lexicographically greater than the prefix but still contains the prefix. + */ + void + validate_successful_calls(int ret_prefix, int exact_prefix, const std::string &key_prefix, + scoped_cursor &cursor_default, int exact_default, const std::string &prefix) + { + const char *k; + std::string k_str; + int ret; + + /* + * The prefix search near call cannot retrieve a key with a smaller value than the prefix we + * searched. + */ + testutil_assert(exact_prefix >= 0); + + /* The key at the prefix cursor should contain the prefix. */ + testutil_assert(key_prefix.substr(0, prefix.size()) == prefix); + + /* Retrieve the key the default cursor is pointing at. */ + const char *key_default; + testutil_check(cursor_default->get_key(cursor_default.get(), &key_default)); + std::string key_default_str = key_default; + + logger::log_msg(LOG_TRACE, + "search_near (normal) exact " + std::to_string(exact_default) + " key " + key_default); + logger::log_msg(LOG_TRACE, + "search_near (prefix) exact " + std::to_string(exact_prefix) + " key " + key_prefix); + + /* Example: */ + /* keys: a, bb, bba. */ + /* Only bb is not visible. */ + /* Default search_near(bb) returns a, exact < 0. */ + /* Prefix search_near(bb) returns bba, exact > 0. */ + if (exact_default < 0) { + /* The key at the default cursor should not contain the prefix. */ + testutil_assert((key_default_str.substr(0, prefix.size()) != prefix)); + + /* + * The prefix cursor should be positioned at a key lexicographically greater than the + * prefix. + */ + testutil_assert(exact_prefix > 0); + + /* + * The next key of the default cursor should be equal to the key pointed by the prefix + * cursor. + */ + testutil_assert(cursor_default->next(cursor_default.get()) == 0); + testutil_check(cursor_default->get_key(cursor_default.get(), &k)); + testutil_assert(k == key_prefix); + } + /* Example: */ + /* keys: a, bb, bba */ + /* Case 1: all keys are visible. */ + /* Default search_near(bb) returns bb, exact = 0 */ + /* Prefix search_near(bb) returns bb, exact = 0 */ + /* Case 2: only bb is not visible. */ + /* Default search_near(bb) returns bba, exact > 0. */ + /* Prefix search_near(bb) returns bba, exact > 0. */ + else { + /* Both cursors should be pointing at the same key. */ + testutil_assert(exact_prefix == exact_default); + testutil_assert(key_default_str == key_prefix); + /* Both cursors should have found the exact key. */ + if (exact_default == 0) + testutil_assert(key_default_str == prefix); + /* Both cursors have found a key that is lexicographically greater than the prefix. */ + else + testutil_assert(key_default_str != prefix); + } + } + + /* + * Validate that no keys with the prefix used for the search have been found. + * To validate this, we can use the exact value set by the default search near. Since the prefix + * search near failed, the exact value set by the default search near call has to be either -1 + * or 1: + * - If it is -1, we need to check the next key, if it exists, is lexicographically greater than + * the prefix we looked for. + * - If it is 1, we need to check the previous keys, if it exists, if lexicographically smaller + * than the prefix we looked for. + */ + void + validate_unsuccessful_prefix_call( + scoped_cursor &cursor_default, const std::string &prefix, int exact_default) + { + int ret; + const char *k; + std::string k_str; + + /* + * The exact value from the default search near call cannot be 0, otherwise the prefix + * search near should be successful too. + */ + testutil_assert(exact_default != 0); + + /* Retrieve the key at the default cursor. */ + const char *key_default; + testutil_check(cursor_default->get_key(cursor_default.get(), &key_default)); + std::string key_default_str = key_default; + + /* The key at the default cursor should not contain the prefix. */ + testutil_assert(key_default_str.substr(0, prefix.size()) != prefix); + + /* Example: */ + /* keys: a, bb, bbb. */ + /* All keys are visible. */ + /* Default search_near(bba) returns bb, exact < 0. */ + /* Prefix search_near(bba) returns WT_NOTFOUND. */ + if (exact_default < 0) { + /* + * The current key of the default cursor should be lexicographically smaller than the + * prefix. + */ + testutil_assert(std::lexicographical_compare( + key_default_str.begin(), key_default_str.end(), prefix.begin(), prefix.end())); + + /* + * The next key of the default cursor should be lexicographically greater than the + * prefix if it exists. + */ + ret = cursor_default->next(cursor_default.get()); + if (ret == 0) { + testutil_check(cursor_default->get_key(cursor_default.get(), &k)); + k_str = k; + testutil_assert(!std::lexicographical_compare( + k_str.begin(), k_str.end(), prefix.begin(), prefix.end())); + } else { + /* End of the table. */ + testutil_assert(ret == WT_NOTFOUND); + } + } + /* Example: */ + /* keys: a, bb, bbb. */ + /* All keys are visible. */ + /* Default search_near(bba) returns bbb, exact > 0. */ + /* Prefix search_near(bba) returns WT_NOTFOUND. */ + else { + /* + * The current key of the default cursor should be lexicographically greater than the + * prefix. + */ + testutil_assert(!std::lexicographical_compare( + key_default_str.begin(), key_default_str.end(), prefix.begin(), prefix.end())); + + /* + * The next key of the default cursor should be lexicographically smaller than the + * prefix if it exists. + */ + ret = cursor_default->prev(cursor_default.get()); + if (ret == 0) { + testutil_check(cursor_default->get_key(cursor_default.get(), &k)); + k_str = k; + testutil_assert(std::lexicographical_compare( + k_str.begin(), k_str.end(), prefix.begin(), prefix.end())); + } else { + /* End of the table. */ + testutil_assert(ret == WT_NOTFOUND); + } + } + } +}; diff --git a/src/third_party/wiredtiger/test/csuite/random_abort/main.c b/src/third_party/wiredtiger/test/csuite/random_abort/main.c index 26789a1ac9b..ce935d26e3f 100644 --- a/src/third_party/wiredtiger/test/csuite/random_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/random_abort/main.c @@ -38,6 +38,7 @@ static char home[1024]; /* Program working dir */ */ static const char *const col_uri = "table:col_main"; static const char *const uri = "table:main"; +static bool compaction; static bool compat; static bool inmem; @@ -218,6 +219,17 @@ thread_run(void *arg) testutil_die(errno, "fprintf"); /* + * If configured, run compaction on database after each epoch of 100000 operations. + */ + if (compaction && i >= 100000 && i % 100000 == 0) { + printf("Running compaction in Thread %" PRIu32 "\n", td->id); + if (columnar_table) + testutil_check(session->compact(session, col_uri, NULL)); + else + testutil_check(session->compact(session, uri, NULL)); + } + + /* * Decide what kind of operation can be performed on the already inserted data. */ if (i % MAX_NUM_OPS == OP_TYPE_DELETE) { @@ -607,18 +619,21 @@ main(int argc, char *argv[]) (void)testutil_set_progname(argv); - compat = inmem = false; + compaction = compat = inmem = false; nth = MIN_TH; rand_th = rand_time = true; timeout = MIN_TIME; verify_only = false; working_dir = "WT_TEST.random-abort"; - while ((ch = __wt_getopt(progname, argc, argv, "Ch:mT:t:v")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "Cch:mT:t:v")) != EOF) switch (ch) { case 'C': compat = true; break; + case 'c': + compaction = true; + break; case 'h': working_dir = __wt_optarg; break; @@ -669,8 +684,9 @@ main(int argc, char *argv[]) printf("Parent: Compatibility %s in-mem log %s\n", compat ? "true" : "false", inmem ? "true" : "false"); printf("Parent: Create %" PRIu32 " threads; sleep %" PRIu32 " seconds\n", nth, timeout); - printf("CONFIG: %s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n", progname, - compat ? " -C" : "", inmem ? " -m" : "", working_dir, nth, timeout); + printf("CONFIG: %s%s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n", progname, + compat ? " -C" : "", compaction ? " -c" : "", inmem ? " -m" : "", working_dir, nth, + timeout); /* * Fork a child to insert as many items. We will then randomly kill the child, run recovery * and make sure all items we wrote exist after recovery runs. diff --git a/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c b/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c index c29460b8f46..8111cd108ac 100644 --- a/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/tiered_abort/main.c @@ -55,15 +55,17 @@ static char home[1024]; /* Program working dir */ * Also each worker thread creates its own textual records file that records the data it * inserted and it records the timestamp that was used for that insertion. */ +#define LOCAL_RETENTION 2 /* Local retention time */ +#define MIN_TIME LOCAL_RETENTION * 8 /* Make sure checkpoint and flush_tier run enough */ +#define MAX_TIME MIN_TIME * 4 + #define BUCKET "bucket" #define INVALID_KEY UINT64_MAX -#define MAX_CKPT_INVL 5 /* Maximum interval between checkpoints */ -#define MAX_FLUSH_INVL 5 /* Maximum interval between flush_tier calls */ -#define MAX_TH 20 /* Maximum configurable threads */ -#define MAX_TIME 40 +#define MAX_CKPT_INVL LOCAL_RETENTION * 3 /* Maximum interval between checkpoints */ +#define MAX_FLUSH_INVL LOCAL_RETENTION * 2 /* Maximum interval between flush_tier calls */ +#define MAX_TH 20 /* Maximum configurable threads */ #define MAX_VAL 1024 #define MIN_TH 5 -#define MIN_TIME 10 #define NUM_INT_THREADS 3 #define RECORDS_FILE "records-%" PRIu32 /* Include worker threads and extra sessions */ @@ -96,7 +98,7 @@ static uint32_t flush_calls = 1; "eviction_updates_target=20,eviction_updates_trigger=90," \ "log=(archive=true,file_max=10M,enabled),session_max=%d," \ "statistics=(fast),statistics_log=(wait=1,json=true)," \ - "tiered_storage=(bucket=%s,bucket_prefix=pfx,name=local_store)" + "tiered_storage=(bucket=%s,bucket_prefix=pfx,local_retention=%d,name=local_store)" #define ENV_CONFIG_TXNSYNC \ ENV_CONFIG_DEF \ ",eviction_dirty_target=20,eviction_dirty_trigger=90" \ @@ -440,8 +442,8 @@ run_workload(uint32_t nth, const char *build_dir) if (chdir(home) != 0) testutil_die(errno, "Child chdir: %s", home); - testutil_check( - __wt_snprintf(envconf, sizeof(envconf), ENV_CONFIG_TXNSYNC, cache_mb, SESSION_MAX, BUCKET)); + testutil_check(__wt_snprintf(envconf, sizeof(envconf), ENV_CONFIG_TXNSYNC, cache_mb, + SESSION_MAX, BUCKET, LOCAL_RETENTION)); testutil_check(__wt_snprintf(extconf, sizeof(extconf), ",extensions=(%s/%s=(early_load=true))", build_dir, WT_STORAGE_LIB)); @@ -490,7 +492,6 @@ run_workload(uint32_t nth, const char *build_dir) testutil_check(__wt_thread_create(NULL, &thr[ts_id], thread_ts_run, &td[ts_id])); } printf("Create %" PRIu32 " writer threads\n", nth); - printf("Create %" PRIu32 " writer threads\n", nth); for (i = 0; i < nth; ++i) { td[i].conn = conn; td[i].start = WT_BILLION * (uint64_t)i; diff --git a/src/third_party/wiredtiger/test/csuite/wt3338_partial_update/main.c b/src/third_party/wiredtiger/test/csuite/wt3338_partial_update/main.c index 8b6b9c5f841..569339eb4af 100644 --- a/src/third_party/wiredtiger/test/csuite/wt3338_partial_update/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt3338_partial_update/main.c @@ -111,94 +111,6 @@ modify_build(void) } /* - * slow_apply_api -- - * Apply a set of modification changes using a different algorithm. - */ -static void -slow_apply_api(WT_ITEM *orig) -{ - static WT_ITEM _tb; - WT_ITEM *ta, *tb, *tmp, _tmp; - size_t len, size; - int i; - - ta = orig; - tb = &_tb; - - /* Mess up anything not initialized in the buffers. */ - if ((ta->memsize - ta->size) > 0) - memset((uint8_t *)ta->mem + ta->size, 0xff, ta->memsize - ta->size); - - if (tb->memsize > 0) - memset((uint8_t *)tb->mem, 0xff, tb->memsize); - - /* - * Process the entries to figure out how large a buffer we need. This is a bit pessimistic - * because we're ignoring replacement bytes, but it's a simpler calculation. - */ - for (size = ta->size, i = 0; i < nentries; ++i) { - if (entries[i].offset >= size) - size = entries[i].offset; - size += entries[i].data.size; - } - - testutil_check(__wt_buf_grow(NULL, ta, size)); - testutil_check(__wt_buf_grow(NULL, tb, size)); - -#if DEBUG - show(ta, "slow-apply start"); -#endif - /* - * From the starting buffer, create a new buffer b based on changes in the entries array. We're - * doing a brute force solution here to test the faster solution implemented in the library. - */ - for (i = 0; i < nentries; ++i) { - /* Take leading bytes from the original, plus any gap bytes. */ - if (entries[i].offset >= ta->size) { - memcpy(tb->mem, ta->mem, ta->size); - if (entries[i].offset > ta->size) - memset((uint8_t *)tb->mem + ta->size, '\0', entries[i].offset - ta->size); - } else if (entries[i].offset > 0) - memcpy(tb->mem, ta->mem, entries[i].offset); - tb->size = entries[i].offset; - - /* Take replacement bytes. */ - if (entries[i].data.size > 0) { - memcpy((uint8_t *)tb->mem + tb->size, entries[i].data.data, entries[i].data.size); - tb->size += entries[i].data.size; - } - - /* Take trailing bytes from the original. */ - len = entries[i].offset + entries[i].size; - if (ta->size > len) { - memcpy((uint8_t *)tb->mem + tb->size, (uint8_t *)ta->mem + len, ta->size - len); - tb->size += ta->size - len; - } - testutil_assert(tb->size <= size); - - /* Swap the buffers and do it again. */ - tmp = ta; - ta = tb; - tb = tmp; - } - ta->data = ta->mem; - tb->data = tb->mem; - - /* - * The final results may not be in the original buffer, in which case we swap them back around. - */ - if (ta != orig) { - _tmp = *ta; - *ta = *tb; - *tb = _tmp; - } - -#if DEBUG - show(ta, "slow-apply finish"); -#endif -} - -/* * compare -- * Compare two results. */ @@ -249,6 +161,7 @@ modify_run(TEST_OPTS *opts) WT_CURSOR *cursor, _cursor; WT_DECL_RET; WT_ITEM *localA, _localA, *localB, _localB; + WT_ITEM modtmp; WT_SESSION_IMPL *session; size_t len; int i, j; @@ -264,7 +177,6 @@ modify_run(TEST_OPTS *opts) /* Set up replacement information. */ modify_repl_init(); - /* We need three WT_ITEMs, one of them part of a fake cursor. */ localA = &_localA; memset(&_localA, 0, sizeof(_localA)); localB = &_localB; @@ -273,6 +185,7 @@ modify_run(TEST_OPTS *opts) memset(&_cursor, 0, sizeof(_cursor)); cursor->session = (WT_SESSION *)session; cursor->value_format = "u"; + memset(&modtmp, 0, sizeof(modtmp)); #define NRUNS 10000 for (i = 0; i < NRUNS; ++i) { @@ -295,7 +208,7 @@ modify_run(TEST_OPTS *opts) modify_build(); testutil_check(__wt_buf_set(session, &cursor->value, localA->data, localA->size)); testutil_check(__wt_modify_apply_api(cursor, entries, nentries)); - slow_apply_api(localA); + testutil_modify_apply(localA, &modtmp, entries, nentries); compare(localB, localA, &cursor->value); /* @@ -324,6 +237,7 @@ modify_run(TEST_OPTS *opts) __wt_buf_free(session, localA); __wt_buf_free(session, localB); __wt_buf_free(session, &cursor->value); + __wt_buf_free(session, &modtmp); } int diff --git a/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c b/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c index 6e388b54535..a93d83b0985 100644 --- a/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c @@ -37,6 +37,7 @@ */ #define NUM_RECORDS 1000000 +#define CHECKPOINT_NUM 3 /* Constants and variables declaration. */ /* @@ -78,11 +79,13 @@ main(int argc, char *argv[]) /* * First run test with WT_TIMING_STRESS_CHECKPOINT_SLOW. */ + printf("Running stress test...\n"); run_test(true, opts->home, opts->uri); /* * Now run test where compact and checkpoint threads are synchronized using condition variable. */ + printf("Running normal test...\n"); testutil_assert(sizeof(home_cv) > strlen(opts->home) + 3); sprintf(home_cv, "%s.CV", opts->home); run_test(false, home_cv, opts->uri); @@ -110,8 +113,8 @@ run_test(bool stress_test, const char *home, const char *uri) if (stress_test) { /* - * Set WT_TIMING_STRESS_CHECKPOINT_SLOW flag. It adds 10 seconds sleep before each - * checkpoint. + * Set WT_TIMING_STRESS_CHECKPOINT_SLOW flag for stress test. It adds 10 seconds sleep + * before each checkpoint. */ set_timing_stress_checkpoint(conn); } @@ -161,17 +164,17 @@ run_test(bool stress_test, const char *home, const char *uri) } testutil_check(session->close(session, NULL)); + session = NULL; + + testutil_check(conn->close(conn, NULL)); + conn = NULL; /* Check if there's at least 10% compaction. */ printf(" - Compressed file size MB: %f\n - Original file size MB: %f\n", file_sz_after / (1024.0 * 1024), file_sz_before / (1024.0 * 1024)); - /* - * FIXME-WT-8055 At the moment the assert below is commented out to prevent evergreen from going - * red. Please enable the assert as soon as the underlying defect is fixed and compact does its - * job well. - */ - /*testutil_assert(file_sz_before * 0.9 > file_sz_after);*/ + /* Make sure the compact operation has reduced the file size by at least 20%. */ + testutil_assert((file_sz_before / 100) * 80 > file_sz_after); } static void * @@ -198,7 +201,9 @@ thread_func_compact(void *arg) /* Perform compact operation. */ testutil_check(session->compact(session, td->uri, NULL)); + testutil_check(session->close(session, NULL)); + session = NULL; return (NULL); } @@ -219,13 +224,18 @@ static void * thread_func_checkpoint(void *arg) { struct thread_data *td; + WT_RAND_STATE rnd; WT_SESSION *session; + uint64_t sleep_sec; + int i; bool signalled; td = (struct thread_data *)arg; testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); + if (td->cond != NULL) { printf("Waiting for the signal...\n"); /* @@ -238,8 +248,22 @@ thread_func_checkpoint(void *arg) printf("Signal received!\n"); } - testutil_check(session->checkpoint(session, NULL)); + /* + * Run several checkpoints. First one without any delay. Others will have a random delay before + * start. + */ + for (i = 0; i < CHECKPOINT_NUM; i++) { + testutil_check(session->checkpoint(session, NULL)); + + if (i < CHECKPOINT_NUM - 1) { + sleep_sec = (uint64_t)__wt_random(&rnd) % 15 + 1; + printf("Sleep %" PRIu64 " sec before next checkpoint.\n", sleep_sec); + __wt_sleep(sleep_sec, 0); + } + } + testutil_check(session->close(session, NULL)); + session = NULL; return (NULL); } @@ -248,27 +272,28 @@ static void populate(WT_SESSION *session, const char *uri) { WT_CURSOR *cursor; - time_t t; + WT_RAND_STATE rnd; uint64_t val; int i, str_len; - srand((u_int)time(&t)); + __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); str_len = sizeof(data_str) / sizeof(data_str[0]); for (i = 0; i < str_len - 1; i++) - data_str[i] = 'a' + rand() % 26; + data_str[i] = 'a' + __wt_random(&rnd) % 26; data_str[str_len - 1] = '\0'; testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); for (i = 0; i < NUM_RECORDS; i++) { cursor->set_key(cursor, i); - val = (uint64_t)rand(); + val = (uint64_t)__wt_random(&rnd); cursor->set_value(cursor, val, val, val, data_str); testutil_check(cursor->insert(cursor)); } testutil_check(cursor->close(cursor)); + cursor = NULL; } static void @@ -286,6 +311,7 @@ remove_records(WT_SESSION *session, const char *uri) } testutil_check(cursor->close(cursor)); + cursor = NULL; } static uint64_t @@ -293,8 +319,7 @@ get_file_size(WT_SESSION *session, const char *uri) { WT_CURSOR *cur_stat; uint64_t val; - char *descr, *str_val; - char stat_uri[128]; + char *descr, *str_val, stat_uri[128]; sprintf(stat_uri, "statistics:%s", uri); testutil_check(session->open_cursor(session, stat_uri, NULL, "statistics=(all)", &cur_stat)); @@ -302,6 +327,7 @@ get_file_size(WT_SESSION *session, const char *uri) testutil_check(cur_stat->search(cur_stat)); testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, &val)); testutil_check(cur_stat->close(cur_stat)); + cur_stat = NULL; return (val); } diff --git a/src/third_party/wiredtiger/test/ctest_helpers.cmake b/src/third_party/wiredtiger/test/ctest_helpers.cmake index 1a8a7b4152e..a6afb901f71 100644 --- a/src/third_party/wiredtiger/test/ctest_helpers.cmake +++ b/src/third_party/wiredtiger/test/ctest_helpers.cmake @@ -153,16 +153,23 @@ function(define_test_variants target) 1 "DEFINE_TEST" "" - "" - "VARIANTS;LABELS" + "DIR_NAME" + "VARIANTS;LABELS;CMDS" ) if (NOT "${DEFINE_TEST_UNPARSED_ARGUMENTS}" STREQUAL "") - message(FATAL_ERROR "Unknown arguments to define_test_variants: ${DEFINE_TEST_VARIANTS_UNPARSED_ARGUMENTS}") + message(FATAL_ERROR "Unknown arguments to define_test_variants: ${DEFINE_TEST_UNPARSED_ARGUMENTS}") endif() if ("${DEFINE_TEST_VARIANTS}" STREQUAL "") message(FATAL_ERROR "Need at least one variant for define_test_variants") endif() + set(dir_prefix) + if(DEFINE_TEST_DIR_NAME) + set(dir_prefix ${CMAKE_CURRENT_BINARY_DIR}/${DEFINE_TEST_DIR_NAME}) + else() + set(dir_prefix ${CMAKE_CURRENT_BINARY_DIR}) + endif() + set(defined_tests) foreach(variant ${DEFINE_TEST_VARIANTS}) list(LENGTH variant variant_length) @@ -182,17 +189,23 @@ function(define_test_variants target) endif() # Create a variant directory to run the test in. add_custom_command(OUTPUT ${curr_variant_name}_test_dir - COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/${curr_variant_name}_test_dir + COMMAND ${CMAKE_COMMAND} -E make_directory ${dir_prefix}/${curr_variant_name}_test_dir ) add_custom_target(create_dir_${curr_variant_name} DEPENDS ${curr_variant_name}_test_dir) # Ensure the variant target is created prior to building the test. add_dependencies(${target} create_dir_${curr_variant_name}) + set(test_cmd) + if(DEFINE_TEST_CMDS) + set(test_cmd ${DEFINE_TEST_CMDS}) + else() + set(test_cmd $<TARGET_FILE:${target}>) + endif() add_test( NAME ${curr_variant_name} - COMMAND $<TARGET_FILE:${target}> ${variant_args} + COMMAND ${test_cmd} ${variant_args} # Run each variant in its own subdirectory, allowing us to execute variants in # parallel. - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${curr_variant_name}_test_dir + WORKING_DIRECTORY ${dir_prefix}/${curr_variant_name}_test_dir ) list(APPEND defined_tests ${curr_variant_name}) endforeach() @@ -201,13 +214,14 @@ function(define_test_variants target) endif() endfunction() -macro(define_c_test) +function(define_c_test) cmake_parse_arguments( + PARSE_ARGV + 0 "C_TEST" - "SMOKE" - "TARGET;DIR_NAME;DEPENDS" - "SOURCES;FLAGS;ARGUMENTS" - ${ARGN} + "" + "TARGET;DIR_NAME;DEPENDS;EXEC_SCRIPT" + "SOURCES;FLAGS;ARGUMENTS;VARIANTS" ) if (NOT "${C_TEST_UNPARSED_ARGUMENTS}" STREQUAL "") message(FATAL_ERROR "Unknown arguments to define_c_test: ${C_TEST_UNPARSED_ARGUMENTS}") @@ -222,6 +236,10 @@ macro(define_c_test) message(FATAL_ERROR "No directory given to define_c_test") endif() + if("${C_TEST_ARGUMENTS}" AND "${C_TEST_VARIANTS}") + message(FATAL_ERROR "Can't pass both ARGUMENTS and VARIANTS, use only one") + endif() + # Check that the csuite dependencies are enabled before compiling and creating the test. eval_dependency("${C_TEST_DEPENDS}" enabled) if(enabled) @@ -236,35 +254,40 @@ macro(define_c_test) # Which while technically valid breaks assumptions in our testing utilities. Wrap the execution in powershell to avoid this. set(exec_wrapper "powershell.exe") endif() - if (C_TEST_SMOKE) - # csuite test comes with a smoke execution wrapper. + set(test_cmd) + if (C_TEST_EXEC_SCRIPT) + # Define the c test to be executed with a script, rather than invoking the binary directly. create_test_executable(${C_TEST_TARGET} SOURCES ${C_TEST_SOURCES} - ADDITIONAL_FILES ${CMAKE_CURRENT_SOURCE_DIR}/${C_TEST_DIR_NAME}/smoke.sh + ADDITIONAL_FILES ${C_TEST_EXEC_SCRIPT} BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/${C_TEST_DIR_NAME} ${additional_executable_args} ) - add_test(NAME ${C_TEST_TARGET} - COMMAND ${exec_wrapper} ${CMAKE_CURRENT_BINARY_DIR}/${C_TEST_DIR_NAME}/smoke.sh ${C_TEST_ARGUMENTS} $<TARGET_FILE:${C_TEST_TARGET}> - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${C_TEST_DIR_NAME} - ) + get_filename_component(exec_script_basename ${C_TEST_EXEC_SCRIPT} NAME) + set(test_cmd ${exec_wrapper} ${exec_script_basename}) else() create_test_executable(${C_TEST_TARGET} SOURCES ${C_TEST_SOURCES} BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/${C_TEST_DIR_NAME} ${additional_executable_args} ) - # Take a CMake-based path and convert it to a platform-specfic path (/ for Unix, \ for Windows). - set(wt_test_home_dir ${CMAKE_CURRENT_BINARY_DIR}/${C_TEST_DIR_NAME}/WT_HOME_${C_TEST_TARGET}) - file(TO_NATIVE_PATH "${wt_test_home_dir}" wt_test_home_dir) - # Ensure each DB home directory is run under the tests working directory. - set(command_args -h ${wt_test_home_dir}) - list(APPEND command_args ${C_TEST_ARGUMENTS}) + set(test_cmd ${exec_wrapper} $<TARGET_FILE:${C_TEST_TARGET}>) + endif() + # Define the ctest target. + if(C_TEST_VARIANTS) + # If we want to define multiple variant executions of the test script/binary. + define_test_variants(${C_TEST_TARGET} + VARIANTS ${C_TEST_VARIANTS} + CMDS ${test_cmd} + DIR_NAME ${C_TEST_DIR_NAME} + LABELS "check;csuite" + ) + else() add_test(NAME ${C_TEST_TARGET} - COMMAND ${exec_wrapper} $<TARGET_FILE:${C_TEST_TARGET}> ${command_args} + COMMAND ${test_cmd} ${C_TEST_ARGUMENTS} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${C_TEST_DIR_NAME} ) + set_tests_properties(${C_TEST_TARGET} PROPERTIES LABELS "check;csuite") endif() - list(APPEND c_tests ${C_TEST_TARGET}) endif() -endmacro(define_c_test) +endfunction(define_c_test) diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index ba9f33806d5..a775a9aec7a 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -317,7 +317,7 @@ functions: if [ ${is_cmake_build|false} = true ]; then . test/evergreen/find_cmake.sh cd cmake_build - ${test_env_vars|} $CTEST -L check ${smp_command|} -VV 2>&1 + ${test_env_vars|} $CTEST -L check ${smp_command|} --output-on-failure 2>&1 else cd build_posix ${test_env_vars|} ${make_command|make} VERBOSE=1 check ${smp_command|} 2>&1 @@ -554,6 +554,35 @@ functions: done done + "generic-perf-test": + # Run a performance test + # Parameterised using the 'perf-test-name' and 'maxruns' variables + - command: shell.exec + params: + working_dir: "wiredtiger/bench/wtperf/wtperf_run_py" + shell: bash + script: | + set -o errexit + set -o verbose + ${virtualenv_binary} -p ${python_binary} venv + source venv/bin/activate + ${pip3_binary} install psutil + ${python_binary} wtperf_run.py -p ../../../cmake_build/bench/wtperf/wtperf -t ../runners/${perf-test-name}.wtperf -v -ho WT_TEST -m ${maxruns} -o out.json + + "generic-perf-test-push-results": + # Push the json results to the 'Files' tab of the task in Evergreen + # Parameterised using the 'perf-test-name' variable + - command: s3.put + params: + aws_secret: ${aws_secret} + aws_key: ${aws_key} + local_file: wiredtiger/bench/wtperf/wtperf_run_py/out.json + bucket: build_external + permissions: public-read + content_type: text/html + display_name: "Test results (JSON)" + remote_file: wiredtiger/${build_variant}/${revision}/perf-test-${perf-test-name}-${build_id}-${execution}/test-results.json + ######################################################################################### # VARIABLES # @@ -968,6 +997,22 @@ tasks: ${test_env_vars|} $(pwd)/test/cppsuite/run -t hs_cleanup -C 'debug_mode=(cursor_copy=true)' -f test/cppsuite/configs/hs_cleanup_default.txt -l 2 + - name: cppsuite-search-near-default + tags: ["pull_request"] + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/" + script: | + set -o errexit + set -o verbose + + ${test_env_vars|} $(pwd)/test/cppsuite/run -t search_near_01 -f test/cppsuite/configs/search_near_01_default.txt -l 2 + ${test_env_vars|} $(pwd)/test/cppsuite/run -t search_near_02 -f test/cppsuite/configs/search_near_02_default.txt -l 2 + - name: cppsuite-base-test-stress depends_on: - name: compile @@ -2366,6 +2411,10 @@ tasks: - func: "random abort test" vars: random_abort_args: -t 40 + # random-abort - run compaction + - func: "random abort test" + vars: + random_abort_args: -c -t 60 # truncated-log - func: "truncated log test" @@ -2378,6 +2427,12 @@ tasks: vars: extra_args: file_type=row + # format test for stressing compaction code path + - func: "format test" + vars: + times: 3 + extra_args: file_type=row compaction=1 verify=1 runs.timer=3 ops.pct.delete=30 + #FIXME-WT-5270: Add wtperf testing from Jenkin "wiredtiger-test-check-long" after fixing WT-5270 - name: time-shift-sensitivity-test @@ -2847,6 +2902,111 @@ tasks: set -o errexit python "../metrixplusplus/metrix++.py" limit --max-limit=std.code.complexity:cyclomatic:95 + ############################# + # Performance Tests for lsm # + ############################# + + - name: perf-test-small-lsm + commands: + - func: "get project" + - func: "compile wiredtiger" + - func: "generic-perf-test" + vars: + perf-test-name: small-lsm + maxruns: 3 + - func: "generic-perf-test-push-results" + vars: + perf-test-name: small-lsm + + - name: perf-test-medium-lsm + commands: + - func: "get project" + - func: "compile wiredtiger" + - func: "generic-perf-test" + vars: + perf-test-name: medium-lsm + maxruns: 1 + - func: "generic-perf-test-push-results" + vars: + perf-test-name: medium-lsm + + - name: perf-test-medium-lsm-compact + commands: + - func: "get project" + - func: "compile wiredtiger" + - func: "generic-perf-test" + vars: + perf-test-name: medium-lsm-compact + maxruns: 1 + - func: "generic-perf-test-push-results" + vars: + perf-test-name: medium-lsm-compact + + - name: perf-test-medium-multi-lsm + commands: + - func: "get project" + - func: "compile wiredtiger" + - func: "generic-perf-test" + vars: + perf-test-name: medium-multi-lsm + maxruns: 1 + - func: "generic-perf-test-push-results" + vars: + perf-test-name: medium-multi-lsm + + ############################### + # Performance Tests for btree # + ############################### + + - name: perf-test-small-btree + commands: + - func: "get project" + - func: "compile wiredtiger" + - func: "generic-perf-test" + vars: + perf-test-name: small-btree + maxruns: 1 + - func: "generic-perf-test-push-results" + vars: + perf-test-name: small-btree + + - name: perf-test-small-btree-backup + commands: + - func: "get project" + - func: "compile wiredtiger" + - func: "generic-perf-test" + vars: + perf-test-name: small-btree-backup + maxruns: 1 + - func: "generic-perf-test-push-results" + vars: + perf-test-name: small-btree-backup + + - name: perf-test-medium-btree + commands: + - func: "get project" + - func: "compile wiredtiger" + - func: "generic-perf-test" + vars: + perf-test-name: medium-btree + maxruns: 3 + - func: "generic-perf-test-push-results" + vars: + perf-test-name: medium-btree + + - name: perf-test-medium-btree-backup + commands: + - func: "get project" + - func: "compile wiredtiger" + - func: "generic-perf-test" + vars: + perf-test-name: medium-btree-backup + maxruns: 3 + - func: "generic-perf-test-push-results" + vars: + perf-test-name: medium-btree-backup + + buildvariants: - name: ubuntu2004 @@ -2990,6 +3150,7 @@ buildvariants: - name: make-check-test - name: cppsuite-base-test-default - name: cppsuite-hs-cleanup-default + - name: cppsuite-search-near-default - name: ubuntu2004-compilers display_name: "! Ubuntu 20.04 Compilers" @@ -3024,6 +3185,33 @@ buildvariants: - name: ".stress-test-4" - name: format-abort-recovery-stress-test +- name: ubuntu2004-perf-tests + display_name: Ubuntu 20.04 Performance tests + run_on: + - ubuntu2004-test + expansions: + test_env_vars: LD_LIBRARY_PATH=$(pwd) WT_BUILDDIR=$(pwd) + posix_configure_flags: -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/mongodbtoolchain_v3_gcc.cmake -DCMAKE_C_FLAGS="-ggdb" -DHAVE_DIAGNOSTIC=1 -DENABLE_ZLIB=1 -DENABLE_SNAPPY=1 -DENABLE_STRICT=1 -DCMAKE_INSTALL_PREFIX=$(pwd)/LOCAL_INSTALL + python_binary: '/opt/mongodbtoolchain/v3/bin/python3' + pip3_binary: '/opt/mongodbtoolchain/v3/bin/pip3' + virtualenv_binary: '/opt/mongodbtoolchain/v3/bin/virtualenv' + smp_command: -j $(echo "`grep -c ^processor /proc/cpuinfo` * 2" | bc) + cmake_generator: Ninja + make_command: ninja + is_cmake_build: true + tasks: + # btree tests + - name: perf-test-small-btree + - name: perf-test-small-btree-backup + - name: perf-test-medium-btree + - name: perf-test-medium-btree-backup + # lsm tests + - name: perf-test-small-lsm + - name: perf-test-medium-lsm + - name: perf-test-medium-lsm-compact + - name: perf-test-medium-multi-lsm + + - name: large-scale-tests display_name: "Large scale tests" batchtime: 480 # 3 times a day @@ -3052,8 +3240,8 @@ buildvariants: test_env_vars: LD_LIBRARY_PATH=$(pwd)/../../.libs PATH=/opt/mongodbtoolchain/v3/bin:$PATH make_command: PATH=/opt/mongodbtoolchain/v3/bin:$PATH make posix_configure_flags: - --enable-silent-rules --enable-python --enable-zlib --enable-snappy - --enable-strict --enable-static + --enable-diagnostic --enable-python --enable-silent-rules --enable-snappy --enable-static + --enable-strict --enable-zlib tasks: - name: compile - name: cppsuite-hs-cleanup-stress diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index cfb4f78ba27..d12c30ed756 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -353,9 +353,6 @@ static CONFIG c[] = { /* 2% */ {"stress.split_7", "stress splits (#7)", C_BOOL, 2, 0, 0, &g.c_timing_stress_split_7, NULL}, - /* 2% */ - {"stress.split_8", "stress splits (#8)", C_BOOL, 2, 0, 0, &g.c_timing_stress_split_8, NULL}, - {"transaction.implicit", "implicit, without timestamps, transactions (percentage)", 0x0, 0, 100, 100, &g.c_txn_implicit, NULL}, diff --git a/src/third_party/wiredtiger/test/format/config_compat.c b/src/third_party/wiredtiger/test/format/config_compat.c index 2926d54ca4b..935cbea5eab 100644 --- a/src/third_party/wiredtiger/test/format/config_compat.c +++ b/src/third_party/wiredtiger/test/format/config_compat.c @@ -181,8 +181,6 @@ static const char *list[] = { "stress.split_6", "timing_stress_split_7=", "stress.split_7", - "timing_stress_split_8=", - "stress.split_8", "transaction-frequency=", "transaction.frequency", "transaction_timestamps=", diff --git a/src/third_party/wiredtiger/test/format/config_compat.sed b/src/third_party/wiredtiger/test/format/config_compat.sed index b90b21332e8..81d7a58f025 100644 --- a/src/third_party/wiredtiger/test/format/config_compat.sed +++ b/src/third_party/wiredtiger/test/format/config_compat.sed @@ -78,7 +78,6 @@ s/^stress.split_4=/timing_stress_split_4=/ s/^stress.split_5=/timing_stress_split_5=/ s/^stress.split_6=/timing_stress_split_6=/ s/^stress.split_7=/timing_stress_split_7=/ -s/^stress.split_8=/timing_stress_split_8=/ s/^transaction.frequency=/transaction-frequency=/ s/^transaction.isolation=/isolation=/ s/^transaction.timestamps=/transaction_timestamps=/ diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-5637 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-5637 index 68338d050ae..8e27f5b5967 100644 --- a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-5637 +++ b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-5637 @@ -91,7 +91,6 @@ stress.split_4=0 stress.split_5=0 stress.split_6=0 stress.split_7=0 -stress.split_8=0 transaction.frequency=100 transaction.isolation=snapshot transaction.rollback_to_stable=0 diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6568 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6568 index 4e695a8d87e..3f3192a5ef2 100644 --- a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6568 +++ b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6568 @@ -90,7 +90,6 @@ stress.split_4=0 stress.split_5=0 stress.split_6=1 stress.split_7=0 -stress.split_8=0 transaction.frequency=100 transaction.isolation=snapshot transaction.rollback_to_stable=0 diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6725 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6725 index 144d9aaadd4..54aafd08f0b 100644 --- a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6725 +++ b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6725 @@ -90,7 +90,6 @@ stress.split_5=0 stress.split_6=0 stress.split_7=0 - stress.split_8=0 transaction.frequency=100 transaction.isolation=snapshot transaction.rollback_to_stable=0 diff --git a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6727 b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6727 index 62b3f7bba5f..587a8e5b978 100644 --- a/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6727 +++ b/src/third_party/wiredtiger/test/format/failure_configs/CONFIG.WT-6727 @@ -87,7 +87,6 @@ stress.split_4=0 stress.split_5=0 stress.split_6=0 stress.split_7=0 -stress.split_8=0 transaction.frequency=100 transaction.isolation=snapshot transaction.timestamps=1 diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index ecfb83a37ce..cce28406706 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -264,7 +264,6 @@ typedef struct { uint32_t c_timing_stress_split_5; uint32_t c_timing_stress_split_6; uint32_t c_timing_stress_split_7; - uint32_t c_timing_stress_split_8; uint32_t c_truncate; uint32_t c_txn_implicit; uint32_t c_txn_timestamps; @@ -402,7 +401,8 @@ typedef struct { uint64_t insert_list[256]; /* column-store inserted records */ u_int insert_list_cnt; - WT_ITEM vprint; /* Temporary buffer for printable values */ + WT_ITEM vprint; /* Temporary buffer for printable values */ + WT_ITEM moda, modb; /* Temporary buffer for modify operations */ #define TINFO_RUNNING 1 /* Running */ #define TINFO_COMPLETE 2 /* Finished */ diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 81d21111ec3..4c24ba9fd45 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -171,6 +171,8 @@ tinfo_teardown(void) tinfo = tinfo_list[i]; __wt_buf_free(NULL, &tinfo->vprint); + __wt_buf_free(NULL, &tinfo->moda); + __wt_buf_free(NULL, &tinfo->modb); /* * Assert records were not removed unless configured to do so, otherwise subsequent runs can @@ -1454,26 +1456,50 @@ modify_build(TINFO *tinfo, WT_MODIFY *entries, int *nentriesp) } /* + * modify -- + * Cursor modify worker function. + */ +static int +modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +{ + WT_MODIFY entries[MAX_MODIFY_ENTRIES]; + int nentries; + bool modify_check; + + /* Periodically verify the WT_CURSOR.modify return. */ + modify_check = positioned && mmrand(&tinfo->rnd, 1, 10) == 1; + if (modify_check) { + testutil_check(cursor->get_value(cursor, &tinfo->moda)); + testutil_check( + __wt_buf_set(CUR2S(cursor), &tinfo->moda, tinfo->moda.data, tinfo->moda.size)); + } + + modify_build(tinfo, entries, &nentries); + WT_RET(cursor->modify(cursor, entries, nentries)); + + testutil_check(cursor->get_value(cursor, tinfo->value)); + if (modify_check) { + testutil_modify_apply(&tinfo->moda, &tinfo->modb, entries, nentries); + testutil_assert(tinfo->moda.size == tinfo->value->size && + memcmp(tinfo->moda.data, tinfo->value->data, tinfo->moda.size) == 0); + } + return (0); +} + +/* * row_modify -- * Modify a row in a row-store file. */ static int row_modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) { - WT_DECL_RET; - WT_MODIFY entries[MAX_MODIFY_ENTRIES]; - int nentries; if (!positioned) { key_gen(tinfo->key, tinfo->keyno); cursor->set_key(cursor, tinfo->key); } - modify_build(tinfo, entries, &nentries); - if ((ret = cursor->modify(cursor, entries, nentries)) != 0) - return (ret); - - testutil_check(cursor->get_value(cursor, tinfo->value)); + WT_RET(modify(tinfo, cursor, positioned)); trace_op(tinfo, "modify %" PRIu64 " {%.*s}, {%s}", tinfo->keyno, (int)tinfo->key->size, (char *)tinfo->key->data, trace_item(tinfo, tinfo->value)); @@ -1488,18 +1514,10 @@ row_modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) static int col_modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) { - WT_DECL_RET; - WT_MODIFY entries[MAX_MODIFY_ENTRIES]; - int nentries; - if (!positioned) cursor->set_key(cursor, tinfo->keyno); - modify_build(tinfo, entries, &nentries); - if ((ret = cursor->modify(cursor, entries, nentries)) != 0) - return (ret); - - testutil_check(cursor->get_value(cursor, tinfo->value)); + WT_RET(modify(tinfo, cursor, positioned)); trace_op(tinfo, "modify %" PRIu64 ", {%s}", tinfo->keyno, trace_item(tinfo, tinfo->value)); diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index d9f0c81a34c..74027748246 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -292,8 +292,6 @@ create_database(const char *home, WT_CONNECTION **connp) CONFIG_APPEND(p, ",split_6"); if (g.c_timing_stress_split_7) CONFIG_APPEND(p, ",split_7"); - if (g.c_timing_stress_split_8) - CONFIG_APPEND(p, ",split_8"); CONFIG_APPEND(p, "]"); /* Extensions. */ @@ -523,8 +521,6 @@ wts_open(const char *home, WT_CONNECTION **connp, WT_SESSION **sessionp, bool al CONFIG_APPEND(p, ",split_6"); if (g.c_timing_stress_split_7) CONFIG_APPEND(p, ",split_7"); - if (g.c_timing_stress_split_8) - CONFIG_APPEND(p, ",split_8"); CONFIG_APPEND(p, "]"); /* If in-memory, there's only a single, shared WT_CONNECTION handle. */ diff --git a/src/third_party/wiredtiger/test/suite/test_backup10.py b/src/third_party/wiredtiger/test/suite/test_backup10.py index 36593c205e2..988b8467e87 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup10.py +++ b/src/third_party/wiredtiger/test/suite/test_backup10.py @@ -56,6 +56,7 @@ class test_backup10(backup_base): def test_backup10(self): log2 = "WiredTigerLog.0000000002" log3 = "WiredTigerLog.0000000003" + log4 = "WiredTigerLog.0000000004" self.session.create(self.uri, "key_format=S,value_format=S") @@ -84,7 +85,8 @@ class test_backup10(backup_base): # We expect that the duplicate logs are a superset of the # original logs. And we expect the difference to be the - # addition of log file 3 only. + # addition of two log files, one switch when opening the backup + # cursor and a switch when opening the duplicate cursor. orig_set = set(orig_logs) dup_set = set(dup_logs) self.assertTrue(dup_set.issuperset(orig_set)) @@ -92,6 +94,7 @@ class test_backup10(backup_base): self.assertEqual(len(diff), 1) self.assertTrue(log3 in dup_set) self.assertFalse(log3 in orig_set) + self.assertFalse(log4 in dup_set) # Test a few error cases now. # - We cannot make multiple duplcate backup cursors. @@ -114,20 +117,6 @@ class test_backup10(backup_base): lambda:self.assertEquals(self.session.open_cursor(None, bkup_c, None), 0), msg) - # Open duplicate backup cursor again now that the first - # one is closed. Test every log file returned is the same - # as the first time. - dupc = self.session.open_cursor(None, bkup_c, config) - while True: - ret = dupc.next() - if ret != 0: - break - newfile = dupc.get_key() - self.assertTrue("WiredTigerLog" in newfile) - self.assertTrue(newfile in dup_logs) - self.assertEqual(ret, wiredtiger.WT_NOTFOUND) - - dupc.close() bkup_c.close() # After the full backup, open and recover the backup database. diff --git a/src/third_party/wiredtiger/test/suite/test_cursor17.py b/src/third_party/wiredtiger/test/suite/test_cursor17.py new file mode 100644 index 00000000000..322b82a65f8 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_cursor17.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_cursor17.py +# Test the largest_key interface under various scenarios. +# +import wttest +import wiredtiger +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet +from wtscenario import make_scenarios + +class test_cursor17(wttest.WiredTigerTestCase): + tablename = 'test_cursor17' + + # Enable the lsm tests once it is supported. + types = [ + ('file-row', dict(type='file:', keyformat='i', valueformat='i', dataset=SimpleDataSet)), + ('table-row', dict(type='table:', keyformat='i', valueformat='i', dataset=SimpleDataSet)), + ('file-var', dict(type='file:', keyformat='r', valueformat='i', dataset=SimpleDataSet)), + ('table-var', dict(type='table:', keyformat='r', valueformat='i', dataset=SimpleDataSet)), + ('file-fix', dict(type='file:', keyformat='r', valueformat='8t', dataset=SimpleDataSet)), + # ('lsm', dict(type='lsm:', keyformat='i', valueformat='i', dataset=SimpleDataSet)), + ('table-r-complex', dict(type='table:', keyformat='r', valueformat=None, + dataset=ComplexDataSet)), + # ('table-i-complex-lsm', dict(type='table:', keyformat='i', valueformat=None, + # dataset=ComplexLSMDataSet)), + ] + + scenarios = make_scenarios(types) + + def populate(self, rownum): + if self.valueformat != None: + self.ds = self.dataset(self, self.type + self.tablename, rownum, key_format=self.keyformat, value_format=self.valueformat) + else: + self.ds = self.dataset(self, self.type + self.tablename, rownum, key_format=self.keyformat) + self.ds.populate() + + def test_globally_deleted_key(self): + self.populate(100) + + # Delete the largest key. + cursor = self.session.open_cursor(self.type + self.tablename, None) + self.session.begin_transaction() + cursor.set_key(100) + self.assertEqual(cursor.remove(), 0) + self.session.commit_transaction() + + # Verify the key is not visible. + self.session.begin_transaction() + cursor.set_key(100) + if self.valueformat != '8t': + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + else: + self.assertEqual(cursor.search(), 0) + self.session.rollback_transaction() + + # Verify the largest key. + self.session.begin_transaction() + self.assertEqual(cursor.largest_key(), 0) + self.assertEqual(cursor.get_key(), 100) + self.session.rollback_transaction() + + # Verify the key is still not visible after the largest call. + self.session.begin_transaction() + cursor.set_key(100) + if self.valueformat != '8t': + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + else: + self.assertEqual(cursor.search(), 0) + self.session.rollback_transaction() + + # Use evict cursor to evict the key from memory. + evict_cursor = self.session.open_cursor(self.type + self.tablename, None, "debug=(release_evict)") + evict_cursor.set_key(100) + if self.valueformat != '8t': + self.assertEquals(evict_cursor.search(), wiredtiger.WT_NOTFOUND) + else: + self.assertEquals(evict_cursor.search(), 0) + evict_cursor.close() + + # Verify the largest key changed. + self.session.begin_transaction() + self.assertEqual(cursor.largest_key(), 0) + if self.valueformat != '8t': + self.assertEqual(cursor.get_key(), 99) + else: + self.assertEquals(cursor.get_key(), 100) + self.session.rollback_transaction() + + def test_uncommitted_insert(self): + self.populate(100) + + session2 = self.setUpSessionOpen(self.conn) + cursor2 = session2.open_cursor(self.type + self.tablename, None) + session2.begin_transaction() + cursor2[101] = self.ds.value(101) + + cursor = self.session.open_cursor(self.type + self.tablename, None) + + # Verify the largest key. + self.session.begin_transaction() + self.assertEqual(cursor.largest_key(), 0) + self.assertEqual(cursor.get_key(), 101) + self.session.rollback_transaction() + + session2.rollback_transaction() + + def test_read_timestamp(self): + self.populate(100) + + cursor = self.session.open_cursor(self.type + self.tablename, None) + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) + # Expect the largest key to throw. + with self.expectedStderrPattern("largest key cannot be called with a read timestamp"): + try: + cursor.largest_key() + except wiredtiger.WiredTigerError as e: + gotException = True + self.pr('got expected exception: ' + str(e)) + self.assertTrue(str(e).find('nvalid argument') >= 0) + self.assertTrue(gotException, msg = 'expected exception') + self.session.rollback_transaction() + + def test_not_positioned(self): + self.populate(100) + + cursor = self.session.open_cursor(self.type + self.tablename, None) + # Verify the largest key. + self.session.begin_transaction() + self.assertEqual(cursor.largest_key(), 0) + self.assertEqual(cursor.get_key(), 100) + + # Call prev + self.assertEqual(cursor.prev(), 0) + self.assertEqual(cursor.get_key(), 100) + + # Verify the largest key again. + self.assertEqual(cursor.largest_key(), 0) + self.assertEqual(cursor.get_key(), 100) + + self.assertEqual(cursor.next(), 0) + self.assertEqual(cursor.get_key(), 1) + self.session.rollback_transaction() + + def test_get_value(self): + self.populate(100) + + cursor = self.session.open_cursor(self.type + self.tablename, None) + # Verify the largest key. + self.session.begin_transaction() + self.assertEqual(cursor.largest_key(), 0) + self.assertEqual(cursor.get_key(), 100) + with self.expectedStderrPattern("requires value be set"): + try: + cursor.get_value() + except wiredtiger.WiredTigerError as e: + gotException = True + self.pr('got expected exception: ' + str(e)) + self.assertTrue(str(e).find('nvalid argument') >= 0) + self.assertTrue(gotException, msg = 'expected exception') + self.session.rollback_transaction() + + def test_empty_table(self): + self.populate(0) + + cursor = self.session.open_cursor(self.type + self.tablename, None) + # Verify the largest key. + self.session.begin_transaction() + self.assertEquals(cursor.largest_key(), wiredtiger.WT_NOTFOUND) + self.session.rollback_transaction() + + def test_fast_truncate(self): + self.populate(100) + + # evict all the pages + evict_cursor = self.session.open_cursor(self.type + self.tablename, None, "debug=(release_evict)") + self.session.begin_transaction() + for i in range(1, 101): + evict_cursor.set_key(i) + self.assertEquals(evict_cursor.search(), 0) + self.session.rollback_transaction() + evict_cursor.close() + + # truncate + cursor = self.session.open_cursor(self.type + self.tablename, None) + self.session.begin_transaction() + cursor.set_key(1) + self.session.truncate(None, cursor, None, None) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) + + # verify the largest key + self.session.begin_transaction() + self.assertEqual(cursor.largest_key(), 0) + self.assertEqual(cursor.get_key(), 100) + self.session.rollback_transaction() + + def test_slow_truncate(self): + self.populate(100) + + # truncate + cursor = self.session.open_cursor(self.type + self.tablename, None) + self.session.begin_transaction() + cursor.set_key(100) + self.session.truncate(None, cursor, None, None) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) + + # verify the largest key + self.session.begin_transaction() + self.assertEqual(cursor.largest_key(), 0) + self.assertEqual(cursor.get_key(), 100) + self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py index 25aaae7ff67..63989155146 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py @@ -173,6 +173,10 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): def test_rollback_to_stable_prepare(self): nrows = 1000 + # FIXME-WT-7250 This test fails because of cache stuck on Windows. + if os.name == "nt": + self.skipTest('rollback_to_stable10 prepare test skipped on Windows') + # Create a table without logging. self.pr("create/populate tables") uri_1 = "table:rollback_to_stable10_1" diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable27.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable27.py new file mode 100644 index 00000000000..ee0499e72da --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable27.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from test_rollback_to_stable01 import test_rollback_to_stable_base +from wiredtiger import stat, Modify, WT_NOTFOUND +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_rollback_to_stable27.py +# +# Test mixing timestamped and non-timestamped updates on the same VLCS RLE cell. +class test_rollback_to_stable27(test_rollback_to_stable_base): + session_config = 'isolation=snapshot' + + # Run it all on row-store as well as a control group: if something odd arises from the + # RLE cell handling it won't happen in row-store. + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + in_memory_values = [ + ('no_inmem', dict(in_memory=False)), + ('inmem', dict(in_memory=True)) + ] + + scenarios = make_scenarios(key_format_values, in_memory_values) + + def conn_config(self): + if self.in_memory: + return 'in_memory=true' + else: + return 'in_memory=false' + + # Evict the page to force reconciliation. + def evict(self, uri, key, check_value): + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + self.session.begin_transaction() + v = evict_cursor[1] + self.assertEqual(v, check_value) + self.assertEqual(evict_cursor.reset(), 0) + self.session.rollback_transaction() + evict_cursor.close() + + def test_rollback_to_stable(self): + nrows = 10 + + # Create a table without logging. + uri = "table:rollback_to_stable27" + ds = SimpleDataSet( + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + ds.populate() + + value_a = "aaaaa" * 10 + value_b = "bbbbb" * 10 + + # Pin oldest and stable to timestamp 10. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) + + # Write aaaaaa to all the keys at time 20. + self.large_updates(uri, value_a, ds, nrows, False, 20) + + # Evict the page to force reconciliation. + self.evict(uri, 1, value_a) + + # Ideally here we'd check to make sure we actually have a single RLE cell, because + # if not the rest of the work isn't going to do much good. Maybe via stats...? + + cursor = self.session.open_cursor(uri) + self.session.begin_transaction() + cursor[7] = value_b + self.session.commit_transaction() + cursor.close() + + # Now roll back. + self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(15)) + self.conn.rollback_to_stable() + + # The only thing we should see (at any time) is value_b at key 7. + cursor = self.session.open_cursor(uri) + for ts in [10, 20, 30]: + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts)) + for k, v in cursor: + self.assertEqual(k, 7) + self.assertEqual(v, value_b) + self.session.rollback_transaction() + cursor.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_search_near01.py b/src/third_party/wiredtiger/test/suite/test_search_near01.py index 412cbd894f0..6df71b438ff 100755 --- a/src/third_party/wiredtiger/test/suite/test_search_near01.py +++ b/src/third_party/wiredtiger/test/suite/test_search_near01.py @@ -98,7 +98,7 @@ class test_search_near01(wttest.WiredTigerTestCase): # range forward, and then the whole range backwards. self.assertGreater(skip_count, key_count * 2) - cursor2.reconfigure("prefix_key=true") + cursor2.reconfigure("prefix_search=true") cursor2.set_key('aa') cursor2.search_near() @@ -196,7 +196,7 @@ class test_search_near01(wttest.WiredTigerTestCase): # range forward, and then the whole range backwards. self.assertGreater(skip_count, key_count * 2) - cursor2.reconfigure("prefix_key=true") + cursor2.reconfigure("prefix_search=true") cursor2.set_key('cc') cursor2.search_near() self.assertEqual(self.get_stat(stat.conn.cursor_search_near_prefix_fast_paths), 2) @@ -304,7 +304,7 @@ class test_search_near01(wttest.WiredTigerTestCase): # range forward, and then the whole range backwards. self.assertGreater(skip_count, key_count) - cursor2.reconfigure("prefix_key=true") + cursor2.reconfigure("prefix_search=true") cursor2.set_key('c') cursor2.search_near() @@ -317,14 +317,14 @@ class test_search_near01(wttest.WiredTigerTestCase): session2.rollback_transaction() session2.begin_transaction('ignore_prepare=true') cursor4 = session2.open_cursor(uri) - cursor4.reconfigure("prefix_key=true") + cursor4.reconfigure("prefix_search=true") cursor4.set_key('c') cursor4.search_near() prefix_skip_count = self.get_stat(stat.conn.cursor_next_skip_lt_100, session2) self.assertEqual(prefix_skip_count - skip_count, 2) skip_count = prefix_skip_count - cursor4.reconfigure("prefix_key=false") + cursor4.reconfigure("prefix_search=false") cursor4.set_key('c') cursor4.search_near() self.assertEqual(self.get_stat(stat.conn.cursor_next_skip_lt_100, session2) - skip_count, 2) diff --git a/src/third_party/wiredtiger/test/suite/test_search_near02.py b/src/third_party/wiredtiger/test/suite/test_search_near02.py index a8377aac078..0f981f84af4 100644 --- a/src/third_party/wiredtiger/test/suite/test_search_near02.py +++ b/src/third_party/wiredtiger/test/suite/test_search_near02.py @@ -44,7 +44,12 @@ class test_search_near02(wttest.WiredTigerTestCase): ('byte_array', dict(key_format='u')), ] - scenarios = make_scenarios(key_format_values) + eviction = [ + ('eviction', dict(eviction=True)), + ('no eviction', dict(eviction=False)), + ] + + scenarios = make_scenarios(key_format_values, eviction) def check_key(self, key): if self.key_format == 'u': @@ -80,16 +85,16 @@ class test_search_near02(wttest.WiredTigerTestCase): cursor[prefix + "zab"] = prefix + "zab" self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(250)) - # Evict the whole range. - for k in range (0, 26): - cursor2.set_key(prefix + l[k]) + if self.eviction: + # Evict the whole range. + for k in range (0, 26): + cursor2.set_key(prefix + l[k]) + self.assertEqual(cursor2.search(), 0) + self.assertEqual(cursor2.reset(), 0) + cursor2.set_key(prefix + "zab") self.assertEqual(cursor2.search(), 0) self.assertEqual(cursor2.reset(), 0) - cursor2.set_key(prefix + "zab") - self.assertEqual(cursor2.search(), 0) - self.assertEqual(cursor2.reset(), 0) - # Start a transaction at timestamp 100, aaz should be the only key that is visible. self.session.begin_transaction('read_timestamp=' + self.timestamp_str(100)) cursor3 = self.session.open_cursor(uri) @@ -113,7 +118,7 @@ class test_search_near02(wttest.WiredTigerTestCase): self.assertEqual(cursor3.get_key(), self.check_key("aaz")) # Enable prefix search. - cursor3.reconfigure("prefix_key=true") + cursor3.reconfigure("prefix_search=true") # The only visible key is aaz. As long we are looking for a key that starts with either "a", # "aa" or "aaz", search near should return back aaz. Otherwise, search near should return @@ -148,7 +153,7 @@ class test_search_near02(wttest.WiredTigerTestCase): self.assertEqual(cursor3.search_near(), wiredtiger.WT_NOTFOUND) # Enable prefix search. - cursor3.reconfigure("prefix_key=true") + cursor3.reconfigure("prefix_search=true") cursor3.set_key("aaz") self.assertEqual(cursor3.search_near(), wiredtiger.WT_NOTFOUND) @@ -176,7 +181,7 @@ class test_search_near02(wttest.WiredTigerTestCase): self.assertEqual(cursor3.get_key(), self.check_key("aazab")) # Enable prefix search. - cursor3.reconfigure("prefix_key=true") + cursor3.reconfigure("prefix_search=true") # Search near for a, should return the closest visible key with a matching prefix: aaa. cursor3.set_key("a") diff --git a/src/third_party/wiredtiger/test/suite/test_search_near03.py b/src/third_party/wiredtiger/test/suite/test_search_near03.py index 073e5c44a13..748ef746cba 100644 --- a/src/third_party/wiredtiger/test/suite/test_search_near03.py +++ b/src/third_party/wiredtiger/test/suite/test_search_near03.py @@ -64,9 +64,9 @@ class test_search_near03(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri) # Check if the format is valid for prefix configuration. if self.valid_key_format(): - self.assertEqual(cursor.reconfigure("prefix_key=true"), 0) + self.assertEqual(cursor.reconfigure("prefix_search=true"), 0) else: msg = '/Invalid argument/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: cursor.reconfigure("prefix_key=true"), msg) + lambda: cursor.reconfigure("prefix_search=true"), msg) cursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered02.py b/src/third_party/wiredtiger/test/suite/test_tiered02.py index bed8e57ef54..2041bcdf767 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered02.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered02.py @@ -83,9 +83,7 @@ class test_tiered02(wttest.WiredTigerTestCase): self.assertEqual(len(got), self.flushed_objects) self.flushed_objects = len(got) - # Test tiered storage with the old prototype way of signaling flushing to the shared - # tier via checkpoints. When flush_tier is working, the checkpoint calls can be - # replaced with flush_tier. + # Test tiered storage with checkpoints and flush_tier calls. def test_tiered(self): self.flushed_objects = 0 args = 'key_format=S' @@ -110,7 +108,7 @@ class test_tiered02(wttest.WiredTigerTestCase): self.close_conn() self.progress('reopen_conn') self.reopen_conn() - # Check what was there before + # Check what was there before. ds = SimpleDataSet(self, self.uri, 10, config=args) ds.check() @@ -152,7 +150,7 @@ class test_tiered02(wttest.WiredTigerTestCase): self.progress('reopen_conn') self.reopen_conn() - # Check what was there before + # Check what was there before. ds = SimpleDataSet(self, self.uri, 200, config=args) ds.check() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered04.py b/src/third_party/wiredtiger/test/suite/test_tiered04.py index efb5630eda6..05fbbc44a5e 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered04.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered04.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import os, wiredtiger, wttest +import os, time, wiredtiger, wttest from wiredtiger import stat StorageSource = wiredtiger.StorageSource # easy access to constants @@ -35,8 +35,11 @@ StorageSource = wiredtiger.StorageSource # easy access to constants class test_tiered04(wttest.WiredTigerTestCase): # If the 'uri' changes all the other names must change with it. - fileuri_base = 'file:test_tiered04-000000000' - objuri = 'object:test_tiered04-0000000001.wtobj' + base = 'test_tiered04-000000000' + fileuri_base = 'file:' + base + obj1file = base + '1.wtobj' + obj2file = base + '2.wtobj' + objuri = 'object:' + base + '1.wtobj' tiereduri = "tiered:test_tiered04" uri = "table:test_tiered04" @@ -53,8 +56,8 @@ class test_tiered04(wttest.WiredTigerTestCase): object_sys_val = 9 * 1024 * 1024 object_uri = "15M" object_uri_val = 15 * 1024 * 1024 - retention = 600 - retention1 = 350 + retention = 3 + retention1 = 600 def conn_config(self): os.mkdir(self.bucket) os.mkdir(self.bucket1) @@ -120,19 +123,50 @@ class test_tiered04(wttest.WiredTigerTestCase): self.pr("flush tier") c = self.session.open_cursor(self.uri) + c1 = self.session.open_cursor(self.uri1) + cn = self.session.open_cursor(self.uri_none) c["0"] = "0" + c1["0"] = "0" + cn["0"] = "0" self.check(c, 1) + self.check(c1, 1) + self.check(cn, 1) c.close() + + # Check the local retention. After a flush_tier call the object file should exist in + # the local database. Then after sleeping long enough it should be removed. + self.session.checkpoint() + self.session.flush_tier(None) + self.pr("Check for ") + self.pr(self.obj1file) + self.assertTrue(os.path.exists(self.obj1file)) + self.assertTrue(os.path.exists(self.obj2file)) + self.pr("Sleep") + time.sleep(self.retention + 1) + # We call flush_tier here because otherwise the internal thread that + # processes the work units won't run for a while. This call will signal + # the internal thread to process the work units. self.session.flush_tier(None) + time.sleep(1) + self.pr("Check removal of ") + self.pr(self.obj1file) + self.assertFalse(os.path.exists(self.obj1file)) c = self.session.open_cursor(self.uri) c["1"] = "1" + c1["1"] = "1" + cn["1"] = "1" self.check(c, 2) c.close() c = self.session.open_cursor(self.uri) c["2"] = "2" + c1["2"] = "2" + cn["2"] = "2" self.check(c, 3) + c1.close() + cn.close() + self.session.checkpoint() self.pr("flush tier again, holding open cursor") self.session.flush_tier(None) @@ -142,7 +176,7 @@ class test_tiered04(wttest.WiredTigerTestCase): c.close() calls = self.get_stat(stat.conn.flush_tier, None) - flush = 2 + flush = 3 self.assertEqual(calls, flush) obj = self.get_stat(stat.conn.tiered_object_size, None) self.assertEqual(obj, self.object_sys_val) @@ -174,26 +208,27 @@ class test_tiered04(wttest.WiredTigerTestCase): self.assertEqual(retain, self.retention) self.session.flush_tier(None) self.session.flush_tier('force=true') + flush += 2 calls = self.get_stat(stat.conn.flush_tier, None) - self.assertEqual(calls, 4) + self.assertEqual(calls, flush) # Test reconfiguration. - new = self.retention * 2 - config = 'tiered_storage=(local_retention=%d)' % new + config = 'tiered_storage=(local_retention=%d)' % self.retention1 self.pr("reconfigure") self.conn.reconfigure(config) retain = self.get_stat(stat.conn.tiered_retention, None) - self.assertEqual(retain, new) - self.pr("reconfigure flush_tier") + self.assertEqual(retain, self.retention1) + # Call flush_tier with its various configuration arguments. It is difficult # to force a timeout or lock contention with a unit test. So just test the # call for now. self.session.flush_tier('timeout=10') self.session.flush_tier('lock_wait=false') self.session.flush_tier('sync=off') + flush += 3 self.pr("reconfigure get stat") calls = self.get_stat(stat.conn.flush_tier, None) - self.assertEqual(calls, 7) + self.assertEqual(calls, flush) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered06.py b/src/third_party/wiredtiger/test/suite/test_tiered06.py index 614619fdef2..f129bafd0c0 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered06.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered06.py @@ -81,7 +81,7 @@ class test_tiered06(wttest.WiredTigerTestCase): # Nothing is in the directory list until a flush. self.assertEquals(fs.fs_directory_list(session, '', ''), []) - # Flushing moves the file into the file system + # Flushing copies the file into the file system. local.ss_flush(session, fs, 'foobar', 'foobar', None) local.ss_flush_finish(session, fs, 'foobar', 'foobar', None) @@ -91,7 +91,7 @@ class test_tiered06(wttest.WiredTigerTestCase): fh = fs.fs_open_file(session, 'foobar', FileSystem.open_file_type_data, FileSystem.open_readonly) inbytes = bytes(1000000) # An empty buffer with a million zero bytes. - fh.fh_read(session, 0, inbytes) # read into the buffer + fh.fh_read(session, 0, inbytes) # Read into the buffer. self.assertEquals(outbytes[0:1000000], inbytes) self.assertEquals(fs.fs_size(session, 'foobar'), len(outbytes)) self.assertEquals(fh.fh_size(session), len(outbytes)) @@ -136,29 +136,29 @@ class test_tiered06(wttest.WiredTigerTestCase): block_size = 4096 f = open('abc', 'wb') - # blocks filled with 'a', etc. + # Create some blocks filled with 'a', etc. a_block = ('a' * block_size).encode() b_block = ('b' * block_size).encode() c_block = ('c' * block_size).encode() file_size = nblocks * block_size - # write all blocks as 'a', but in reverse order + # Write all blocks as 'a', but in reverse order. for pos in range(file_size - block_size, 0, -block_size): f.seek(pos) f.write(a_block) - # write the even blocks as 'b', forwards + # Write the even blocks as 'b', forwards. for pos in range(0, file_size, block_size * 2): f.seek(pos) f.write(b_block) - # write every third block as 'c', backwards + # Write every third block as 'c', backwards. for pos in range(file_size - block_size, 0, -block_size * 3): f.seek(pos) f.write(c_block) f.close() - # Flushing moves the file into the file system + # Flushing copies the file into the file system. local.ss_flush(session, fs, 'abc', 'abc', None) local.ss_flush_finish(session, fs, 'abc', 'abc', None) @@ -172,7 +172,7 @@ class test_tiered06(wttest.WiredTigerTestCase): in_block = bytes(block_size) fh = fs.fs_open_file(session, 'abc', FileSystem.open_file_type_data, FileSystem.open_readonly) - # Do some spot checks, reading non-sequentially + # Do some spot checks, reading non-sequentially. fh.fh_read(session, 500 * block_size, in_block) # divisible by 2, not 3 self.assertEquals(in_block, b_block) fh.fh_read(session, 333 * block_size, in_block) # divisible by 3, not 2 @@ -208,7 +208,7 @@ class test_tiered06(wttest.WiredTigerTestCase): cachedir1 = "./cache1" cachedir2 = "./cache2" - # Add a suffix to each in a list + # Add a suffix to each in a list. def suffix(self, lst, sfx): return [x + '.' + sfx for x in lst] @@ -221,7 +221,7 @@ class test_tiered06(wttest.WiredTigerTestCase): # Check for data files in the WiredTiger home directory. def check_home(self, expect): - # Get list of all .wt files in home, prune out the WiredTiger produced ones + # Get list of all .wt files in home, prune out the WiredTiger produced ones. got = sorted(list(os.listdir(self.home))) got = [x for x in got if not x.startswith('WiredTiger') and x.endswith('.wt')] expect = sorted(self.suffix(expect, 'wt')) @@ -229,7 +229,7 @@ class test_tiered06(wttest.WiredTigerTestCase): # Check that objects are "in the cloud" after a flush. # Using the local storage module, they are actually going to be in either - # objectdir1 or objectdir2 + # objectdir1 or objectdir2. def check_objects(self, expect1, expect2): got = sorted(list(os.listdir(self.objectdir1))) expect = sorted(self.suffix(expect1, 'wtobj')) @@ -252,7 +252,7 @@ class test_tiered06(wttest.WiredTigerTestCase): f.write('hello') def test_local_file_systems(self): - # Test using various buckets, hosts + # Test using various buckets, hosts. session = self.session local = self.conn.get_storage_source('local_store') @@ -310,12 +310,12 @@ class test_tiered06(wttest.WiredTigerTestCase): self.check_caches([], []) self.check_objects(['beagle'], []) - # Bad file to flush + # Bad file to flush. errmsg = '/No such file/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: local.ss_flush(session, fs1, 'bad.wt', 'bad.wtobj'), errmsg) - # It's okay to flush again, nothing changes + # It's okay to flush again, nothing changes. local.ss_flush(session, fs1, 'beagle.wt', 'beagle.wtobj') self.check_home(['beagle', 'bird', 'bison', 'bat', 'cat', 'cougar', 'coyote', 'cub']) self.check_dirlist(fs1, '', ['beagle']) @@ -323,15 +323,15 @@ class test_tiered06(wttest.WiredTigerTestCase): self.check_caches([], []) self.check_objects(['beagle'], []) - # When we flush_finish, the local file will move to the cache directory + # When we flush_finish, the local file will be in both the local and cache directory. local.ss_flush_finish(session, fs1, 'beagle.wt', 'beagle.wtobj') - self.check_home(['bird', 'bison', 'bat', 'cat', 'cougar', 'coyote', 'cub']) + self.check_home(['beagle', 'bird', 'bison', 'bat', 'cat', 'cougar', 'coyote', 'cub']) self.check_dirlist(fs1, '', ['beagle']) self.check_dirlist(fs2, '', []) self.check_caches(['beagle'], []) self.check_objects(['beagle'], []) - # Do a some more in each file ssytem + # Do a some more in each file system. local.ss_flush(session, fs1, 'bison.wt', 'bison.wtobj') local.ss_flush(session, fs2, 'cat.wt', 'cat.wtobj') local.ss_flush(session, fs1, 'bat.wt', 'bat.wtobj') @@ -339,13 +339,13 @@ class test_tiered06(wttest.WiredTigerTestCase): local.ss_flush(session, fs2, 'cub.wt', 'cub.wtobj') local.ss_flush_finish(session, fs1, 'bat.wt', 'bat.wtobj') - self.check_home(['bird', 'bison', 'cougar', 'coyote', 'cub']) + self.check_home(['beagle', 'bird', 'bison', 'bat', 'cat', 'cougar', 'coyote', 'cub']) self.check_dirlist(fs1, '', ['beagle', 'bat', 'bison']) self.check_dirlist(fs2, '', ['cat', 'cub']) self.check_caches(['beagle', 'bat'], ['cat']) self.check_objects(['beagle', 'bat', 'bison'], ['cat', 'cub']) - # Test directory listing prefixes + # Test directory listing prefixes. self.check_dirlist(fs1, '', ['beagle', 'bat', 'bison']) self.check_dirlist(fs1, 'ba', ['bat']) self.check_dirlist(fs1, 'be', ['beagle']) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp22.py b/src/third_party/wiredtiger/test/suite/test_timestamp22.py index 55d6329af2f..c45eea97ac6 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp22.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp22.py @@ -44,6 +44,7 @@ class test_timestamp22(wttest.WiredTigerTestCase): rand = suite_random.suite_random() oldest_ts = 0 stable_ts = 0 + last_durable = 0 SUCCESS = 'success' FAILURE = 'failure' @@ -147,6 +148,12 @@ class test_timestamp22(wttest.WiredTigerTestCase): else: # It's possible this will succeed, we'll check below. this_commit_ts = self.gen_ts(commit_ts) + + # OOD does not work with prepared updates. Hence, the commit ts should always be + # greater than the last durable ts. + if this_commit_ts <= self.last_durable: + this_commit_ts = self.last_durable + 1 + config += ',commit_timestamp=' + self.timestamp_str(this_commit_ts) if this_commit_ts >= 0: @@ -288,6 +295,8 @@ class test_timestamp22(wttest.WiredTigerTestCase): with self.expect(ok_commit, 'commit'): session.commit_transaction(commit_config) self.commit_value = value + if do_prepare: + self.last_durable = durable_ts if needs_rollback: # Rollback this one transaction, and continue the loop self.report('rollback_transaction') @@ -422,6 +431,12 @@ class test_timestamp22(wttest.WiredTigerTestCase): read_ts = self.gen_ts(iternum) else: read_ts = -1 # no read_timestamp used in txn + + # OOD does not work with prepared updates. Hence, the commit ts should always be + # greater than the last durable ts. + if commit_ts <= self.last_durable: + commit_ts = self.last_durable + 1 + if do_prepare: # If we doing a prepare, we must abide by some additional rules. # If we don't we'll immediately panic diff --git a/src/third_party/wiredtiger/test/utility/Makefile.am b/src/third_party/wiredtiger/test/utility/Makefile.am index a2923eb41a8..4bdbc775a59 100644 --- a/src/third_party/wiredtiger/test/utility/Makefile.am +++ b/src/third_party/wiredtiger/test/utility/Makefile.am @@ -1,4 +1,4 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include -libtest_util_la_SOURCES = misc.c parse_opts.c thread.c +libtest_util_la_SOURCES = misc.c modify.c parse_opts.c thread.c noinst_LTLIBRARIES = libtest_util.la diff --git a/src/third_party/wiredtiger/test/utility/modify.c b/src/third_party/wiredtiger/test/utility/modify.c new file mode 100644 index 00000000000..254dd58e3b4 --- /dev/null +++ b/src/third_party/wiredtiger/test/utility/modify.c @@ -0,0 +1,115 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * testutil_modify_apply -- + * Implement a modify using a completely separate algorithm as a check on the internal library + * algorithms. + */ +void +testutil_modify_apply(WT_ITEM *value, WT_ITEM *workspace, WT_MODIFY *entries, int nentries) +{ + WT_ITEM *ta, *tb, *tmp, _tmp; + size_t len, size; + int i; + + /* + * Passed a value and array of modifications, plus a temporary buffer for an additional work + * space. + * + * Process the entries to figure out the largest possible buffer we need. This is pessimistic + * because we're ignoring replacement bytes, but it's a simpler calculation. + */ + for (size = value->size, i = 0; i < nentries; ++i) { + if (entries[i].offset >= size) + size = entries[i].offset; + size += entries[i].data.size; + } + + /* Grow the buffers. */ + testutil_check(__wt_buf_grow(NULL, value, size)); + testutil_check(__wt_buf_grow(NULL, workspace, size)); + + /* + * Overwrite anything not initialized in the original buffer, and overwrite the entire workspace + * buffer. + */ + if ((value->memsize - value->size) > 0) + memset((uint8_t *)value->mem + value->size, 0xff, value->memsize - value->size); + if (workspace->memsize > 0) + memset((uint8_t *)workspace->mem, 0xff, workspace->memsize); + + ta = value; + tb = workspace; + + /* + * From the starting buffer, create a new buffer b based on changes in the entries array. We're + * doing a brute force solution here to test the faster solution implemented in the library. + */ + for (i = 0; i < nentries; ++i) { + /* Take leading bytes from the original, plus any gap bytes. */ + if (entries[i].offset >= ta->size) { + memcpy(tb->mem, ta->mem, ta->size); + if (entries[i].offset > ta->size) + memset((uint8_t *)tb->mem + ta->size, '\0', entries[i].offset - ta->size); + } else if (entries[i].offset > 0) + memcpy(tb->mem, ta->mem, entries[i].offset); + tb->size = entries[i].offset; + + /* Take replacement bytes. */ + if (entries[i].data.size > 0) { + memcpy((uint8_t *)tb->mem + tb->size, entries[i].data.data, entries[i].data.size); + tb->size += entries[i].data.size; + } + + /* Take trailing bytes from the original. */ + len = entries[i].offset + entries[i].size; + if (ta->size > len) { + memcpy((uint8_t *)tb->mem + tb->size, (uint8_t *)ta->mem + len, ta->size - len); + tb->size += ta->size - len; + } + testutil_assert(tb->size <= size); + + /* Swap the buffers and do it again. */ + tmp = ta; + ta = tb; + tb = tmp; + } + ta->data = ta->mem; + tb->data = tb->mem; + + /* + * The final results may not be in the original buffer, in which case we swap them back around. + */ + if (ta != value) { + _tmp = *ta; + *ta = *tb; + *tb = _tmp; + } +} diff --git a/src/third_party/wiredtiger/test/utility/test_util.h b/src/third_party/wiredtiger/test/utility/test_util.h index 24e37e5acb9..54444684ddf 100644 --- a/src/third_party/wiredtiger/test/utility/test_util.h +++ b/src/third_party/wiredtiger/test/utility/test_util.h @@ -266,21 +266,22 @@ void op_create(void *); void op_create_unique(void *); void op_cursor(void *); void op_drop(void *); +bool testutil_is_flag_set(const char *); +void testutil_build_dir(TEST_OPTS *, char *, int); void testutil_clean_work_dir(const char *); void testutil_cleanup(TEST_OPTS *); void testutil_copy_data(const char *); -bool testutil_is_flag_set(const char *); -void testutil_build_dir(TEST_OPTS *, char *, int); +void testutil_copy_file(WT_SESSION *, const char *); +void testutil_create_backup_directory(const char *); void testutil_make_work_dir(const char *); +void testutil_modify_apply(WT_ITEM *, WT_ITEM *, WT_MODIFY *, int); int testutil_parse_opts(int, char *const *, TEST_OPTS *); void testutil_print_command_line(int argc, char *const *argv); void testutil_progress(TEST_OPTS *, const char *); -void testutil_timestamp_parse(const char *, uint64_t *); -void testutil_create_backup_directory(const char *); -void testutil_copy_file(WT_SESSION *, const char *); #ifndef _WIN32 void testutil_sleep_wait(uint32_t, pid_t); #endif +void testutil_timestamp_parse(const char *, uint64_t *); void testutil_work_dir_from_path(char *, size_t, const char *); WT_THREAD_RET thread_append(void *); |