diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2021-11-01 17:42:31 +1100 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-11-01 07:08:29 +0000 |
| commit | b5c7d4ea2fb8ce7318139806dc9a320cbbc3b419 (patch) | |
| tree | 0b5e43fa59ac71a02233b6c6a29c547ed9b45cb8 | |
| parent | d410535b8fb7fd6f93033bc66f3f653b8216428b (diff) | |
| download | mongo-b5c7d4ea2fb8ce7318139806dc9a320cbbc3b419.tar.gz | |
Import wiredtiger: dfc1385ec8b2aa0d86dfe66890a7e3b95b729d43 from branch mongodb-master
ref: 854df2c974..dfc1385ec8
for: 5.2.0
WT-3445 Add multiple tables to format tester.
36 files changed, 3006 insertions, 2236 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index b7913c04a0a..ffe26ac16bd 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -510,47 +510,49 @@ table_meta = format_meta + table_only_config # Connection runtime config, shared by conn.reconfigure and wiredtiger_open connection_runtime_config = [ - Config('block_cache', '', r'''block cache configuration options''', - type='category', subconfig=[ - Config('cache_on_checkpoint', 'true', r''' - Cache blocks that are written by a checkpoint''', - type='boolean'), - Config('cache_on_writes', 'true', r''' - cache newly generated blocks''', - type='boolean'), - Config('enabled', 'false', r''' - enable block cache''', - type='boolean'), - Config('blkcache_eviction_aggression', '1800', r''' - how many seconds an unused block remains in the cache before it is evicted''', - min='1', max='7200'), - Config('full_target', '95', r''' - a fraction of cache that must be full before eviction - will remove unused blocks''', - min='30', max='100'), - Config('size', '0', r''' - maximum memory to allocate for the block cache''', - min='0', max='6155GB'), - Config('hashsize', '0', r''' - number of buckets in the hashtable that keeps track of blocks.''', - min='512', max='256K'), - Config('max_percent_overhead', '10', r''' - maximum tolerated overhead expressed as the number of - blocks added and removed as percent of blocks looked up; - cache population and eviction will be suppressed if the overhead - exceeds the supplied threshold''', - min='1', max='500'), - Config('nvram_path', '', r''' - the absolute path to the file system mounted on the NVRAM device''',), - Config('percent_file_in_dram', '50', r''' - bypass cache if that percent of file fits in DRAM''', - min='0', max='100'), - Config('system_ram', '0', r''' - amount of DRAM expected to be available on the system''', - min='0', max='1024GB'), - Config('type', '', r''' - cache location: DRAM or NVRAM'''), - ]), + Config('block_cache', '', r''' + block cache configuration options''', + type='category', subconfig=[ + Config('cache_on_checkpoint', 'true', r''' + cache blocks written by a checkpoint''', + type='boolean'), + Config('cache_on_writes', 'true', r''' + cache blocks as they are written (other than checkpoint blocks)''', + type='boolean'), + Config('enabled', 'false', r''' + enable block cache''', + type='boolean'), + Config('blkcache_eviction_aggression', '1800', r''' + seconds an unused block remains in the cache before it is evicted''', + min='1', max='7200'), + Config('full_target', '95', r''' + the fraction of the block cache that must be full before eviction + will remove unused blocks''', + min='30', max='100'), + Config('size', '0', r''' + maximum memory to allocate for the block cache''', + min='0', max='10TB'), + Config('hashsize', '0', r''' + number of buckets in the hashtable that keeps track of blocks''', + min='512', max='256K'), + Config('max_percent_overhead', '10', r''' + maximum tolerated overhead expressed as the number of blocks added + and removed as percent of blocks looked up; cache population + and eviction will be suppressed if the overhead exceeds the + supplied threshold''', + min='1', max='500'), + Config('nvram_path', '', r''' + the absolute path to the file system mounted on the NVRAM device'''), + Config('percent_file_in_dram', '50', r''' + bypass cache for a file if the set percentage of the file fits in system DRAM + (as specified by block_cache.system_ram)''', + min='0', max='100'), + Config('system_ram', '0', r''' + the bytes of system DRAM available for caching filesystem blocks''', + min='0', max='1024GB'), + Config('type', '', r''' + cache location: DRAM or NVRAM'''), + ]), Config('cache_size', '100MB', r''' maximum heap memory to allocate for the cache. A database should configure either \c cache_size or \c shared_cache but not both''', diff --git a/src/third_party/wiredtiger/dist/s_clang-format.list b/src/third_party/wiredtiger/dist/s_clang-format.list index 51c807aa2a4..f50d772a061 100644 --- a/src/third_party/wiredtiger/dist/s_clang-format.list +++ b/src/third_party/wiredtiger/dist/s_clang-format.list @@ -17,3 +17,4 @@ src/os_posix/os_getopt.c src/support/hash_city.c src/support/hash_fnv.c src/support/stat.c +test/format/config_def.c diff --git a/src/third_party/wiredtiger/dist/s_longlines b/src/third_party/wiredtiger/dist/s_longlines index 16fa1252131..9d38a481fbc 100755 --- a/src/third_party/wiredtiger/dist/s_longlines +++ b/src/third_party/wiredtiger/dist/s_longlines @@ -12,6 +12,7 @@ l=`(cd .. && -e '/checksum\/zseries/d' \ -e '/config\/config_def\.c/d' \ -e '/dist\/stat_data\.py/d' \ + -e '/format\/config_def\.c/d' \ -e '/include\/extern\.h/d' \ -e '/include\/extern_posix\.h/d' \ -e '/include\/extern_win\.h/d' \ diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 9bcfd210a09..a2f43166fc2 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -144,6 +144,7 @@ FALLOC FALLTHROUGH FH FIXME +FLCS FLD FLSv FLv @@ -446,6 +447,7 @@ Unordered Uryyb VALGRIND VARCHAR +VLCS VLDB VMSG VPM diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index baa8361c895..0381b75fc61 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "854df2c974559fddb3412f919d6421376f3b0ae7" + "commit": "dfc1385ec8b2aa0d86dfe66890a7e3b95b729d43" } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 25aceda326b..fbe1981fade 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -42,7 +42,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_block_cache_subconfigs[] = {"max_percent_overhead", "int", NULL, "min=1,max=500", NULL, 0}, {"nvram_path", "string", NULL, NULL, NULL, 0}, {"percent_file_in_dram", "int", NULL, "min=0,max=100", NULL, 0}, - {"size", "int", NULL, "min=0,max=6155GB", NULL, 0}, + {"size", "int", NULL, "min=0,max=10TB", NULL, 0}, {"system_ram", "int", NULL, "min=0,max=1024GB", NULL, 0}, {"type", "string", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}}; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 1cea9a9aef3..2bbebb4c5ce 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -2015,32 +2015,35 @@ struct __wt_connection { * @config{block_cache = (, block cache configuration options., a set of related * configuration options defined below.} * @config{ - * blkcache_eviction_aggression, how many seconds an unused block remains in the cache - * before it is evicted., an integer between 1 and 7200; default \c 1800.} - * @config{ cache_on_checkpoint, Cache blocks that are written by a + * blkcache_eviction_aggression, seconds an unused block remains in the cache before it is + * evicted., an integer between 1 and 7200; default \c 1800.} + * @config{ cache_on_checkpoint, cache blocks written by a * checkpoint., a boolean flag; default \c true.} * @config{ - * cache_on_writes, cache newly generated blocks., a boolean flag; default \c true.} - * @config{ enabled, enable block cache., a boolean flag; default \c - * false.} - * @config{ full_target, a fraction of cache that must be - * full before eviction will remove unused blocks., an integer between 30 and 100; default - * \c 95.} - * @config{ hashsize, number of buckets in the hashtable that - * keeps track of blocks., an integer between 512 and 256K; default \c 0.} - * @config{ max_percent_overhead, maximum tolerated overhead - * expressed as the number of blocks added and removed as percent of blocks looked up; cache - * population and eviction will be suppressed if the overhead exceeds the supplied - * threshold., an integer between 1 and 500; default \c 10.} - * @config{ nvram_path, the absolute path to the file system mounted - * on the NVRAM device., a string; default empty.} + * cache_on_writes, cache blocks as they are written (other than checkpoint blocks)., a + * boolean flag; default \c true.} + * @config{ enabled, enable block + * cache., a boolean flag; default \c false.} + * @config{ full_target, + * the fraction of the block cache that must be full before eviction will remove unused + * blocks., an integer between 30 and 100; default \c 95.} + * @config{ + * hashsize, number of buckets in the hashtable that keeps track of blocks., an integer + * between 512 and 256K; default \c 0.} * @config{ - * percent_file_in_dram, bypass cache if that percent of file fits in DRAM., an integer - * between 0 and 100; default \c 50.} - * @config{ size, maximum memory - * to allocate for the block cache., an integer between 0 and 6155GB; default \c 0.} - * @config{ system_ram, amount of DRAM expected to be available on - * the system., an integer between 0 and 1024GB; default \c 0.} + * max_percent_overhead, maximum tolerated overhead expressed as the number of blocks added + * and removed as percent of blocks looked up; cache population and eviction will be + * suppressed if the overhead exceeds the supplied threshold., an integer between 1 and 500; + * default \c 10.} + * @config{ nvram_path, the absolute path to the file + * system mounted on the NVRAM device., a string; default empty.} + * @config{ percent_file_in_dram, bypass cache for a file if the set + * percentage of the file fits in system DRAM (as specified by block_cache.system_ram)., an + * integer between 0 and 100; default \c 50.} + * @config{ size, maximum + * memory to allocate for the block cache., an integer between 0 and 10TB; default \c 0.} + * @config{ system_ram, the bytes of system DRAM available for + * caching filesystem blocks., an integer between 0 and 1024GB; default \c 0.} * @config{ type, cache location: DRAM or NVRAM., a string; default * empty.} * @config{ ),,} @@ -2703,16 +2706,18 @@ struct __wt_connection { * @configstart{wiredtiger_open, see dist/api_data.py} * @config{block_cache = (, block cache configuration options., a set of related configuration * options defined below.} - * @config{ blkcache_eviction_aggression, how many - * seconds an unused block remains in the cache before it is evicted., an integer between 1 and - * 7200; default \c 1800.} - * @config{ cache_on_checkpoint, Cache blocks that - * are written by a checkpoint., a boolean flag; default \c true.} - * @config{ - * cache_on_writes, cache newly generated blocks., a boolean flag; default \c true.} - * @config{ enabled, enable block cache., a boolean flag; default \c false.} - * @config{ full_target, a fraction of cache that must be full before - * eviction will remove unused blocks., an integer between 30 and 100; default \c 95.} + * @config{ blkcache_eviction_aggression, seconds an + * unused block remains in the cache before it is evicted., an integer between 1 and 7200; default + * \c 1800.} + * @config{ cache_on_checkpoint, cache blocks written by a + * checkpoint., a boolean flag; default \c true.} + * @config{ cache_on_writes, + * cache blocks as they are written (other than checkpoint blocks)., a boolean flag; default \c + * true.} + * @config{ enabled, enable block cache., a boolean flag; default \c + * false.} + * @config{ full_target, the fraction of the block cache that must be + * full before eviction will remove unused blocks., an integer between 30 and 100; default \c 95.} * @config{ hashsize, number of buckets in the hashtable that keeps track of * blocks., an integer between 512 and 256K; default \c 0.} * @config{ @@ -2722,12 +2727,15 @@ struct __wt_connection { * @config{ nvram_path, the absolute path to the file system mounted on the * NVRAM device., a string; default empty.} * @config{ percent_file_in_dram, - * bypass cache if that percent of file fits in DRAM., an integer between 0 and 100; default \c 50.} + * bypass cache for a file if the set percentage of the file fits in system DRAM (as specified by + * block_cache.system_ram)., an integer between 0 and 100; default \c 50.} * @config{ size, maximum memory to allocate for the block cache., an integer - * between 0 and 6155GB; default \c 0.} - * @config{ system_ram, amount of DRAM - * expected to be available on the system., an integer between 0 and 1024GB; default \c 0.} - * @config{ type, cache location: DRAM or NVRAM., a string; default empty.} + * between 0 and 10TB; default \c 0.} + * @config{ system_ram, the bytes of + * system DRAM available for caching filesystem blocks., an integer between 0 and 1024GB; default \c + * 0.} + * @config{ type, cache location: DRAM or NVRAM., a string; default + * empty.} * @config{ ),,} * @config{buffer_alignment, in-memory alignment (in bytes) for buffers used for I/O. The default * value of -1 indicates a platform-specific alignment value should be used (4KB on Linux systems diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index 6b85d469c96..6639f019709 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -206,7 +206,7 @@ __rollback_has_stable_update(WT_UPDATE *upd) { while (upd != NULL && (upd->type == WT_UPDATE_INVALID || upd->txnid == WT_TXN_ABORTED)) upd = upd->next; - return upd != NULL; + return (upd != NULL); } /* @@ -1428,7 +1428,7 @@ __rollback_to_stable_btree_apply( wt_timestamp_t max_durable_ts, newest_start_durable_ts, newest_stop_durable_ts; size_t addr_size; uint64_t rollback_txnid, write_gen; - uint32_t btree_id, handle_open_flags; + uint32_t btree_id; char ts_string[2][WT_TS_INT_STRING_SIZE]; bool dhandle_allocated, durable_ts_found, has_txn_updates_gt_than_ckpt_snap, perform_rts; bool prepared_updates; @@ -1529,16 +1529,7 @@ __rollback_to_stable_btree_apply( if (perform_rts || max_durable_ts > rollback_timestamp || prepared_updates || !durable_ts_found || has_txn_updates_gt_than_ckpt_snap) { - /* - * MongoDB does not close all open handles before calling rollback-to-stable; otherwise, - * don't permit that behavior, the application is likely making a mistake. - */ -#ifdef WT_STANDALONE_BUILD - handle_open_flags = WT_DHANDLE_DISCARD | WT_DHANDLE_EXCLUSIVE; -#else - handle_open_flags = 0; -#endif - ret = __wt_session_get_dhandle(session, uri, NULL, NULL, handle_open_flags); + ret = __wt_session_get_dhandle(session, uri, NULL, NULL, 0); if (ret != 0) WT_ERR_MSG(session, ret, "%s: unable to open handle%s", uri, ret == EBUSY ? ", error indicates handle is unavailable due to concurrent use" : ""); diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index a6e6b543c99..bbf198aa017 100755 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -384,7 +384,7 @@ functions: set -o errexit set -o verbose for i in $(seq ${times|1}); do - ./t -1 -c ${config|../../../test/format/CONFIG.stress} ${extra_args|} || ( [ -f RUNDIR/CONFIG ] && cat RUNDIR/CONFIG ) 2>&1 + ./t -c ${config|../../../test/format/CONFIG.stress} ${extra_args|} || ( [ -f RUNDIR/CONFIG ] && cat RUNDIR/CONFIG ) 2>&1 done "format test script": command: shell.exec diff --git a/src/third_party/wiredtiger/test/format/CONFIG.stress b/src/third_party/wiredtiger/test/format/CONFIG.stress index a57a4f2df51..5ea078899f3 100644 --- a/src/third_party/wiredtiger/test/format/CONFIG.stress +++ b/src/third_party/wiredtiger/test/format/CONFIG.stress @@ -2,7 +2,7 @@ btree.huffman_value=0 cache.minimum=20 runs.rows=1000000:5000000 +runs.tables=3:10 runs.threads=4:32 runs.timer=6:30 runs.type=row,var -runs=100 diff --git a/src/third_party/wiredtiger/test/format/Makefile.am b/src/third_party/wiredtiger/test/format/Makefile.am index 771e41fd662..8897feee112 100644 --- a/src/third_party/wiredtiger/test/format/Makefile.am +++ b/src/third_party/wiredtiger/test/format/Makefile.am @@ -4,8 +4,8 @@ AM_CPPFLAGS +=-I$(top_srcdir)/test/utility noinst_PROGRAMS = t t_SOURCES =\ - alter.c backup.c bulk.c checkpoint.c compact.c config.c config_compat.c hs.c import.c kv.c ops.c \ - random.c salvage.c snap.c t.c trace.c util.c wts.c + alter.c backup.c bulk.c checkpoint.c compact.c config.c config_compat.c config_def.c hs.c \ + import.c kv.c ops.c random.c salvage.c snap.c t.c trace.c util.c wts.c t_LDADD = $(top_builddir)/test/utility/libtest_util.la t_LDADD +=$(top_builddir)/libwiredtiger.la diff --git a/src/third_party/wiredtiger/test/format/alter.c b/src/third_party/wiredtiger/test/format/alter.c index 7c3dc4f02d9..2a87f0654ec 100644 --- a/src/third_party/wiredtiger/test/format/alter.c +++ b/src/third_party/wiredtiger/test/format/alter.c @@ -35,6 +35,7 @@ WT_THREAD_RET alter(void *arg) { + TABLE *table; WT_CONNECTION *conn; WT_DECL_RET; WT_SESSION *session; @@ -60,10 +61,10 @@ alter(void *arg) testutil_check(__wt_snprintf( buf, sizeof(buf), "access_pattern_hint=%s", access_value ? "random" : "none")); access_value = !access_value; - /* - * Alter can return EBUSY if concurrent with other operations. - */ - while ((ret = session->alter(session, g.uri, buf)) != 0 && ret != EBUSY) + + /* Alter can return EBUSY if concurrent with other operations. */ + table = table_select(NULL); + while ((ret = session->alter(session, table->uri, buf)) != 0 && ret != EBUSY) testutil_die(ret, "session.alter"); while (period > 0 && !g.workers_finished) { --period; diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c index 8ec113dd6d8..3d53b5324f7 100644 --- a/src/third_party/wiredtiger/test/format/backup.c +++ b/src/third_party/wiredtiger/test/format/backup.c @@ -71,12 +71,8 @@ check_copy(void) testutil_check(__wt_snprintf(path, len, "%s/BACKUP", g.home)); wts_open(path, &conn, &session, true); - /* - * Verify can return EBUSY if the handle isn't available. Don't yield and retry, in the case of - * LSM, the handle may not be available for a long time. - */ - ret = session->verify(session, g.uri, NULL); - testutil_assertfmt(ret == 0 || ret == EBUSY, "WT_SESSION.verify: %s: %s", path, g.uri); + /* Verify the objects. */ + tables_apply(wts_verify, conn); wts_close(&conn, &session); @@ -114,7 +110,7 @@ active_files_print(ACTIVE_FILES *active, const char *msg) if (active == NULL) return; - fprintf(stderr, "Active files: %s, %d entries\n", msg, (int)active->count); + fprintf(stderr, "Active files: %s, %" PRIu32 " entries\n", msg, active->count); for (i = 0; i < active->count; i++) fprintf(stderr, " %s\n", active->names[i]); } @@ -167,7 +163,7 @@ active_files_remove_missing(ACTIVE_FILES *prev, ACTIVE_FILES *cur) { uint32_t curpos, prevpos; int cmp; - char filename[1024]; + char filename[MAX_FORMAT_PATH]; if (prev == NULL) return; @@ -244,7 +240,7 @@ copy_blocks(WT_SESSION *session, WT_CURSOR *bkup_c, const char *name) ssize_t rdsize; uint64_t offset, size, this_size, total, type; int rfd, wfd1, wfd2; - char config[512], *tmp; + char config[MAX_FORMAT_PATH], *tmp; bool first_pass; tmp_sz = 0; @@ -366,7 +362,7 @@ restore_backup_info(WT_SESSION *session, ACTIVE_FILES *active) uint32_t i; char buf[512], *path; - testutil_assert(g.c_backup_incr_flag == INCREMENTAL_BLOCK); + testutil_assert(g.backup_incr_flag == INCREMENTAL_BLOCK); len = strlen(g.home) + strlen(BACKUP_INFO_FILE) + 2; path = dmalloc(len); testutil_check(__wt_snprintf(path, len, "%s/%s", g.home, BACKUP_INFO_FILE)); @@ -440,7 +436,7 @@ save_backup_info(ACTIVE_FILES *active, uint64_t id) uint32_t i; char *from_path, *to_path; - if (g.c_backup_incr_flag != INCREMENTAL_BLOCK) + if (g.backup_incr_flag != INCREMENTAL_BLOCK) return; len = strlen(g.home) + strlen(BACKUP_INFO_FILE_TMP) + 2; from_path = dmalloc(len); @@ -498,7 +494,7 @@ backup(void *arg) * that we can take an incremental backup and use the older id as a source identifier. We force * that only if the restore function was successful in restoring the backup information. */ - if (g.reopen && g.c_backup_incr_flag == INCREMENTAL_BLOCK && + if (g.reopen && g.backup_incr_flag == INCREMENTAL_BLOCK && restore_backup_info(session, &active[0]) == RESTORE_SUCCESS) { incr_full = false; full = false; @@ -529,7 +525,7 @@ backup(void *arg) break; } - if (g.c_backup_incr_flag == INCREMENTAL_BLOCK) { + if (g.backup_incr_flag == INCREMENTAL_BLOCK) { /* * If we're doing a full backup as the start of the incremental backup, only send in an * identifier for this one. Also set the block granularity. @@ -541,7 +537,7 @@ backup(void *arg) active_prev = NULL; testutil_check(__wt_snprintf(cfg, sizeof(cfg), "incremental=(enabled,granularity=%" PRIu32 "K,this_id=%" PRIu64 ")", - g.c_backup_incr_granularity, g.backup_id)); + GV(BACKUP_INCR_GRANULARITY), g.backup_id)); full = true; incr_full = false; } else { @@ -562,7 +558,7 @@ backup(void *arg) config = cfg; /* Free up the old active file list we're going to overwrite. */ active_files_free(active_now); - } else if (g.c_logging && g.c_backup_incr_flag == INCREMENTAL_LOG) { + } else if (GV(LOGGING) && g.backup_incr_flag == INCREMENTAL_LOG) { if (incr_full) { config = NULL; full = true; @@ -595,7 +591,7 @@ backup(void *arg) while ((ret = backup_cursor->next(backup_cursor)) == 0) { testutil_check(backup_cursor->get_key(backup_cursor, &key)); - if (g.c_backup_incr_flag == INCREMENTAL_BLOCK) { + if (g.backup_incr_flag == INCREMENTAL_BLOCK) { if (full) testutil_copy_file(session, key); else @@ -609,7 +605,7 @@ backup(void *arg) testutil_die(ret, "backup-cursor"); /* After a log-based incremental backup, truncate the log files. */ - if (g.c_backup_incr_flag == INCREMENTAL_LOG) + if (g.backup_incr_flag == INCREMENTAL_LOG) testutil_check(session->truncate(session, "log:", backup_cursor, NULL, NULL)); testutil_check(backup_cursor->close(backup_cursor)); @@ -628,9 +624,9 @@ backup(void *arg) */ if (full) { incremental = 1; - if (g.c_backup_incr_flag == INCREMENTAL_LOG) - incremental = g.c_logging_archive ? 1 : mmrand(NULL, 1, 8); - else if (g.c_backup_incr_flag == INCREMENTAL_BLOCK) + if (g.backup_incr_flag == INCREMENTAL_LOG) + incremental = GV(LOGGING_ARCHIVE) ? 1 : mmrand(NULL, 1, 8); + else if (g.backup_incr_flag == INCREMENTAL_BLOCK) incremental = mmrand(NULL, 1, 8); } if (--incremental == 0) { diff --git a/src/third_party/wiredtiger/test/format/bulk.c b/src/third_party/wiredtiger/test/format/bulk.c index d1864741a77..def8241f045 100644 --- a/src/third_party/wiredtiger/test/format/bulk.c +++ b/src/third_party/wiredtiger/test/format/bulk.c @@ -73,8 +73,12 @@ bulk_rollback_transaction(WT_SESSION *session) testutil_check(session->rollback_transaction(session, NULL)); } +/* + * wts_load -- + * Bulk load a table. + */ void -wts_load(void) +wts_load(TABLE *table, void *arg) { WT_CONNECTION *conn; WT_CURSOR *cursor; @@ -82,41 +86,38 @@ wts_load(void) WT_ITEM key, value; WT_SESSION *session; uint32_t committed_keyno, keyno, v; + char track_buf[128]; bool is_bulk; - conn = g.wts_conn; + (void)arg; /* unused argument */ - testutil_check(conn->open_session(conn, NULL, NULL, &session)); + conn = g.wts_conn; - trace_msg("%s", "=============== bulk load start"); + testutil_check(__wt_snprintf(track_buf, sizeof(track_buf), "table %u bulk load", table->id)); + trace_msg("=============== %s bulk load start", table->uri); /* * No bulk load with custom collators, the order of insertion will not match the collation * order. */ is_bulk = true; - if (g.c_reverse) + if (TV(BTREE_REVERSE)) is_bulk = false; - /* - * open_cursor can return EBUSY if concurrent with a metadata operation, retry in that case. - */ - while ((ret = session->open_cursor( - session, g.uri, NULL, is_bulk ? "bulk,append" : NULL, &cursor)) == EBUSY) - __wt_yield(); - testutil_check(ret); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + wiredtiger_open_cursor(session, table->uri, is_bulk ? "bulk,append" : NULL, &cursor); /* Set up the key/value buffers. */ key_gen_init(&key); val_gen_init(&value); - if (g.c_txn_timestamps) + if (g.transaction_timestamps_config) bulk_begin_transaction(session); - for (committed_keyno = keyno = 0; ++keyno <= g.c_rows;) { - val_gen(NULL, &value, keyno); + for (committed_keyno = keyno = 0; ++keyno <= table->rows_current;) { + val_gen(table, NULL, &value, keyno); - switch (g.type) { + switch (table->type) { case FIX: if (!is_bulk) cursor->set_key(cursor, keyno); @@ -132,7 +133,7 @@ wts_load(void) trace_msg("bulk %" PRIu32 " {%.*s}", keyno, (int)value.size, (char *)value.data); break; case ROW: - key_gen(&key, keyno); + key_gen(table, &key, keyno); cursor->set_key(cursor, &key); cursor->set_value(cursor, &value); if (g.trace_all) @@ -150,7 +151,7 @@ wts_load(void) if ((ret = cursor->insert(cursor)) != 0) { testutil_assert(ret == WT_CACHE_FULL || ret == WT_ROLLBACK); - if (g.c_txn_timestamps) { + if (g.transaction_timestamps_config) { bulk_rollback_transaction(session); bulk_begin_transaction(session); } @@ -161,13 +162,13 @@ wts_load(void) * simply modify the values because they have to equal 100 when the database is reopened * (we are going to rewrite the CONFIG file, too). */ - if (g.c_insert_pct > 5) { - g.c_delete_pct += g.c_insert_pct - 5; - g.c_insert_pct = 5; + if (TV(OPS_PCT_INSERT) > 5) { + TV(OPS_PCT_DELETE) += TV(OPS_PCT_INSERT) - 5; + TV(OPS_PCT_INSERT) = 5; } - v = g.c_write_pct / 2; - g.c_delete_pct += v; - g.c_write_pct -= v; + v = TV(OPS_PCT_WRITE) / 2; + TV(OPS_PCT_DELETE) += v; + TV(OPS_PCT_WRITE) -= v; break; } @@ -179,9 +180,9 @@ wts_load(void) */ if ((keyno < 5000 && keyno % 10 == 0) || keyno % 5000 == 0) { /* Report on progress. */ - track("bulk load", keyno, NULL); + track(track_buf, keyno); - if (g.c_txn_timestamps) { + if (g.transaction_timestamps_config) { bulk_commit_transaction(session); committed_keyno = keyno; bulk_begin_transaction(session); @@ -189,27 +190,23 @@ wts_load(void) } } - if (g.c_txn_timestamps) + if (g.transaction_timestamps_config) bulk_commit_transaction(session); /* * Ideally, the insert loop runs until the number of rows plus one, in which case row counts are - * correct. If the loop exited early, reset the counters and rewrite the CONFIG file (so reopens - * aren't surprised). + * correct. If the loop exited early, reset the table's row count and rewrite the CONFIG file + * (so reopens aren't surprised). */ - if (keyno != g.c_rows + 1) { - g.c_rows = g.c_txn_timestamps ? committed_keyno : (keyno - 1); - testutil_assert(g.c_rows > 0); - g.rows = g.c_rows; + if (keyno != table->rows_current + 1) { + table->rows_current = g.transaction_timestamps_config ? committed_keyno : (keyno - 1); + testutil_assert(table->rows_current > 0); config_print(false); } - testutil_check(cursor->close(cursor)); - - trace_msg("%s", "=============== bulk load stop"); - testutil_check(session->close(session, NULL)); + trace_msg("=============== %s bulk load stop", table->uri); key_gen_teardown(&key); val_gen_teardown(&value); diff --git a/src/third_party/wiredtiger/test/format/checkpoint.c b/src/third_party/wiredtiger/test/format/checkpoint.c index dac538348a6..b2eef25bea0 100644 --- a/src/third_party/wiredtiger/test/format/checkpoint.c +++ b/src/third_party/wiredtiger/test/format/checkpoint.c @@ -35,7 +35,7 @@ void wts_checkpoints(void) { - char config[1024]; + char config[128]; /* * Configuring WiredTiger library checkpoints is done separately, rather than as part of the @@ -45,12 +45,12 @@ wts_checkpoints(void) * checkpoint running in the tree, and the cache can get stuck. That workload is unlikely enough * we're not going to fix it in the library, so configure it away by delaying checkpoint start. */ - if (g.c_checkpoint_flag != CHECKPOINT_WIREDTIGER) + if (g.checkpoint_config != CHECKPOINT_WIREDTIGER) return; testutil_check( __wt_snprintf(config, sizeof(config), ",checkpoint=(wait=%" PRIu32 ",log_size=%" PRIu32 ")", - g.c_checkpoint_wait, MEGABYTE(g.c_checkpoint_log_size))); + GV(CHECKPOINT_WAIT), MEGABYTE(GV(CHECKPOINT_LOG_SIZE)))); testutil_check(g.wts_conn->reconfigure(g.wts_conn, config)); } @@ -67,11 +67,13 @@ checkpoint(void *arg) u_int secs; char config_buf[64]; const char *ckpt_config; - bool backup_locked; + bool backup_locked, named_checkpoints; (void)arg; + conn = g.wts_conn; testutil_check(conn->open_session(conn, NULL, NULL, &session)); + named_checkpoints = !g.lsm_config; for (secs = mmrand(NULL, 1, 10); !g.workers_finished;) { if (secs > 0) { @@ -88,7 +90,7 @@ checkpoint(void *arg) */ ckpt_config = NULL; backup_locked = false; - if (!DATASOURCE("lsm")) + if (named_checkpoints) switch (mmrand(NULL, 1, 20)) { case 1: /* diff --git a/src/third_party/wiredtiger/test/format/compact.c b/src/third_party/wiredtiger/test/format/compact.c index 4de1d898bcc..f4b53f21f75 100644 --- a/src/third_party/wiredtiger/test/format/compact.c +++ b/src/third_party/wiredtiger/test/format/compact.c @@ -29,12 +29,13 @@ #include "format.h" /* - * compaction -- + * compact -- * Periodically do a compaction operation. */ WT_THREAD_RET compact(void *arg) { + TABLE *table; WT_CONNECTION *conn; WT_DECL_RET; WT_SESSION *session; @@ -66,7 +67,8 @@ compact(void *arg) * Compact returns ETIMEDOUT if the compaction doesn't finish in in some number of seconds. * We don't configure a timeout and occasionally exceed the default of 1200 seconds. */ - ret = session->compact(session, g.uri, NULL); + table = table_select(NULL); + ret = session->compact(session, table->uri, NULL); if (ret != 0 && ret != EBUSY && ret != ETIMEDOUT && ret != WT_ROLLBACK && ret != WT_CACHE_FULL) testutil_die(ret, "session.compact"); diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 6cf7c795774..4eb81bb7c19 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -27,7 +27,6 @@ */ #include "format.h" -#include "config.h" static void config_backup_incr(void); static void config_backup_incr_granularity(void); @@ -35,81 +34,124 @@ static void config_backup_incr_log_compatibility_check(void); static void config_backward_compatible(void); static void config_cache(void); static void config_checkpoint(void); -static void config_checksum(void); -static void config_compression(const char *); +static void config_checksum(TABLE *); +static void config_compression(TABLE *, const char *); static void config_directio(void); static void config_encryption(void); static const char *config_file_type(u_int); -static bool config_fix(void); +static bool config_explicit(TABLE *, const char *); +static bool config_fix(TABLE *); static void config_in_memory(void); static void config_in_memory_reset(void); -static int config_is_perm(const char *); -static void config_lsm_reset(void); +static void config_lsm_reset(TABLE *); static void config_map_backup_incr(const char *, u_int *); static void config_map_checkpoint(const char *, u_int *); -static void config_map_checksum(const char *, u_int *); -static void config_map_compression(const char *, u_int *); -static void config_map_encryption(const char *, u_int *); static void config_map_file_type(const char *, u_int *); -static void config_pct(void); -static void config_prefix(void); -static void config_reset(void); +static void config_pct(TABLE *); static void config_transaction(void); /* - * We currently disable random LSM testing, that is, it can be specified explicitly but we won't - * randomly choose LSM as a data_source configuration. - */ -#define DISABLE_RANDOM_LSM_TESTING 1 - -/* - * config_final -- - * Final run initialization. + * config_random -- + * Do random configuration on the remaining global or table space. */ -void -config_final(void) +static void +config_random(TABLE *table, bool table_only) { - config_print(false); + CONFIG *cp; + CONFIGV *v; + char buf[128]; - g.rows = g.c_rows; /* Set the key count. */ + for (cp = configuration_list; cp->name != NULL; ++cp) { + if (F_ISSET(cp, C_IGNORE)) + continue; + if (table_only && !F_ISSET(cp, C_TABLE)) + continue; + if (!table_only && F_ISSET(cp, C_TABLE)) + continue; + + /* + * Don't randomly configure runs.tables if we read a CONFIG file, that prevents us from + * turning old-style CONFIG files into multi-table tests. + */ + if (cp->off == V_GLOBAL_RUNS_TABLES && !g.multi_table_config) + continue; + + v = &table->v[cp->off]; + if (v->set) + continue; - key_init(); /* Initialize key/value information. */ - val_init(); + /* Configure key prefixes only rarely, 5% if the length isn't set explicitly. */ + if (cp->off == V_TABLE_BTREE_PREFIX_LEN && mmrand(NULL, 1, 100) > 5) + continue; + + /* + * Boolean flags are 0 or 1, where the variable's "min" value is the percent chance the flag + * is "on" (so "on" if random rolled <= N, otherwise "off"). + */ + if (F_ISSET(cp, C_BOOL)) + testutil_check(__wt_snprintf( + buf, sizeof(buf), "%s=%s", cp->name, mmrand(NULL, 1, 100) <= cp->min ? "on" : "off")); + else + testutil_check(__wt_snprintf( + buf, sizeof(buf), "%s=%" PRIu32, cp->name, mmrand(NULL, cp->min, cp->maxrand))); + config_single(table, buf, false); + } } /* - * config -- - * Initialize the configuration itself. + * config_promote -- + * Promote a base value to a table. */ -void -config_run(void) +static void +config_promote(TABLE *table, CONFIG *cp, CONFIGV *v) { - CONFIG *cp; char buf[128]; - /* Clear any temporary values. */ - config_reset(); + if (F_ISSET(cp, C_STRING)) + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s=%s", cp->name, v->vstr)); + else + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s=%" PRIu32, cp->name, v->v)); + config_single(table, buf, true); +} - /* Periodically run in-memory. */ - config_in_memory(); +/* + * We currently disable random LSM testing, that is, it can be specified explicitly but we won't + * randomly choose LSM as a data_source configuration. + */ +#define DISABLE_RANDOM_LSM_TESTING 1 + +/* + * config_table_am -- + * Configure the table's access methods (type and source). + */ +static void +config_table_am(TABLE *table) +{ + char buf[128]; /* - * Choose a file format and a data source: they're interrelated (LSM is only compatible with - * row-store) and other items depend on them. + * The runs.type configuration allows more than a single type, for example, choosing from either + * RS and VLCS but not FLCS. If there's no table value but there was a global value, re-evaluate + * the original global specification, not the choice set for the global table. */ - if (!config_is_perm("runs.type")) { - if (config_is_perm("runs.source") && DATASOURCE("lsm")) - config_single("runs.type=row", false); + if (!table->v[V_TABLE_RUNS_TYPE].set && tables[0]->v[V_TABLE_RUNS_TYPE].set) { + testutil_check(__wt_snprintf(buf, sizeof(buf), "runs.type=%s", g.runs_type)); + config_single(table, buf, true); + } + + if (!config_explicit(table, "runs.type")) { + if (config_explicit(table, "runs.source") && DATASOURCE(table, "lsm")) + config_single(table, "runs.type=row", false); else switch (mmrand(NULL, 1, 10)) { case 1: case 2: case 3: /* 30% */ - config_single("runs.type=var", false); + config_single(table, "runs.type=var", false); break; case 4: /* 10% */ - if (config_fix()) { - config_single("runs.type=fix", false); + if (config_fix(table)) { + config_single(table, "runs.type=fix", false); break; } /* FALLTHROUGH */ /* 60% */ @@ -119,118 +161,182 @@ config_run(void) case 8: case 9: case 10: - config_single("runs.type=row", false); + config_single(table, "runs.type=row", false); break; } } - if (!config_is_perm("runs.source")) { - config_single("runs.source=table", false); + if (!config_explicit(table, "runs.source")) switch (mmrand(NULL, 1, 5)) { case 1: /* 20% */ - config_single("runs.source=file", false); + config_single(table, "runs.source=file", false); break; case 2: /* 20% */ #if !defined(DISABLE_RANDOM_LSM_TESTING) /* - * LSM requires a row-store and backing disk. + * LSM requires a row-store and backing disk. Don't configure LSM if in-memory, + * timestamps or truncation are configured, they result in cache problems. * - * Configuring truncation or timestamps results in LSM cache problems, don't configure - * LSM if those set. - * - * XXX Remove the timestamp test when WT-4162 resolved. + * FIXME WT-4162: Remove the timestamp test when WT-4162 resolved. */ - if (g.type != ROW || g.c_in_memory) + if (table->type != ROW || GV(RUNS_IN_MEMORY)) + break; + if (config_explicit(table, "transaction.timestamps") && TV(TRANSACTION_TIMESTAMPS)) break; - if (config_is_perm("transaction.timestamps") && g.c_txn_timestamps) + if (GV(BACKUP) && config_explicit(table, "backup.incremental") && + g.backup_incr_flag == INCREMENTAL_BLOCK) break; - if (config_is_perm("ops.truncate") && g.c_truncate) + if (config_explicit(table, "ops.truncate") && TV(OPS_TRUNCATE)) break; - config_single("runs.source=lsm", false); + config_single(table, "runs.source=lsm", false); #endif - break; + /* FALLTHROUGH */ case 3: case 4: case 5: /* 60% */ + config_single(table, "runs.source=table", false); break; } - } - /* If data_source and file_type were both "permanent", we may still have a mismatch. */ - if (DATASOURCE("lsm") && g.type != ROW) - testutil_die( - EINVAL, "%s: lsm data_source is only compatible with row file_type\n", progname); + /* If data_source and file_type were both set explicitly, we may still have a mismatch. */ + if (DATASOURCE(table, "lsm") && table->type != ROW) + testutil_die(EINVAL, "%s: lsm data_source is only compatible with row file_type", progname); +} + +/* + * config_table -- + * Finish initialization of a single table. + */ +static void +config_table(TABLE *table, void *arg) +{ + CONFIG *cp; + + (void)arg; /* unused argument */ + + /* + * Choose a file format and a data source: they're interrelated (LSM is only compatible with + * row-store) and other items depend on them. + */ + config_table_am(table); /* * Build the top-level object name: we're overloading data_source in our configuration, LSM * objects are "tables", but files are tested as well. */ - g.uri = dmalloc(256); - strcpy(g.uri, DATASOURCE("file") ? "file:" : "table:"); - strcat(g.uri, WT_NAME); + if (ntables == 0) + testutil_check(__wt_snprintf(table->uri, sizeof(table->uri), "%s", + DATASOURCE(table, "file") ? "file:wt" : "table:wt")); + else + testutil_check(__wt_snprintf(table->uri, sizeof(table->uri), + DATASOURCE(table, "file") ? "file:F%05u" : "table:T%05u", table->id)); + testutil_check( + __wt_snprintf(table->track_prefix, sizeof(table->track_prefix), "table %u", table->id)); + + /* + * For any values set in the base configuration, export them to this table (where this table + * doesn't already have a value set). + */ + if (ntables != 0) + for (cp = configuration_list; cp->name != NULL; ++cp) + if (F_ISSET(cp, C_TABLE) && !table->v[cp->off].set && tables[0]->v[cp->off].set) + config_promote(table, cp, &tables[0]->v[cp->off]); /* Fill in random values for the rest of the run. */ - for (cp = c; cp->name != NULL; ++cp) { - if (F_ISSET(cp, C_IGNORE | C_PERM | C_TEMP)) - continue; + config_random(table, true); - /* - * Boolean flags are 0 or 1, where the variable's "min" value is the percent chance the flag - * is "on" (so "on" if random rolled <= N, otherwise "off"). - */ - if (F_ISSET(cp, C_BOOL)) - testutil_check(__wt_snprintf( - buf, sizeof(buf), "%s=%s", cp->name, mmrand(NULL, 1, 100) <= cp->min ? "on" : "off")); - else - testutil_check(__wt_snprintf( - buf, sizeof(buf), "%s=%" PRIu32, cp->name, mmrand(NULL, cp->min, cp->maxrand))); - config_single(buf, false); + /* Page sizes are configured using powers-of-two or megabytes, convert them. */ + table->max_intl_page = 1U << TV(BTREE_INTERNAL_PAGE_MAX); + table->max_leaf_page = 1U << TV(BTREE_LEAF_PAGE_MAX); + table->max_mem_page = MEGABYTE(TV(BTREE_MEMORY_PAGE_MAX)); + + /* + * Key/value minimum/maximum are related, correct unless specified by the configuration. Key + * sizes are a row-store consideration: column-store doesn't store keys, a constant of 8 will + * reserve a small amount of additional space. + */ + if (table->type == ROW) { + if (!config_explicit(table, "btree.key_min") && TV(BTREE_KEY_MIN) > TV(BTREE_KEY_MAX)) + TV(BTREE_KEY_MIN) = TV(BTREE_KEY_MAX); + if (!config_explicit(table, "btree.key_max") && TV(BTREE_KEY_MAX) < TV(BTREE_KEY_MIN)) + TV(BTREE_KEY_MAX) = TV(BTREE_KEY_MIN); + if (TV(BTREE_KEY_MIN) > TV(BTREE_KEY_MAX)) + testutil_die(EINVAL, "btree.key_min may not be larger than btree.key_max"); + } else + TV(BTREE_KEY_MIN) = TV(BTREE_KEY_MAX) = 8; + if (!config_explicit(table, "btree.value_min") && TV(BTREE_VALUE_MIN) > TV(BTREE_VALUE_MAX)) + TV(BTREE_VALUE_MIN) = TV(BTREE_VALUE_MAX); + if (!config_explicit(table, "btree.value_max") && TV(BTREE_VALUE_MAX) < TV(BTREE_VALUE_MIN)) + TV(BTREE_VALUE_MAX) = TV(BTREE_VALUE_MIN); + if (TV(BTREE_VALUE_MIN) > TV(BTREE_VALUE_MAX)) + testutil_die(EINVAL, "btree.value_min may not be larger than btree.value_max"); + + /* + * If common key prefixes are configured, add prefix compression if no explicit choice was made + * and track the largest common key prefix in the run. + */ + if (TV(BTREE_PREFIX_LEN) != 0) { + if (TV(BTREE_PREFIX_COMPRESSION) == 0 && + !config_explicit(table, "btree.prefix_compression")) + config_single(table, "btree.prefix_compression=on", false); + g.prefix_len_max = WT_MAX(g.prefix_len_max, TV(BTREE_PREFIX_LEN)); } - /* Only row-store tables support collation order. */ - if (g.type != ROW) - config_single("btree.reverse=off", false); - - /* First, transaction configuration, it configures other features. */ - config_transaction(); - - /* Simple selection. */ - config_backup_incr(); - config_checkpoint(); - config_checksum(); - config_compression("btree.compression"); - config_compression("logging.compression"); - config_encryption(); - config_prefix(); - - /* Configuration based on the configuration already chosen. */ - config_directio(); - config_pct(); - config_cache(); - - /* Give in-memory, LSM and backward compatible configurations a final review. */ - if (g.c_in_memory != 0) + config_checksum(table); + config_compression(table, "btree.compression"); + config_pct(table); + + /* The number of rows in the table can change, get a local copy of the starting value. */ + table->rows_current = TV(RUNS_ROWS); + + /* Column-store tables require special row insert resolution. */ + if (table->type != ROW) + g.column_store_config = true; + + /* Only row-store tables support a collation order. */ + if (table->type != ROW) + config_single(table, "btree.reverse=off", false); + + /* Give LSM a final review and flag if there's at least one LSM data source. */ + if (DATASOURCE(table, "lsm")) { + g.lsm_config = true; + config_lsm_reset(table); + } +} + +/* + * config_run -- + * Run initialization. + */ +void +config_run(void) +{ + config_in_memory(); /* Periodically run in-memory. */ + + config_random(tables[0], false); /* Configure the remaining global name space. */ + + tables_apply(config_table, NULL); /* Configure the tables. */ + + /* Order can be important, don't shuffle without careful consideration. */ + config_transaction(); /* Transactions */ + config_backup_incr(); /* Incremental backup */ + config_checkpoint(); /* Checkpoints */ + config_compression(NULL, "logging.compression"); /* Logging compression */ + config_directio(); /* Direct I/O */ + config_encryption(); /* Encryption */ + + /* If doing an in-memory run, make sure we haven't configured something that won't work. */ + if (GV(RUNS_IN_MEMORY)) config_in_memory_reset(); - if (DATASOURCE("lsm")) - config_lsm_reset(); - config_backward_compatible(); /* - * Key/value minimum/maximum are related, correct unless specified by the configuration. + * If built in a branch that doesn't support all current options, or creating a database for + * such an environment, strip out configurations that won't work. */ - if (!config_is_perm("btree.key_min") && g.c_key_min > g.c_key_max) - g.c_key_min = g.c_key_max; - if (!config_is_perm("btree.key_max") && g.c_key_max < g.c_key_min) - g.c_key_max = g.c_key_min; - if (g.c_key_min > g.c_key_max) - testutil_die(EINVAL, "key_min may not be larger than key_max"); - - if (!config_is_perm("btree.value_min") && g.c_value_min > g.c_value_max) - g.c_value_min = g.c_value_max; - if (!config_is_perm("btree.value_max") && g.c_value_max < g.c_value_min) - g.c_value_max = g.c_value_min; - if (g.c_value_min > g.c_value_max) - testutil_die(EINVAL, "value_min may not be larger than value_max"); + if (g.backward_compatible) + config_backward_compatible(); + + config_cache(); /* Cache */ /* * Run-length is configured by a number of operations and a timer. @@ -244,14 +350,14 @@ config_run(void) * operations but the rest of the configuration means operations take a long time to complete * (for example, a small cache and many worker threads), don't let it run forever. */ - if (config_is_perm("runs.timer")) { - if (!config_is_perm("runs.ops")) - config_single("runs.ops=0", false); + if (config_explicit(NULL, "runs.timer")) { + if (!config_explicit(NULL, "runs.ops")) + config_single(NULL, "runs.ops=0", false); } else { - if (!config_is_perm("runs.ops")) - config_single("runs.timer=30", false); + if (!config_explicit(NULL, "runs.ops")) + config_single(NULL, "runs.timer=30", false); else - config_single("runs.timer=360", false); + config_single(NULL, "runs.timer=360", false); } } @@ -263,10 +369,10 @@ static void config_backup_incr(void) { /* Incremental backup requires backup. */ - if (g.c_backups == 0) { - if (!config_is_perm("backup.incremental")) - config_single("backup.incremental=off", false); - if (g.c_backup_incr_flag != INCREMENTAL_OFF) + if (GV(BACKUP) == 0) { + if (!config_explicit(NULL, "backup.incremental")) + config_single(NULL, "backup.incremental=off", false); + if (g.backup_incr_flag != INCREMENTAL_OFF) testutil_die(EINVAL, "backup.incremental requires backups be configured"); return; } @@ -275,10 +381,10 @@ config_backup_incr(void) * Incremental backup using log files is incompatible with logging archival. Testing log file * archival doesn't seem as useful as testing backup, let the backup configuration override. */ - if (config_is_perm("backup.incremental")) { - if (g.c_backup_incr_flag == INCREMENTAL_LOG) + if (config_explicit(NULL, "backup.incremental")) { + if (g.backup_incr_flag == INCREMENTAL_LOG) config_backup_incr_log_compatibility_check(); - if (g.c_backup_incr_flag == INCREMENTAL_BLOCK) + if (g.backup_incr_flag == INCREMENTAL_BLOCK) config_backup_incr_granularity(); return; } @@ -291,15 +397,15 @@ config_backup_incr(void) case 1: /* 30% full backup only */ case 2: case 3: - config_single("backup.incremental=off", false); + config_single(NULL, "backup.incremental=off", false); break; case 4: /* 30% log based incremental */ case 5: case 6: - if (!g.c_logging_archive || !config_is_perm("logging.archive")) { - if (g.c_logging_archive) - config_single("logging.archive=0", false); - config_single("backup.incremental=log", false); + if (!GV(LOGGING_ARCHIVE) || !config_explicit(NULL, "logging.archive")) { + if (GV(LOGGING_ARCHIVE)) + config_single(NULL, "logging.archive=0", false); + config_single(NULL, "backup.incremental=log", false); break; } /* FALLTHROUGH */ @@ -307,7 +413,7 @@ config_backup_incr(void) case 8: case 9: case 10: - config_single("backup.incremental=block", false); + config_single(NULL, "backup.incremental=block", false); config_backup_incr_granularity(); break; } @@ -323,7 +429,7 @@ config_backup_incr_granularity(void) uint32_t granularity, i; char confbuf[128]; - if (config_is_perm("backup.incr_granularity")) + if (config_explicit(NULL, "backup.incr_granularity")) return; /* @@ -353,8 +459,27 @@ config_backup_incr_granularity(void) } testutil_check( - __wt_snprintf(confbuf, sizeof(confbuf), "backup.incr_granularity=%u", granularity)); - config_single(confbuf, false); + __wt_snprintf(confbuf, sizeof(confbuf), "backup.incr_granularity=%" PRIu32, granularity)); + config_single(NULL, confbuf, false); +} + +/* + * config_backward_compatible_table -- + * Backward compatibility configuration, per table. + */ +static void +config_backward_compatible_table(TABLE *table, void *arg) +{ + (void)arg; /* unused argument */ + +#undef BC_CHECK +#define BC_CHECK(name, flag) \ + if (TV(flag)) { \ + if (config_explicit(table, name)) \ + testutil_die(EINVAL, "%s not supported in backward compatibility mode", name); \ + config_single(table, #name "=off", false); \ + } + BC_CHECK("btree.prefix_len", BTREE_PREFIX_LEN); } /* @@ -364,51 +489,22 @@ config_backup_incr_granularity(void) static void config_backward_compatible(void) { - bool backward_compatible; - - /* - * If built in a branch that doesn't support all current options, or creating a database for - * such an environment, strip out configurations that won't work. - */ - backward_compatible = g.backward_compatible; -#if WIREDTIGER_VERSION_MAJOR < 10 - backward_compatible = true; -#endif - if (!backward_compatible) - return; - - if (g.c_mmap_all) { - if (config_is_perm("disk.mmap_all")) - testutil_die(EINVAL, "disk.mmap_all not supported in backward compatibility mode"); - config_single("disk.mmap_all=off", false); - } - - if (g.c_timing_stress_checkpoint_reserved_txnid_delay) { - if (config_is_perm("stress.checkpoint_reserved_txnid_delay")) - testutil_die(EINVAL, - "stress.checkpoint_reserved_txnid_delay not supported in backward compatibility " - "mode"); - config_single("stress.checkpoint_reserved_txnid_delay=off", false); - } - - if (g.c_timing_stress_hs_sweep) { - if (config_is_perm("stress.hs_sweep")) - testutil_die(EINVAL, "stress.hs_sweep not supported in backward compatibility mode"); - config_single("stress.hs_sweep=off", false); +#undef BC_CHECK +#define BC_CHECK(name, flag) \ + if (GV(flag)) { \ + if (config_explicit(NULL, name)) \ + testutil_die(EINVAL, "%s not supported in backward compatibility mode", name); \ + config_single(NULL, #name "=off", false); \ } - if (g.c_timing_stress_hs_checkpoint_delay) { - if (config_is_perm("stress.hs_checkpoint_delay")) - testutil_die( - EINVAL, "stress.hs_checkpoint_delay not supported in backward compatibility mode"); - config_single("stress.hs_checkpoint_delay=off", false); - } + BC_CHECK("disk.mmap_all", DISK_MMAP_ALL); + BC_CHECK("block_cache", BLOCK_CACHE); + BC_CHECK("stress.checkpoint_reserved_txnid_delay", STRESS_CHECKPOINT_RESERVED_TXNID_DELAY); + BC_CHECK("stress.hs_checkpoint_delay", STRESS_HS_CHECKPOINT_DELAY); + BC_CHECK("stress.hs_search", STRESS_HS_SEARCH); + BC_CHECK("stress.hs_sweep", STRESS_HS_SWEEP); - if (g.c_timing_stress_hs_search) { - if (config_is_perm("stress.hs_search")) - testutil_die(EINVAL, "stress.hs_search not supported in backward compatibility mode"); - config_single("stress.hs_search=off", false); - } + tables_apply(config_backward_compatible_table, NULL); } /* @@ -418,22 +514,38 @@ config_backward_compatible(void) static void config_cache(void) { - uint32_t required, workers; - - /* Page sizes are powers-of-two for bad historic reasons. */ - g.intl_page_max = 1U << g.c_intl_page_max; - g.leaf_page_max = 1U << g.c_leaf_page_max; + uint64_t cache; + uint32_t workers; - /* Check if a minimum cache size has been specified. */ - if (config_is_perm("cache")) { - if (config_is_perm("cache.minimum") && g.c_cache_minimum != 0 && - g.c_cache < g.c_cache_minimum) + /* Check if both min and max cache sizes have been specified and if they're consistent. */ + if (config_explicit(NULL, "cache")) { + if (config_explicit(NULL, "cache.minimum") && GV(CACHE) < GV(CACHE_MINIMUM)) testutil_die(EINVAL, "minimum cache set larger than cache (%" PRIu32 " > %" PRIu32 ")", - g.c_cache_minimum, g.c_cache); + GV(CACHE_MINIMUM), GV(CACHE)); return; } - g.c_cache = WT_MAX(g.c_cache, g.c_cache_minimum); + GV(CACHE) = GV(CACHE_MINIMUM); + + /* + * If it's an in-memory run, size the cache at 2x the maximum initial data set. This calculation + * is done in bytes, convert to megabytes before testing against the cache. + */ + if (GV(RUNS_IN_MEMORY)) { + cache = table_sumv(V_TABLE_BTREE_KEY_MAX) + table_sumv(V_TABLE_BTREE_VALUE_MAX); + cache *= table_sumv(V_TABLE_RUNS_ROWS); + cache *= 2; + cache /= WT_MEGABYTE; /* NOT in MB units, convert for cache test */ + if (GV(CACHE) < cache) + GV(CACHE) = (uint32_t)cache; + } + + /* Sum the number of workers. */ + workers = GV(RUNS_THREADS); + if (GV(OPS_HS_CURSOR)) + ++workers; + if (GV(OPS_RANDOM_CURSOR)) + ++workers; /* * Maximum internal/leaf page size sanity. @@ -442,33 +554,34 @@ config_cache(void) * cache with pinned pages, that is, every thread consuming an internal page and a leaf page (or * a pair of leaf pages for cursor movements). * - * Maximum memory pages are in units of MB. - * * This code is what dramatically increases the cache size when there are lots of threads, it * grows the cache to several megabytes per thread. */ - workers = g.c_threads; - if (g.c_hs_cursor) - ++workers; - if (g.c_random_cursor) - ++workers; - g.c_cache = WT_MAX(g.c_cache, 2 * workers * g.c_memory_page_max); + cache = table_sumv(V_TABLE_BTREE_MEMORY_PAGE_MAX); /* in MB units, no conversion to cache */ + cache *= workers; + cache *= 2; + if (GV(CACHE) < cache) + GV(CACHE) = (uint32_t)cache; /* - * Ensure cache size sanity for LSM runs. An LSM tree open requires 3 - * chunks plus a page for each participant in up to three concurrent - * merges. Integrate a thread count into that calculation by requiring - * 3 chunks/pages per configured thread. That might be overkill, but - * LSM runs are more sensitive to small caches than other runs, and a - * generous cache avoids stalls we're not interested in chasing. + * Ensure cache size sanity for LSM runs. An LSM tree open requires 3 chunks plus a page for + * each participant in up to three concurrent merges. Integrate a thread count into that + * calculation by requiring 3 chunks/pages per configured thread. That might be overkill, but + * LSM runs are more sensitive to small caches than other runs, and a generous cache avoids + * stalls we're not interested in chasing. */ - if (DATASOURCE("lsm")) { - required = WT_LSM_TREE_MINIMUM_SIZE( - g.c_chunk_size * WT_MEGABYTE, workers * g.c_merge_max, workers * g.leaf_page_max); - required = (required + (WT_MEGABYTE - 1)) / WT_MEGABYTE; - if (g.c_cache < required) - g.c_cache = required; + if (g.lsm_config) { + cache = WT_LSM_TREE_MINIMUM_SIZE(table_sumv(V_TABLE_LSM_CHUNK_SIZE) * WT_MEGABYTE, + workers * table_sumv(V_TABLE_LSM_MERGE_MAX), + workers * table_sumv(V_TABLE_BTREE_LEAF_PAGE_MAX) * WT_MEGABYTE); + cache = (cache + (WT_MEGABYTE - 1)) / WT_MEGABYTE; + if (GV(CACHE) < cache) + GV(CACHE) = (uint32_t)cache; } + + /* Give any block cache 20% of the total cache size, over and above the cache. */ + if (GV(BLOCK_CACHE) != 0) + GV(BLOCK_CACHE_SIZE) = (GV(CACHE) + 4) / 5; } /* @@ -479,19 +592,19 @@ static void config_checkpoint(void) { /* Choose a checkpoint mode if nothing was specified. */ - if (!config_is_perm("checkpoint")) + if (!config_explicit(NULL, "checkpoint")) switch (mmrand(NULL, 1, 20)) { case 1: case 2: case 3: case 4: /* 20% */ - config_single("checkpoint=wiredtiger", false); + config_single(NULL, "checkpoint=wiredtiger", false); break; case 5: /* 5 % */ - config_single("checkpoint=off", false); + config_single(NULL, "checkpoint=off", false); break; default: /* 75% */ - config_single("checkpoint=on", false); + config_single(NULL, "checkpoint=on", false); break; } } @@ -501,25 +614,25 @@ config_checkpoint(void) * Checksum configuration. */ static void -config_checksum(void) +config_checksum(TABLE *table) { /* Choose a checksum mode if nothing was specified. */ - if (!config_is_perm("disk.checksum")) + if (!config_explicit(table, "disk.checksum")) switch (mmrand(NULL, 1, 10)) { case 1: case 2: case 3: case 4: /* 40% */ - config_single("disk.checksum=on", false); + config_single(table, "disk.checksum=on", false); break; case 5: /* 10% */ - config_single("disk.checksum=off", false); + config_single(table, "disk.checksum=off", false); break; case 6: /* 10% */ - config_single("disk.checksum=uncompressed", false); + config_single(table, "disk.checksum=uncompressed", false); break; default: /* 40% */ - config_single("disk.checksum=unencrypted", false); + config_single(table, "disk.checksum=unencrypted", false); break; } } @@ -529,31 +642,26 @@ config_checksum(void) * Compression configuration. */ static void -config_compression(const char *conf_name) +config_compression(TABLE *table, const char *conf_name) { char confbuf[128]; const char *cstr; /* Return if already specified. */ - if (config_is_perm(conf_name)) + if (config_explicit(table, conf_name)) return; - /* - * Don't configure a compression engine for logging if logging isn't configured (it won't break, - * but it's confusing). - */ - cstr = "none"; - if (strcmp(conf_name, "logging.compression") == 0 && g.c_logging == 0) { - testutil_check(__wt_snprintf(confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr)); - config_single(confbuf, false); + /* Ignore logging compression if we're not doing logging. */ + if (strcmp(conf_name, "logging.compression") == 0 && GV(LOGGING) == 0) { + config_single(NULL, "logging.compression=none", false); return; } /* - * Select a compression type from the list of built-in engines. - * - * Listed percentages are only correct if all of the possible engines are compiled in. + * Select a compression type from the list of built-in engines. Listed percentages are only + * correct if all of the possible engines are compiled in. */ + cstr = "none"; switch (mmrand(NULL, 1, 20)) { #ifdef HAVE_BUILTIN_EXTENSION_LZ4 case 1: @@ -566,9 +674,9 @@ config_compression(const char *conf_name) case 4: case 5: case 6: - case 7: /* 30% snappy */ + case 7: case 8: - case 9: + case 9: /* 30% snappy */ cstr = "snappy"; break; #endif @@ -596,11 +704,11 @@ config_compression(const char *conf_name) } testutil_check(__wt_snprintf(confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr)); - config_single(confbuf, false); + config_single(table, confbuf, false); } /* - * config_directio + * config_directio -- * Direct I/O configuration. */ static void @@ -610,26 +718,24 @@ config_directio(void) * We don't roll the dice and set direct I/O, it has to be set explicitly. For that reason, any * incompatible "permanent" option set with direct I/O is a configuration error. */ - if (!g.c_direct_io) + if (!GV(DISK_DIRECT_IO)) return; /* * Direct I/O may not work with backups, doing copies through the buffer cache after configuring * direct I/O in Linux won't work. If direct I/O is configured, turn off backups. */ - if (g.c_backups) { - if (config_is_perm("backup")) + if (GV(BACKUP)) { + if (config_explicit(NULL, "backup")) testutil_die(EINVAL, "direct I/O is incompatible with backup configurations"); - config_single("backup=off", false); + config_single(NULL, "backup=off", false); } - /* - * Direct I/O may not work with imports for the same reason as for backups. - */ - if (g.c_import) { - if (config_is_perm("import")) + /* Direct I/O may not work with imports for the same reason as for backups. */ + if (GV(IMPORT)) { + if (config_explicit(NULL, "import")) testutil_die(EINVAL, "direct I/O is incompatible with import configurations"); - config_single("import=0", false); + config_single(NULL, "import=0", false); } /* @@ -637,10 +743,10 @@ config_directio(void) * the presence of shared cache configurations (including mmap), but we've seen file corruption * and it doesn't make much sense (the library disallows the combination). */ - if (g.c_mmap_all != 0) { - if (config_is_perm("disk.mmap_all")) + if (GV(DISK_MMAP_ALL) != 0) { + if (config_explicit(NULL, "disk.mmap_all")) testutil_die(EINVAL, "direct I/O is incompatible with mmap_all configurations"); - config_single("disk.mmap_all=off", false); + config_single(NULL, "disk.mmap_all=off", false); } /* @@ -649,10 +755,10 @@ config_directio(void) * format just hung, and the 15-minute timeout isn't effective. We could play games to handle * child process termination, but it's not worth the effort. */ - if (g.c_salvage) { - if (config_is_perm("ops.salvage")) + if (GV(OPS_SALVAGE)) { + if (config_explicit(NULL, "ops.salvage")) testutil_die(EINVAL, "direct I/O is incompatible with salvage configurations"); - config_single("ops.salvage=off", false); + config_single(NULL, "ops.salvage=off", false); } } @@ -668,7 +774,7 @@ config_encryption(void) /* * Encryption: choose something if encryption wasn't specified. */ - if (!config_is_perm("disk.encryption")) { + if (!config_explicit(NULL, "disk.encryption")) { cstr = "disk.encryption=none"; switch (mmrand(NULL, 1, 10)) { case 1: @@ -686,7 +792,7 @@ config_encryption(void) break; } - config_single(cstr, false); + config_single(NULL, cstr, false); } } @@ -695,12 +801,10 @@ config_encryption(void) * Fixed-length column-store configuration. */ static bool -config_fix(void) +config_fix(TABLE *table) { - /* Fixed-length column stores don't support the history store table, so no modify operations. */ - if (config_is_perm("ops.pct.modify")) - return (false); - return (true); + /* Fixed-length column stores don't support modify operations. */ + return (!config_explicit(table, "ops.pct.modify")); } /* @@ -716,29 +820,27 @@ config_in_memory(void) * don't have to configure in-memory every time we configure something like LSM, that's too * painful. */ - if (config_is_perm("backup")) + if (config_explicit(NULL, "backup")) return; - if (config_is_perm("btree.compression")) + if (config_explicit(NULL, "btree.compression")) return; - if (config_is_perm("checkpoint")) + if (config_explicit(NULL, "checkpoint")) return; - if (config_is_perm("format.abort")) + if (config_explicit(NULL, "format.abort")) return; - if (config_is_perm("import")) + if (config_explicit(NULL, "import")) return; - if (config_is_perm("logging")) + if (config_explicit(NULL, "logging")) return; - if (config_is_perm("ops.hs_cursor")) + if (config_explicit(NULL, "ops.hs_cursor")) return; - if (config_is_perm("ops.salvage")) + if (config_explicit(NULL, "ops.salvage")) return; - if (config_is_perm("ops.verify")) - return; - if (config_is_perm("runs.source") && DATASOURCE("lsm")) + if (config_explicit(NULL, "ops.verify")) return; - if (!config_is_perm("runs.in_memory") && mmrand(NULL, 1, 20) == 1) - g.c_in_memory = 1; + if (!config_explicit(NULL, "runs.in_memory") && mmrand(NULL, 1, 20) == 1) + config_single(NULL, "runs.in_memory=1", false); } /* @@ -748,54 +850,38 @@ config_in_memory(void) static void config_in_memory_reset(void) { - uint32_t cache; - /* Turn off a lot of stuff. */ - if (!config_is_perm("backup")) - config_single("backup=off", false); - if (!config_is_perm("btree.compression")) - config_single("btree.compression=none", false); - if (!config_is_perm("checkpoint")) - config_single("checkpoint=off", false); - if (!config_is_perm("import")) - config_single("import=off", false); - if (!config_is_perm("logging")) - config_single("logging=off", false); - if (!config_is_perm("ops.alter")) - config_single("ops.alter=off", false); - if (!config_is_perm("ops.hs_cursor")) - config_single("ops.hs_cursor=off", false); - if (!config_is_perm("ops.salvage")) - config_single("ops.salvage=off", false); - if (!config_is_perm("ops.verify")) - config_single("ops.verify=off", false); + if (!config_explicit(NULL, "backup")) + config_single(NULL, "backup=off", false); + if (!config_explicit(NULL, "btree.compression")) + config_single(NULL, "btree.compression=none", false); + if (!config_explicit(NULL, "checkpoint")) + config_single(NULL, "checkpoint=off", false); + if (!config_explicit(NULL, "import")) + config_single(NULL, "import=off", false); + if (!config_explicit(NULL, "logging")) + config_single(NULL, "logging=off", false); + if (!config_explicit(NULL, "ops.alter")) + config_single(NULL, "ops.alter=off", false); + if (!config_explicit(NULL, "ops.hs_cursor")) + config_single(NULL, "ops.hs_cursor=off", false); + if (!config_explicit(NULL, "ops.salvage")) + config_single(NULL, "ops.salvage=off", false); + if (!config_explicit(NULL, "ops.verify")) + config_single(NULL, "ops.verify=off", false); /* * Keep keys/values small, overflow items aren't an issue for in-memory configurations and it * keeps us from overflowing the cache. */ - if (!config_is_perm("btree.key_max")) - config_single("btree.key_max=32", false); - if (!config_is_perm("btree.value_max")) - config_single("btree.value_max=80", false); - - /* - * Size the cache relative to the initial data set, use 2x the base size as a minimum. - */ - if (!config_is_perm("cache")) { - cache = g.c_value_max; - if (g.type == ROW) - cache += g.c_key_max; - cache *= g.c_rows; - cache *= 2; - cache /= WT_MEGABYTE; - if (g.c_cache < cache) - g.c_cache = cache; - } + if (!config_explicit(NULL, "btree.key_max")) + config_single(NULL, "btree.key_max=32", false); + if (!config_explicit(NULL, "btree.value_max")) + config_single(NULL, "btree.value_max=80", false); } /* - * config_backup_incr_compatibility_check -- + * config_backup_incr_log_compatibility_check -- * Backup incremental log compatibility check. */ static void @@ -805,10 +891,10 @@ config_backup_incr_log_compatibility_check(void) * Incremental backup using log files is incompatible with logging archival. Disable logging * archival if log incremental backup is set. */ - if (g.c_logging_archive && config_is_perm("logging.archive")) + if (GV(LOGGING_ARCHIVE) && config_explicit(NULL, "logging.archive")) testutil_die(EINVAL, "backup.incremental=log is incompatible with logging.archive"); - if (g.c_logging_archive) - config_single("logging.archive=0", false); + if (GV(LOGGING_ARCHIVE)) + config_single(NULL, "logging.archive=0", false); } /* @@ -816,45 +902,39 @@ config_backup_incr_log_compatibility_check(void) * LSM configuration review. */ static void -config_lsm_reset(void) +config_lsm_reset(TABLE *table) { /* * Turn off truncate for LSM runs (some configurations with truncate always result in a * timeout). */ - if (!config_is_perm("ops.truncate")) - config_single("ops.truncate=off", false); + if (config_explicit(table, "ops.truncate")) { + if (DATASOURCE(table, "lsm")) + testutil_die(EINVAL, "LSM (currently) incompatible with truncate configurations"); + config_single(table, "ops.truncate=off", false); + } /* - * LSM doesn't currently play nicely with timestamps, don't choose the pair unless forced to. If - * we turn off timestamps, make sure we turn off prepare as well, it requires timestamps. Remove - * this code with WT-4162. + * Turn off prepare and timestamps for LSM runs (prepare requires timestamps). + * + * FIXME: WT-4162. */ - if (!config_is_perm("ops.prepare") && !config_is_perm("transaction.timestamps")) { - config_single("ops.prepare=off", false); - config_single("transaction.timestamps=off", false); - } + if (config_explicit(table, "ops.prepare")) + testutil_die(EINVAL, "LSM (currently) incompatible with prepare configurations"); + config_single(table, "ops.prepare=off", false); + if (config_explicit(table, "transaction.timestamps")) + testutil_die(EINVAL, "LSM (currently) incompatible with timestamp configurations"); + config_single(table, "transaction.timestamps=off", false); /* * LSM does not work with block-based incremental backup, change the incremental backup - * mechanism if block based in configured. + * mechanism if configured to be block based. */ - if (g.c_backups) { - if (config_is_perm("backup.incremental") && g.c_backup_incr_flag == INCREMENTAL_BLOCK) - testutil_die(EINVAL, "LSM does not work with backup.incremental=block configuration."); - - if (g.c_backup_incr_flag == INCREMENTAL_BLOCK) - switch (mmrand(NULL, 1, 2)) { - case 1: - /* 50% */ - config_single("backup.incremental=off", false); - break; - case 2: - /* 50% */ - config_single("backup.incremental=log", false); - config_backup_incr_log_compatibility_check(); - break; - } + if (GV(BACKUP)) { + if (config_explicit(table, "backup.incremental")) + testutil_die( + EINVAL, "LSM (currently) incompatible with incremental backup configurations"); + config_single(NULL, "backup.incremental=log", false); } } @@ -863,29 +943,39 @@ config_lsm_reset(void) * Configure operation percentages. */ static void -config_pct(void) +config_pct(TABLE *table) { - static struct { + struct { const char *name; /* Operation */ uint32_t *vp; /* Value store */ u_int order; /* Order of assignment */ - } list[] = { - {"ops.pct.delete", &g.c_delete_pct, 0}, - {"ops.pct.insert", &g.c_insert_pct, 0}, -#define CONFIG_MODIFY_ENTRY 2 - {"ops.pct.modify", &g.c_modify_pct, 0}, - {"ops.pct.read", &g.c_read_pct, 0}, - {"ops.pct.write", &g.c_write_pct, 0}, - }; + } list[5]; u_int i, max_order, max_slot, n, pct; +#define CONFIG_MODIFY_ENTRY 2 + list[0].name = "ops.pct.delete"; + list[0].vp = &TV(OPS_PCT_DELETE); + list[0].order = 0; + list[1].name = "ops.pct.insert"; + list[1].vp = &TV(OPS_PCT_INSERT); + list[1].order = 0; + list[2].name = "ops.pct.modify"; + list[2].vp = &TV(OPS_PCT_MODIFY); + list[2].order = 0; + list[3].name = "ops.pct.read"; + list[3].vp = &TV(OPS_PCT_READ); + list[3].order = 0; + list[4].name = "ops.pct.write"; + list[4].vp = &TV(OPS_PCT_WRITE); + list[4].order = 0; + /* * Walk the list of operations, checking for an illegal configuration and creating a random * order in the list. */ pct = 0; for (i = 0; i < WT_ELEMENTS(list); ++i) - if (config_is_perm(list[i].name)) + if (config_explicit(table, list[i].name)) pct += *list[i].vp; else list[i].order = mmrand(NULL, 1, 1000); @@ -893,8 +983,8 @@ config_pct(void) testutil_die(EINVAL, "operation percentages do not total to 100%%"); /* Cursor modify isn't possible for fixed-length column store. */ - if (g.type == FIX) { - if (config_is_perm("ops.pct.modify") && g.c_modify_pct != 0) + if (table->type == FIX) { + if (config_explicit(table, "ops.pct.modify") && TV(OPS_PCT_MODIFY) != 0) testutil_die(EINVAL, "WT_CURSOR.modify not supported by fixed-length column store"); list[CONFIG_MODIFY_ENTRY].order = 0; *list[CONFIG_MODIFY_ENTRY].vp = 0; @@ -927,21 +1017,9 @@ config_pct(void) pct -= *list[max_slot].vp; } - testutil_assert( - g.c_delete_pct + g.c_insert_pct + g.c_modify_pct + g.c_read_pct + g.c_write_pct == 100); -} - -/* - * config_prefix -- - * Prefix configuration. - */ -static void -config_prefix(void) -{ - /* Add prefix compression if prefixes are configured and no explicit choice was made. */ - if (g.c_prefix != 0 && g.c_prefix_compression == 0 && - !config_is_perm("btree.prefix_compression")) - config_single("btree.prefix_compression=on", false); + testutil_assert(TV(OPS_PCT_DELETE) + TV(OPS_PCT_INSERT) + TV(OPS_PCT_MODIFY) + + TV(OPS_PCT_READ) + TV(OPS_PCT_WRITE) == + 100); } /* @@ -952,21 +1030,21 @@ static void config_transaction(void) { /* Transaction prepare requires timestamps and is incompatible with logging. */ - if (g.c_prepare && config_is_perm("ops.prepare")) { - if (g.c_logging && config_is_perm("logging")) - testutil_die(EINVAL, "prepare is incompatible with logging"); - if (!g.c_txn_timestamps && config_is_perm("transaction.timestamps")) + if (GV(OPS_PREPARE) && config_explicit(NULL, "ops.prepare")) { + if (!GV(TRANSACTION_TIMESTAMPS) && config_explicit(NULL, "transaction.timestamps")) testutil_die(EINVAL, "prepare requires transaction timestamps"); + if (GV(LOGGING) && config_explicit(NULL, "logging")) + testutil_die(EINVAL, "prepare is incompatible with logging"); } /* Transaction timestamps are incompatible with implicit transactions. */ - if (g.c_txn_timestamps && config_is_perm("transaction.timestamps")) { - if (g.c_txn_implicit && config_is_perm("transaction.implicit")) + if (GV(TRANSACTION_TIMESTAMPS) && config_explicit(NULL, "transaction.timestamps")) { + if (GV(TRANSACTION_IMPLICIT) && config_explicit(NULL, "transaction.implicit")) testutil_die( EINVAL, "transaction.timestamps is incompatible with implicit transactions"); /* FIXME-WT-6431: temporarily disable salvage with timestamps. */ - if (g.c_salvage && config_is_perm("ops.salvage")) + if (GV(OPS_SALVAGE) && config_explicit(NULL, "ops.salvage")) testutil_die(EINVAL, "transaction.timestamps is incompatible with salvage"); } @@ -978,24 +1056,28 @@ config_transaction(void) * time we check logging, logging must have been required by the run if both logging and prepare * are still set, so we can just turn off prepare in that case). */ - if (g.c_prepare) { - if (!config_is_perm("logging")) - config_single("logging=off", false); - if (!config_is_perm("transaction.timestamps")) - config_single("transaction.timestamps=on", false); + if (GV(OPS_PREPARE)) { + if (!config_explicit(NULL, "logging")) + config_single(NULL, "logging=off", false); + if (!config_explicit(NULL, "transaction.timestamps")) + config_single(NULL, "transaction.timestamps=on", false); } - if (g.c_txn_timestamps) { - if (!config_is_perm("transaction.implicit")) - config_single("transaction.implicit=0", false); - if (!config_is_perm("ops.salvage")) - config_single("ops.salvage=off", false); + if (GV(TRANSACTION_TIMESTAMPS)) { + if (!config_explicit(NULL, "transaction.implicit")) + config_single(NULL, "transaction.implicit=0", false); + if (!config_explicit(NULL, "ops.salvage")) + config_single(NULL, "ops.salvage=off", false); } - if (g.c_logging) - config_single("ops.prepare=off", false); - if (g.c_txn_implicit) - config_single("transaction.timestamps=off", false); - if (g.c_salvage) - config_single("transaction.timestamps=off", false); + if (GV(LOGGING)) + config_single(NULL, "ops.prepare=off", false); + if (GV(TRANSACTION_IMPLICIT)) + config_single(NULL, "transaction.timestamps=off", false); + if (GV(OPS_SALVAGE)) + config_single(NULL, "transaction.timestamps=off", false); + + /* Transaction timestamps configures format behavior, flag it. */ + if (GV(TRANSACTION_TIMESTAMPS)) + g.transaction_timestamps_config = true; } /* @@ -1023,13 +1105,67 @@ config_error(void) fprintf(stderr, "\n"); fprintf(stderr, "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=\n"); fprintf(stderr, "Configuration names:\n"); - for (max_name = 0, cp = c; cp->name != NULL; ++cp) + for (max_name = 0, cp = configuration_list; cp->name != NULL; ++cp) max_name = WT_MAX(max_name, strlen(cp->name)); - for (cp = c; cp->name != NULL; ++cp) + for (cp = configuration_list; cp->name != NULL; ++cp) fprintf(stderr, "%*s: %s\n", (int)max_name, cp->name, cp->desc); } /* + * config_print_one -- + * Print out a single configuration setting. + */ +static void +config_print_one(FILE *fp, CONFIG *cp, CONFIGV *v, const char *prefix) +{ + if (F_ISSET(cp, C_STRING)) + fprintf(fp, "%s%s=%s\n", prefix, cp->name, v->vstr == NULL ? "" : v->vstr); + else + fprintf(fp, "%s%s=%" PRIu32 "\n", prefix, cp->name, v->v); +} + +/* + * config_print_table -- + * Print per-table information. + */ +static void +config_print_table(FILE *fp, TABLE *table) +{ + CONFIG *cp; + CONFIGV *v, *gv; + char buf[128]; + bool lsm; + + testutil_check(__wt_snprintf(buf, sizeof(buf), "table%u.", table->id)); + fprintf(fp, "############################################\n"); + fprintf(fp, "# TABLE PARAMETERS: table %u\n", table->id); + fprintf(fp, "############################################\n"); + + lsm = DATASOURCE(table, "lsm"); + for (cp = configuration_list; cp->name != NULL; ++cp) { + /* Skip global items. */ + if (!F_ISSET(cp, C_TABLE)) + continue; + /* Skip mismatched objects and configurations. */ + if (!lsm && F_ISSET(cp, C_TYPE_LSM)) + continue; + if (!C_TYPE_MATCH(cp, table->type)) + continue; + + gv = &tables[0]->v[cp->off]; + v = &table->v[cp->off]; + + /* Skip entries that match any global setting. */ + if (gv->set && v->v == gv->v && + ((v->vstr == NULL && gv->vstr == NULL) || + (v->vstr != NULL && gv->vstr != NULL && strcmp(v->vstr, gv->vstr) == 0))) + continue; + + config_print_one(fp, cp, v, buf); + } +} + +/* * config_print -- * Print configuration information. */ @@ -1037,7 +1173,9 @@ void config_print(bool error_display) { CONFIG *cp; + CONFIGV *gv; FILE *fp; + uint32_t i; /* Reopening an existing database should leave the existing CONFIG file. */ if (g.reopen) @@ -1049,17 +1187,31 @@ config_print(bool error_display) testutil_die(errno, "fopen: %s", g.home_config); fprintf(fp, "############################################\n"); - fprintf(fp, "# RUN PARAMETERS: V2\n"); + fprintf(fp, "# RUN PARAMETERS: V3\n"); fprintf(fp, "############################################\n"); - /* Display configuration values. */ - for (cp = c; cp->name != NULL; ++cp) - if (F_ISSET(cp, C_STRING)) - fprintf(fp, "%s=%s\n", cp->name, *cp->vstr == NULL ? "" : *cp->vstr); - else - fprintf(fp, "%s=%" PRIu32 "\n", cp->name, *cp->v); + /* Display global configuration values. */ + for (cp = configuration_list; cp->name != NULL; ++cp) { + /* Skip mismatched objects and configurations. */ + if (!g.lsm_config && F_ISSET(cp, C_TYPE_LSM)) + continue; + /* Skip table count if tables not configured (implying an old-style CONFIG file). */ + if (ntables == 0 && cp->off == V_GLOBAL_RUNS_TABLES) + continue; - fprintf(fp, "############################################\n"); + /* + * Otherwise, print if we never configured any tables, if the global item was explicitly + * configured, or this isn't a table option. + */ + gv = &tables[0]->v[cp->off]; + if (ntables == 0 || gv->set || !F_ISSET(cp, C_TABLE)) + config_print_one(fp, cp, gv, ""); + } + + /* Display per-table configuration values. */ + if (ntables != 0) + for (i = 1; i <= ntables; ++i) + config_print_table(fp, tables[i]); /* Flush so we're up-to-date on error. */ (void)fflush(fp); @@ -1078,6 +1230,16 @@ config_file(const char *name) FILE *fp; char buf[256], *p, *t; + /* + * Turn off multi-table configuration for all configuration files, for backward compatibility. + * This doesn't stop multiple table configurations, using either "runs.tables" or an explicit + * mention of a table, it only prevents CONFIG files without a table reference from configuring + * tables. This should only affect putting some non-table-specific configurations into a file + * and running that file as a CONFIG, expecting a multi-table test, and means old-style CONFIG + * files don't suddenly turn into multiple table tests. + */ + g.multi_table_config = false; + if ((fp = fopen(name, "r")) == NULL) testutil_die(errno, "fopen: %s", name); @@ -1092,7 +1254,7 @@ config_file(const char *name) *p = '\0'; break; } - if (*p == '#') { /* Comment, skip the line */ + if (*p == '#') { /* Comment */ t = p; break; } @@ -1104,7 +1266,7 @@ config_file(const char *name) } if (*t == '\0' || *t == '#') continue; - config_single(t, true); + config_single(NULL, t, true); } fclose_and_clear(&fp); } @@ -1116,55 +1278,35 @@ config_file(const char *name) void config_clear(void) { - CONFIG *cp; + u_int i, j, slots; - /* Clear all allocated configuration data. */ - for (cp = c; cp->name != NULL; ++cp) - if (cp->vstr != NULL) { - free((void *)*cp->vstr); - *cp->vstr = NULL; - } - free(g.uri); - g.uri = NULL; -} - -/* - * config_reset -- - * Clear per-run configuration values. - */ -static void -config_reset(void) -{ - CONFIG *cp; + /* Clear all allocated configuration data in the tables array. */ + slots = ntables == 0 ? 1 : ntables; + for (i = 0; i < slots; ++i) { + free(tables[i]->val_base); - /* Clear temporary allocated configuration data. */ - for (cp = c; cp->name != NULL; ++cp) { - F_CLR(cp, C_TEMP); - if (!F_ISSET(cp, C_PERM) && cp->vstr != NULL) { - free((void *)*cp->vstr); - *cp->vstr = NULL; - } + for (j = 0; j < V_ELEMENT_COUNT; ++j) + free(tables[i]->v[j].vstr); + free(tables[i]); } - free(g.uri); - g.uri = NULL; } /* - * config_find - * Find a specific configuration entry. + * config_find -- + * Find a specific configuration entry. */ static CONFIG * config_find(const char *s, size_t len, bool fatal) { CONFIG *cp; - for (cp = c; cp->name != NULL; ++cp) + for (cp = configuration_list; cp->name != NULL; ++cp) if (strncmp(s, cp->name, len) == 0 && cp->name[len] == '\0') return (cp); /* Optionally ignore unknown keywords, it makes it easier to run old CONFIG files. */ if (fatal) - testutil_die(EINVAL, "%s: %s: unknown required configuration keyword\n", progname, s); + testutil_die(EINVAL, "%s: %s: unknown required configuration keyword", progname, s); fprintf(stderr, "%s: %s: WARNING, ignoring unknown configuration keyword\n", progname, s); return (NULL); @@ -1190,70 +1332,109 @@ config_value(const char *config, const char *p, int match) } /* + * config_table_extend -- + * Extend the tables array as necessary. + */ +static void +config_table_extend(u_int ntable) +{ + u_int i; + + if (ntable <= ntables) + return; + + /* + * Allocate any new tables structures. (We do it this way, rather than reallocating the whole + * tables array, because our caller doesn't know we're extending the list of tables, and is + * likely holding pointers into the current list of tables. Reallocating the whole array would + * require handling reallocation in our caller, and it's not worth the effort.) + * + * This might be the first extension, reset the base table's ID (for debugging, we should never + * be using a table with ID 0). + */ + for (i = 0; i <= ntable; ++i) { + if (tables[i] == NULL) + tables[i] = dcalloc(1, sizeof(TABLE)); + tables[i]->id = i; + } + ntables = ntable; +} + +/* * config_single -- * Set a single configuration structure value. */ void -config_single(const char *s, bool perm) +config_single(TABLE *table, const char *s, bool explicit) { enum { RANGE_FIXED, RANGE_NONE, RANGE_WEIGHTED } range; CONFIG *cp; + CONFIGV *v; uint32_t steps, v1, v2; + u_long ntable; u_int i; const char *equalp, *vp1, *vp2; + char *endptr; while (__wt_isspace((u_char)*s)) ++s; + /* + * If configuring a single table, the table argument will be non-NULL. The configuration itself + * may include a table reference, in which case we extend the table as necessary and select the + * table. + */ + if (table == NULL) { + table = tables[0]; + if (strncmp(s, "table", strlen("table")) == 0) { + errno = 0; + ntable = strtoul(s + strlen("table"), &endptr, 10); + testutil_assert(errno == 0 && endptr[0] == '.'); + config_table_extend((uint32_t)ntable); + table = tables[ntable]; + + s = endptr + 1; + } + } + + /* Process backward compatibility configuration. */ config_compat(&s); if ((equalp = strchr(s, '=')) == NULL) - testutil_die(EINVAL, "%s: %s: illegal configuration value\n", progname, s); + testutil_die(EINVAL, "%s: %s: configuration missing \'=\' character", progname, s); + /* Find the configuration value, and assert it's not a table/global mismatch. */ if ((cp = config_find(s, (size_t)(equalp - s), false)) == NULL) return; + testutil_assert(F_ISSET(cp, C_TABLE) || table == tables[0]); - F_SET(cp, perm ? C_PERM : C_TEMP); ++equalp; + v = &table->v[cp->off]; if (F_ISSET(cp, C_STRING)) { - /* - * Free the previous setting if a configuration has been passed in twice. - */ - if (*cp->vstr != NULL) { - free(*cp->vstr); - *cp->vstr = NULL; - } - - if (strncmp(s, "backup.incremental", strlen("backup.incremental")) == 0) { - config_map_backup_incr(equalp, &g.c_backup_incr_flag); - *cp->vstr = dstrdup(equalp); - } else if (strncmp(s, "checkpoint", strlen("checkpoint")) == 0) { - config_map_checkpoint(equalp, &g.c_checkpoint_flag); - *cp->vstr = dstrdup(equalp); - } else if (strncmp(s, "disk.checksum", strlen("disk.checksum")) == 0) { - config_map_checksum(equalp, &g.c_checksum_flag); - *cp->vstr = dstrdup(equalp); - } else if (strncmp(s, "btree.compression", strlen("btree.compression")) == 0) { - config_map_compression(equalp, &g.c_compression_flag); - *cp->vstr = dstrdup(equalp); - } else if (strncmp(s, "runs.source", strlen("runs.source")) == 0 && + if (strncmp(s, "backup.incremental", strlen("backup.incremental")) == 0) + config_map_backup_incr(equalp, &g.backup_incr_flag); + else if (strncmp(s, "checkpoint", strlen("checkpoint")) == 0) + config_map_checkpoint(equalp, &g.checkpoint_config); + else if (strncmp(s, "runs.source", strlen("runs.source")) == 0 && strncmp("file", equalp, strlen("file")) != 0 && strncmp("lsm", equalp, strlen("lsm")) != 0 && strncmp("table", equalp, strlen("table")) != 0) { - testutil_die(EINVAL, "Invalid data source option: %s\n", equalp); - } else if (strncmp(s, "disk.encryption", strlen("disk.encryption")) == 0) { - config_map_encryption(equalp, &g.c_encryption_flag); - *cp->vstr = dstrdup(equalp); + testutil_die(EINVAL, "Invalid data source option: %s", equalp); } else if (strncmp(s, "runs.type", strlen("runs.type")) == 0) { - config_map_file_type(equalp, &g.type); - *cp->vstr = dstrdup(config_file_type(g.type)); - } else if (strncmp(s, "logging.compression", strlen("logging.compression")) == 0) { - config_map_compression(equalp, &g.c_logging_compression_flag); - *cp->vstr = dstrdup(equalp); - } else - *cp->vstr = dstrdup(equalp); + /* Save any global configuration for later table configuration. */ + if (table == tables[0]) + testutil_check(__wt_snprintf(g.runs_type, sizeof(g.runs_type), "%s", equalp)); + + config_map_file_type(equalp, &table->type); + equalp = config_file_type(table->type); + } + /* Free the previous setting if a configuration has been passed in twice. */ + free(v->vstr); + + v->vstr = dstrdup(equalp); + v->set = explicit; return; } @@ -1268,7 +1449,8 @@ config_single(const char *s, bool perm) testutil_die(EINVAL, "%s: %s: value of boolean not 0 or 1", progname, s); } - *cp->v = v1; + v->v = v1; + v->set = explicit; return; } @@ -1287,19 +1469,22 @@ config_single(const char *s, bool perm) range = RANGE_WEIGHTED; } + /* + * Get the value and check the range; zero is optionally an out-of-band "don't set this + * variable" value. + */ v1 = config_value(s, vp1, range == RANGE_NONE ? '\0' : (range == RANGE_FIXED ? '-' : ':')); - if (v1 < cp->min || v1 > cp->maxset) - testutil_die(EINVAL, "%s: %s: value outside min/max values of %" PRIu32 "-%" PRIu32 "\n", + if (!(v1 == 0 && F_ISSET(cp, C_ZERO_NOTSET)) && (v1 < cp->min || v1 > cp->maxset)) + testutil_die(EINVAL, "%s: %s: value outside min/max values of %" PRIu32 "-%" PRIu32, progname, s, cp->min, cp->maxset); if (range != RANGE_NONE) { v2 = config_value(s, vp2, '\0'); if (v2 < cp->min || v2 > cp->maxset) - testutil_die(EINVAL, - "%s: %s: value outside min/max values of %" PRIu32 "-%" PRIu32 "\n", progname, s, - cp->min, cp->maxset); + testutil_die(EINVAL, "%s: %s: value outside min/max values of %" PRIu32 "-%" PRIu32, + progname, s, cp->min, cp->maxset); if (v1 > v2) - testutil_die(EINVAL, "%s: %s: illegal numeric range\n", progname, s); + testutil_die(EINVAL, "%s: %s: illegal numeric range", progname, s); if (range == RANGE_FIXED) v1 = mmrand(NULL, (u_int)v1, (u_int)v2); @@ -1318,7 +1503,11 @@ config_single(const char *s, bool perm) } } - *cp->v = v1; + v->v = v1; + v->set = explicit; + + if (strncmp(s, "runs.tables", strlen("runs.tables")) == 0) + config_table_extend((uint32_t)v1); } /* @@ -1418,77 +1607,38 @@ config_map_checkpoint(const char *s, u_int *vp) } /* - * config_map_checksum -- - * Map a checksum configuration to a flag. - */ -static void -config_map_checksum(const char *s, u_int *vp) -{ - if (strcmp(s, "on") == 0) - *vp = CHECKSUM_ON; - else if (strcmp(s, "off") == 0) - *vp = CHECKSUM_OFF; - else if (strcmp(s, "uncompressed") == 0) - *vp = CHECKSUM_UNCOMPRESSED; - else if (strcmp(s, "unencrypted") == 0) - *vp = CHECKSUM_UNENCRYPTED; - else - testutil_die(EINVAL, "illegal checksum configuration: %s", s); -} - -/* - * config_map_compression -- - * Map a compression configuration to a flag. - */ -static void -config_map_compression(const char *s, u_int *vp) -{ - if (strcmp(s, "none") == 0) - *vp = COMPRESS_NONE; - else if (strcmp(s, "lz4") == 0) - *vp = COMPRESS_LZ4; - else if (strcmp(s, "lz4-noraw") == 0) /* CONFIG compatibility */ - *vp = COMPRESS_LZ4; - else if (strcmp(s, "snappy") == 0) - *vp = COMPRESS_SNAPPY; - else if (strcmp(s, "zlib") == 0) - *vp = COMPRESS_ZLIB; - else if (strcmp(s, "zlib-noraw") == 0) /* CONFIG compatibility */ - *vp = COMPRESS_ZLIB; - else if (strcmp(s, "zstd") == 0) - *vp = COMPRESS_ZSTD; - else - testutil_die(EINVAL, "illegal compression configuration: %s", s); -} - -/* - * config_map_encryption -- - * Map a encryption configuration to a flag. - */ -static void -config_map_encryption(const char *s, u_int *vp) -{ - if (strcmp(s, "none") == 0) - *vp = ENCRYPT_NONE; - else if (strcmp(s, "rotn-7") == 0) - *vp = ENCRYPT_ROTN_7; - else if (strcmp(s, "sodium") == 0) - *vp = ENCRYPT_SODIUM; - else - testutil_die(EINVAL, "illegal encryption configuration: %s", s); -} - -/* - * config_is_perm - * Return if a specific configuration entry was permanently set. + * config_explicit -- + * Return if a configuration entry is explicitly set (as opposed to being randomly set). */ -static int -config_is_perm(const char *s) +static bool +config_explicit(TABLE *table, const char *s) { CONFIG *cp; + u_int i; + /* Look up the configuration option. */ cp = config_find(s, strlen(s), true); - return (F_ISSET(cp, C_PERM) ? 1 : 0); + + /* + * If it's a global option, assert our caller didn't ask for a table value, and return if it's + * set in the base values. + */ + if (!F_ISSET(cp, C_TABLE)) { + testutil_assert(table == NULL); + return (tables[0]->v[cp->off].set); + } + + /* If checking a single table, the table argument is non-NULL. */ + if (table != NULL) + return (table->v[cp->off].set); + + /* Otherwise, check if it's set in any table. */ + if (ntables == 0) + return (tables[0]->v[cp->off].set); + for (i = 1; i < ntables; ++i) + if (tables[i]->v[cp->off].set) + return (true); + return (false); } /* diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index d12c30ed756..471fccd1365 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -1,373 +1,137 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ +/* DO NOT EDIT: automatically built by format/config.sh. */ + +#define C_TYPE_MATCH(cp, type) \ + (!F_ISSET(cp, (C_TYPE_FIX | C_TYPE_ROW | C_TYPE_VAR)) || \ + ((type) == FIX && F_ISSET(cp, C_TYPE_FIX)) || ((type) == ROW && F_ISSET(cp, C_TYPE_ROW)) || \ + ((type) == VAR && F_ISSET(cp, C_TYPE_VAR))) -/* - * Configuration for the wts program is an array of string-based parameters. This is the structure - * used to declare them. - */ typedef struct { const char *name; /* Configuration item */ const char *desc; /* Configuration description */ -/* Value is a boolean, yes if roll of 1-to-100 is <= CONFIG->min. */ -#define C_BOOL 0x01u - -/* Not a simple randomization, handle outside the main loop. */ -#define C_IGNORE 0x02u - -/* Value was set from command-line or file, ignore for all runs. */ -#define C_PERM 0x04u - -/* Value isn't random for this run, ignore just for this run. */ -#define C_TEMP 0x08u - -/* Value is a string. */ -#define C_STRING 0x20u - u_int flags; +#define C_BOOL 0x001u /* Boolean (true if roll of 1-to-100 is <= CONFIG->min) */ +#define C_IGNORE 0x002u /* Not a simple randomization, configured specially */ +#define C_STRING 0x004u /* String (rather than integral) */ +#define C_TABLE 0x008u /* Value is per table, not global */ +#define C_TYPE_FIX 0x010u /* Value is only relevant to FLCS */ +#define C_TYPE_LSM 0x020u /* Value is only relevant to LSM */ +#define C_TYPE_ROW 0x040u /* Value is only relevant to RS */ +#define C_TYPE_VAR 0x080u /* Value is only relevant to VLCS */ +#define C_ZERO_NOTSET 0x100u /* Ignore zero values */ + uint32_t flags; uint32_t min; /* Minimum value */ uint32_t maxrand; /* Maximum value randomly chosen */ uint32_t maxset; /* Maximum value explicitly set */ - uint32_t *v; /* Value for this run */ - char **vstr; /* Value for string options */ -} CONFIG; - -#define COMPRESSION_LIST " (none | lz4 | snappy | zlib | zstd)" - -static CONFIG c[] = { - /* 2% */ - {"assert.read_timestamp", "assert read_timestamp", C_BOOL, 2, 0, 0, &g.c_assert_read_timestamp, - NULL}, - - /* 2% */ - {"assert.write_timestamp", "set write_timestamp_usage and assert write_timestamp", C_BOOL, 2, 0, - 0, &g.c_assert_write_timestamp, NULL}, - - /* 20% */ - {"backup", "configure backups", C_BOOL, 20, 0, 0, &g.c_backups, NULL}, - - {"backup.incremental", "backup type (block | log | off)", C_IGNORE | C_STRING, 0, 0, 0, NULL, - &g.c_backup_incremental}, - - {"backup.incr_granularity", "incremental backup block granularity (KB)", 0x0, 4, 16384, 16384, - &g.c_backup_incr_granularity, NULL}, - - {"btree.bitcnt", "fixed-length column-store object size (number of bits)", 0x0, 1, 8, 8, - &g.c_bitcnt, NULL}, - - {"btree.compression", "compression type" COMPRESSION_LIST, C_IGNORE | C_STRING, 0, 0, 0, NULL, - &g.c_compression}, - - /* 20% */ - {"btree.dictionary", "configure dictionary compressed values", C_BOOL, 20, 0, 0, &g.c_dictionary, - NULL}, - - /* 20% */ - {"btree.huffman_value", "configure huffman encoded values", C_BOOL, 20, 0, 0, &g.c_huffman_value, - NULL}, - - /* 95% */ - {"btree.internal_key_truncation", "truncate internal keys", C_BOOL, 95, 0, 0, - &g.c_internal_key_truncation, NULL}, - - {"btree.internal_page_max", "btree internal node maximum size", 0x0, 9, 17, 27, - &g.c_intl_page_max, NULL}, - - {"btree.key_max", "maximum key size", 0x0, 20, 128, MEGABYTE(10), &g.c_key_max, NULL}, - - /* - * A minimum key size of 11 is necessary. Row-store keys have a leading 10-digit number and the - * 11 guarantees we never see a key that we can't convert to a numeric value without formatting - * it first because there's a trailing non-digit character in every key. - */ - {"btree.key_min", "minimum key size", 0x0, 11, 32, 256, &g.c_key_min, NULL}, - - {"btree.leaf_page_max", "btree leaf node maximum size", 0x0, 9, 17, 27, &g.c_leaf_page_max, NULL}, - - {"btree.memory_page_max", "maximum cache page size", 0x0, 1, 10, 128, &g.c_memory_page_max, NULL}, - - {"btree.prefix", "common key prefix", C_BOOL, 3, 0, 0, &g.c_prefix, NULL}, - - /* 80% */ - {"btree.prefix_compression", "configure prefix compressed keys", C_BOOL, 80, 0, 0, - &g.c_prefix_compression, NULL}, - - {"btree.prefix_compression_min", "minimum gain before prefix compression is used (bytes)", 0x0, 0, - 8, 256, &g.c_prefix_compression_min, NULL}, - - {"btree.repeat_data_pct", "duplicate values (percentage)", 0x0, 0, 90, 90, &g.c_repeat_data_pct, - NULL}, - - /* 10% */ - {"btree.reverse", "reverse order collation", C_BOOL, 10, 0, 0, &g.c_reverse, NULL}, - - {"btree.split_pct", "page split size as a percentage of the maximum page size", 0x0, 50, 100, 100, - &g.c_split_pct, NULL}, - - {"btree.value_max", "maximum value size", 0x0, 32, 4096, MEGABYTE(10), &g.c_value_max, NULL}, - - {"btree.value_min", "minimum value size", 0x0, 0, 20, 4096, &g.c_value_min, NULL}, - - {"cache", "cache size (MB)", 0x0, 1, 100, 100 * 1024, &g.c_cache, NULL}, - - {"cache.evict_max", "maximum number of eviction workers", 0x0, 0, 5, 100, &g.c_evict_max, NULL}, - - {"cache.minimum", "minimum cache size (MB)", C_IGNORE, 0, 0, 100 * 1024, &g.c_cache_minimum, - NULL}, - - {"checkpoint", "checkpoint type (on | off | wiredtiger)", C_IGNORE | C_STRING, 0, 0, 0, NULL, - &g.c_checkpoint}, - - {"checkpoint.log_size", "MB of log to wait if wiredtiger checkpoints configured", 0x0, 20, 200, - 1024, &g.c_checkpoint_log_size, NULL}, - - {"checkpoint.wait", "seconds to wait if wiredtiger checkpoints configured", 0x0, 5, 100, 3600, - &g.c_checkpoint_wait, NULL}, - - {"disk.checksum", "checksum type (on | off | uncompressed | unencrypted)", C_IGNORE | C_STRING, 0, - 0, 0, NULL, &g.c_checksum}, - - /* 5% */ - {"disk.data_extend", "configure data file extension", C_BOOL, 5, 0, 0, &g.c_data_extend, NULL}, - - /* 0% */ - {"disk.direct_io", "configure direct I/O for data objects", C_IGNORE | C_BOOL, 0, 0, 1, - &g.c_direct_io, NULL}, - - {"disk.encryption", "encryption type (none | rotn-7)", C_IGNORE | C_STRING, 0, 0, 0, NULL, - &g.c_encryption}, - - /* 10% */ - {"disk.firstfit", "configure first-fit allocation", C_BOOL, 10, 0, 0, &g.c_firstfit, NULL}, - - /* 90% */ - {"disk.mmap", "configure mmap operations (reads only)", C_BOOL, 90, 0, 0, &g.c_mmap, NULL}, - - /* 5% */ - {"disk.mmap_all", "configure mmap operations (read and write)", C_BOOL, 5, 0, 0, &g.c_mmap_all, - NULL}, - - /* 0% */ - {"format.abort", "drop core during timed run", C_BOOL, 0, 0, 0, &g.c_abort, NULL}, - - /* 75% */ - {"format.independent_thread_rng", "configure independent thread RNG space", C_BOOL, 75, 0, 0, - &g.c_independent_thread_rng, NULL}, - - {"format.major_timeout", "long-running operations timeout (minutes)", C_IGNORE, 0, 0, 1000, - &g.c_major_timeout, NULL}, - - /* - * 0% - * FIXME-WT-7418 and FIXME-WT-7510: Temporarily disable import until WT_ROLLBACK error and - * wt_copy_and_sync error is fixed. It should be (C_BOOL, 20, 0, 0). - */ - {"import", "import table from newly created database", C_BOOL, 0, 0, 0, &g.c_import, NULL}, - - /* 50% */ - {"logging", "configure logging", C_BOOL, 50, 0, 0, &g.c_logging, NULL}, - - /* 50% */ - {"logging.archive", "configure log file archival", C_BOOL, 50, 0, 0, &g.c_logging_archive, NULL}, - - {"logging.compression", "logging compression type" COMPRESSION_LIST, C_IGNORE | C_STRING, 0, 0, 0, - NULL, &g.c_logging_compression}, - - {"logging.file_max", "maximum log file size (KB)", 0x0, 100, 512000, 2097152, - &g.c_logging_file_max, NULL}, - - /* 50% */ - {"logging.prealloc", "configure log file pre-allocation", C_BOOL, 50, 0, 0, &g.c_logging_prealloc, - NULL}, - - /* 90% */ - {"lsm.auto_throttle", "throttle LSM inserts", C_BOOL, 90, 0, 0, &g.c_auto_throttle, NULL}, - - /* 95% */ - {"lsm.bloom", "configure bloom filters", C_BOOL, 95, 0, 0, &g.c_bloom, NULL}, - {"lsm.bloom_bit_count", "number of bits per item for bloom filters", 0x0, 4, 64, 1000, - &g.c_bloom_bit_count, NULL}, - - {"lsm.bloom_hash_count", "number of hash values per item for bloom filters", 0x0, 4, 32, 100, - &g.c_bloom_hash_count, NULL}, - - /* 10% */ - {"lsm.bloom_oldest", "configure bloom_oldest=true", C_BOOL, 10, 0, 0, &g.c_bloom_oldest, NULL}, - - {"lsm.chunk_size", "LSM chunk size (MB)", 0x0, 1, 10, 100, &g.c_chunk_size, NULL}, - - {"lsm.merge_max", "maximum number of chunks to include in an LSM merge operation", 0x0, 4, 20, - 100, &g.c_merge_max, NULL}, - - {"lsm.worker_threads", "number of LSM worker threads", 0x0, 3, 4, 20, &g.c_lsm_worker_threads, - NULL}, - - /* 10% */ - {"ops.alter", "configure table alterations", C_BOOL, 10, 0, 0, &g.c_alter, NULL}, - - /* 10% */ - {"ops.compaction", "configure compaction", C_BOOL, 10, 0, 0, &g.c_compact, NULL}, - - /* 50% */ - {"ops.hs_cursor", "configure history store cursor reads", C_BOOL, 50, 0, 0, &g.c_hs_cursor, NULL}, - - {"ops.pct.delete", "delete operations (percentage)", C_IGNORE, 0, 0, 100, &g.c_delete_pct, NULL}, - - {"ops.pct.insert", "insert operations (percentage)", C_IGNORE, 0, 0, 100, &g.c_insert_pct, NULL}, - - {"ops.pct.modify", "modify operations (percentage)", C_IGNORE, 0, 0, 100, &g.c_modify_pct, NULL}, - - {"ops.pct.read", "read operations (percentage)", C_IGNORE, 0, 0, 100, &g.c_read_pct, NULL}, - - {"ops.pct.write", "update operations (percentage)", C_IGNORE, 0, 0, 100, &g.c_write_pct, NULL}, - - /* 5% */ - {"ops.prepare", "configure transaction prepare", C_BOOL, 5, 0, 0, &g.c_prepare, NULL}, - - /* 10% */ - {"ops.random_cursor", "configure random cursor reads", C_BOOL, 10, 0, 0, &g.c_random_cursor, - NULL}, - - /* 100% */ - {"ops.salvage", "configure salvage", C_BOOL, 100, 1, 0, &g.c_salvage, NULL}, - - /* 100% */ - {"ops.truncate", "configure truncation", C_BOOL, 100, 0, 0, &g.c_truncate, NULL}, - - /* 100% */ - {"ops.verify", "configure verify", C_BOOL, 100, 1, 0, &g.c_verify, NULL}, - - {"quiet", "quiet run (same as -q)", C_IGNORE | C_BOOL, 0, 0, 1, &g.c_quiet, NULL}, - - {"runs", "number of runs", C_IGNORE, 0, 0, UINT_MAX, &g.c_runs, NULL}, - - {"runs.in_memory", "configure in-memory", C_IGNORE | C_BOOL, 0, 0, 1, &g.c_in_memory, NULL}, - - {"runs.ops", "operations per run", 0x0, 0, M(2), M(100), &g.c_ops, NULL}, - - {"runs.rows", "number of rows", 0x0, 10, M(1), M(100), &g.c_rows, NULL}, - - {"runs.source", "data source type (file | lsm | table)", C_IGNORE | C_STRING, 0, 0, 0, NULL, - &g.c_data_source}, - - {"runs.threads", "number of worker threads", 0x0, 1, 32, 128, &g.c_threads, NULL}, - - {"runs.timer", "run time (minutes)", C_IGNORE, 0, 0, UINT_MAX, &g.c_timer, NULL}, - - {"runs.type", "object type (fix | row | var)", C_IGNORE | C_STRING, 0, 0, 0, NULL, - &g.c_file_type}, - - {"runs.verify_failure_dump", "configure page dump on repeatable read error", C_IGNORE | C_BOOL, 0, - 0, 1, &g.c_verify_failure_dump, NULL}, - - /* 20% */ - {"statistics", "configure statistics", C_BOOL, 20, 0, 0, &g.c_statistics, NULL}, - - /* 5% */ - {"statistics.server", "configure statistics server thread", C_BOOL, 5, 0, 0, - &g.c_statistics_server, NULL}, - - /* 2% */ - {"stress.aggressive_sweep", "stress aggressive sweep", C_BOOL, 2, 0, 0, - &g.c_timing_stress_aggressive_sweep, NULL}, - - /* 2% */ - {"stress.checkpoint", "stress checkpoints", C_BOOL, 2, 0, 0, &g.c_timing_stress_checkpoint, NULL}, - - /* 2% */ - {"stress.checkpoint_reserved_txnid_delay", "stress checkpoint invisible transaction id delay", - C_BOOL, 2, 0, 0, &g.c_timing_stress_checkpoint_reserved_txnid_delay, NULL}, - - /* 2% */ - {"stress.checkpoint_prepare", "stress checkpoint prepare", C_BOOL, 2, 0, 0, - &g.c_timing_stress_checkpoint_prepare, NULL}, - - /* 30% */ - {"stress.failpoint_hs_delete_key_from_ts", "stress failpoint history store delete key from ts", - C_BOOL, 30, 0, 0, &g.c_timing_stress_failpoint_hs_delete_key_from_ts, NULL}, - - /* 30% */ - {"stress.failpoint_hs_insert_1", "stress failpoint history store insert (#1)", C_BOOL, 30, 0, 0, - &g.c_timing_stress_failpoint_hs_insert_1, NULL}, - - /* 30% */ - {"stress.failpoint_hs_insert_2", "stress failpoint history store insert (#2)", C_BOOL, 30, 0, 0, - &g.c_timing_stress_failpoint_hs_insert_2, NULL}, - - /* 2% */ - {"stress.hs_checkpoint_delay", "stress history store checkpoint delay", C_BOOL, 2, 0, 0, - &g.c_timing_stress_hs_checkpoint_delay, NULL}, - - /* 2% */ - {"stress.hs_search", "stress history store search", C_BOOL, 2, 0, 0, &g.c_timing_stress_hs_search, - NULL}, - - /* 2% */ - {"stress.hs_sweep", "stress history store sweep", C_BOOL, 2, 0, 0, &g.c_timing_stress_hs_sweep, - NULL}, - - /* 2% */ - {"stress.split_1", "stress splits (#1)", C_BOOL, 2, 0, 0, &g.c_timing_stress_split_1, NULL}, - - /* 2% */ - {"stress.split_2", "stress splits (#2)", C_BOOL, 2, 0, 0, &g.c_timing_stress_split_2, NULL}, - - /* 2% */ - {"stress.split_3", "stress splits (#3)", C_BOOL, 2, 0, 0, &g.c_timing_stress_split_3, NULL}, - - /* 2% */ - {"stress.split_4", "stress splits (#4)", C_BOOL, 2, 0, 0, &g.c_timing_stress_split_4, NULL}, - - /* 2% */ - {"stress.split_5", "stress splits (#5)", C_BOOL, 2, 0, 0, &g.c_timing_stress_split_5, NULL}, - - /* 2% */ - {"stress.split_6", "stress splits (#6)", C_BOOL, 2, 0, 0, &g.c_timing_stress_split_6, NULL}, - - /* 2% */ - {"stress.split_7", "stress splits (#7)", C_BOOL, 2, 0, 0, &g.c_timing_stress_split_7, NULL}, - - {"transaction.implicit", "implicit, without timestamps, transactions (percentage)", 0x0, 0, 100, - 100, &g.c_txn_implicit, NULL}, - - /* 70% */ - {"transaction.timestamps", "all transactions (or none), have timestamps", C_BOOL, 80, 0, 0, - &g.c_txn_timestamps, NULL}, - - {"wiredtiger.config", "wiredtiger_open API configuration string", C_IGNORE | C_STRING, 0, 0, 0, - NULL, &g.c_config_open}, - - /* 80% */ - {"wiredtiger.rwlock", "configure wiredtiger read/write mutexes", C_BOOL, 80, 0, 0, &g.c_wt_mutex, - NULL}, - - {"wiredtiger.leak_memory", "configure memory leaked on shutdown", C_BOOL, 0, 0, 0, - &g.c_leak_memory, NULL}, + u_int off; /* Value offset */ +} CONFIG; - {NULL, NULL, 0x0, 0, 0, 0, NULL, NULL}}; +#define V_MAX_TABLES_CONFIG 1000 + +#define V_GLOBAL_ASSERT_READ_TIMESTAMP 0 +#define V_GLOBAL_ASSERT_WRITE_TIMESTAMP 1 +#define V_GLOBAL_BACKUP 2 +#define V_GLOBAL_BACKUP_INCREMENTAL 3 +#define V_GLOBAL_BACKUP_INCR_GRANULARITY 4 +#define V_GLOBAL_BLOCK_CACHE 5 +#define V_GLOBAL_BLOCK_CACHE_CACHE_ON_CHECKPOINT 6 +#define V_GLOBAL_BLOCK_CACHE_CACHE_ON_WRITES 7 +#define V_GLOBAL_BLOCK_CACHE_SIZE 8 +#define V_TABLE_BTREE_BITCNT 9 +#define V_TABLE_BTREE_COMPRESSION 10 +#define V_TABLE_BTREE_DICTIONARY 11 +#define V_TABLE_BTREE_HUFFMAN_VALUE 12 +#define V_TABLE_BTREE_INTERNAL_KEY_TRUNCATION 13 +#define V_TABLE_BTREE_INTERNAL_PAGE_MAX 14 +#define V_TABLE_BTREE_KEY_MAX 15 +#define V_TABLE_BTREE_KEY_MIN 16 +#define V_TABLE_BTREE_LEAF_PAGE_MAX 17 +#define V_TABLE_BTREE_MEMORY_PAGE_MAX 18 +#define V_TABLE_BTREE_PREFIX_LEN 19 +#define V_TABLE_BTREE_PREFIX_COMPRESSION 20 +#define V_TABLE_BTREE_PREFIX_COMPRESSION_MIN 21 +#define V_TABLE_BTREE_REPEAT_DATA_PCT 22 +#define V_TABLE_BTREE_REVERSE 23 +#define V_TABLE_BTREE_SPLIT_PCT 24 +#define V_TABLE_BTREE_VALUE_MAX 25 +#define V_TABLE_BTREE_VALUE_MIN 26 +#define V_GLOBAL_CACHE 27 +#define V_GLOBAL_CACHE_EVICT_MAX 28 +#define V_GLOBAL_CACHE_MINIMUM 29 +#define V_GLOBAL_CHECKPOINT 30 +#define V_GLOBAL_CHECKPOINT_LOG_SIZE 31 +#define V_GLOBAL_CHECKPOINT_WAIT 32 +#define V_TABLE_DISK_CHECKSUM 33 +#define V_GLOBAL_DISK_DATA_EXTEND 34 +#define V_GLOBAL_DISK_DIRECT_IO 35 +#define V_GLOBAL_DISK_ENCRYPTION 36 +#define V_TABLE_DISK_FIRSTFIT 37 +#define V_GLOBAL_DISK_MMAP 38 +#define V_GLOBAL_DISK_MMAP_ALL 39 +#define V_GLOBAL_FORMAT_ABORT 40 +#define V_GLOBAL_FORMAT_INDEPENDENT_THREAD_RNG 41 +#define V_GLOBAL_FORMAT_MAJOR_TIMEOUT 42 +#define V_GLOBAL_IMPORT 43 +#define V_GLOBAL_LOGGING 44 +#define V_GLOBAL_LOGGING_ARCHIVE 45 +#define V_GLOBAL_LOGGING_COMPRESSION 46 +#define V_GLOBAL_LOGGING_FILE_MAX 47 +#define V_GLOBAL_LOGGING_PREALLOC 48 +#define V_TABLE_LSM_AUTO_THROTTLE 49 +#define V_TABLE_LSM_BLOOM 50 +#define V_TABLE_LSM_BLOOM_BIT_COUNT 51 +#define V_TABLE_LSM_BLOOM_HASH_COUNT 52 +#define V_TABLE_LSM_BLOOM_OLDEST 53 +#define V_TABLE_LSM_CHUNK_SIZE 54 +#define V_TABLE_LSM_MERGE_MAX 55 +#define V_GLOBAL_LSM_WORKER_THREADS 56 +#define V_GLOBAL_OPS_ALTER 57 +#define V_GLOBAL_OPS_COMPACTION 58 +#define V_GLOBAL_OPS_HS_CURSOR 59 +#define V_TABLE_OPS_PCT_DELETE 60 +#define V_TABLE_OPS_PCT_INSERT 61 +#define V_TABLE_OPS_PCT_MODIFY 62 +#define V_TABLE_OPS_PCT_READ 63 +#define V_TABLE_OPS_PCT_WRITE 64 +#define V_GLOBAL_OPS_PREPARE 65 +#define V_GLOBAL_OPS_RANDOM_CURSOR 66 +#define V_GLOBAL_OPS_SALVAGE 67 +#define V_TABLE_OPS_TRUNCATE 68 +#define V_GLOBAL_OPS_VERIFY 69 +#define V_GLOBAL_QUIET 70 +#define V_GLOBAL_RUNS_IN_MEMORY 71 +#define V_GLOBAL_RUNS_OPS 72 +#define V_TABLE_RUNS_ROWS 73 +#define V_TABLE_RUNS_SOURCE 74 +#define V_GLOBAL_RUNS_TABLES 75 +#define V_GLOBAL_RUNS_THREADS 76 +#define V_GLOBAL_RUNS_TIMER 77 +#define V_TABLE_RUNS_TYPE 78 +#define V_GLOBAL_RUNS_VERIFY_FAILURE_DUMP 79 +#define V_GLOBAL_STATISTICS 80 +#define V_GLOBAL_STATISTICS_SERVER 81 +#define V_GLOBAL_STRESS_AGGRESSIVE_SWEEP 82 +#define V_GLOBAL_STRESS_CHECKPOINT 83 +#define V_GLOBAL_STRESS_CHECKPOINT_RESERVED_TXNID_DELAY 84 +#define V_GLOBAL_STRESS_CHECKPOINT_PREPARE 85 +#define V_GLOBAL_STRESS_FAILPOINT_HS_DELETE_KEY_FROM_TS 86 +#define V_GLOBAL_STRESS_FAILPOINT_HS_INSERT_1 87 +#define V_GLOBAL_STRESS_FAILPOINT_HS_INSERT_2 88 +#define V_GLOBAL_STRESS_HS_CHECKPOINT_DELAY 89 +#define V_GLOBAL_STRESS_HS_SEARCH 90 +#define V_GLOBAL_STRESS_HS_SWEEP 91 +#define V_GLOBAL_STRESS_SPLIT_1 92 +#define V_GLOBAL_STRESS_SPLIT_2 93 +#define V_GLOBAL_STRESS_SPLIT_3 94 +#define V_GLOBAL_STRESS_SPLIT_4 95 +#define V_GLOBAL_STRESS_SPLIT_5 96 +#define V_GLOBAL_STRESS_SPLIT_6 97 +#define V_GLOBAL_STRESS_SPLIT_7 98 +#define V_GLOBAL_TRANSACTION_IMPLICIT 99 +#define V_GLOBAL_TRANSACTION_TIMESTAMPS 100 +#define V_GLOBAL_WIREDTIGER_CONFIG 101 +#define V_GLOBAL_WIREDTIGER_RWLOCK 102 +#define V_GLOBAL_WIREDTIGER_LEAK_MEMORY 103 + +#define V_ELEMENT_COUNT 104 diff --git a/src/third_party/wiredtiger/test/format/config.sh b/src/third_party/wiredtiger/test/format/config.sh new file mode 100644 index 00000000000..dce73364530 --- /dev/null +++ b/src/third_party/wiredtiger/test/format/config.sh @@ -0,0 +1,296 @@ +#! /bin/sh + +# This script creates format's config.h and config_def.c files. To change format's configuration, +# modify this file and then run it as a script. + +fc="config_def.c" +fh="config.h" + +cat<<END_OF_HEADER_FILE_PREFIX>$fh +/* DO NOT EDIT: automatically built by format/config.sh. */ + +#define C_TYPE_MATCH(cp, type) \\ + (!F_ISSET(cp, (C_TYPE_FIX | C_TYPE_ROW | C_TYPE_VAR)) || \\ + ((type) == FIX && F_ISSET(cp, C_TYPE_FIX)) || ((type) == ROW && F_ISSET(cp, C_TYPE_ROW)) || \\ + ((type) == VAR && F_ISSET(cp, C_TYPE_VAR))) + +typedef struct { + const char *name; /* Configuration item */ + const char *desc; /* Configuration description */ + +#define C_BOOL 0x001u /* Boolean (true if roll of 1-to-100 is <= CONFIG->min) */ +#define C_IGNORE 0x002u /* Not a simple randomization, configured specially */ +#define C_STRING 0x004u /* String (rather than integral) */ +#define C_TABLE 0x008u /* Value is per table, not global */ +#define C_TYPE_FIX 0x010u /* Value is only relevant to FLCS */ +#define C_TYPE_LSM 0x020u /* Value is only relevant to LSM */ +#define C_TYPE_ROW 0x040u /* Value is only relevant to RS */ +#define C_TYPE_VAR 0x080u /* Value is only relevant to VLCS */ +#define C_ZERO_NOTSET 0x100u /* Ignore zero values */ + uint32_t flags; + + uint32_t min; /* Minimum value */ + uint32_t maxrand; /* Maximum value randomly chosen */ + uint32_t maxset; /* Maximum value explicitly set */ + + u_int off; /* Value offset */ +} CONFIG; + +#define V_MAX_TABLES_CONFIG 1000 + +END_OF_HEADER_FILE_PREFIX + +n=0 +while IFS= read -r line; do + case "$line" in + '{"'*) + tag=`echo "$line" | + sed -e 's/{"//' \ + -e 's/",.*//' \ + -e 's/\./_/g' | + tr '[:lower:]' '[:upper:]'` + prefix="GLOBAL" + if `echo "$line" | grep 'C_TABLE' > /dev/null`; then + prefix="TABLE" + fi + def="V_""$prefix""_""$tag" + echo "$line" | + sed -e "s/}/, $def},/" \ + -e 's/\(^.*",\) \(.*\)/ \1\n \2/' + + echo "#define $def $n" >> $fh + + n=`expr $n + 1` + ;; + *) + echo "$line" + esac +done<<END_OF_INPUT>$fc +/* DO NOT EDIT: automatically built by format/config.sh. */ + +#include "format.h" + +CONFIG configuration_list[] = { +{"assert.read_timestamp", "assert read_timestamp", C_BOOL, 2, 0, 0} + +{"assert.write_timestamp", "set write_timestamp_usage and assert write_timestamp", C_BOOL, 2, 0, 0} + +{"backup", "configure backups", C_BOOL, 20, 0, 0} + +{"backup.incremental", "backup type (block | log | off)", C_IGNORE | C_STRING, 0, 0, 0} + +{"backup.incr_granularity", "incremental backup block granularity (KB)", 0x0, 4, 16384, 16384} + +{"block_cache", "enable the block cache", C_BOOL, 10, 0, 0} + +{"block_cache.cache_on_checkpoint", "block cache: cache checkpoint writes", C_BOOL, 30, 0, 0} + +{"block_cache.cache_on_writes", "block cache: populate the cache on writes", C_BOOL, 60, 0, 0} + +{"block_cache.size", "block cache size (MB)", 0x0, 1, 100, 100 * 1024} + +{"btree.bitcnt", "fixed-length column-store object size (number of bits)", C_TABLE | C_TYPE_FIX, 1, 8, 8} + +{"btree.compression", "data compression (none | lz4 | snappy | zlib | zstd)", C_IGNORE | C_STRING | C_TABLE, 0, 0, 0} + +{"btree.dictionary", "configure dictionary compressed values", C_BOOL | C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 20, 0, 0} + +{"btree.huffman_value", "configure huffman encoded values", C_BOOL | C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 20, 0, 0} + +{"btree.internal_key_truncation", "truncate internal keys", C_BOOL | C_TABLE, 95, 0, 0} + +{"btree.internal_page_max", "btree internal node maximum size", C_TABLE, 9, 17, 27} + +{"btree.key_max", "maximum key size", C_TABLE | C_TYPE_ROW, 20, 128, MEGABYTE(10)} + +{"btree.key_min", "minimum key size", C_TABLE | C_TYPE_ROW, KEY_LEN_CONFIG_MIN, 32, 256} + +{"btree.leaf_page_max", "btree leaf node maximum size", C_TABLE, 9, 17, 27} + +{"btree.memory_page_max", "maximum cache page size", C_TABLE, 1, 10, 128} + +{"btree.prefix_len", "common key prefix", C_TABLE | C_TYPE_ROW | C_ZERO_NOTSET, PREFIX_LEN_CONFIG_MIN, PREFIX_LEN_CONFIG_MAX, PREFIX_LEN_CONFIG_MAX} + +{"btree.prefix_compression", "configure prefix compressed keys", C_BOOL | C_TABLE | C_TYPE_ROW, 80, 0, 0} + +{"btree.prefix_compression_min", "minimum gain before prefix compression is used (bytes)", C_TABLE | C_TYPE_ROW, 0, 8, 256} + +{"btree.repeat_data_pct", "duplicate values (percentage)", C_TABLE | C_TYPE_VAR, 0, 90, 90} + +{"btree.reverse", "reverse order collation", C_BOOL | C_TABLE | C_TYPE_ROW, 10, 0, 0} + +{"btree.split_pct", "page split size as a percentage of the maximum page size", C_TABLE, 50, 100, 100} + +{"btree.value_max", "maximum value size", C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 32, 4096, MEGABYTE(10)} + +{"btree.value_min", "minimum value size", C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 0, 20, 4096} + +{"cache", "cache size (MB)", 0x0, 1, 100, 100 * 1024} + +{"cache.evict_max", "maximum number of eviction workers", 0x0, 0, 5, 100} + +{"cache.minimum", "minimum cache size (MB)", C_IGNORE, 0, 0, 100 * 1024} + +{"checkpoint", "checkpoint type (on | off | wiredtiger)", C_IGNORE | C_STRING, 0, 0, 0} + +{"checkpoint.log_size", "MB of log to wait if wiredtiger checkpoints configured", 0x0, 20, 200, 1024} + +{"checkpoint.wait", "seconds to wait if wiredtiger checkpoints configured", 0x0, 5, 100, 3600} + +{"disk.checksum", "checksum type (on | off | uncompressed | unencrypted)", C_IGNORE | C_STRING | C_TABLE, 0, 0, 0} + +{"disk.data_extend", "configure data file extension", C_BOOL, 5, 0, 0} + +{"disk.direct_io", "configure direct I/O for data objects", C_BOOL | C_IGNORE, 0, 0, 1} + +{"disk.encryption", "encryption type (none | rotn-7)", C_IGNORE | C_STRING, 0, 0, 0} + +{"disk.firstfit", "configure first-fit allocation", C_BOOL | C_TABLE, 10, 0, 0} + +{"disk.mmap", "configure mmap operations (reads only)", C_BOOL, 90, 0, 0} + +{"disk.mmap_all", "configure mmap operations (read and write)", C_BOOL, 5, 0, 0} + +{"format.abort", "drop core during timed run", C_BOOL, 0, 0, 0} + +{"format.independent_thread_rng", "configure independent thread RNG space", C_BOOL, 75, 0, 0} + +{"format.major_timeout", "long-running operations timeout (minutes)", C_IGNORE, 0, 0, 1000} + +/* + * 0% + * FIXME-WT-7510: Temporarily disable import until WT_ROLLBACK error and wt_copy_and_sync error is + * fixed. It should be (C_BOOL, 20, 0, 0). + */ +{"import", "import table from newly created database", C_BOOL, 0, 0, 0} + +{"logging", "configure logging", C_BOOL, 50, 0, 0} + +{"logging.archive", "configure log file archival", C_BOOL, 50, 0, 0} + +{"logging.compression", "logging compression (none | lz4 | snappy | zlib | zstd)", C_IGNORE | C_STRING, 0, 0, 0} + +{"logging.file_max", "maximum log file size (KB)", 0x0, 100, 512000, 2097152} + +{"logging.prealloc", "configure log file pre-allocation", C_BOOL, 50, 0, 0} + +{"lsm.auto_throttle", "throttle LSM inserts", C_BOOL | C_TABLE | C_TYPE_LSM, 90, 0, 0} + +{"lsm.bloom", "configure bloom filters", C_BOOL | C_TABLE | C_TYPE_LSM, 95, 0, 0} + +{"lsm.bloom_bit_count", "number of bits per item for bloom filters", C_TABLE | C_TYPE_LSM, 4, 64, 1000} + +{"lsm.bloom_hash_count", "number of hash values per item for bloom filters", C_TABLE | C_TYPE_LSM, 4, 32, 100} + +{"lsm.bloom_oldest", "configure bloom_oldest=true", C_BOOL | C_TABLE | C_TYPE_LSM, 10, 0, 0} + +{"lsm.chunk_size", "LSM chunk size (MB)", C_TABLE | C_TYPE_LSM, 1, 10, 100} + +{"lsm.merge_max", "maximum number of chunks to include in an LSM merge operation", C_TABLE | C_TYPE_LSM, 4, 20, 100} + +{"lsm.worker_threads", "number of LSM worker threads", C_TYPE_LSM, 3, 4, 20} + +{"ops.alter", "configure table alterations", C_BOOL, 10, 0, 0} + +{"ops.compaction", "configure compaction", C_BOOL, 10, 0, 0} + +{"ops.hs_cursor", "configure history store cursor reads", C_BOOL, 50, 0, 0} + +{"ops.pct.delete", "delete operations (percentage)", C_IGNORE | C_TABLE, 0, 0, 100} + +{"ops.pct.insert", "insert operations (percentage)", C_IGNORE | C_TABLE, 0, 0, 100} + +{"ops.pct.modify", "modify operations (percentage)", C_IGNORE | C_TABLE, 0, 0, 100} + +{"ops.pct.read", "read operations (percentage)", C_IGNORE | C_TABLE, 0, 0, 100} + +{"ops.pct.write", "update operations (percentage)", C_IGNORE | C_TABLE, 0, 0, 100} + +{"ops.prepare", "configure transaction prepare", C_BOOL, 5, 0, 0} + +{"ops.random_cursor", "configure random cursor reads", C_BOOL, 10, 0, 0} + +{"ops.salvage", "configure salvage", C_BOOL, 100, 1, 0} + +{"ops.truncate", "configure truncation", C_BOOL | C_TABLE, 100, 0, 0} + +{"ops.verify", "configure verify", C_BOOL, 100, 1, 0} + +{"quiet", "quiet run (same as -q)", C_BOOL | C_IGNORE, 0, 0, 1} + +{"runs.in_memory", "configure in-memory", C_BOOL | C_IGNORE, 0, 0, 1} + +{"runs.ops", "operations per run", 0x0, 0, M(2), M(100)} + +{"runs.rows", "number of rows", C_TABLE, 10, M(1), M(100)} + +{"runs.source", "data source type (file | lsm | table)", C_IGNORE | C_STRING | C_TABLE, 0, 0, 0} + +{"runs.tables", "number of tables", 0x0, 1, 32, V_MAX_TABLES_CONFIG} + +{"runs.threads", "number of worker threads", 0x0, 1, 32, 128} + +{"runs.timer", "run time (minutes)", C_IGNORE, 0, 0, UINT_MAX} + +{"runs.type", "object type (fix | row | var)", C_IGNORE | C_STRING | C_TABLE, 0, 0, 0} + +{"runs.verify_failure_dump", "configure page dump on repeatable read error", C_BOOL | C_IGNORE, 0, 0, 1} + +{"statistics", "configure statistics", C_BOOL, 20, 0, 0} + +{"statistics.server", "configure statistics server thread", C_BOOL, 5, 0, 0} + +{"stress.aggressive_sweep", "stress aggressive sweep", C_BOOL, 2, 0, 0} + +{"stress.checkpoint", "stress checkpoints", C_BOOL, 2, 0, 0} + +{"stress.checkpoint_reserved_txnid_delay", "stress checkpoint invisible transaction id delay", C_BOOL, 2, 0, 0} + +{"stress.checkpoint_prepare", "stress checkpoint prepare", C_BOOL, 2, 0, 0} + +{"stress.failpoint_hs_delete_key_from_ts", "stress failpoint history store delete key from ts", C_BOOL, 30, 0, 0} + +{"stress.failpoint_hs_insert_1", "stress failpoint history store insert (#1)", C_BOOL, 30, 0, 0} + +{"stress.failpoint_hs_insert_2", "stress failpoint history store insert (#2)", C_BOOL, 30, 0, 0} + +{"stress.hs_checkpoint_delay", "stress history store checkpoint delay", C_BOOL, 2, 0, 0} + +{"stress.hs_search", "stress history store search", C_BOOL, 2, 0, 0} + +{"stress.hs_sweep", "stress history store sweep", C_BOOL, 2, 0, 0} + +{"stress.split_1", "stress splits (#1)", C_BOOL, 2, 0, 0} + +{"stress.split_2", "stress splits (#2)", C_BOOL, 2, 0, 0} + +{"stress.split_3", "stress splits (#3)", C_BOOL, 2, 0, 0} + +{"stress.split_4", "stress splits (#4)", C_BOOL, 2, 0, 0} + +{"stress.split_5", "stress splits (#5)", C_BOOL, 2, 0, 0} + +{"stress.split_6", "stress splits (#6)", C_BOOL, 2, 0, 0} + +{"stress.split_7", "stress splits (#7)", C_BOOL, 2, 0, 0} + +{"transaction.implicit", "implicit, without timestamps, transactions (percentage)", 0, 0, 100, 100} + +{"transaction.timestamps", "all transactions (or none), have timestamps", C_BOOL, 80, 0, 0} + +{"wiredtiger.config", "wiredtiger_open API configuration string", C_IGNORE | C_STRING, 0, 0, 0} + +{"wiredtiger.rwlock", "configure wiredtiger read/write mutexes", C_BOOL, 80, 0, 0} + +{"wiredtiger.leak_memory", "leak memory on wiredtiger shutdown", C_BOOL, 0, 0, 0} +END_OF_INPUT + +( +echo +echo ' {NULL, NULL, 0x0, 0, 0, 0, 0}' +echo '};') >> $fc + +( +echo +echo "#define V_ELEMENT_COUNT $n") >> $fh + diff --git a/src/third_party/wiredtiger/test/format/config_def.c b/src/third_party/wiredtiger/test/format/config_def.c new file mode 100644 index 00000000000..22d650f9326 --- /dev/null +++ b/src/third_party/wiredtiger/test/format/config_def.c @@ -0,0 +1,324 @@ +/* DO NOT EDIT: automatically built by format/config.sh. */ + +#include "format.h" + +CONFIG configuration_list[] = { + {"assert.read_timestamp", "assert read_timestamp", + C_BOOL, 2, 0, 0, V_GLOBAL_ASSERT_READ_TIMESTAMP}, + + {"assert.write_timestamp", "set write_timestamp_usage and assert write_timestamp", + C_BOOL, 2, 0, 0, V_GLOBAL_ASSERT_WRITE_TIMESTAMP}, + + {"backup", "configure backups", + C_BOOL, 20, 0, 0, V_GLOBAL_BACKUP}, + + {"backup.incremental", "backup type (block | log | off)", + C_IGNORE | C_STRING, 0, 0, 0, V_GLOBAL_BACKUP_INCREMENTAL}, + + {"backup.incr_granularity", "incremental backup block granularity (KB)", + 0x0, 4, 16384, 16384, V_GLOBAL_BACKUP_INCR_GRANULARITY}, + + {"block_cache", "enable the block cache", + C_BOOL, 10, 0, 0, V_GLOBAL_BLOCK_CACHE}, + + {"block_cache.cache_on_checkpoint", "block cache: cache checkpoint writes", + C_BOOL, 30, 0, 0, V_GLOBAL_BLOCK_CACHE_CACHE_ON_CHECKPOINT}, + + {"block_cache.cache_on_writes", "block cache: populate the cache on writes", + C_BOOL, 60, 0, 0, V_GLOBAL_BLOCK_CACHE_CACHE_ON_WRITES}, + + {"block_cache.size", "block cache size (MB)", + 0x0, 1, 100, 100 * 1024, V_GLOBAL_BLOCK_CACHE_SIZE}, + + {"btree.bitcnt", "fixed-length column-store object size (number of bits)", + C_TABLE | C_TYPE_FIX, 1, 8, 8, V_TABLE_BTREE_BITCNT}, + + {"btree.compression", "data compression (none | lz4 | snappy | zlib | zstd)", + C_IGNORE | C_STRING | C_TABLE, 0, 0, 0, V_TABLE_BTREE_COMPRESSION}, + + {"btree.dictionary", "configure dictionary compressed values", + C_BOOL | C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 20, 0, 0, V_TABLE_BTREE_DICTIONARY}, + + {"btree.huffman_value", "configure huffman encoded values", + C_BOOL | C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 20, 0, 0, V_TABLE_BTREE_HUFFMAN_VALUE}, + + {"btree.internal_key_truncation", "truncate internal keys", + C_BOOL | C_TABLE, 95, 0, 0, V_TABLE_BTREE_INTERNAL_KEY_TRUNCATION}, + + {"btree.internal_page_max", "btree internal node maximum size", + C_TABLE, 9, 17, 27, V_TABLE_BTREE_INTERNAL_PAGE_MAX}, + + {"btree.key_max", "maximum key size", + C_TABLE | C_TYPE_ROW, 20, 128, MEGABYTE(10), V_TABLE_BTREE_KEY_MAX}, + + {"btree.key_min", "minimum key size", + C_TABLE | C_TYPE_ROW, KEY_LEN_CONFIG_MIN, 32, 256, V_TABLE_BTREE_KEY_MIN}, + + {"btree.leaf_page_max", "btree leaf node maximum size", + C_TABLE, 9, 17, 27, V_TABLE_BTREE_LEAF_PAGE_MAX}, + + {"btree.memory_page_max", "maximum cache page size", + C_TABLE, 1, 10, 128, V_TABLE_BTREE_MEMORY_PAGE_MAX}, + + {"btree.prefix_len", "common key prefix", + C_TABLE | C_TYPE_ROW | C_ZERO_NOTSET, PREFIX_LEN_CONFIG_MIN, PREFIX_LEN_CONFIG_MAX, PREFIX_LEN_CONFIG_MAX, V_TABLE_BTREE_PREFIX_LEN}, + + {"btree.prefix_compression", "configure prefix compressed keys", + C_BOOL | C_TABLE | C_TYPE_ROW, 80, 0, 0, V_TABLE_BTREE_PREFIX_COMPRESSION}, + + {"btree.prefix_compression_min", "minimum gain before prefix compression is used (bytes)", + C_TABLE | C_TYPE_ROW, 0, 8, 256, V_TABLE_BTREE_PREFIX_COMPRESSION_MIN}, + + {"btree.repeat_data_pct", "duplicate values (percentage)", + C_TABLE | C_TYPE_VAR, 0, 90, 90, V_TABLE_BTREE_REPEAT_DATA_PCT}, + + {"btree.reverse", "reverse order collation", + C_BOOL | C_TABLE | C_TYPE_ROW, 10, 0, 0, V_TABLE_BTREE_REVERSE}, + + {"btree.split_pct", "page split size as a percentage of the maximum page size", + C_TABLE, 50, 100, 100, V_TABLE_BTREE_SPLIT_PCT}, + + {"btree.value_max", "maximum value size", + C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 32, 4096, MEGABYTE(10), V_TABLE_BTREE_VALUE_MAX}, + + {"btree.value_min", "minimum value size", + C_TABLE | C_TYPE_ROW | C_TYPE_VAR, 0, 20, 4096, V_TABLE_BTREE_VALUE_MIN}, + + {"cache", "cache size (MB)", + 0x0, 1, 100, 100 * 1024, V_GLOBAL_CACHE}, + + {"cache.evict_max", "maximum number of eviction workers", + 0x0, 0, 5, 100, V_GLOBAL_CACHE_EVICT_MAX}, + + {"cache.minimum", "minimum cache size (MB)", + C_IGNORE, 0, 0, 100 * 1024, V_GLOBAL_CACHE_MINIMUM}, + + {"checkpoint", "checkpoint type (on | off | wiredtiger)", + C_IGNORE | C_STRING, 0, 0, 0, V_GLOBAL_CHECKPOINT}, + + {"checkpoint.log_size", "MB of log to wait if wiredtiger checkpoints configured", + 0x0, 20, 200, 1024, V_GLOBAL_CHECKPOINT_LOG_SIZE}, + + {"checkpoint.wait", "seconds to wait if wiredtiger checkpoints configured", + 0x0, 5, 100, 3600, V_GLOBAL_CHECKPOINT_WAIT}, + + {"disk.checksum", "checksum type (on | off | uncompressed | unencrypted)", + C_IGNORE | C_STRING | C_TABLE, 0, 0, 0, V_TABLE_DISK_CHECKSUM}, + + {"disk.data_extend", "configure data file extension", + C_BOOL, 5, 0, 0, V_GLOBAL_DISK_DATA_EXTEND}, + + {"disk.direct_io", "configure direct I/O for data objects", + C_BOOL | C_IGNORE, 0, 0, 1, V_GLOBAL_DISK_DIRECT_IO}, + + {"disk.encryption", "encryption type (none | rotn-7)", + C_IGNORE | C_STRING, 0, 0, 0, V_GLOBAL_DISK_ENCRYPTION}, + + {"disk.firstfit", "configure first-fit allocation", + C_BOOL | C_TABLE, 10, 0, 0, V_TABLE_DISK_FIRSTFIT}, + + {"disk.mmap", "configure mmap operations (reads only)", + C_BOOL, 90, 0, 0, V_GLOBAL_DISK_MMAP}, + + {"disk.mmap_all", "configure mmap operations (read and write)", + C_BOOL, 5, 0, 0, V_GLOBAL_DISK_MMAP_ALL}, + + {"format.abort", "drop core during timed run", + C_BOOL, 0, 0, 0, V_GLOBAL_FORMAT_ABORT}, + + {"format.independent_thread_rng", "configure independent thread RNG space", + C_BOOL, 75, 0, 0, V_GLOBAL_FORMAT_INDEPENDENT_THREAD_RNG}, + + {"format.major_timeout", "long-running operations timeout (minutes)", + C_IGNORE, 0, 0, 1000, V_GLOBAL_FORMAT_MAJOR_TIMEOUT}, + +/* + * 0% + * FIXME-WT-7510: Temporarily disable import until WT_ROLLBACK error and wt_copy_and_sync error is + * fixed. It should be (C_BOOL, 20, 0, 0). + */ + {"import", "import table from newly created database", + C_BOOL, 0, 0, 0, V_GLOBAL_IMPORT}, + + {"logging", "configure logging", + C_BOOL, 50, 0, 0, V_GLOBAL_LOGGING}, + + {"logging.archive", "configure log file archival", + C_BOOL, 50, 0, 0, V_GLOBAL_LOGGING_ARCHIVE}, + + {"logging.compression", "logging compression (none | lz4 | snappy | zlib | zstd)", + C_IGNORE | C_STRING, 0, 0, 0, V_GLOBAL_LOGGING_COMPRESSION}, + + {"logging.file_max", "maximum log file size (KB)", + 0x0, 100, 512000, 2097152, V_GLOBAL_LOGGING_FILE_MAX}, + + {"logging.prealloc", "configure log file pre-allocation", + C_BOOL, 50, 0, 0, V_GLOBAL_LOGGING_PREALLOC}, + + {"lsm.auto_throttle", "throttle LSM inserts", + C_BOOL | C_TABLE | C_TYPE_LSM, 90, 0, 0, V_TABLE_LSM_AUTO_THROTTLE}, + + {"lsm.bloom", "configure bloom filters", + C_BOOL | C_TABLE | C_TYPE_LSM, 95, 0, 0, V_TABLE_LSM_BLOOM}, + + {"lsm.bloom_bit_count", "number of bits per item for bloom filters", + C_TABLE | C_TYPE_LSM, 4, 64, 1000, V_TABLE_LSM_BLOOM_BIT_COUNT}, + + {"lsm.bloom_hash_count", "number of hash values per item for bloom filters", + C_TABLE | C_TYPE_LSM, 4, 32, 100, V_TABLE_LSM_BLOOM_HASH_COUNT}, + + {"lsm.bloom_oldest", "configure bloom_oldest=true", + C_BOOL | C_TABLE | C_TYPE_LSM, 10, 0, 0, V_TABLE_LSM_BLOOM_OLDEST}, + + {"lsm.chunk_size", "LSM chunk size (MB)", + C_TABLE | C_TYPE_LSM, 1, 10, 100, V_TABLE_LSM_CHUNK_SIZE}, + + {"lsm.merge_max", "maximum number of chunks to include in an LSM merge operation", + C_TABLE | C_TYPE_LSM, 4, 20, 100, V_TABLE_LSM_MERGE_MAX}, + + {"lsm.worker_threads", "number of LSM worker threads", + C_TYPE_LSM, 3, 4, 20, V_GLOBAL_LSM_WORKER_THREADS}, + + {"ops.alter", "configure table alterations", + C_BOOL, 10, 0, 0, V_GLOBAL_OPS_ALTER}, + + {"ops.compaction", "configure compaction", + C_BOOL, 10, 0, 0, V_GLOBAL_OPS_COMPACTION}, + + {"ops.hs_cursor", "configure history store cursor reads", + C_BOOL, 50, 0, 0, V_GLOBAL_OPS_HS_CURSOR}, + + {"ops.pct.delete", "delete operations (percentage)", + C_IGNORE | C_TABLE, 0, 0, 100, V_TABLE_OPS_PCT_DELETE}, + + {"ops.pct.insert", "insert operations (percentage)", + C_IGNORE | C_TABLE, 0, 0, 100, V_TABLE_OPS_PCT_INSERT}, + + {"ops.pct.modify", "modify operations (percentage)", + C_IGNORE | C_TABLE, 0, 0, 100, V_TABLE_OPS_PCT_MODIFY}, + + {"ops.pct.read", "read operations (percentage)", + C_IGNORE | C_TABLE, 0, 0, 100, V_TABLE_OPS_PCT_READ}, + + {"ops.pct.write", "update operations (percentage)", + C_IGNORE | C_TABLE, 0, 0, 100, V_TABLE_OPS_PCT_WRITE}, + + {"ops.prepare", "configure transaction prepare", + C_BOOL, 5, 0, 0, V_GLOBAL_OPS_PREPARE}, + + {"ops.random_cursor", "configure random cursor reads", + C_BOOL, 10, 0, 0, V_GLOBAL_OPS_RANDOM_CURSOR}, + + {"ops.salvage", "configure salvage", + C_BOOL, 100, 1, 0, V_GLOBAL_OPS_SALVAGE}, + + {"ops.truncate", "configure truncation", + C_BOOL | C_TABLE, 100, 0, 0, V_TABLE_OPS_TRUNCATE}, + + {"ops.verify", "configure verify", + C_BOOL, 100, 1, 0, V_GLOBAL_OPS_VERIFY}, + + {"quiet", "quiet run (same as -q)", + C_BOOL | C_IGNORE, 0, 0, 1, V_GLOBAL_QUIET}, + + {"runs.in_memory", "configure in-memory", + C_BOOL | C_IGNORE, 0, 0, 1, V_GLOBAL_RUNS_IN_MEMORY}, + + {"runs.ops", "operations per run", + 0x0, 0, M(2), M(100), V_GLOBAL_RUNS_OPS}, + + {"runs.rows", "number of rows", + C_TABLE, 10, M(1), M(100), V_TABLE_RUNS_ROWS}, + + {"runs.source", "data source type (file | lsm | table)", + C_IGNORE | C_STRING | C_TABLE, 0, 0, 0, V_TABLE_RUNS_SOURCE}, + + {"runs.tables", "number of tables", + 0x0, 1, 32, V_MAX_TABLES_CONFIG, V_GLOBAL_RUNS_TABLES}, + + {"runs.threads", "number of worker threads", + 0x0, 1, 32, 128, V_GLOBAL_RUNS_THREADS}, + + {"runs.timer", "run time (minutes)", + C_IGNORE, 0, 0, UINT_MAX, V_GLOBAL_RUNS_TIMER}, + + {"runs.type", "object type (fix | row | var)", + C_IGNORE | C_STRING | C_TABLE, 0, 0, 0, V_TABLE_RUNS_TYPE}, + + {"runs.verify_failure_dump", "configure page dump on repeatable read error", + C_BOOL | C_IGNORE, 0, 0, 1, V_GLOBAL_RUNS_VERIFY_FAILURE_DUMP}, + + {"statistics", "configure statistics", + C_BOOL, 20, 0, 0, V_GLOBAL_STATISTICS}, + + {"statistics.server", "configure statistics server thread", + C_BOOL, 5, 0, 0, V_GLOBAL_STATISTICS_SERVER}, + + {"stress.aggressive_sweep", "stress aggressive sweep", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_AGGRESSIVE_SWEEP}, + + {"stress.checkpoint", "stress checkpoints", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_CHECKPOINT}, + + {"stress.checkpoint_reserved_txnid_delay", "stress checkpoint invisible transaction id delay", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_CHECKPOINT_RESERVED_TXNID_DELAY}, + + {"stress.checkpoint_prepare", "stress checkpoint prepare", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_CHECKPOINT_PREPARE}, + + {"stress.failpoint_hs_delete_key_from_ts", "stress failpoint history store delete key from ts", + C_BOOL, 30, 0, 0, V_GLOBAL_STRESS_FAILPOINT_HS_DELETE_KEY_FROM_TS}, + + {"stress.failpoint_hs_insert_1", "stress failpoint history store insert (#1)", + C_BOOL, 30, 0, 0, V_GLOBAL_STRESS_FAILPOINT_HS_INSERT_1}, + + {"stress.failpoint_hs_insert_2", "stress failpoint history store insert (#2)", + C_BOOL, 30, 0, 0, V_GLOBAL_STRESS_FAILPOINT_HS_INSERT_2}, + + {"stress.hs_checkpoint_delay", "stress history store checkpoint delay", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_HS_CHECKPOINT_DELAY}, + + {"stress.hs_search", "stress history store search", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_HS_SEARCH}, + + {"stress.hs_sweep", "stress history store sweep", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_HS_SWEEP}, + + {"stress.split_1", "stress splits (#1)", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_1}, + + {"stress.split_2", "stress splits (#2)", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_2}, + + {"stress.split_3", "stress splits (#3)", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_3}, + + {"stress.split_4", "stress splits (#4)", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_4}, + + {"stress.split_5", "stress splits (#5)", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_5}, + + {"stress.split_6", "stress splits (#6)", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_6}, + + {"stress.split_7", "stress splits (#7)", + C_BOOL, 2, 0, 0, V_GLOBAL_STRESS_SPLIT_7}, + + {"transaction.implicit", "implicit, without timestamps, transactions (percentage)", + 0, 0, 100, 100, V_GLOBAL_TRANSACTION_IMPLICIT}, + + {"transaction.timestamps", "all transactions (or none), have timestamps", + C_BOOL, 80, 0, 0, V_GLOBAL_TRANSACTION_TIMESTAMPS}, + + {"wiredtiger.config", "wiredtiger_open API configuration string", + C_IGNORE | C_STRING, 0, 0, 0, V_GLOBAL_WIREDTIGER_CONFIG}, + + {"wiredtiger.rwlock", "configure wiredtiger read/write mutexes", + C_BOOL, 80, 0, 0, V_GLOBAL_WIREDTIGER_RWLOCK}, + + {"wiredtiger.leak_memory", "leak memory on wiredtiger shutdown", + C_BOOL, 0, 0, 0, V_GLOBAL_WIREDTIGER_LEAK_MEMORY}, + + {NULL, NULL, 0x0, 0, 0, 0, 0} +}; diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index cce28406706..a0224d35d2d 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -80,12 +80,12 @@ #undef MEGABYTE #define MEGABYTE(v) ((v)*WT_MEGABYTE) +/* Format isn't careful about path buffers, an easy to fix hard-coded length. */ +#define MAX_FORMAT_PATH 1024 + #define BACKUP_INFO_FILE "BACKUP_INFO" /* Format's backup information for restart */ #define BACKUP_INFO_FILE_TMP "BACKUP_INFO.TMP" /* Format's backup information for restart */ #define BACKUP_MAX_COPY MEGABYTE(64) /* Maximum size we'll read/write at a time */ -#define WT_NAME "wt" /* Object name */ - -#define DATASOURCE(v) (strcmp(v, g.c_data_source) == 0 ? 1 : 0) #define FORMAT_OPERATION_REPS 3 /* 3 thread operations sets */ @@ -105,45 +105,109 @@ typedef struct { #define LOCK_INITIALIZED(lock) ((lock)->lock_type != LOCK_NONE) +#include "config.h" +extern CONFIG configuration_list[]; + typedef struct { - char tidbuf[128]; /* thread ID in printable form */ + uint32_t v; /* integral value */ + char *vstr; /* string value */ + bool set; /* value explicitly set */ +} CONFIGV; + +typedef enum { FIX, ROW, VAR } table_type; +typedef struct { + u_int id; /* table ID */ + char uri[32]; /* table URI */ + table_type type; /* table type */ + char track_prefix[32]; /* table track message prefix */ + uint32_t max_intl_page; /* page size configurations converted to bytes */ + uint32_t max_leaf_page; + uint32_t max_mem_page; + + uint32_t rows_current; /* current row count */ + + uint64_t truncate_cnt; /* truncation operation counter */ + + uint32_t key_rand_len[1031]; /* key: lengths */ + char *val_base; /* value: base/original */ + uint32_t val_dup_data_len; /* value: length of duplicate data items */ + + CONFIGV v[V_ELEMENT_COUNT]; /* table configuration */ +} TABLE; + +/* + * We read the configuration in a single pass, which means we don't know the table count until the + * end, and it can be extended at any time. Start out with a single table, which contains all of the + * global/default values, stored in the first slot of the tables array. If tables are added during + * configuration, they are separately allocated, but we continue to use the first (base) table slot + * for non-specific table or global configurations. In other words, the base information and the + * only table's information are both in tables' slot 0 to start. If additional tables are + * configured, the per-table information for each table is stored in tables slots 1-N. The number of + * tables starts at 0, and if any tables are configured, it's incremented: in other words, if the + * number of tables is 0, all of the information is in tables' slot 0. If the number of tables is + * greater than 1, all of the base information is in tables slot 0, and tables slot 1 holds table + * #1's specific information, slot #2 holds table #2's specific information and so on. This allows + * general and table-specific information to be configured in any order, and as part of the final + * table configuration, if there's more than a single table, the information in tables' slot 0 is + * propagated out to the additional table slots. + */ +extern TABLE *tables[V_MAX_TABLES_CONFIG + 1]; /* Table array */ +extern u_int ntables; + +/* + * Global and table-specific macros to retrieve configuration information. All of the tables contain + * all of the possible configuration entries, but the first table slot contains all of the global + * configuration information. The offset names a prefixed with "V_GLOBAL" and "V_TABLE" to reduce + * the chance of a coding error retrieving the wrong configuration item. + */ +#define GV(off) (tables[0]->v[V_GLOBAL_##off].v) +#define GVS(off) (tables[0]->v[V_GLOBAL_##off].vstr) +#define TV(off) (table->v[V_TABLE_##off].v) +#define TVS(off) (table->v[V_TABLE_##off].vstr) + +#define DATASOURCE(table, ds) (strcmp((table)->v[V_TABLE_RUNS_SOURCE].vstr, ds) == 0) + +typedef struct { WT_CONNECTION *wts_conn; WT_CONNECTION *wts_conn_inmemory; WT_SESSION *wts_session; - char *uri; /* Object name */ - bool backward_compatible; /* Backward compatibility testing */ + bool configured; /* Configuration completed */ bool reopen; /* Reopen an existing database */ bool workers_finished; /* Operations completed */ char *home; /* Home directory */ char *home_config; /* Run CONFIG file path */ char *home_hsdump; /* HS dump filename */ - char *home_init; /* Initialize home command */ char *home_key; /* Key file filename */ - char *home_log; /* Operation log file path */ char *home_pagedump; /* Page dump filename */ - char *home_rand; /* RNG log file path */ char *home_stats; /* Statistics file path */ char *config_open; /* Command-line configuration */ - uint32_t run_cnt; /* Run counter */ - bool trace; /* trace operations */ bool trace_all; /* trace all operations */ bool trace_local; /* write trace to the primary database */ + char tidbuf[128]; /* thread ID in printable form */ WT_CONNECTION *trace_conn; /* optional tracing database */ WT_SESSION *trace_session; RWLOCK backup_lock; /* Backup running */ uint64_t backup_id; /* Block incremental id */ +#define INCREMENTAL_BLOCK 1 +#define INCREMENTAL_LOG 2 +#define INCREMENTAL_OFF 3 + u_int backup_incr_flag; /* Incremental backup configuration */ WT_RAND_STATE rnd; /* Global RNG state */ - uint32_t rts_no_check; /* track unsuccessful RTS checking */ + u_int rts_no_check; /* track unsuccessful RTS checking */ + + uint64_t timestamp; /* Counter for timestamps */ + uint64_t oldest_timestamp; /* Last timestamp used for oldest */ + uint64_t stable_timestamp; /* Last timestamp used for stable */ /* * Prepare will return an error if the prepare timestamp is less than any active read timestamp. @@ -153,6 +217,7 @@ typedef struct { * that requires locking out transactional ops that set a timestamp. */ RWLOCK ts_lock; + /* * Lock to prevent the stable timestamp from moving during the commit of prepared transactions. * Otherwise, it may panic if the stable timestamp is moved to greater than or equal to the @@ -160,163 +225,38 @@ typedef struct { */ RWLOCK prepare_commit_lock; - uint64_t timestamp; /* Counter for timestamps */ - uint64_t oldest_timestamp; /* Last timestamp used for oldest */ - uint64_t stable_timestamp; /* Last timestamp used for stable */ - - uint64_t truncate_cnt; /* Counter for truncation */ - /* - * Single-thread failure. Always use pthread lock rather than WT lock in case WT library is - * misbehaving. + * Single-thread failure. Not a WiredTiger library lock because it's set up before configuring + * anything. */ pthread_rwlock_t death_lock; + WT_CURSOR *page_dump_cursor; /* Snapshot isolation read failed, modifies failure handling. */ - uint32_t c_abort; /* Config values */ - uint32_t c_alter; - uint32_t c_assert_read_timestamp; - uint32_t c_assert_write_timestamp; - uint32_t c_auto_throttle; - char *c_backup_incremental; - uint32_t c_backup_incr_granularity; - uint32_t c_backups; - uint32_t c_bitcnt; - uint32_t c_bloom; - uint32_t c_bloom_bit_count; - uint32_t c_bloom_hash_count; - uint32_t c_bloom_oldest; - uint32_t c_cache; - uint32_t c_cache_minimum; - char *c_checkpoint; - uint32_t c_checkpoint_log_size; - uint32_t c_checkpoint_wait; - char *c_checksum; - uint32_t c_chunk_size; - uint32_t c_compact; - char *c_compression; - char *c_config_open; - uint32_t c_data_extend; - char *c_data_source; - uint32_t c_delete_pct; - uint32_t c_dictionary; - uint32_t c_direct_io; - char *c_encryption; - uint32_t c_evict_max; - char *c_file_type; - uint32_t c_firstfit; - uint32_t c_hs_cursor; - uint32_t c_huffman_value; - uint32_t c_import; - uint32_t c_in_memory; - uint32_t c_independent_thread_rng; - uint32_t c_insert_pct; - uint32_t c_internal_key_truncation; - uint32_t c_intl_page_max; - uint32_t c_key_max; - uint32_t c_key_min; - uint32_t c_leaf_page_max; - uint32_t c_leak_memory; - uint32_t c_logging; - uint32_t c_logging_archive; - char *c_logging_compression; - uint32_t c_logging_file_max; - uint32_t c_logging_prealloc; - uint32_t c_lsm_worker_threads; - uint32_t c_major_timeout; - uint32_t c_memory_page_max; - uint32_t c_merge_max; - uint32_t c_mmap; - uint32_t c_mmap_all; - uint32_t c_modify_pct; - uint32_t c_ops; - uint32_t c_prefix; - uint32_t c_prefix_compression; - uint32_t c_prefix_compression_min; - uint32_t c_prepare; - uint32_t c_quiet; - uint32_t c_random_cursor; - uint32_t c_read_pct; - uint32_t c_repeat_data_pct; - uint32_t c_reverse; - uint32_t c_rows; - uint32_t c_runs; - uint32_t c_salvage; - uint32_t c_split_pct; - uint32_t c_statistics; - uint32_t c_statistics_server; - uint32_t c_threads; - uint32_t c_timer; - uint32_t c_timing_stress_aggressive_sweep; - uint32_t c_timing_stress_checkpoint; - uint32_t c_timing_stress_checkpoint_reserved_txnid_delay; - uint32_t c_timing_stress_failpoint_hs_delete_key_from_ts; - uint32_t c_timing_stress_failpoint_hs_insert_1; - uint32_t c_timing_stress_failpoint_hs_insert_2; - uint32_t c_timing_stress_hs_checkpoint_delay; - uint32_t c_timing_stress_hs_search; - uint32_t c_timing_stress_hs_sweep; - uint32_t c_timing_stress_checkpoint_prepare; - uint32_t c_timing_stress_split_1; - uint32_t c_timing_stress_split_2; - uint32_t c_timing_stress_split_3; - uint32_t c_timing_stress_split_4; - uint32_t c_timing_stress_split_5; - uint32_t c_timing_stress_split_6; - uint32_t c_timing_stress_split_7; - uint32_t c_truncate; - uint32_t c_txn_implicit; - uint32_t c_txn_timestamps; - uint32_t c_value_max; - uint32_t c_value_min; - uint32_t c_verify; - uint32_t c_verify_failure_dump; - uint32_t c_write_pct; - uint32_t c_wt_mutex; - -#define FIX 1 -#define ROW 2 -#define VAR 3 - u_int type; /* File type's flag value */ + /* Any runs.type configuration. */ + char runs_type[64]; -#define INCREMENTAL_BLOCK 1 -#define INCREMENTAL_LOG 2 -#define INCREMENTAL_OFF 3 - u_int c_backup_incr_flag; /* Incremental backup flag value */ + /* + * The minimum key size: A minimum key size of 11 is necessary, row-store keys have a leading + * 10-digit number and the 11 guarantees we never see a key we can't immediately convert to a + * numeric value without modification (there's a trailing non-digit character after every key). + * + * Range of common key prefix selection and the maximum table prefix length. + */ +#define KEY_LEN_CONFIG_MIN 11 +#define PREFIX_LEN_CONFIG_MIN 15 +#define PREFIX_LEN_CONFIG_MAX 80 + uint32_t prefix_len_max; + + bool column_store_config; /* At least one column-store table configured */ + bool lsm_config; /* At least one LSM data source configured */ + bool multi_table_config; /* If configuring multiple tables */ + bool transaction_timestamps_config; /* If transaction timestamps configured on any table */ #define CHECKPOINT_OFF 1 #define CHECKPOINT_ON 2 #define CHECKPOINT_WIREDTIGER 3 - u_int c_checkpoint_flag; /* Checkpoint flag value */ - -#define CHECKSUM_OFF 1 -#define CHECKSUM_ON 2 -#define CHECKSUM_UNCOMPRESSED 3 -#define CHECKSUM_UNENCRYPTED 4 - u_int c_checksum_flag; /* Checksum flag value */ - -#define COMPRESS_NONE 1 -#define COMPRESS_LZ4 2 -#define COMPRESS_SNAPPY 3 -#define COMPRESS_ZLIB 4 -#define COMPRESS_ZSTD 5 - u_int c_compression_flag; /* Compression flag value */ - u_int c_logging_compression_flag; /* Log compression flag value */ - -#define ENCRYPT_NONE 1 -#define ENCRYPT_ROTN_7 2 -#define ENCRYPT_SODIUM 3 - u_int c_encryption_flag; /* Encryption flag value */ - -/* The page must be a multiple of the allocation size, and 512 always works. */ -#define BLOCK_ALLOCATION_SIZE 512 - uint32_t intl_page_max; /* Maximum page sizes */ - uint32_t leaf_page_max; - - uint64_t rows; /* Total rows */ - - uint32_t prefix_len; /* Common key prefix length */ - uint32_t key_rand_len[1031]; /* Key lengths */ + u_int checkpoint_config; /* Checkpoint configuration */ } GLOBAL; extern GLOBAL g; @@ -330,6 +270,7 @@ typedef struct { thread_op op; /* Operation */ uint64_t opid; /* Operation ID */ + uint32_t id; /* Table ID */ uint64_t keyno; /* Row number */ uint64_t ts; /* Read/commit timestamp */ @@ -373,7 +314,14 @@ typedef struct { uint64_t update; WT_SESSION *session; /* WiredTiger session */ - WT_CURSOR *cursor; /* WiredTiger cursor */ + WT_CURSOR **cursors; /* WiredTiger cursors, maps one-to-one to tables */ + WT_CURSOR *cursor; /* Current cursor */ + TABLE *table; /* Current table */ + + struct col_insert { + uint32_t insert_list[256]; /* Inserted column-store records, maps one-to-one to tables */ + u_int insert_list_cnt; + } * col_insert; WT_SESSION *trace; /* WiredTiger operations tracing session */ @@ -398,9 +346,6 @@ typedef struct { #define snap_first s->snap_state_first #define snap_list s->snap_state_list - uint64_t insert_list[256]; /* column-store inserted records */ - u_int insert_list_cnt; - WT_ITEM vprint; /* Temporary buffer for printable values */ WT_ITEM moda, modb; /* Temporary buffer for modify operations */ @@ -426,17 +371,16 @@ void config_clear(void); void config_compat(const char **); void config_error(void); void config_file(const char *); -void config_final(void); void config_print(bool); void config_run(void); -void config_single(const char *, bool); +void config_single(TABLE *, const char *, bool); void create_database(const char *home, WT_CONNECTION **connp); void fclose_and_clear(FILE **); bool fp_readv(FILE *, char *, uint32_t *); -void key_gen_common(WT_ITEM *, uint64_t, const char *); +void key_gen_common(TABLE *, WT_ITEM *, uint64_t, const char *); void key_gen_init(WT_ITEM *); void key_gen_teardown(WT_ITEM *); -void key_init(void); +void key_init(TABLE *, void *); void lock_destroy(WT_SESSION *, RWLOCK *); void lock_init(WT_SESSION *, RWLOCK *); void operations(u_int, bool); @@ -445,37 +389,40 @@ void set_alarm(u_int); void set_core_off(void); void set_oldest_timestamp(void); void snap_init(TINFO *); -void snap_teardown(TINFO *); void snap_op_init(TINFO *, uint64_t, bool); -void snap_repeat_rollback(WT_CURSOR *, TINFO **, size_t); -void snap_repeat_single(WT_CURSOR *, TINFO *); -int snap_repeat_txn(WT_CURSOR *, TINFO *); +void snap_repeat_rollback(TINFO **, size_t); +void snap_repeat_single(TINFO *); +int snap_repeat_txn(TINFO *); void snap_repeat_update(TINFO *, bool); +void snap_teardown(TINFO *); void snap_track(TINFO *, thread_op); void timestamp_init(void); void timestamp_once(WT_SESSION *, bool, bool); void timestamp_teardown(WT_SESSION *); -int trace_config(const char *); +void trace_config(const char *); void trace_init(void); void trace_ops_init(TINFO *); void trace_teardown(void); -void track(const char *, uint64_t, TINFO *); -void val_gen(WT_RAND_STATE *, WT_ITEM *, uint64_t); +void track(const char *, uint64_t); +void track_ops(TINFO *); +void val_gen(TABLE *, WT_RAND_STATE *, WT_ITEM *, uint64_t); void val_gen_init(WT_ITEM *); void val_gen_teardown(WT_ITEM *); -void val_init(void); +void val_init(TABLE *, void *); void wts_checkpoints(void); void wts_close(WT_CONNECTION **, WT_SESSION **); -void wts_create(const char *); +void wts_create_database(void); +void wts_create_home(void); void wts_dump(const char *, bool); -void wts_load(void); +void wts_load(TABLE *, void *); void wts_open(const char *, WT_CONNECTION **, WT_SESSION **, bool); -void wts_read_scan(void); +void wts_read_scan(TABLE *, void *); void wts_reopen(void); -void wts_salvage(void); +void wts_salvage(TABLE *, void *); void wts_stats(void); -void wts_verify(WT_CONNECTION *, const char *); +void wts_verify(TABLE *, void *); +/* Backward compatibility to older versions of the WiredTiger library. */ #if !defined(CUR2S) #define CUR2S(c) ((WT_SESSION_IMPL *)((WT_CURSOR *)c)->session) #endif diff --git a/src/third_party/wiredtiger/test/format/format.i b/src/third_party/wiredtiger/test/format/format.i index f0645ffb956..eb1d406ca79 100644 --- a/src/third_party/wiredtiger/test/format/format.i +++ b/src/third_party/wiredtiger/test/format/format.i @@ -147,6 +147,136 @@ random_sleep(WT_RAND_STATE *rnd, u_int max_seconds) } } +/* + * tables_apply - + * Call an underlying function on all tables. + */ +static inline void +tables_apply(void (*func)(TABLE *, void *), void *arg) +{ + u_int i; + + if (ntables == 0) + func(tables[0], arg); + else + for (i = 1; i <= ntables; ++i) + func(tables[i], arg); +} + +/* + * table_maxv -- + * Return the maximum value for a table configuration variable. + */ +static inline uint32_t +table_maxv(u_int off) +{ + uint32_t v; + u_int i; + + if (ntables == 0) + return (tables[0]->v[off].v); + + for (v = 0, i = 1; i <= ntables; ++i) + v = WT_MAX(v, tables[i]->v[off].v); + return (v); +} + +/* + * table_sumv -- + * Return the summed value for a table configuration variable. + */ +static inline uint32_t +table_sumv(u_int off) +{ + uint32_t v; + u_int i; + + if (ntables == 0) + return (tables[0]->v[off].v); + + for (v = 0, i = 1; i <= ntables; ++i) + v += tables[i]->v[off].v; + return (v); +} + +/* + * table_select -- + * Randomly select a table. + */ +static inline TABLE * +table_select(TINFO *tinfo) +{ + if (ntables == 0) + return (tables[0]); + + return (tables[mmrand(tinfo == NULL ? NULL : &tinfo->rnd, 1, ntables)]); +} + +/* + * table_select_type -- + * Randomly select a table of a specific type. + */ +static inline TABLE * +table_select_type(table_type type) +{ + u_int i; + + if (ntables == 0) + return (tables[0]->type == type ? tables[0] : NULL); + + for (i = mmrand(NULL, 1, ntables);; ++i) { + if (i > ntables) + i = 1; + if (tables[i]->type == type) + break; + } + return (tables[i]); +} + +/* + * wiredtiger_open_cursor -- + * Open a WiredTiger cursor. + */ +static inline void +wiredtiger_open_cursor( + WT_SESSION *session, const char *uri, const char *config, WT_CURSOR **cursorp) +{ + WT_DECL_RET; + + /* WT_SESSION.open_cursor can return EBUSY if concurrent with a metadata operation, retry. */ + while ((ret = session->open_cursor(session, uri, NULL, config, cursorp)) == EBUSY) + __wt_yield(); + testutil_checkfmt(ret, "%s", uri); +} + +/* + * table_cursor -- + * Return the cursor for a table, opening a new one if necessary. + */ +static inline WT_CURSOR * +table_cursor(TINFO *tinfo, u_int id) +{ + TABLE *table; + const char *config; + + testutil_assert(id > 0); + + /* The table ID is 1-based, the cursor array is 0-based. */ + table = tables[ntables == 0 ? 0 : id]; + --id; + + if (tinfo->cursors[id] == NULL) { + /* Configure "append", in the case of column stores, we append when inserting new rows. */ + config = table->type == ROW ? NULL : "append"; + wiredtiger_open_cursor(tinfo->session, table->uri, config, &tinfo->cursors[id]); + } + return (tinfo->cursors[id]); +} + +/* + * wiredtiger_begin_transaction -- + * Start a WiredTiger transaction. + */ static inline void wiredtiger_begin_transaction(WT_SESSION *session, const char *config) { @@ -166,9 +296,9 @@ wiredtiger_begin_transaction(WT_SESSION *session, const char *config) * Generate a key for lookup. */ static inline void -key_gen(WT_ITEM *key, uint64_t keyno) +key_gen(TABLE *table, WT_ITEM *key, uint64_t keyno) { - key_gen_common(key, keyno, "00"); + key_gen_common(table, key, keyno, "00"); } /* @@ -176,12 +306,12 @@ key_gen(WT_ITEM *key, uint64_t keyno) * Generate a key for insertion. */ static inline void -key_gen_insert(WT_RAND_STATE *rnd, WT_ITEM *key, uint64_t keyno) +key_gen_insert(TABLE *table, WT_RAND_STATE *rnd, WT_ITEM *key, uint64_t keyno) { static const char *const suffix[15] = { "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15"}; - key_gen_common(key, keyno, suffix[mmrand(rnd, 0, 14)]); + key_gen_common(table, key, keyno, suffix[mmrand(rnd, 0, 14)]); } /* @@ -193,11 +323,9 @@ lock_try_writelock(WT_SESSION *session, RWLOCK *lock) { testutil_assert(LOCK_INITIALIZED(lock)); - if (lock->lock_type == LOCK_WT) { + if (lock->lock_type == LOCK_WT) return (__wt_try_writelock((WT_SESSION_IMPL *)session, &lock->l.wt)); - } else { - return (pthread_rwlock_trywrlock(&lock->l.pthread)); - } + return (pthread_rwlock_trywrlock(&lock->l.pthread)); } /* @@ -209,11 +337,10 @@ lock_writelock(WT_SESSION *session, RWLOCK *lock) { testutil_assert(LOCK_INITIALIZED(lock)); - if (lock->lock_type == LOCK_WT) { + if (lock->lock_type == LOCK_WT) __wt_writelock((WT_SESSION_IMPL *)session, &lock->l.wt); - } else { + else testutil_check(pthread_rwlock_wrlock(&lock->l.pthread)); - } } /* @@ -225,11 +352,10 @@ lock_writeunlock(WT_SESSION *session, RWLOCK *lock) { testutil_assert(LOCK_INITIALIZED(lock)); - if (lock->lock_type == LOCK_WT) { + if (lock->lock_type == LOCK_WT) __wt_writeunlock((WT_SESSION_IMPL *)session, &lock->l.wt); - } else { + else testutil_check(pthread_rwlock_unlock(&lock->l.pthread)); - } } /* @@ -241,11 +367,10 @@ lock_readlock(WT_SESSION *session, RWLOCK *lock) { testutil_assert(LOCK_INITIALIZED(lock)); - if (lock->lock_type == LOCK_WT) { + if (lock->lock_type == LOCK_WT) __wt_readlock((WT_SESSION_IMPL *)session, &lock->l.wt); - } else { + else testutil_check(pthread_rwlock_rdlock(&lock->l.pthread)); - } } /* @@ -257,11 +382,10 @@ lock_readunlock(WT_SESSION *session, RWLOCK *lock) { testutil_assert(LOCK_INITIALIZED(lock)); - if (lock->lock_type == LOCK_WT) { + if (lock->lock_type == LOCK_WT) __wt_readunlock((WT_SESSION_IMPL *)session, &lock->l.wt); - } else { + else testutil_check(pthread_rwlock_unlock(&lock->l.pthread)); - } } #define trace_msg(fmt, ...) \ @@ -275,16 +399,16 @@ lock_readunlock(WT_SESSION *session, RWLOCK *lock) (uintmax_t)__ts.tv_nsec / WT_THOUSAND, g.tidbuf, __VA_ARGS__)); \ } \ } while (0) -#define trace_op(tinfo, fmt, ...) \ - do { \ - if (g.trace) { \ - struct timespec __ts; \ - WT_SESSION *__s = (tinfo)->trace; \ - __wt_epoch((WT_SESSION_IMPL *)__s, &__ts); \ - testutil_check( \ - __s->log_printf(__s, "[%" PRIuMAX ":%" PRIuMAX "][%s] " fmt, (uintmax_t)__ts.tv_sec, \ - (uintmax_t)__ts.tv_nsec / WT_THOUSAND, tinfo->tidbuf, __VA_ARGS__)); \ - } \ +#define trace_op(tinfo, fmt, ...) \ + do { \ + if (g.trace) { \ + struct timespec __ts; \ + WT_SESSION *__s = (tinfo)->trace; \ + __wt_epoch((WT_SESSION_IMPL *)__s, &__ts); \ + testutil_check(__s->log_printf(__s, "[%" PRIuMAX ":%" PRIuMAX "][%s]:%s " fmt, \ + (uintmax_t)__ts.tv_sec, (uintmax_t)__ts.tv_nsec / WT_THOUSAND, (tinfo)->tidbuf, \ + (tinfo)->table->uri, __VA_ARGS__)); \ + } \ } while (0) /* diff --git a/src/third_party/wiredtiger/test/format/format.sh b/src/third_party/wiredtiger/test/format/format.sh index 461875e33bb..8a04266ce86 100755 --- a/src/third_party/wiredtiger/test/format/format.sh +++ b/src/third_party/wiredtiger/test/format/format.sh @@ -45,17 +45,16 @@ smoke_base_2="$smoke_base_1 leaf_page_max=9 internal_page_max=9" smoke_list=( # Three access methods. "$smoke_base_1 file_type=row" - # Temporarily disabled + # Temporarily disabled: FIXME FLCS # "$smoke_base_1 file_type=fix" - # "$smoke_base_1 file_type=var" + "$smoke_base_1 file_type=var" # Huffman value encoding. "$smoke_base_1 file_type=row huffman_value=1" - # Temporarily disabled - # "$smoke_base_1 file_type=var huffman_value=1" + "$smoke_base_1 file_type=var huffman_value=1" # LSM - # Temporarily disabled + # Temporarily disabled: FIXME LSM # "$smoke_base_1 file_type=row runs.source=lsm" # Force the statistics server. @@ -64,8 +63,7 @@ smoke_list=( # Overflow testing. "$smoke_base_2 file_type=row key_min=256" "$smoke_base_2 file_type=row key_min=256 value_min=256" - # Temporarily disabled - # "$smoke_base_2 file_type=var value_min=256" + "$smoke_base_2 file_type=var value_min=256" ) smoke_next=0 @@ -392,18 +390,36 @@ resolve() echo "$name: original directory copied into $dir.RECOVER" echo) >> $log - # Everything is a table unless explicitly a file. - uri="table:wt" - grep 'runs.source=file' $dir/CONFIG > /dev/null && uri="file:wt" + # Verify the objects. In current format, it's a list of files named with a + # leading F or tables named with a leading T. Historically, it was a file + # or table named "wt". + verify_failed=0 + for i in $(ls $dir | sed -e 's/.*\///'); do + case $i in + F*) uri="file:$i";; + T*) uri="table:${i%.wt}";; + wt) uri="file:wt";; + wt.wt) uri="table:wt";; + *) continue;; + esac + + # Use the wt utility to recover & verify the object. + echo "verify: $wt_binary -m -R -h $dir verify $uri" >> $log + if $($wt_binary -m -R -h $dir verify $uri >> $log 2>&1); then + continue + fi + + verify_failed=1 + break + done - # Use the wt utility to recover & verify the object. - if $($wt_binary -m -R -h $dir verify $uri >> $log 2>&1); then - rm -rf $dir $dir.RECOVER $log - success=$(($success + 1)) - verbose "$name: job in $dir successfully completed" + if [[ $verify_failed -eq 0 ]]; then + rm -rf $dir $dir.RECOVER $log + success=$(($success + 1)) + verbose "$name: job in $dir successfully completed" else - echo "$name: job in $dir failed abort/recovery testing" - report_failure $dir + echo "$name: job in $dir failed abort/recovery testing" + report_failure $dir fi continue } @@ -498,7 +514,7 @@ format() live_record_binary="$live_record_binary --save-on=error" fi - cmd="$live_record_binary $format_binary -c "$config" -h "$dir" -1 $args quiet=1" + cmd="$live_record_binary $format_binary -c "$config" -h "$dir" $args quiet=1" echo "$name: $cmd" # Disassociate the command from the shell script so we can exit and let the command diff --git a/src/third_party/wiredtiger/test/format/import.c b/src/third_party/wiredtiger/test/format/import.c index 0b85515b806..1d22782ca3e 100644 --- a/src/third_party/wiredtiger/test/format/import.c +++ b/src/third_party/wiredtiger/test/format/import.c @@ -104,18 +104,14 @@ import(void *arg) copy_file_into_directory(import_session, "import.wt"); /* Perform import with either repair or file metadata. */ - memset(buf, 0, sizeof(buf)); import_value = mmrand(NULL, 0, 1); - if (import_value == 0) { + if (import_value == 0) testutil_check(__wt_snprintf(buf, sizeof(buf), "import=(enabled,repair=true)")); - if ((ret = session->create(session, IMPORT_URI, buf)) != 0) - testutil_die(ret, "session.import", ret); - } else { + else testutil_check(__wt_snprintf(buf, sizeof(buf), "%s,import=(enabled,repair=false,file_metadata=(%s))", table_config, file_config)); - if ((ret = session->create(session, IMPORT_URI, buf)) != 0) - testutil_die(ret, "session.import", ret); - } + if ((ret = session->create(session, IMPORT_URI, buf)) != 0) + testutil_die(ret, "session.import", ret); verify_import(session); diff --git a/src/third_party/wiredtiger/test/format/kv.c b/src/third_party/wiredtiger/test/format/kv.c index 32788b86ffb..54ed9732fb4 100644 --- a/src/third_party/wiredtiger/test/format/kv.c +++ b/src/third_party/wiredtiger/test/format/kv.c @@ -33,11 +33,16 @@ * Initialize the keys for a run. */ void -key_init(void) +key_init(TABLE *table, void *arg) { FILE *fp; size_t i; uint32_t max; + char buf[MAX_FORMAT_PATH]; + + (void)arg; /* unused argument */ + + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s.%u", g.home_key, table->id)); /* * The key is a variable length item with a leading 10-digit value. Since we have to be able @@ -48,10 +53,10 @@ key_init(void) * Read in the values during reopen. */ if (g.reopen) { - if ((fp = fopen(g.home_key, "r")) == NULL) - testutil_die(errno, "%s", g.home_key); - for (i = 0; i < WT_ELEMENTS(g.key_rand_len); ++i) - fp_readv(fp, g.home_key, &g.key_rand_len[i]); + if ((fp = fopen(buf, "r")) == NULL) + testutil_die(errno, "%s", buf); + for (i = 0; i < WT_ELEMENTS(table->key_rand_len); ++i) + fp_readv(fp, buf, &table->key_rand_len[i]); fclose_and_clear(&fp); return; } @@ -62,23 +67,19 @@ key_init(void) * Focus on relatively small items, admitting the possibility of larger items. Pick a size close * to the minimum most of the time, only create a larger item 1 in 20 times. */ - for (i = 0; i < WT_ELEMENTS(g.key_rand_len); ++i) { - max = g.c_key_max; - if (i % 20 != 0 && max > g.c_key_min + 20) - max = g.c_key_min + 20; - g.key_rand_len[i] = mmrand(NULL, g.c_key_min, max); + for (i = 0; i < WT_ELEMENTS(table->key_rand_len); ++i) { + max = TV(BTREE_KEY_MAX); + if (i % 20 != 0 && max > TV(BTREE_KEY_MIN) + 20) + max = TV(BTREE_KEY_MIN) + 20; + table->key_rand_len[i] = mmrand(NULL, TV(BTREE_KEY_MIN), max); } /* Write out the values for a subsequent reopen. */ - if ((fp = fopen(g.home_key, "w")) == NULL) - testutil_die(errno, "%s", g.home_key); - for (i = 0; i < WT_ELEMENTS(g.key_rand_len); ++i) - fprintf(fp, "%" PRIu32 "\n", g.key_rand_len[i]); + if ((fp = fopen(buf, "w")) == NULL) + testutil_die(errno, "%s", buf); + for (i = 0; i < WT_ELEMENTS(table->key_rand_len); ++i) + fprintf(fp, "%" PRIu32 "\n", table->key_rand_len[i]); fclose_and_clear(&fp); - - /* Fill in the common key prefix length (which is added to the key min/max). */ - if (g.c_prefix != 0) - g.prefix_len = mmrand(NULL, 15, 80); } /* @@ -91,7 +92,7 @@ key_gen_init(WT_ITEM *key) size_t i, len; char *p; - len = WT_MAX(KILOBYTE(100), g.c_key_max + g.prefix_len); + len = WT_MAX(KILOBYTE(100), table_maxv(V_TABLE_BTREE_KEY_MAX) + g.prefix_len_max + 10); p = dmalloc(len); for (i = 0; i < len; ++i) p[i] = "abcdefghijklmnopqrstuvwxyz"[i % 26]; @@ -113,20 +114,22 @@ key_gen_teardown(WT_ITEM *key) memset(key, 0, sizeof(*key)); } +#define COMMON_PREFIX_CHAR 'C' + /* * key_gen_common -- * Row-store key generation code shared between normal and insert key generation. */ void -key_gen_common(WT_ITEM *key, uint64_t keyno, const char *const suffix) +key_gen_common(TABLE *table, WT_ITEM *key, uint64_t keyno, const char *const suffix) { + size_t i; uint64_t n; + uint32_t prefix_len; char *p; const char *bucket; - testutil_assert(g.type == ROW); - - p = key->mem; + testutil_assert(table->type == ROW); /* * The workload we're trying to mimic with a prefix is a long common prefix followed by a record @@ -138,16 +141,32 @@ key_gen_common(WT_ITEM *key, uint64_t keyno, const char *const suffix) * lexicographically, meaning the common key prefix will grow and shrink by a few bytes as the * number increments, which is a good thing for testing. */ - if (g.prefix_len > 0) { - bucket = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; - for (n = keyno; n > 0; n >>= 1) { - if (*bucket == 'z') - break; - ++bucket; + p = key->mem; + prefix_len = TV(BTREE_PREFIX_LEN); + if (g.prefix_len_max != 0) { + /* + * Not all tables have prefixes and prefixes may be of different lengths. If any table has a + * prefix, check if we need to reset the leading bytes in the key to their original values. + * It's an ugly test, but it avoids rewriting the key in a performance path. The variable is + * the largest prefix in the run, and the hard-coded 20 gets us past the key appended to + * that prefix. + */ + if (p[1] == COMMON_PREFIX_CHAR) { + for (i = 0; i < g.prefix_len_max + 20; ++i) + p[i] = "abcdefghijklmnopqrstuvwxyz"[i % 26]; + p = key->mem; + } + if (prefix_len != 0) { + bucket = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + for (n = keyno; n > 0; n >>= 1) { + if (*bucket == 'z') + break; + ++bucket; + } + p[0] = *bucket; + memset(p + 1, COMMON_PREFIX_CHAR, prefix_len - 1); + p += prefix_len; } - p[0] = *bucket; - memset(p + 1, 'C', g.prefix_len - 1); - p += g.prefix_len; } /* @@ -166,19 +185,19 @@ key_gen_common(WT_ITEM *key, uint64_t keyno, const char *const suffix) * the cache. Handle that here, use a really big key 1 in 2500 times. */ key->data = key->mem; - key->size = g.prefix_len; - key->size += keyno % 2500 == 0 && g.c_key_max < KILOBYTE(80) ? + key->size = prefix_len; + key->size += keyno % 2500 == 0 && TV(BTREE_KEY_MAX) < KILOBYTE(80) ? KILOBYTE(80) : - g.key_rand_len[keyno % WT_ELEMENTS(g.key_rand_len)]; + table->key_rand_len[keyno % WT_ELEMENTS(table->key_rand_len)]; testutil_assert(key->size <= key->memsize); } -static char *val_base; /* Base/original value */ -static uint32_t val_dup_data_len; /* Length of duplicate data items */ -static uint32_t val_len; /* Length of data items */ - +/* + * val_len -- + * Select and return the length for a value. + */ static inline uint32_t -value_len(WT_RAND_STATE *rnd, uint64_t keyno, uint32_t min, uint32_t max) +val_len(WT_RAND_STATE *rnd, uint64_t keyno, uint32_t min, uint32_t max) { /* * Focus on relatively small items, admitting the possibility of larger items. Pick a size close @@ -193,15 +212,22 @@ value_len(WT_RAND_STATE *rnd, uint64_t keyno, uint32_t min, uint32_t max) return (mmrand(rnd, min, max)); } +/* + * val_init -- + * Initialize the value structures for a table. + */ void -val_init(void) +val_init(TABLE *table, void *arg) { size_t i; + uint32_t len; + + (void)arg; /* unused argument */ /* Discard any previous value initialization. */ - free(val_base); - val_base = NULL; - val_dup_data_len = val_len = 0; + free(table->val_base); + table->val_base = NULL; + table->val_dup_data_len = 0; /* * Set initial buffer contents to recognizable text. @@ -209,23 +235,35 @@ val_init(void) * Add a few extra bytes in order to guarantee we can always offset into the buffer by a few * extra bytes, used to generate different data for column-store run-length encoded files. */ - val_len = WT_MAX(KILOBYTE(100), g.c_value_max) + 20; - val_base = dmalloc(val_len); - for (i = 0; i < val_len; ++i) - val_base[i] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"[i % 26]; + len = WT_MAX(KILOBYTE(100), table_maxv(V_TABLE_BTREE_VALUE_MAX)) + 20; + table->val_base = dmalloc(len); + for (i = 0; i < len; ++i) + table->val_base[i] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"[i % 26]; - val_dup_data_len = value_len(NULL, (uint64_t)mmrand(NULL, 1, 20), g.c_value_min, g.c_value_max); + table->val_dup_data_len = + val_len(NULL, (uint64_t)mmrand(NULL, 1, 20), TV(BTREE_VALUE_MIN), TV(BTREE_VALUE_MAX)); } +/* + * val_gen_init -- + * Initialize a single value structure. + */ void val_gen_init(WT_ITEM *value) { - value->mem = dmalloc(val_len); - value->memsize = val_len; + uint32_t len; + + len = WT_MAX(KILOBYTE(100), table_maxv(V_TABLE_BTREE_VALUE_MAX)) + 20; + value->mem = dmalloc(len); + value->memsize = len; value->data = value->mem; value->size = 0; } +/* + * val_gen_teardown -- + * Discard a single value structure. + */ void val_gen_teardown(WT_ITEM *value) { @@ -233,8 +271,12 @@ val_gen_teardown(WT_ITEM *value) memset(value, 0, sizeof(*value)); } +/* + * val_gen -- + * Generate a new value. + */ void -val_gen(WT_RAND_STATE *rnd, WT_ITEM *value, uint64_t keyno) +val_gen(TABLE *table, WT_RAND_STATE *rnd, WT_ITEM *value, uint64_t keyno) { char *p; @@ -244,8 +286,8 @@ val_gen(WT_RAND_STATE *rnd, WT_ITEM *value, uint64_t keyno) /* * Fixed-length records: take the low N bits from the last digit of the record number. */ - if (g.type == FIX) { - switch (g.c_bitcnt) { + if (table->type == FIX) { + switch (TV(BTREE_BITCNT)) { case 8: p[0] = (char)mmrand(rnd, 1, 0xff); break; @@ -289,14 +331,14 @@ val_gen(WT_RAND_STATE *rnd, WT_ITEM *value, uint64_t keyno) * Data items have unique leading numbers by default and random lengths; variable-length * column-stores use a duplicate data value to test RLE. */ - if (g.type == VAR && mmrand(rnd, 1, 100) < g.c_repeat_data_pct) { - value->size = val_dup_data_len; - memcpy(p, val_base, value->size); + if (table->type == VAR && mmrand(rnd, 1, 100) < TV(BTREE_REPEAT_DATA_PCT)) { + value->size = table->val_dup_data_len; + memcpy(p, table->val_base, value->size); (void)strcpy(p, "DUPLICATEV"); p[10] = '/'; } else { - value->size = value_len(rnd, keyno, g.c_value_min, g.c_value_max); - memcpy(p, val_base, value->size); + value->size = val_len(rnd, keyno, TV(BTREE_VALUE_MIN), TV(BTREE_VALUE_MAX)); + memcpy(p, table->val_base, value->size); u64_to_string_zf(keyno, p, 11); p[10] = '/'; } diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 4c24ba9fd45..6d079eb12aa 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -28,23 +28,23 @@ #include "format.h" -static int col_insert(TINFO *, WT_CURSOR *); -static void col_insert_resolve(TINFO *); -static int col_modify(TINFO *, WT_CURSOR *, bool); -static int col_remove(TINFO *, WT_CURSOR *, bool); -static int col_reserve(TINFO *, WT_CURSOR *, bool); -static int col_truncate(TINFO *, WT_CURSOR *); -static int col_update(TINFO *, WT_CURSOR *, bool); -static int nextprev(TINFO *, WT_CURSOR *, bool); +static int col_insert(TINFO *); +static void col_insert_resolve(TABLE *, void *); +static int col_modify(TINFO *, bool); +static int col_remove(TINFO *, bool); +static int col_reserve(TINFO *, bool); +static int col_truncate(TINFO *); +static int col_update(TINFO *, bool); +static int nextprev(TINFO *, bool); static WT_THREAD_RET ops(void *); -static int read_row(TINFO *, WT_CURSOR *); -static int read_row_worker(WT_CURSOR *, TINFO *, uint64_t, WT_ITEM *, WT_ITEM *, bool); -static int row_insert(TINFO *, WT_CURSOR *, bool); -static int row_modify(TINFO *, WT_CURSOR *, bool); -static int row_remove(TINFO *, WT_CURSOR *, bool); -static int row_reserve(TINFO *, WT_CURSOR *, bool); -static int row_truncate(TINFO *, WT_CURSOR *); -static int row_update(TINFO *, WT_CURSOR *, bool); +static int read_row(TINFO *); +static int read_row_worker(TINFO *, TABLE *, WT_CURSOR *, uint64_t, WT_ITEM *, WT_ITEM *, bool); +static int row_insert(TINFO *, bool); +static int row_modify(TINFO *, bool); +static int row_remove(TINFO *, bool); +static int row_reserve(TINFO *, bool); +static int row_truncate(TINFO *); +static int row_update(TINFO *, bool); static char modify_repl[256]; @@ -113,13 +113,16 @@ tinfo_init(void) /* Allocate the thread structures separately to minimize false sharing. */ if (tinfo_list == NULL) { - tinfo_list = dcalloc((size_t)g.c_threads + 1, sizeof(TINFO *)); - for (i = 0; i < g.c_threads; ++i) { + tinfo_list = dcalloc((size_t)GV(RUNS_THREADS) + 1, sizeof(TINFO *)); + for (i = 0; i < GV(RUNS_THREADS); ++i) { tinfo_list[i] = dcalloc(1, sizeof(TINFO)); tinfo = tinfo_list[i]; tinfo->id = (int)i + 1; + tinfo->cursors = dcalloc(WT_MAX(ntables, 1), sizeof(tinfo->cursors[0])); + tinfo->col_insert = dcalloc(WT_MAX(ntables, 1), sizeof(tinfo->col_insert[0])); + /* Set up the default key and value buffers. */ tinfo->key = &tinfo->_key; key_gen_init(tinfo->key); @@ -133,7 +136,7 @@ tinfo_init(void) } /* Cleanup for each new run. */ - for (i = 0; i < g.c_threads; ++i) { + for (i = 0; i < GV(RUNS_THREADS); ++i) { tinfo = tinfo_list[i]; tinfo->ops = 0; @@ -147,10 +150,8 @@ tinfo_init(void) tinfo->update = 0; tinfo->session = NULL; - tinfo->cursor = NULL; - - memset(tinfo->insert_list, 0, sizeof(tinfo->insert_list)); - tinfo->insert_list_cnt = 0; + memset(tinfo->cursors, 0, WT_MAX(ntables, 1) * sizeof(tinfo->cursors[0])); + memset(tinfo->col_insert, 0, WT_MAX(ntables, 1) * sizeof(tinfo->col_insert[0])); tinfo->state = TINFO_RUNNING; tinfo->quit = false; @@ -167,19 +168,16 @@ tinfo_teardown(void) TINFO *tinfo; u_int i; - for (i = 0; i < g.c_threads; ++i) { + for (i = 0; i < GV(RUNS_THREADS); ++i) { tinfo = tinfo_list[i]; + free(tinfo->cursors); + free(tinfo->col_insert); + __wt_buf_free(NULL, &tinfo->vprint); __wt_buf_free(NULL, &tinfo->moda); __wt_buf_free(NULL, &tinfo->modb); - /* - * Assert records were not removed unless configured to do so, otherwise subsequent runs can - * incorrectly report scan errors. - */ - testutil_assert(g.c_delete_pct != 0 || tinfo->remove == 0); - snap_teardown(tinfo); key_gen_teardown(tinfo->key); val_gen_teardown(tinfo->value); @@ -192,25 +190,21 @@ tinfo_teardown(void) } /* - * tinfo_rollback_to_stable -- + * rollback_to_stable -- * Do a rollback to stable and verify operations. */ static void -tinfo_rollback_to_stable(WT_SESSION *session) +rollback_to_stable(void) { - WT_CURSOR *cursor; - - /* Rollback-to-stable only makes sense for timestamps and on-disk stores. */ - if (g.c_txn_timestamps == 0 || g.c_in_memory != 0) + /* Rollback-to-stable only makes sense for timestamps. */ + if (!g.transaction_timestamps_config) return; trace_msg("%-10s ts=%" PRIu64, "rts", g.stable_timestamp); testutil_check(g.wts_conn->rollback_to_stable(g.wts_conn, NULL)); /* Check the saved snap operations for consistency. */ - testutil_check(session->open_cursor(session, g.uri, NULL, NULL, &cursor)); - snap_repeat_rollback(cursor, tinfo_list, g.c_threads); - testutil_check(cursor->close(cursor)); + snap_repeat_rollback(tinfo_list, GV(RUNS_THREADS)); } /* @@ -252,12 +246,12 @@ operations(u_int ops_seconds, bool lastrun) * Calculate how many fourth-of-a-second sleeps until the timer expires. If the timer expires * and threads don't return in 15 minutes, assume there is something hung, and force the quit. */ - if (g.c_ops == 0) + if (GV(RUNS_OPS) == 0) thread_ops = -1; else { - if (g.c_ops < g.c_threads) - g.c_ops = g.c_threads; - thread_ops = g.c_ops / g.c_threads; + if (GV(RUNS_OPS) < GV(RUNS_THREADS)) + GV(RUNS_OPS) = GV(RUNS_THREADS); + thread_ops = GV(RUNS_OPS) / GV(RUNS_THREADS); } if (ops_seconds == 0) fourths = quit_fourths = -1; @@ -273,35 +267,35 @@ operations(u_int ops_seconds, bool lastrun) tinfo_init(); trace_msg("%s", "=============== thread ops start"); - for (i = 0; i < g.c_threads; ++i) { + for (i = 0; i < GV(RUNS_THREADS); ++i) { tinfo = tinfo_list[i]; testutil_check(__wt_thread_create(NULL, &tinfo->tid, ops, tinfo)); } /* Start optional special-purpose threads. */ - if (g.c_alter) + if (GV(OPS_ALTER)) testutil_check(__wt_thread_create(NULL, &alter_tid, alter, NULL)); - if (g.c_backups) + if (GV(BACKUP)) testutil_check(__wt_thread_create(NULL, &backup_tid, backup, NULL)); - if (g.c_compact) + if (GV(OPS_COMPACTION)) testutil_check(__wt_thread_create(NULL, &compact_tid, compact, NULL)); - if (g.c_hs_cursor) + if (GV(OPS_HS_CURSOR)) testutil_check(__wt_thread_create(NULL, &hs_tid, hs_cursor, NULL)); - if (g.c_import) + if (GV(IMPORT)) testutil_check(__wt_thread_create(NULL, &import_tid, import, NULL)); - if (g.c_random_cursor) + if (GV(OPS_RANDOM_CURSOR)) testutil_check(__wt_thread_create(NULL, &random_tid, random_kv, NULL)); - if (g.c_txn_timestamps) + if (g.transaction_timestamps_config) testutil_check(__wt_thread_create(NULL, ×tamp_tid, timestamp, tinfo_list)); - if (g.c_checkpoint_flag == CHECKPOINT_ON) + if (g.checkpoint_config == CHECKPOINT_ON) testutil_check(__wt_thread_create(NULL, &checkpoint_tid, checkpoint, NULL)); /* Spin on the threads, calculating the totals. */ for (;;) { /* Clear out the totals each pass. */ memset(&total, 0, sizeof(total)); - for (i = 0, running = false; i < g.c_threads; ++i) { + for (i = 0, running = false; i < GV(RUNS_THREADS); ++i) { tinfo = tinfo_list[i]; total.commit += tinfo->commit; total.insert += tinfo->insert; @@ -332,12 +326,12 @@ operations(u_int ops_seconds, bool lastrun) /* * On the last execution, optionally drop core for recovery testing. */ - if (lastrun && g.c_abort) + if (lastrun && GV(FORMAT_ABORT)) random_failure(); tinfo->quit = true; } } - track("ops", 0ULL, &total); + track_ops(&total); if (!running) break; __wt_sleep(0, 250000); /* 1/4th of a second */ @@ -363,21 +357,21 @@ operations(u_int ops_seconds, bool lastrun) /* Wait for the special-purpose threads. */ g.workers_finished = true; - if (g.c_alter) + if (GV(OPS_ALTER)) testutil_check(__wt_thread_join(NULL, &alter_tid)); - if (g.c_backups) + if (GV(BACKUP)) testutil_check(__wt_thread_join(NULL, &backup_tid)); - if (g.c_checkpoint_flag == CHECKPOINT_ON) + if (g.checkpoint_config == CHECKPOINT_ON) testutil_check(__wt_thread_join(NULL, &checkpoint_tid)); - if (g.c_compact) + if (GV(OPS_COMPACTION)) testutil_check(__wt_thread_join(NULL, &compact_tid)); - if (g.c_hs_cursor) + if (GV(OPS_HS_CURSOR)) testutil_check(__wt_thread_join(NULL, &hs_tid)); - if (g.c_import) + if (GV(IMPORT)) testutil_check(__wt_thread_join(NULL, &import_tid)); - if (g.c_random_cursor) + if (GV(OPS_RANDOM_CURSOR)) testutil_check(__wt_thread_join(NULL, &random_tid)); - if (g.c_txn_timestamps) + if (g.transaction_timestamps_config) testutil_check(__wt_thread_join(NULL, ×tamp_tid)); g.workers_finished = false; @@ -390,7 +384,7 @@ operations(u_int ops_seconds, bool lastrun) * close/re-open pair. Note we are not advancing the oldest timestamp, otherwise we wouldn't be * able to replay operations from after rollback-to-stable completes. */ - tinfo_rollback_to_stable(session); + rollback_to_stable(); if (lastrun) { tinfo_teardown(); @@ -496,12 +490,12 @@ commit_transaction(TINFO *tinfo, bool prepared) uint64_t ts; char buf[64]; - ++tinfo->commit; - session = tinfo->session; + ++tinfo->commit; + ts = 0; /* -Wconditional-uninitialized */ - if (g.c_txn_timestamps) { + if (g.transaction_timestamps_config) { if (prepared) lock_readlock(session, &g.prepare_commit_lock); @@ -628,47 +622,35 @@ prepare_transaction(TINFO *tinfo) } while (0) /* - * ops_open_session -- - * Create a new session/cursor pair for the thread. + * ops_session_open -- + * Create a new session for the thread. */ static void -ops_open_session(TINFO *tinfo) +ops_session_open(TINFO *tinfo) { WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_DECL_RET; WT_SESSION *session; conn = g.wts_conn; - /* Close any open session/cursor. */ + /* Close any open session (which closes all open cursors as well). */ if ((session = tinfo->session) != NULL) testutil_check(session->close(session, NULL)); + tinfo->session = NULL; + memset(tinfo->cursors, 0, WT_MAX(ntables, 1) * sizeof(tinfo->cursors[0])); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - /* - * Configure "append", in the case of column stores, we append when inserting new rows. - * - * WT_SESSION.open_cursor can return EBUSY if concurrent with a metadata operation, retry. - */ - while ((ret = session->open_cursor(session, g.uri, NULL, "append", &cursor)) == EBUSY) - __wt_yield(); - testutil_checkfmt(ret, "%s", g.uri); - - tinfo->session = session; - tinfo->cursor = cursor; + testutil_check(conn->open_session(conn, NULL, NULL, &tinfo->session)); } /* Isolation configuration. */ typedef enum { + ISOLATION_IMPLICIT, ISOLATION_READ_COMMITTED, ISOLATION_READ_UNCOMMITTED, ISOLATION_SNAPSHOT } iso_level_t; -/* When in an explicit snapshot isolation transaction, track operations for later - * repetition. */ +/* When in an explicit snapshot isolation transaction, track operations for later repetition. */ #define SNAP_TRACK(tinfo, op) \ do { \ if (intxn && iso_level == ISOLATION_SNAPSHOT) \ @@ -682,14 +664,15 @@ typedef enum { static WT_THREAD_RET ops(void *arg) { + struct col_insert *cip; TINFO *tinfo; - WT_CURSOR *cursor; + TABLE *table; WT_DECL_RET; WT_SESSION *session; iso_level_t iso_level; thread_op op; - uint64_t max_rows, reset_op, session_op, truncate_op; - uint32_t range, rnd; + uint64_t reset_op, session_op, truncate_op; + uint32_t max_rows, range, rnd; u_int i, j; const char *iso_config; bool greater_than, intxn, next, positioned, prepared; @@ -703,29 +686,28 @@ ops(void *arg) * pound on the same key/value pairs, that is, by making them traverse the same RNG space. 75% * of the time we run in independent RNG space. */ - if (g.c_independent_thread_rng) + if (GV(FORMAT_INDEPENDENT_THREAD_RNG)) __wt_random_init_seed(NULL, &tinfo->rnd); else __wt_random_init(&tinfo->rnd); iso_level = ISOLATION_SNAPSHOT; /* -Wconditional-uninitialized */ - /* Set the first operation where we'll create sessions and cursors. */ - cursor = NULL; + /* Set the first operation where we'll create a new session and cursors. */ session = NULL; session_op = 0; /* Set the first operation where we'll reset the session. */ reset_op = mmrand(&tinfo->rnd, 100, 10000); /* Set the first operation where we'll truncate a range. */ - truncate_op = g.c_truncate == 0 ? UINT64_MAX : mmrand(&tinfo->rnd, 100, 10000); + truncate_op = mmrand(&tinfo->rnd, 100, 10000); /* Initialize operation trace. */ trace_ops_init(tinfo); for (intxn = false; !tinfo->quit; ++tinfo->ops) { /* Periodically open up a new session and cursors. */ - if (tinfo->ops > session_op || session == NULL || cursor == NULL) { + if (tinfo->ops > session_op || session == NULL) { /* * We can't swap sessions/cursors if in a transaction, resolve any running transaction. */ @@ -734,13 +716,11 @@ ops(void *arg) intxn = false; } - ops_open_session(tinfo); + ops_session_open(tinfo); + session = tinfo->session; /* Pick the next session/cursor close/open. */ session_op += mmrand(&tinfo->rnd, 100, 5000); - - session = tinfo->session; - cursor = tinfo->cursor; } /* @@ -752,20 +732,27 @@ ops(void *arg) testutil_check(session->reset(session)); /* Pick the next reset operation. */ - reset_op += mmrand(&tinfo->rnd, 20000, 50000); + reset_op += mmrand(&tinfo->rnd, 40000, 60000); } + /* Select a table, acquire a cursor. */ + tinfo->table = table = table_select(tinfo); + tinfo->cursor = table_cursor(tinfo, table->id); + /* - * If not in a transaction and in a timestamp world, occasionally repeat a timestamped - * operation. + * If not in a transaction and in a timestamp world, occasionally repeat timestamped + * operations. */ - if (!intxn && g.c_txn_timestamps && mmrand(&tinfo->rnd, 1, 15) == 1) { + if (!intxn && g.transaction_timestamps_config && mmrand(&tinfo->rnd, 1, 15) == 1) { ++tinfo->search; - snap_repeat_single(cursor, tinfo); + snap_repeat_single(tinfo); } - /* If not in a transaction and in a timestamp world, start a transaction. */ - if (!intxn && g.c_txn_timestamps) { + /* + * If not in a transaction and in a timestamp world, start a transaction (which is always at + * snapshot-isolation). + */ + if (!intxn && g.transaction_timestamps_config) { iso_level = ISOLATION_SNAPSHOT; begin_transaction_ts(tinfo); intxn = true; @@ -773,47 +760,52 @@ ops(void *arg) /* * If not in a transaction and not in a timestamp world, start a transaction some percentage - * of the time. + * of the time, otherwise it's an implicit transaction. */ - if (!intxn && mmrand(&tinfo->rnd, 1, 100) < g.c_txn_implicit) { - iso_level = ISOLATION_SNAPSHOT; - iso_config = "isolation=snapshot"; + if (!intxn && !g.transaction_timestamps_config) { + iso_level = ISOLATION_IMPLICIT; + + if (mmrand(&tinfo->rnd, 1, 100) < GV(TRANSACTION_IMPLICIT)) { + iso_level = ISOLATION_SNAPSHOT; + iso_config = "isolation=snapshot"; + + /* Occasionally do reads at an isolation level lower than snapshot. */ + switch (mmrand(NULL, 1, 20)) { + case 1: + iso_level = ISOLATION_READ_COMMITTED; /* 5% */ + iso_config = "isolation=read-committed"; + break; + case 2: + iso_level = ISOLATION_READ_UNCOMMITTED; /* 5% */ + iso_config = "isolation=read-uncommitted"; + break; + } - /* Occasionally do reads at an isolation level lower than snapshot. */ - switch (mmrand(NULL, 1, 20)) { - case 1: - iso_level = ISOLATION_READ_COMMITTED; /* 5% */ - iso_config = "isolation=read-committed"; - break; - case 2: - iso_level = ISOLATION_READ_UNCOMMITTED; /* 5% */ - iso_config = "isolation=read-uncommitted"; - break; + begin_transaction(tinfo, iso_config); + intxn = true; } - - begin_transaction(tinfo, iso_config); - intxn = true; } /* - * Select an operation: all updates must be in snapshot isolation, modify must be in an - * explicit transaction. + * Select an operation: updates cannot happen at lower isolation levels and modify must be + * in an explicit transaction. */ op = READ; - if (iso_level == ISOLATION_SNAPSHOT) { + if (iso_level == ISOLATION_IMPLICIT || iso_level == ISOLATION_SNAPSHOT) { i = mmrand(&tinfo->rnd, 1, 100); - if (i < g.c_delete_pct && tinfo->ops > truncate_op) { + if (TV(OPS_TRUNCATE) && i < TV(OPS_PCT_DELETE) && tinfo->ops > truncate_op) { op = TRUNCATE; /* Pick the next truncate operation. */ truncate_op += mmrand(&tinfo->rnd, 20000, 100000); - } else if (i < g.c_delete_pct) + } else if (i < TV(OPS_PCT_DELETE)) op = REMOVE; - else if (i < g.c_delete_pct + g.c_insert_pct) + else if (i < TV(OPS_PCT_DELETE) + TV(OPS_PCT_INSERT)) op = INSERT; - else if (intxn && i < g.c_delete_pct + g.c_insert_pct + g.c_modify_pct) + else if (intxn && i < TV(OPS_PCT_DELETE) + TV(OPS_PCT_INSERT) + TV(OPS_PCT_MODIFY)) op = MODIFY; - else if (i < g.c_delete_pct + g.c_insert_pct + g.c_modify_pct + g.c_write_pct) + else if (i < + TV(OPS_PCT_DELETE) + TV(OPS_PCT_INSERT) + TV(OPS_PCT_MODIFY) + TV(OPS_PCT_WRITE)) op = UPDATE; } @@ -821,7 +813,7 @@ ops(void *arg) * Select a row. Column-store extends the object, explicitly read the maximum row count and * then use a local variable so the value won't change inside the loop. */ - max_rows = (volatile uint64_t)g.rows; + WT_ORDERED_READ(max_rows, table->rows_current); tinfo->keyno = mmrand(&tinfo->rnd, 1, (u_int)max_rows); /* @@ -832,7 +824,7 @@ ops(void *arg) positioned = false; if (op != READ && mmrand(&tinfo->rnd, 1, 5) == 1) { ++tinfo->search; - ret = read_row(tinfo, cursor); + ret = read_row(tinfo); if (ret == 0) { positioned = true; SNAP_TRACK(tinfo, READ); @@ -841,17 +833,18 @@ ops(void *arg) } /* - * Optionally reserve a row, it's an update so it requires snapshot isolation. Reserving a - * row before a read isn't all that sensible, but not unexpected, either. + * If we're in a transaction, optionally reserve a row: it's an update so cannot be done at + * lower isolation levels. Reserving a row in an implicit transaction will work, but doesn't + * make sense. Reserving a row before a read isn't sensible either, but it's not unexpected. */ if (intxn && iso_level == ISOLATION_SNAPSHOT && mmrand(&tinfo->rnd, 0, 20) == 1) { - switch (g.type) { + switch (table->type) { case ROW: - ret = row_reserve(tinfo, cursor, positioned); + ret = row_reserve(tinfo, positioned); break; case FIX: case VAR: - ret = col_reserve(tinfo, cursor, positioned); + ret = col_reserve(tinfo, positioned); break; } if (ret == 0) { @@ -864,9 +857,9 @@ ops(void *arg) /* Perform the operation. */ switch (op) { case INSERT: - switch (g.type) { + switch (table->type) { case ROW: - ret = row_insert(tinfo, cursor, positioned); + ret = row_insert(tinfo, positioned); break; case FIX: case VAR: @@ -874,10 +867,11 @@ ops(void *arg) * We can only append so many new records, once we reach that limit, update a record * instead of inserting. */ - if (tinfo->insert_list_cnt >= WT_ELEMENTS(tinfo->insert_list)) + cip = &tinfo->col_insert[table->id - 1]; + if (cip->insert_list_cnt >= WT_ELEMENTS(cip->insert_list)) goto update_instead_of_chosen_op; - ret = col_insert(tinfo, cursor); + ret = col_insert(tinfo); break; } @@ -891,12 +885,16 @@ ops(void *arg) break; case MODIFY: ++tinfo->update; - switch (g.type) { + switch (table->type) { + case FIX: + testutil_die( + 0, "%s", "fixed-length column-store does not support modify operations"); + /* NOTREACHED */ case ROW: - ret = row_modify(tinfo, cursor, positioned); + ret = row_modify(tinfo, positioned); break; case VAR: - ret = col_modify(tinfo, cursor, positioned); + ret = col_modify(tinfo, positioned); break; } if (ret == 0) { @@ -907,7 +905,7 @@ ops(void *arg) break; case READ: ++tinfo->search; - ret = read_row(tinfo, cursor); + ret = read_row(tinfo); if (ret == 0) { positioned = true; SNAP_TRACK(tinfo, READ); @@ -915,13 +913,13 @@ ops(void *arg) READ_OP_FAILED(true); break; case REMOVE: - switch (g.type) { + switch (table->type) { case ROW: - ret = row_remove(tinfo, cursor, positioned); + ret = row_remove(tinfo, positioned); break; case FIX: case VAR: - ret = col_remove(tinfo, cursor, positioned); + ret = col_remove(tinfo, positioned); break; } if (ret == 0) { @@ -936,11 +934,11 @@ ops(void *arg) break; case TRUNCATE: /* - * A maximum of 2 truncation operations at a time, more than that can lead to serious - * thrashing. + * A maximum of 2 truncation operations at a time in an object, more than that can lead + * to serious thrashing. */ - if (__wt_atomic_addv64(&g.truncate_cnt, 1) > 2) { - (void)__wt_atomic_subv64(&g.truncate_cnt, 1); + if (__wt_atomic_addv64(&table->truncate_cnt, 1) > 2) { + (void)__wt_atomic_subv64(&table->truncate_cnt, 1); goto update_instead_of_chosen_op; } @@ -960,7 +958,7 @@ ops(void *arg) range = max_rows < 20 ? 0 : mmrand(&tinfo->rnd, 0, (u_int)max_rows / 20); tinfo->last = tinfo->keyno; if (greater_than) { - if (g.c_reverse) { + if (TV(BTREE_REVERSE)) { if (tinfo->keyno <= range) tinfo->last = 0; else @@ -971,7 +969,7 @@ ops(void *arg) tinfo->last = 0; } } else { - if (g.c_reverse) { + if (TV(BTREE_REVERSE)) { tinfo->keyno += range; if (tinfo->keyno > max_rows) tinfo->keyno = 0; @@ -982,16 +980,16 @@ ops(void *arg) tinfo->keyno -= range; } } - switch (g.type) { + switch (table->type) { case ROW: - ret = row_truncate(tinfo, cursor); + ret = row_truncate(tinfo); break; case FIX: case VAR: - ret = col_truncate(tinfo, cursor); + ret = col_truncate(tinfo); break; } - (void)__wt_atomic_subv64(&g.truncate_cnt, 1); + (void)__wt_atomic_subv64(&table->truncate_cnt, 1); /* Truncate never leaves the cursor positioned. */ positioned = false; @@ -1004,13 +1002,13 @@ ops(void *arg) case UPDATE: update_instead_of_chosen_op: ++tinfo->update; - switch (g.type) { + switch (table->type) { case ROW: - ret = row_update(tinfo, cursor, positioned); + ret = row_update(tinfo, positioned); break; case FIX: case VAR: - ret = col_update(tinfo, cursor, positioned); + ret = col_update(tinfo, positioned); break; } if (ret == 0) { @@ -1022,8 +1020,8 @@ update_instead_of_chosen_op: } /* If we have pending inserts, try and update the total rows. */ - if (tinfo->insert_list_cnt > 0) - col_insert_resolve(tinfo); + if (g.column_store_config) + tables_apply(col_insert_resolve, tinfo); /* * The cursor is positioned if we did any operation other than insert, do a small number of @@ -1033,7 +1031,7 @@ update_instead_of_chosen_op: next = mmrand(&tinfo->rnd, 0, 1) == 1; j = mmrand(&tinfo->rnd, 1, 100); for (i = 0; i < j; ++i) { - if ((ret = nextprev(tinfo, cursor, next)) == 0) + if ((ret = nextprev(tinfo, next)) == 0) continue; READ_OP_FAILED(true); @@ -1042,7 +1040,7 @@ update_instead_of_chosen_op: } /* Reset the cursor: there is no reason to keep pages pinned. */ - testutil_check(cursor->reset(cursor)); + testutil_check(tinfo->cursor->reset(tinfo->cursor)); /* * No post-operation work is needed outside of a transaction. If in a transaction, add more @@ -1052,21 +1050,24 @@ update_instead_of_chosen_op: continue; /* - * Ending a transaction. If the transaction was configured for snapshot isolation, repeat - * the operations and confirm the results are unchanged. + * Ending a transaction. If an explicit transaction was configured for snapshot isolation, + * repeat the operations and confirm the results are unchanged. */ if (intxn && iso_level == ISOLATION_SNAPSHOT) { __wt_yield(); /* Encourage races */ - ret = snap_repeat_txn(cursor, tinfo); + ret = snap_repeat_txn(tinfo); testutil_assert(ret == 0 || ret == WT_ROLLBACK || ret == WT_CACHE_FULL); if (ret == WT_ROLLBACK || ret == WT_CACHE_FULL) goto rollback; } - /* If prepare configured, prepare the transaction 10% of the time. */ + /* + * If prepare configured, prepare the transaction 10% of the time. Note prepare requires a + * timestamped world, which means we're in a snapshot-isolation transaction by definition. + */ prepared = false; - if (g.c_prepare && mmrand(&tinfo->rnd, 1, 10) == 1) { + if (GV(OPS_PREPARE) && mmrand(&tinfo->rnd, 1, 10) == 1) { if ((ret = prepare_transaction(tinfo)) != 0) WRITE_OP_FAILED(false); @@ -1096,11 +1097,10 @@ rollback: intxn = false; } - if (session != NULL) { + if (session != NULL) testutil_check(session->close(session, NULL)); - tinfo->cursor = NULL; - tinfo->session = NULL; - } + tinfo->session = NULL; + memset(tinfo->cursors, 0, WT_MAX(ntables, 1) * sizeof(tinfo->cursors[0])); tinfo->state = TINFO_COMPLETE; return (WT_THREAD_RET_VALUE); @@ -1111,7 +1111,7 @@ rollback: * Read and verify a subset of the elements in a file. */ void -wts_read_scan(void) +wts_read_scan(TABLE *table, void *arg) { WT_CONNECTION *conn; WT_CURSOR *cursor; @@ -1119,14 +1119,15 @@ wts_read_scan(void) WT_ITEM key, value; WT_SESSION *session; uint64_t keyno; + uint32_t max_rows; - conn = g.wts_conn; + conn = (WT_CONNECTION *)arg; /* * We're not configuring transactions or read timestamps, if there's a diagnostic check, skip * the scan. */ - if (g.c_assert_read_timestamp) + if (GV(ASSERT_READ_TIMESTAMP)) return; /* Set up the default key/value buffers. */ @@ -1135,20 +1136,16 @@ wts_read_scan(void) /* Open a session and cursor pair. */ testutil_check(conn->open_session(conn, NULL, NULL, &session)); - /* - * open_cursor can return EBUSY if concurrent with a metadata operation, retry in that case. - */ - while ((ret = session->open_cursor(session, g.uri, NULL, NULL, &cursor)) == EBUSY) - __wt_yield(); - testutil_check(ret); + wiredtiger_open_cursor(session, table->uri, NULL, &cursor); /* Check a random subset of the records using the key. */ - for (keyno = 0; keyno < g.rows;) { + WT_ORDERED_READ(max_rows, table->rows_current); + for (keyno = 0; keyno < max_rows;) { keyno += mmrand(NULL, 1, 1000); - if (keyno > g.rows) - keyno = g.rows; + if (keyno > max_rows) + keyno = max_rows; - switch (ret = read_row_worker(cursor, NULL, keyno, &key, &value, false)) { + switch (ret = read_row_worker(NULL, table, cursor, keyno, &key, &value, false)) { case 0: case WT_NOTFOUND: case WT_ROLLBACK: @@ -1171,20 +1168,20 @@ wts_read_scan(void) * Read and verify a single element in a row- or column-store file. */ static int -read_row_worker( - WT_CURSOR *cursor, TINFO *tinfo, uint64_t keyno, WT_ITEM *key, WT_ITEM *value, bool sn) +read_row_worker(TINFO *tinfo, TABLE *table, WT_CURSOR *cursor, uint64_t keyno, WT_ITEM *key, + WT_ITEM *value, bool sn) { uint8_t bitfield; int exact, ret; /* Retrieve the key/value pair by key. */ - switch (g.type) { + switch (table->type) { case FIX: case VAR: cursor->set_key(cursor, keyno); break; case ROW: - key_gen(key, keyno); + key_gen(table, key, keyno); cursor->set_key(cursor, key); break; } @@ -1197,7 +1194,7 @@ read_row_worker( ret = read_op(cursor, SEARCH, NULL); switch (ret) { case 0: - if (g.type == FIX) { + if (table->type == FIX) { testutil_check(cursor->get_value(cursor, &bitfield)); *(uint8_t *)(value->data) = bitfield; value->size = 1; @@ -1211,7 +1208,7 @@ read_row_worker( * The WiredTiger cursor has lost its position though, so we return not-found, the cursor * movement can't continue. */ - if (g.type == FIX) { + if (table->type == FIX) { *(uint8_t *)(value->data) = 0; value->size = 1; } @@ -1222,7 +1219,7 @@ read_row_worker( /* Log the operation */ if (ret == 0) - switch (g.type) { + switch (table->type) { case FIX: if (tinfo == NULL && g.trace_all) trace_msg("read %" PRIu64 " {0x%02x}", keyno, ((char *)value->data)[0]); @@ -1247,11 +1244,11 @@ read_row_worker( * Read and verify a single element in a row- or column-store file. */ static int -read_row(TINFO *tinfo, WT_CURSOR *cursor) +read_row(TINFO *tinfo) { /* 25% of the time we call search-near. */ - return (read_row_worker( - cursor, tinfo, tinfo->keyno, tinfo->key, tinfo->value, mmrand(&tinfo->rnd, 0, 3) == 1)); + return (read_row_worker(tinfo, tinfo->table, tinfo->cursor, tinfo->keyno, tinfo->key, + tinfo->value, mmrand(&tinfo->rnd, 0, 3) == 1)); } /* @@ -1259,22 +1256,26 @@ read_row(TINFO *tinfo, WT_CURSOR *cursor) * Read and verify the next/prev element in a row- or column-store file. */ static int -nextprev(TINFO *tinfo, WT_CURSOR *cursor, bool next) +nextprev(TINFO *tinfo, bool next) { + TABLE *table; + WT_CURSOR *cursor; WT_DECL_RET; WT_ITEM key, value; - uint64_t keyno, keyno_prev; + uint64_t keyno; uint8_t bitfield; int cmp; const char *which; - bool incrementing, record_gaps; + bool incrementing; + table = tinfo->table; + cursor = tinfo->cursor; keyno = 0; which = next ? "next" : "prev"; switch (ret = read_op(cursor, next ? NEXT : PREV, NULL)) { case 0: - switch (g.type) { + switch (table->type) { case FIX: if ((ret = cursor->get_key(cursor, &keyno)) == 0 && (ret = cursor->get_value(cursor, &bitfield)) == 0) { @@ -1294,64 +1295,26 @@ nextprev(TINFO *tinfo, WT_CURSOR *cursor, bool next) if (ret != 0) testutil_die(ret, "nextprev: get_key/get_value"); - /* Check that keys are never returned out-of-order. */ - /* - * XXX WT-3889 LSM has a bug that prevents cursor order checks from working, skip the test - * for now. - */ - if (DATASOURCE("lsm")) - break; - /* - * Compare the returned key with the previously returned key, and assert the order is - * correct. If not deleting keys, and the rows aren't in the column-store insert name space, - * also assert we don't skip groups of records (that's a page-split bug symptom). Note a - * previous run that performed salvage might have corrupted a chunk of space such that - * records were removed. If this is a reopen of an existing database, assume salvage might - * have happened. + * Check that keys are never returned out-of-order by comparing the returned key with the + * previously returned key, and assert the order is correct. */ - record_gaps = g.c_delete_pct != 0 || g.reopen; - switch (g.type) { + switch (table->type) { case FIX: case VAR: - if (tinfo->keyno > g.c_rows || keyno > g.c_rows) - record_gaps = true; - if (!next) { - if (tinfo->keyno < keyno || (!record_gaps && keyno != tinfo->keyno - 1)) - goto order_error_col; - } else if (tinfo->keyno > keyno || (!record_gaps && keyno != tinfo->keyno + 1)) - goto order_error_col; - if (0) { -order_error_col: + if ((next && tinfo->keyno > keyno) || (!next && tinfo->keyno < keyno)) testutil_die( 0, "%s returned %" PRIu64 " then %" PRIu64, which, tinfo->keyno, keyno); - } - tinfo->keyno = keyno; break; case ROW: - incrementing = (next && !g.c_reverse) || (!next && g.c_reverse); + incrementing = (next && !TV(BTREE_REVERSE)) || (!next && TV(BTREE_REVERSE)); cmp = memcmp(tinfo->key->data, key.data, WT_MIN(tinfo->key->size, key.size)); if (incrementing) { if (cmp > 0 || (cmp == 0 && tinfo->key->size < key.size)) goto order_error_row; } else if (cmp < 0 || (cmp == 0 && tinfo->key->size > key.size)) goto order_error_row; - if (!record_gaps) { - /* - * Convert the keys to record numbers and then compare less-than-or-equal. (It's not - * less-than, row-store inserts new rows in-between rows by appending a new suffix - * to the row's key.) Keys are strings with terminating '/' values, so absent key - * corruption, we can simply do the underlying string conversion on the key string. - */ - keyno_prev = strtoul((char *)tinfo->key->data + g.prefix_len, NULL, 10); - keyno = strtoul((char *)key.data + g.prefix_len, NULL, 10); - if (incrementing) { - if (keyno_prev != keyno && keyno_prev + 1 != keyno) - goto order_error_row; - } else if (keyno_prev != keyno && keyno_prev - 1 != keyno) - goto order_error_row; - } if (0) { order_error_row: #ifdef HAVE_DIAGNOSTIC @@ -1366,13 +1329,12 @@ order_error_row: } break; case WT_NOTFOUND: - break; default: return (ret); } - if (g.trace_all && ret == 0) - switch (g.type) { + if (g.trace_all) + switch (table->type) { case FIX: trace_op(tinfo, "%s %" PRIu64 " {0x%02x}", which, keyno, ((char *)value.data)[0]); break; @@ -1393,12 +1355,15 @@ order_error_row: * Reserve a row in a row-store file. */ static int -row_reserve(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +row_reserve(TINFO *tinfo, bool positioned) { + WT_CURSOR *cursor; WT_DECL_RET; + cursor = tinfo->cursor; + if (!positioned) { - key_gen(tinfo->key, tinfo->keyno); + key_gen(tinfo->table, tinfo->key, tinfo->keyno); cursor->set_key(cursor, tinfo->key); } @@ -1416,10 +1381,13 @@ row_reserve(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) * Reserve a row in a column-store file. */ static int -col_reserve(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +col_reserve(TINFO *tinfo, bool positioned) { + WT_CURSOR *cursor; WT_DECL_RET; + cursor = tinfo->cursor; + if (!positioned) cursor->set_key(cursor, tinfo->keyno); @@ -1491,11 +1459,14 @@ modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) * Modify a row in a row-store file. */ static int -row_modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +row_modify(TINFO *tinfo, bool positioned) { + WT_CURSOR *cursor; + + cursor = tinfo->cursor; if (!positioned) { - key_gen(tinfo->key, tinfo->keyno); + key_gen(tinfo->table, tinfo->key, tinfo->keyno); cursor->set_key(cursor, tinfo->key); } @@ -1512,8 +1483,12 @@ row_modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) * Modify a row in a column-store file. */ static int -col_modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +col_modify(TINFO *tinfo, bool positioned) { + WT_CURSOR *cursor; + + cursor = tinfo->cursor; + if (!positioned) cursor->set_key(cursor, tinfo->keyno); @@ -1529,12 +1504,13 @@ col_modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) * Truncate rows in a row-store file. */ static int -row_truncate(TINFO *tinfo, WT_CURSOR *cursor) +row_truncate(TINFO *tinfo) { - WT_CURSOR *c2; + WT_CURSOR *cursor, *c2; WT_DECL_RET; WT_SESSION *session; + cursor = tinfo->cursor; session = cursor->session; /* @@ -1544,28 +1520,26 @@ row_truncate(TINFO *tinfo, WT_CURSOR *cursor) c2 = NULL; if (tinfo->keyno == 0) { - key_gen(tinfo->key, tinfo->last); + key_gen(tinfo->table, tinfo->key, tinfo->last); cursor->set_key(cursor, tinfo->key); - ret = session->truncate(session, NULL, NULL, cursor, NULL); + WT_RET(session->truncate(session, NULL, NULL, cursor, NULL)); } else if (tinfo->last == 0) { - key_gen(tinfo->key, tinfo->keyno); + key_gen(tinfo->table, tinfo->key, tinfo->keyno); cursor->set_key(cursor, tinfo->key); - ret = session->truncate(session, NULL, cursor, NULL, NULL); + WT_RET(session->truncate(session, NULL, cursor, NULL, NULL)); } else { - key_gen(tinfo->key, tinfo->keyno); + key_gen(tinfo->table, tinfo->key, tinfo->keyno); cursor->set_key(cursor, tinfo->key); - testutil_check(session->open_cursor(session, g.uri, NULL, NULL, &c2)); - key_gen(tinfo->lastkey, tinfo->last); + testutil_check(session->open_cursor(session, tinfo->table->uri, NULL, NULL, &c2)); + key_gen(tinfo->table, tinfo->lastkey, tinfo->last); cursor->set_key(c2, tinfo->lastkey); ret = session->truncate(session, NULL, cursor, c2, NULL); testutil_check(c2->close(c2)); + WT_RET(ret); } - if (ret != 0) - return (ret); - trace_op(tinfo, "truncate %" PRIu64 ", %" PRIu64, "truncate", tinfo->keyno, tinfo->last); return (0); @@ -1576,12 +1550,13 @@ row_truncate(TINFO *tinfo, WT_CURSOR *cursor) * Truncate rows in a column-store file. */ static int -col_truncate(TINFO *tinfo, WT_CURSOR *cursor) +col_truncate(TINFO *tinfo) { - WT_CURSOR *c2; + WT_CURSOR *cursor, *c2; WT_DECL_RET; WT_SESSION *session; + cursor = tinfo->cursor; session = cursor->session; /* @@ -1599,7 +1574,7 @@ col_truncate(TINFO *tinfo, WT_CURSOR *cursor) } else { cursor->set_key(cursor, tinfo->keyno); - testutil_check(session->open_cursor(session, g.uri, NULL, NULL, &c2)); + testutil_check(session->open_cursor(session, tinfo->table->uri, NULL, NULL, &c2)); cursor->set_key(c2, tinfo->last); ret = session->truncate(session, NULL, cursor, c2, NULL); @@ -1618,15 +1593,18 @@ col_truncate(TINFO *tinfo, WT_CURSOR *cursor) * Update a row in a row-store file. */ static int -row_update(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +row_update(TINFO *tinfo, bool positioned) { + WT_CURSOR *cursor; WT_DECL_RET; + cursor = tinfo->cursor; + if (!positioned) { - key_gen(tinfo->key, tinfo->keyno); + key_gen(tinfo->table, tinfo->key, tinfo->keyno); cursor->set_key(cursor, tinfo->key); } - val_gen(&tinfo->rnd, tinfo->value, tinfo->keyno); + val_gen(tinfo->table, &tinfo->rnd, tinfo->value, tinfo->keyno); cursor->set_value(cursor, tinfo->value); if ((ret = cursor->update(cursor)) != 0) @@ -1643,14 +1621,19 @@ row_update(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) * Update a row in a column-store file. */ static int -col_update(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +col_update(TINFO *tinfo, bool positioned) { + TABLE *table; + WT_CURSOR *cursor; WT_DECL_RET; + table = tinfo->table; + cursor = tinfo->cursor; + if (!positioned) cursor->set_key(cursor, tinfo->keyno); - val_gen(&tinfo->rnd, tinfo->value, tinfo->keyno); - if (g.type == FIX) + val_gen(table, &tinfo->rnd, tinfo->value, tinfo->keyno); + if (table->type == FIX) cursor->set_value(cursor, *(uint8_t *)tinfo->value->data); else cursor->set_value(cursor, tinfo->value); @@ -1658,7 +1641,7 @@ col_update(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) if ((ret = cursor->update(cursor)) != 0) return (ret); - if (g.type == FIX) + if (table->type == FIX) trace_op(tinfo, "update %" PRIu64 " {0x%02" PRIx8 "}", tinfo->keyno, ((uint8_t *)tinfo->value->data)[0]); else @@ -1672,19 +1655,22 @@ col_update(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) * Insert a row in a row-store file. */ static int -row_insert(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +row_insert(TINFO *tinfo, bool positioned) { + WT_CURSOR *cursor; WT_DECL_RET; + cursor = tinfo->cursor; + /* * If we positioned the cursor already, it's a test of an update using the insert method. * Otherwise, generate a unique key and insert. */ if (!positioned) { - key_gen_insert(&tinfo->rnd, tinfo->key, tinfo->keyno); + key_gen_insert(tinfo->table, &tinfo->rnd, tinfo->key, tinfo->keyno); cursor->set_key(cursor, tinfo->key); } - val_gen(&tinfo->rnd, tinfo->value, tinfo->keyno); + val_gen(tinfo->table, &tinfo->rnd, tinfo->value, tinfo->keyno); cursor->set_value(cursor, tinfo->value); if ((ret = cursor->insert(cursor)) != 0) @@ -1702,11 +1688,18 @@ row_insert(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) * Resolve newly inserted records. */ static void -col_insert_resolve(TINFO *tinfo) +col_insert_resolve(TABLE *table, void *arg) { - uint64_t v, *p; + struct col_insert *cip; + TINFO *tinfo; + uint32_t max_rows, *p; u_int i; + tinfo = arg; + cip = &tinfo->col_insert[table->id - 1]; + if (cip->insert_list_cnt == 0) + return; + /* * We don't want to ignore column-store records we insert, which requires we update the "last * row" so other threads consider them. Threads allocating record numbers can race with other @@ -1722,17 +1715,17 @@ col_insert_resolve(TINFO *tinfo) * Process the existing records and advance the last row count until we can't go further. */ do { - WT_ORDERED_READ(v, g.rows); - for (i = 0, p = tinfo->insert_list; i < WT_ELEMENTS(tinfo->insert_list); ++i, ++p) { - if (*p == v + 1) { - testutil_assert(__wt_atomic_casv64(&g.rows, v, v + 1)); + WT_ORDERED_READ(max_rows, table->rows_current); + for (i = 0, p = cip->insert_list; i < WT_ELEMENTS(cip->insert_list); ++i, ++p) { + if (*p == max_rows + 1) { + testutil_assert(__wt_atomic_casv32(&table->rows_current, max_rows, max_rows + 1)); *p = 0; - --tinfo->insert_list_cnt; + --cip->insert_list_cnt; break; } - testutil_assert(*p == 0 || *p > v); + testutil_assert(*p == 0 || *p > max_rows); } - } while (tinfo->insert_list_cnt > 0 && i < WT_ELEMENTS(tinfo->insert_list)); + } while (cip->insert_list_cnt > 0 && i < WT_ELEMENTS(cip->insert_list)); } /* @@ -1742,16 +1735,18 @@ col_insert_resolve(TINFO *tinfo) static void col_insert_add(TINFO *tinfo) { + struct col_insert *cip; u_int i; - /* Add the inserted record to the array. */ - for (i = 0; i < WT_ELEMENTS(tinfo->insert_list); ++i) - if (tinfo->insert_list[i] == 0) { - tinfo->insert_list[i] = tinfo->keyno; - ++tinfo->insert_list_cnt; + /* Add the inserted record to the insert array. */ + cip = &tinfo->col_insert[tinfo->table->id - 1]; + for (i = 0; i < WT_ELEMENTS(cip->insert_list); ++i) + if (cip->insert_list[i] == 0) { + cip->insert_list[i] = (uint32_t)tinfo->keyno; + ++cip->insert_list_cnt; break; } - testutil_assert(i < WT_ELEMENTS(tinfo->insert_list)); + testutil_assert(i < WT_ELEMENTS(cip->insert_list)); } /* @@ -1759,12 +1754,19 @@ col_insert_add(TINFO *tinfo) * Insert an element in a column-store file. */ static int -col_insert(TINFO *tinfo, WT_CURSOR *cursor) +col_insert(TINFO *tinfo) { + TABLE *table; + WT_CURSOR *cursor; WT_DECL_RET; + uint64_t max_rows; - val_gen(&tinfo->rnd, tinfo->value, g.rows + 1); - if (g.type == FIX) + table = tinfo->table; + cursor = tinfo->cursor; + + WT_ORDERED_READ(max_rows, table->rows_current); + val_gen(table, &tinfo->rnd, tinfo->value, max_rows + 1); + if (table->type == FIX) cursor->set_value(cursor, *(uint8_t *)tinfo->value->data); else cursor->set_value(cursor, tinfo->value); @@ -1777,7 +1779,7 @@ col_insert(TINFO *tinfo, WT_CURSOR *cursor) col_insert_add(tinfo); /* Extend the object. */ - if (g.type == FIX) + if (table->type == FIX) trace_op(tinfo, "insert %" PRIu64 " {0x%02" PRIx8 "}", tinfo->keyno, ((uint8_t *)tinfo->value->data)[0]); else @@ -1791,12 +1793,15 @@ col_insert(TINFO *tinfo, WT_CURSOR *cursor) * Remove an row from a row-store file. */ static int -row_remove(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +row_remove(TINFO *tinfo, bool positioned) { + WT_CURSOR *cursor; WT_DECL_RET; + cursor = tinfo->cursor; + if (!positioned) { - key_gen(tinfo->key, tinfo->keyno); + key_gen(tinfo->table, tinfo->key, tinfo->keyno); cursor->set_key(cursor, tinfo->key); } @@ -1817,10 +1822,13 @@ row_remove(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) * Remove a row from a column-store file. */ static int -col_remove(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) +col_remove(TINFO *tinfo, bool positioned) { + WT_CURSOR *cursor; WT_DECL_RET; + cursor = tinfo->cursor; + if (!positioned) cursor->set_key(cursor, tinfo->keyno); diff --git a/src/third_party/wiredtiger/test/format/random.c b/src/third_party/wiredtiger/test/format/random.c index f5726ff8ea3..fba5e062079 100644 --- a/src/third_party/wiredtiger/test/format/random.c +++ b/src/third_party/wiredtiger/test/format/random.c @@ -35,6 +35,7 @@ WT_THREAD_RET random_kv(void *arg) { + TABLE *table; WT_CONNECTION *conn; WT_CURSOR *cursor; WT_DECL_RET; @@ -47,13 +48,19 @@ random_kv(void *arg) (void)(arg); /* Unused parameter */ - conn = g.wts_conn; - - /* Random cursor ops are only supported on row-store. */ - if (g.type != ROW) + /* Random cursor ops are only supported on row-store, make sure there's a row-store table. */ + if (ntables == 0 && tables[0]->type != ROW) return (WT_THREAD_RET_VALUE); + else { + for (i = 1; i < ntables; ++i) + if (tables[i]->type == ROW) + break; + if (i == ntables) + return (WT_THREAD_RET_VALUE); + } /* Open a session. */ + conn = g.wts_conn; testutil_check(conn->open_session(conn, NULL, NULL, &session)); for (simple = false;;) { @@ -61,12 +68,9 @@ random_kv(void *arg) config = simple ? "next_random=true" : "next_random=true,next_random_sample_size=37"; simple = !simple; - /* - * open_cursor can return EBUSY if concurrent with a metadata operation, retry in that case. - */ - while ((ret = session->open_cursor(session, g.uri, NULL, config, &cursor)) == EBUSY) - __wt_yield(); - testutil_check(ret); + /* Select a table and open a cursor. */ + table = table_select_type(ROW); + wiredtiger_open_cursor(session, table->uri, config, &cursor); /* This is just a smoke-test, get some key/value pairs. */ for (i = mmrand(NULL, 0, 1000); i > 0; --i) { diff --git a/src/third_party/wiredtiger/test/format/salvage.c b/src/third_party/wiredtiger/test/format/salvage.c index a383c121862..26589e18a3f 100644 --- a/src/third_party/wiredtiger/test/format/salvage.c +++ b/src/third_party/wiredtiger/test/format/salvage.c @@ -29,124 +29,126 @@ #include "format.h" /* + * uri_path -- + * Return the path to an object file, and optionally, the object name. + */ +static void +uri_path(TABLE *table, char **object_namep, char *buf, size_t len) +{ + char *p; + + /* + * It's a little tricky: if the data source is a file, we're looking for the table URI, if the + * data source is a table, we're looking for the table URI with a trailing ".wt". + */ + p = strchr(table->uri, ':'); + testutil_assert(p != NULL); + ++p; + + testutil_check(__wt_snprintf(buf, len, "%s/%s", g.home, p)); + if (object_namep != NULL) + *object_namep = strrchr(buf, '/') + 1; + if (!access(buf, F_OK)) + return; + testutil_check(__wt_snprintf(buf, len, "%s/%s.wt", g.home, p)); + if (object_namep != NULL) + *object_namep = strrchr(buf, '/') + 1; + if (!access(buf, F_OK)) + return; + testutil_die(0, "%s: unable to find file for salvage", table->uri); +} + +/* * corrupt -- * Corrupt the file in a random way. */ -static int -corrupt(void) +static void +corrupt(TABLE *table) { struct stat sb; FILE *fp; wt_off_t offset; size_t len, nw; - int fd, ret; - char copycmd[2 * 1024], path[1024]; + int fd; + char buf[MAX_FORMAT_PATH * 2], *object_name, path[MAX_FORMAT_PATH]; const char *smash; - /* - * If it's a single Btree file (not LSM), open the file, and corrupt roughly 2% of the file at a - * random spot, including the beginning of the file and overlapping the end. - * - * It's a little tricky: if the data source is a file, we're looking for "wt", if the data - * source is a table, we're looking for "wt.wt". - */ - testutil_check(__wt_snprintf(path, sizeof(path), "%s/%s", g.home, WT_NAME)); - if ((fd = open(path, O_RDWR)) != -1) { - testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), - "cp %s/%s %s/SALVAGE.copy/%s.corrupted", g.home, WT_NAME, g.home, WT_NAME)); - goto found; - } - testutil_check(__wt_snprintf(path, sizeof(path), "%s/%s.wt", g.home, WT_NAME)); - if ((fd = open(path, O_RDWR)) != -1) { - testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), - "cp %s/%s.wt %s/SALVAGE.copy/%s.wt.corrupted", g.home, WT_NAME, g.home, WT_NAME)); - goto found; - } - return (0); + uri_path(table, &object_name, path, sizeof(path)); -found: - if (fstat(fd, &sb) == -1) - testutil_die(errno, "salvage-corrupt: fstat"); + fd = open(path, O_RDWR); + testutil_assert(fd != -1); - offset = mmrand(NULL, 0, (u_int)sb.st_size); - len = (size_t)(20 + (sb.st_size / 100) * 2); - testutil_check(__wt_snprintf(path, sizeof(path), "%s/SALVAGE.corrupt", g.home)); - if ((fp = fopen(path, "w")) == NULL) - testutil_die(errno, "salvage-corrupt: open: %s", path); + /* + * Corrupt a chunk of the file at a random spot, including the first bytes of the file and + * possibly overlapping the end. The length of the corruption is roughly 2% of the file, not + * exceeding a megabyte (so we aren't just corrupting the whole file). + */ + testutil_check(fstat(fd, &sb)); + offset = mmrand(NULL, 0, (u_int)sb.st_size - 1024); + len = (size_t)(sb.st_size * 2) / 100; + len += 4 * 1024; + len = WT_MIN(len, WT_MEGABYTE); + + /* Log the corruption offset and length. */ + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s/SALVAGE.corrupt", g.home)); + testutil_assert((fp = fopen(buf, "w")) != NULL); (void)fprintf(fp, "salvage-corrupt: offset %" PRIuMAX ", length %" WT_SIZET_FMT "\n", (uintmax_t)offset, len); fclose_and_clear(&fp); - if (lseek(fd, offset, SEEK_SET) == -1) - testutil_die(errno, "salvage-corrupt: lseek"); - + testutil_assert(lseek(fd, offset, SEEK_SET) != -1); smash = "!!! memory corrupted by format to test salvage "; for (; len > 0; len -= nw) { nw = (size_t)(len > strlen(smash) ? strlen(smash) : len); - if (write(fd, smash, nw) == -1) - testutil_die(errno, "salvage-corrupt: write"); + testutil_assert(write(fd, smash, nw) != -1); } - if (close(fd) == -1) - testutil_die(errno, "salvage-corrupt: close"); + testutil_check(close(fd)); - /* - * Save a copy of the corrupted file so we can replay the salvage step as necessary. - */ - if ((ret = system(copycmd)) != 0) - testutil_die(ret, "salvage corrupt copy step failed"); - - return (1); + /* Save a copy of the corrupted file so we can replay the salvage step as necessary. */ + testutil_check(__wt_snprintf( + buf, sizeof(buf), "cp %s %s/SALVAGE.copy/%s.corrupted", path, g.home, object_name)); + testutil_check(system(buf)); } -/* - * Salvage command, save the interesting files so we can replay the salvage command as necessary. - * - * Redirect the "cd" command to /dev/null so chatty cd implementations don't add the new working - * directory to our output. - */ -#define SALVAGE_COPY_CMD \ - "cd %s > /dev/null && " \ - "rm -rf SALVAGE.copy && mkdir SALVAGE.copy && " \ - "cp WiredTiger* wt* SALVAGE.copy/" +/* Salvage command, save the interesting files so we can replay the salvage command as necessary. */ +#define SALVAGE_COPY_CMD \ + "rm -rf %s/SALVAGE.copy && mkdir %s/SALVAGE.copy && cp %s/WiredTiger* %s %s/SALVAGE.copy/" /* * wts_salvage -- * Salvage testing. */ void -wts_salvage(void) +wts_salvage(TABLE *table, void *arg) { WT_CONNECTION *conn; - WT_DECL_RET; WT_SESSION *session; - size_t len; - char *cmd; + char buf[MAX_FORMAT_PATH * 5], path[MAX_FORMAT_PATH]; - if (g.c_salvage == 0) - return; + (void)arg; /* unused argument */ - track("salvage", 0ULL, NULL); + if (GV(OPS_SALVAGE) == 0 || DATASOURCE(table, "lsm")) + return; /* Save a copy of the interesting files so we can replay the salvage step as necessary. */ - len = strlen(g.home) + strlen(SALVAGE_COPY_CMD) + 1; - cmd = dmalloc(len); - testutil_check(__wt_snprintf(cmd, len, SALVAGE_COPY_CMD, g.home)); - if ((ret = system(cmd)) != 0) - testutil_die(ret, "salvage copy (\"%s\"), failed", cmd); - free(cmd); + uri_path(table, NULL, path, sizeof(path)); + testutil_check( + __wt_snprintf(buf, sizeof(buf), SALVAGE_COPY_CMD, g.home, g.home, g.home, path, g.home)); + testutil_check(system(buf)); /* Salvage, then verify. */ wts_open(g.home, &conn, &session, true); - testutil_check(session->salvage(session, g.uri, "force=true")); - wts_verify(conn, "post-salvage verify"); + session->app_private = table->track_prefix; + testutil_check(session->salvage(session, table->uri, "force=true")); + wts_verify(table, conn); wts_close(&conn, &session); /* Corrupt the file randomly, salvage, then verify. */ - if (corrupt()) { - wts_open(g.home, &conn, &session, false); - testutil_check(session->salvage(session, g.uri, "force=true")); - wts_verify(conn, "post-corrupt-salvage verify"); - wts_close(&conn, &session); - } + corrupt(table); + wts_open(g.home, &conn, &session, false); + testutil_check(session->salvage(session, table->uri, "force=true")); + wts_verify(table, conn); + + wts_close(&conn, &session); } diff --git a/src/third_party/wiredtiger/test/format/smoke.sh b/src/third_party/wiredtiger/test/format/smoke.sh index 151d1be37b3..ab2096dcd12 100755 --- a/src/third_party/wiredtiger/test/format/smoke.sh +++ b/src/third_party/wiredtiger/test/format/smoke.sh @@ -3,14 +3,16 @@ set -e # Smoke-test format as part of running "make check". -args="-1 -c . " +args="-c . " args="$args btree.compression=none " args="$args cache.minimum=40 " args="$args logging_compression=none" -args="$args runs.ops=500000 " args="$args runs.rows=100000 " args="$args runs.source=table " -args="$args runs.threads=4 " +args="$args runs.tables=3 " +args="$args runs.threads=6 " +args="$args runs.timer=1 " +args="$args transaction.timestamps=1 " # Temporarily disable LSM and FLCS. # $TEST_WRAPPER ./t $args runs.type=fix diff --git a/src/third_party/wiredtiger/test/format/snap.c b/src/third_party/wiredtiger/test/format/snap.c index e7709754523..298b82ec9a5 100644 --- a/src/third_party/wiredtiger/test/format/snap.c +++ b/src/third_party/wiredtiger/test/format/snap.c @@ -45,7 +45,7 @@ snap_init(TINFO *tinfo) * we can the secondary snap list to see the state of keys/values seen and updated at the time * of the rollback. */ - if (g.c_txn_timestamps) { + if (g.transaction_timestamps_config) { tinfo->s = &tinfo->snap_states[1]; tinfo->snap_list = dcalloc(SNAP_LIST_SIZE, sizeof(SNAP_OPS)); tinfo->snap_end = &tinfo->snap_list[SNAP_LIST_SIZE]; @@ -77,7 +77,7 @@ snap_teardown(TINFO *tinfo) } /* - * snap_clear -- + * snap_clear_one -- * Clear a single snap entry. */ static void @@ -110,7 +110,7 @@ snap_op_init(TINFO *tinfo, uint64_t read_ts, bool repeatable_reads) ++tinfo->opid; - if (g.c_txn_timestamps) { + if (g.transaction_timestamps_config) { /* * If the stable timestamp has changed and we've advanced beyond it, preserve the current * snapshot history up to this point, we'll use it verify rollback_to_stable. Switch our @@ -150,13 +150,14 @@ snap_track(TINFO *tinfo, thread_op op) snap = tinfo->snap_current; snap->op = op; snap->opid = tinfo->opid; + snap->id = tinfo->table->id; snap->keyno = tinfo->keyno; snap->ts = WT_TS_NONE; snap->repeatable = false; snap->last = op == TRUNCATE ? tinfo->last : 0; snap->ksize = snap->vsize = 0; - if (op == INSERT && g.type == ROW) { + if (op == INSERT && tinfo->table->type == ROW) { ip = tinfo->key; if (snap->kmemsize < ip->size) { snap->kdata = drealloc(snap->kdata, ip->size); @@ -196,11 +197,11 @@ snap_track(TINFO *tinfo, thread_op op) * Display a single data/size pair, with a tag. */ static void -print_item_data(const char *tag, const uint8_t *data, size_t size) +print_item_data(TABLE *table, const char *tag, const uint8_t *data, size_t size) { WT_ITEM tmp; - if (g.type == FIX) { + if (table->type == FIX) { fprintf(stderr, "%s {0x%02x}\n", tag, data[0]); return; } @@ -216,8 +217,10 @@ print_item_data(const char *tag, const uint8_t *data, size_t size) * Repeat a read and verify the contents. */ static int -snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) +snap_verify(TINFO *tinfo, SNAP_OPS *snap) { + TABLE *table; + WT_CURSOR *cursor; WT_DECL_RET; WT_ITEM *key, *value; uint64_t keyno; @@ -225,26 +228,28 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) testutil_assert(snap->op != TRUNCATE); + table = tables[ntables == 0 ? 0 : snap->id]; + cursor = table_cursor(tinfo, snap->id); + keyno = snap->keyno; key = tinfo->key; value = tinfo->value; - keyno = snap->keyno; /* * Retrieve the key/value pair by key. Row-store inserts have a unique generated key we saved, * else generate the key from the key number. */ - if (snap->op == INSERT && g.type == ROW) { + if (snap->op == INSERT && table->type == ROW) { key->data = snap->kdata; key->size = snap->ksize; cursor->set_key(cursor, key); } else { - switch (g.type) { + switch (table->type) { case FIX: case VAR: cursor->set_key(cursor, keyno); break; case ROW: - key_gen(key, keyno); + key_gen(table, key, keyno); cursor->set_key(cursor, key); break; } @@ -252,7 +257,7 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) switch (ret = read_op(cursor, SEARCH, NULL)) { case 0: - if (g.type == FIX) { + if (table->type == FIX) { testutil_check(cursor->get_value(cursor, &bitfield)); *(uint8_t *)(value->data) = bitfield; value->size = 1; @@ -279,15 +284,22 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) * In fixed length stores, zero values at the end of the key space are returned as not-found, * and not-found row reads are saved as zero values. Map back-and-forth for simplicity. */ - if (g.type == FIX) { + if (table->type == FIX) { if (ret == WT_NOTFOUND && snap->vsize == 1 && *(uint8_t *)snap->vdata == 0) return (0); if (snap->op == REMOVE && value->size == 1 && *(uint8_t *)value->data == 0) return (0); } - /* Things went pear-shaped. */ - switch (g.type) { + /* + * Things went pear-shaped. + * + * Dump the WiredTiger handle ID, it's useful in selecting trace records from the log. We have + * an open cursor on the handle, so while this is pretty ugly, I don't think it's unsafe. + */ + fprintf(stderr, "%s: WiredTiger trace ID: %u\n", table->uri, + (u_int)((WT_BTREE *)((WT_CURSOR_BTREE *)cursor)->dhandle->handle)->id); + switch (table->type) { case FIX: fprintf(stderr, "snapshot-isolation: %" PRIu64 " search: expected {0x%02x}, found {0x%02x}\n", keyno, @@ -301,11 +313,11 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) if (snap->op == REMOVE) fprintf(stderr, "expected {deleted}\n"); else - print_item_data("expected", snap->vdata, snap->vsize); + print_item_data(table, "expected", snap->vdata, snap->vsize); if (ret == WT_NOTFOUND) fprintf(stderr, " found {deleted}\n"); else - print_item_data(" found", value->data, value->size); + print_item_data(table, " found", value->data, value->size); break; case VAR: fprintf(stderr, "snapshot-isolation %" PRIu64 " search mismatch\n", keyno); @@ -313,11 +325,11 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) if (snap->op == REMOVE) fprintf(stderr, "expected {deleted}\n"); else - print_item_data("expected", snap->vdata, snap->vsize); + print_item_data(table, "expected", snap->vdata, snap->vsize); if (ret == WT_NOTFOUND) fprintf(stderr, " found {deleted}\n"); else - print_item_data(" found", value->data, value->size); + print_item_data(table, " found", value->data, value->size); break; } @@ -344,31 +356,40 @@ snap_ts_clear(TINFO *tinfo, uint64_t ts) } /* - * snap_repeat_ok_match -- - * Compare two operations and see if they modified the same record. + * snap_repeat_match -- + * Compare two operations and return if they modified the same record. */ static bool -snap_repeat_ok_match(SNAP_OPS *current, SNAP_OPS *a) +snap_repeat_match(SNAP_OPS *current, SNAP_OPS *a) { + TABLE *table; + bool reverse; + /* Reads are never a problem, there's no modification. */ if (a->op == READ) - return (true); + return (false); - /* Check for a matching single record modification. */ - if (a->keyno == current->keyno) + /* Check if the operations were in the same table. */ + if (a->id != current->id) return (false); + /* Check for a matching single record insert, modify, remove or update. */ + if (a->keyno == current->keyno) + return (true); + /* Truncates are slightly harder, make sure the ranges don't overlap. */ + table = tables[ntables == 0 ? 0 : a->id]; + reverse = TV(BTREE_REVERSE) != 0; if (a->op == TRUNCATE) { - if (g.c_reverse && (a->keyno == 0 || a->keyno >= current->keyno) && + if (reverse && (a->keyno == 0 || a->keyno >= current->keyno) && (a->last == 0 || a->last <= current->keyno)) - return (false); - if (!g.c_reverse && (a->keyno == 0 || a->keyno <= current->keyno) && + return (true); + if (!reverse && (a->keyno == 0 || a->keyno <= current->keyno) && (a->last == 0 || a->last >= current->keyno)) - return (false); + return (true); } - return (true); + return (false); } /* @@ -401,7 +422,7 @@ snap_repeat_ok_commit(TINFO *tinfo, SNAP_OPS *current) if (p->opid != tinfo->opid) break; - if (!snap_repeat_ok_match(current, p)) + if (snap_repeat_match(current, p)) return (false); } @@ -414,7 +435,7 @@ snap_repeat_ok_commit(TINFO *tinfo, SNAP_OPS *current) if (p->opid != tinfo->opid) break; - if (!snap_repeat_ok_match(current, p)) + if (snap_repeat_match(current, p)) return (false); } return (true); @@ -444,7 +465,7 @@ snap_repeat_ok_rollback(TINFO *tinfo, SNAP_OPS *current) if (p->opid != tinfo->opid) break; - if (!snap_repeat_ok_match(current, p)) + if (snap_repeat_match(current, p)) return (false); } return (true); @@ -455,7 +476,7 @@ snap_repeat_ok_rollback(TINFO *tinfo, SNAP_OPS *current) * Repeat each operation done within a snapshot isolation transaction. */ int -snap_repeat_txn(WT_CURSOR *cursor, TINFO *tinfo) +snap_repeat_txn(TINFO *tinfo) { SNAP_OPS *current; @@ -478,7 +499,7 @@ snap_repeat_txn(WT_CURSOR *cursor, TINFO *tinfo) * other threads of control committing in our past, until the transaction resolves. */ if (snap_repeat_ok_commit(tinfo, current)) - WT_RET(snap_verify(cursor, tinfo, current)); + WT_RET(snap_verify(tinfo, current)); } return (0); @@ -532,7 +553,7 @@ snap_repeat_update(TINFO *tinfo, bool committed) * Repeat one operation. */ static void -snap_repeat(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) +snap_repeat(TINFO *tinfo, SNAP_OPS *snap) { WT_DECL_RET; WT_SESSION *session; @@ -540,7 +561,7 @@ snap_repeat(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) u_int max_retry; char buf[64]; - session = cursor->session; + session = tinfo->session; trace_op(tinfo, "repeat %" PRIu64 " ts=%" PRIu64 " {%s}", snap->keyno, snap->ts, trace_bytes(tinfo, snap->vdata, snap->vsize)); @@ -563,7 +584,7 @@ snap_repeat(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) * matter to us). Persist after rollback, as a repeatable read we should succeed, yield to * let eviction catch up. */ - if ((ret = snap_verify(cursor, tinfo, snap)) == 0) + if ((ret = snap_verify(tinfo, snap)) == 0) break; testutil_assert(ret == WT_ROLLBACK); @@ -579,7 +600,7 @@ snap_repeat(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) * Repeat an historic operation. */ void -snap_repeat_single(WT_CURSOR *cursor, TINFO *tinfo) +snap_repeat_single(TINFO *tinfo) { SNAP_OPS *snap; u_int v; @@ -602,7 +623,7 @@ snap_repeat_single(WT_CURSOR *cursor, TINFO *tinfo) if (count == 0) return; - snap_repeat(cursor, tinfo, snap); + snap_repeat(tinfo, snap); } /* @@ -610,20 +631,27 @@ snap_repeat_single(WT_CURSOR *cursor, TINFO *tinfo) * Repeat all known operations after a rollback. */ void -snap_repeat_rollback(WT_CURSOR *cursor, TINFO **tinfo_array, size_t tinfo_count) +snap_repeat_rollback(TINFO **tinfo_array, size_t tinfo_count) { SNAP_OPS *snap; SNAP_STATE *state; TINFO *tinfo, **tinfop; + WT_SESSION *push_session, *session; uint32_t count; size_t i, statenum; - char buf[100]; + char buf[64]; count = 0; + session = NULL; - track("rollback_to_stable: checking", 0ULL, NULL); + track("rollback_to_stable: checking", 0ULL); for (i = 0, tinfop = tinfo_array; i < tinfo_count; ++i, ++tinfop) { tinfo = *tinfop; + if ((push_session = tinfo->session) == NULL) { + if (session == NULL) + testutil_check(g.wts_conn->open_session(g.wts_conn, NULL, NULL, &session)); + tinfo->session = session; + } /* * For this thread, walk through both sets of snaps ("states"), looking for entries that are @@ -636,28 +664,33 @@ snap_repeat_rollback(WT_CURSOR *cursor, TINFO **tinfo_array, size_t tinfo_count) state = &tinfo->snap_states[statenum]; for (snap = state->snap_state_list; snap < state->snap_state_end; ++snap) { if (snap->repeatable && snap->ts <= g.stable_timestamp) { - snap_repeat(cursor, tinfo, snap); + snap_repeat(tinfo, snap); ++count; if (count % 100 == 0) { testutil_check(__wt_snprintf( buf, sizeof(buf), "rollback_to_stable: %" PRIu32 " ops repeated", count)); - track(buf, 0ULL, NULL); + track(buf, 0ULL); } } snap_clear_one(snap); } } + + tinfo->session = push_session; } /* Show the final result and check that we're accomplishing some checking. */ testutil_check( __wt_snprintf(buf, sizeof(buf), "rollback_to_stable: %" PRIu32 " ops repeated", count)); - track(buf, 0ULL, NULL); + track(buf, 0ULL); if (count == 0) { #define WARN_RTS_NO_CHECK 5 if (++g.rts_no_check >= WARN_RTS_NO_CHECK) - fprintf(stderr, - "Warning: %" PRIu32 " consecutive runs with no rollback_to_stable checking\n", count); + fprintf( + stderr, "Warning: %u consecutive runs with no rollback_to_stable checking\n", count); } else g.rts_no_check = 0; + + if (session == NULL) + testutil_check(session->close(session, NULL)); } diff --git a/src/third_party/wiredtiger/test/format/t.c b/src/third_party/wiredtiger/test/format/t.c index 3656635e873..b8a926670fd 100644 --- a/src/third_party/wiredtiger/test/format/t.c +++ b/src/third_party/wiredtiger/test/format/t.c @@ -30,17 +30,22 @@ GLOBAL g; +TABLE *tables[V_MAX_TABLES_CONFIG + 1]; /* Table array */ +u_int ntables; + static void format_die(void); static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); extern int __wt_optind; extern char *__wt_optarg; +static void signal_handler(int signo) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void signal_timer(int signo) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); + /* * signal_handler -- * Generic signal handler, report the signal and exit. */ -static void signal_handler(int signo) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void signal_handler(int signo) { @@ -53,7 +58,6 @@ signal_handler(int signo) * signal_timer -- * Alarm signal handler. */ -static void signal_timer(int signo) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void signal_timer(int signo) { @@ -61,7 +65,7 @@ signal_timer(int signo) * Direct I/O configurations can result in really long run times depending on how the test * machine is configured. If a direct I/O run timed out, don't bother dropping core. */ - if (g.c_direct_io) { + if (GV(DISK_DIRECT_IO)) { fprintf(stderr, "format direct I/O configuration timed out\n"); fprintf(stderr, "format caught signal %d, exiting with error\n", signo); fflush(stderr); @@ -118,9 +122,6 @@ format_process_env(void) (void)signal(SIGTERM, signal_handler); #endif - /* Initialize lock to ensure single threading during failure handling */ - testutil_check(pthread_rwlock_init(&g.death_lock, NULL)); - #if 0 /* Configure the GNU malloc for debugging. */ (void)setenv("MALLOC_CHECK_", "2", 1); @@ -135,13 +136,13 @@ format_process_env(void) * TIMED_MAJOR_OP -- * Set a timer and perform a major operation (for example, verify or salvage). */ -#define TIMED_MAJOR_OP(call) \ - do { \ - if (g.c_major_timeout != 0) \ - set_alarm(g.c_major_timeout * 60); \ - call; \ - if (g.c_major_timeout != 0) \ - set_alarm(0); \ +#define TIMED_MAJOR_OP(call) \ + do { \ + if (GV(FORMAT_MAJOR_TIMEOUT) != 0) \ + set_alarm(GV(FORMAT_MAJOR_TIMEOUT) * 60); \ + call; \ + if (GV(FORMAT_MAJOR_TIMEOUT) != 0) \ + set_alarm(0); \ } while (0) int @@ -151,7 +152,7 @@ main(int argc, char *argv[]) u_int ops_seconds; int ch, reps; const char *config, *home; - bool one_flag, quiet_flag; + bool quiet_flag; custom_die = format_die; /* Local death handler. */ @@ -165,15 +166,21 @@ main(int argc, char *argv[]) format_process_env(); - __wt_random_init_seed(NULL, &g.rnd); /* Initialize the RNG. */ + /* + * If built in a branch that doesn't support all current options, configure for backward + * compatibility. + */ +#if WIREDTIGER_VERSION_MAJOR < 10 + g.backward_compatible = true; +#endif /* Set values from the command line. */ home = NULL; - one_flag = quiet_flag = false; + quiet_flag = false; while ((ch = __wt_getopt(progname, argc, argv, "1BC:c:h:qRrT:t")) != EOF) switch (ch) { - case '1': /* One run and quit */ - one_flag = true; + case '1': + /* Ignored for backward compatibility. */ break; case 'B': /* Backward compatibility */ g.backward_compatible = true; @@ -194,11 +201,8 @@ main(int argc, char *argv[]) g.reopen = true; break; case 'T': /* Trace specifics. */ - if (trace_config(__wt_optarg) != 0) { - fprintf(stderr, "unexpected trace configuration \"%s\"\n", __wt_optarg); - usage(); - } - /* FALLTHROUGH */ + trace_config(__wt_optarg); + /* FALLTHROUGH */ case 't': /* Trace */ g.trace = true; break; @@ -207,14 +211,29 @@ main(int argc, char *argv[]) } argv += __wt_optind; + __wt_random_init_seed(NULL, &g.rnd); /* Initialize the RNG. */ + + /* Printable thread ID. */ + testutil_check(__wt_thread_str(g.tidbuf, sizeof(g.tidbuf))); + + /* Initialize lock to ensure single threading during failure handling */ + testutil_check(pthread_rwlock_init(&g.death_lock, NULL)); + + /* + * Initialize the tables array and default to multi-table testing if not in backward-compatible + * mode. + */ + tables[0] = dcalloc(1, sizeof(TABLE)); + tables[0]->id = 1; + g.multi_table_config = !g.backward_compatible; + /* Set up paths. */ path_setup(home); /* - * If it's a reopen, use the already existing home directory's CONFIG file. - * - * If we weren't given a configuration file, set values from "CONFIG", if it exists. Small hack - * to ignore any CONFIG file named ".", that just makes it possible to ignore any local CONFIG + * If it's a reopen, use the already existing home directory's CONFIG file. Otherwise, if we + * weren't given a configuration file, set values from "CONFIG", if it exists. Small hack to + * ignore any CONFIG file named ".", that just makes it possible to ignore any local CONFIG * file, used when running checks. */ if (g.reopen) { @@ -233,16 +252,58 @@ main(int argc, char *argv[]) * which can lead to a lot of hurt if you're not careful. */ for (; *argv != NULL; ++argv) - config_single(*argv, true); + config_single(NULL, *argv, true); /* - * Let the command line -1 and -q flags override values configured from other sources. - * Regardless, don't go all verbose if we're not talking to a terminal. + * Let the command line -q flag override values configured from other sources. Regardless, don't + * go all verbose if we're not talking to a terminal. */ - if (one_flag) - g.c_runs = 1; if (quiet_flag || !isatty(1)) - g.c_quiet = 1; + GV(QUIET) = 1; + + /* Configure the run. */ + config_run(); + g.configured = true; + + /* Initialize locks to single-thread backups and timestamps. */ + lock_init(g.wts_session, &g.backup_lock); + lock_init(g.wts_session, &g.ts_lock); + lock_init(g.wts_session, &g.prepare_commit_lock); + + __wt_seconds(NULL, &start); + track("starting up", 0ULL); + + /* Create and open, or reopen the database. */ + if (g.reopen) { + if (GV(RUNS_IN_MEMORY)) + testutil_die(0, "reopen impossible after in-memory run"); + wts_open(g.home, &g.wts_conn, &g.wts_session, true); + timestamp_init(); + set_oldest_timestamp(); + } else { + wts_create_home(); + config_print(false); + wts_create_database(); + wts_open(g.home, &g.wts_conn, &g.wts_session, true); + timestamp_init(); + } + + trace_init(); /* Initialize operation tracing. */ + + /* + * Initialize key/value information. Load and verify initial records (at least a brief scan if + * not doing a full verify). + */ + tables_apply(key_init, NULL); + tables_apply(val_init, NULL); + if (!g.reopen) + TIMED_MAJOR_OP(tables_apply(wts_load, NULL)); + TIMED_MAJOR_OP(tables_apply(wts_verify, g.wts_conn)); + if (GV(OPS_VERIFY) == 0) + TIMED_MAJOR_OP(tables_apply(wts_read_scan, g.wts_conn)); + + /* Optionally start checkpoints. */ + wts_checkpoints(); /* * Calculate how long each operations loop should run. Take any timer value and convert it to @@ -253,101 +314,58 @@ main(int argc, char *argv[]) * even if we run out of time, otherwise it won't get done. So, in summary pick a reasonable * time and then don't check for timer expiration once the main operations loop completes. */ - ops_seconds = g.c_timer == 0 ? 0 : ((g.c_timer * 60) - 15) / FORMAT_OPERATION_REPS; - - testutil_check(__wt_thread_str(g.tidbuf, sizeof(g.tidbuf))); - - while (++g.run_cnt <= g.c_runs || g.c_runs == 0) { - __wt_seconds(NULL, &start); - track("starting up", 0ULL, NULL); - - config_run(); - - if (g.reopen) { - config_final(); - wts_open(g.home, &g.wts_conn, &g.wts_session, true); - timestamp_init(); - set_oldest_timestamp(); - } else { - wts_create(g.home); - config_final(); - wts_open(g.home, &g.wts_conn, &g.wts_session, true); - timestamp_init(); - - trace_init(); - } - - /* Initialize locks to single-thread backups, failures, and timestamp updates. */ - lock_init(g.wts_session, &g.backup_lock); - lock_init(g.wts_session, &g.ts_lock); - lock_init(g.wts_session, &g.prepare_commit_lock); - - if (!g.reopen) - TIMED_MAJOR_OP(wts_load()); /* Load and verify initial records */ - - TIMED_MAJOR_OP(wts_verify(g.wts_conn, "verify")); - TIMED_MAJOR_OP(wts_read_scan()); - - wts_checkpoints(); + ops_seconds = GV(RUNS_TIMER) == 0 ? 0 : ((GV(RUNS_TIMER) * 60) - 15) / FORMAT_OPERATION_REPS; + for (reps = 1; reps <= FORMAT_OPERATION_REPS; ++reps) + operations(ops_seconds, reps == FORMAT_OPERATION_REPS); - /* Operations. */ - for (reps = 1; reps <= FORMAT_OPERATION_REPS; ++reps) - operations(ops_seconds, reps == FORMAT_OPERATION_REPS); + /* Copy out the run's statistics. */ + TIMED_MAJOR_OP(wts_stats()); - /* Copy out the run's statistics. */ - TIMED_MAJOR_OP(wts_stats()); - - /* - * Verify the objects. Verify closes the underlying handle and discards the statistics, read - * them first. - */ - TIMED_MAJOR_OP(wts_verify(g.wts_conn, "post-ops verify")); - - lock_destroy(g.wts_session, &g.backup_lock); - lock_destroy(g.wts_session, &g.ts_lock); - lock_destroy(g.wts_session, &g.prepare_commit_lock); - - track("shutting down", 0ULL, NULL); - wts_close(&g.wts_conn, &g.wts_session); + /* + * Verify the objects. Verify closes the underlying handle and discards the statistics, read + * them first. + */ + TIMED_MAJOR_OP(tables_apply(wts_verify, g.wts_conn)); - /* - * Salvage testing. - */ - TIMED_MAJOR_OP(wts_salvage()); + track("shutting down", 0ULL); + wts_close(&g.wts_conn, &g.wts_session); - trace_teardown(); + /* Salvage testing. */ + TIMED_MAJOR_OP(tables_apply(wts_salvage, NULL)); - /* Overwrite the progress line with a completion line. */ - if (!g.c_quiet) - printf("\r%78s\r", " "); - __wt_seconds(NULL, &now); - printf("%4" PRIu32 ": %s, %s (%" PRIu64 " seconds)\n", g.run_cnt, g.c_data_source, - g.c_file_type, now - start); - fflush(stdout); - } + trace_teardown(); - config_print(false); + /* Overwrite the progress line with a completion line. */ + if (!GV(QUIET)) + printf("\r%78s\r", " "); + __wt_seconds(NULL, &now); + printf("%s: successful run completed (%" PRIu64 " seconds)\n ", progname, now - start); + fflush(stdout); config_clear(); - printf("%s: successful run completed\n", progname); + lock_destroy(g.wts_session, &g.backup_lock); + lock_destroy(g.wts_session, &g.ts_lock); + lock_destroy(g.wts_session, &g.prepare_commit_lock); return (EXIT_SUCCESS); } /* - * die -- + * format_die -- * Report an error, dumping the configuration. */ static void format_die(void) { /* - * Turn off progress reports so we don't obscure the error message. The lock we're about to - * acquire will act as a barrier to schedule the write. This is really a "best effort" more than - * a guarantee, there's too much stuff in flight to be sure. + * Turn off progress reports and tracing so we don't obscure the error message or drop core when + * using a session that's being closed. The lock we're about to acquire will act as a barrier to + * schedule the write. This is really a "best effort" more than a guarantee, there's too much + * stuff in flight to be sure. */ - g.c_quiet = 1; + GV(QUIET) = 1; + g.trace = 0; /* * Single-thread error handling, our caller exits after calling us (we never release the lock). @@ -359,14 +377,17 @@ format_die(void) fflush(stderr); fflush(stdout); + /* Display the configuration that failed. */ + if (g.configured) + config_print(true); + /* Flush the logs, they may contain debugging information. */ - trace_teardown(); - if (g.c_logging && g.wts_session != NULL) + if (GV(LOGGING) && g.wts_session != NULL) testutil_check(g.wts_session->log_flush(g.wts_session, "sync=off")); - /* Display the configuration that failed. */ - if (g.run_cnt) - config_print(true); + /* Now about to close shared resources, give them a chance to empty. */ + __wt_sleep(2, 0); + trace_teardown(); #ifdef HAVE_DIAGNOSTIC /* @@ -379,7 +400,7 @@ format_die(void) * which can potentially be very large. If it becomes a problem, this can be modified to just * dump out the page this key is on. */ - if (g.c_verify_failure_dump && g.page_dump_cursor != NULL) { + if (GV(RUNS_VERIFY_FAILURE_DUMP) && g.page_dump_cursor != NULL) { set_core_off(); fprintf(stderr, "snapshot-isolation error: Dumping page to %s\n", g.home_pagedump); diff --git a/src/third_party/wiredtiger/test/format/trace.c b/src/third_party/wiredtiger/test/format/trace.c index 24971032279..b365505a05c 100644 --- a/src/third_party/wiredtiger/test/format/trace.c +++ b/src/third_party/wiredtiger/test/format/trace.c @@ -31,10 +31,13 @@ #define TRACE_DIR "OPS.TRACE" #define TRACE_INIT_CMD "rm -rf %s/" TRACE_DIR " && mkdir %s/" TRACE_DIR -int +/* + * trace_config -- + * Configure operation tracing. + */ +void trace_config(const char *config) { - WT_DECL_RET; char *copy, *p; copy = dstrdup(config); @@ -53,15 +56,16 @@ trace_config(const char *config) } for (p = copy; *p != '\0'; ++p) - if (*p != ',' && !__wt_isspace((u_char)*p)) { - ret = EINVAL; - break; - } + if (*p != ',' && !__wt_isspace((u_char)*p)) + testutil_assertfmt(0, "unexpected trace configuration \"%s\"\n", config); free(copy); - return (ret); } +/* + * trace_init -- + * Initialize operation tracing. + */ void trace_init(void) { @@ -76,7 +80,7 @@ trace_init(void) /* Write traces to a separate database by default, optionally write traces to the primary. */ if (g.trace_local) { - if (!g.c_logging) + if (!GV(LOGGING)) testutil_die(EINVAL, "operation logging to the primary database requires logging be configured for that " "database"); @@ -107,6 +111,10 @@ trace_init(void) g.trace_session = session; } +/* + * trace_teardown -- + * Close operation tracing. + */ void trace_teardown(void) { @@ -115,12 +123,14 @@ trace_teardown(void) conn = g.trace_conn; g.trace_conn = NULL; - if (!g.trace || g.trace_local || conn == NULL) - return; - - testutil_check(conn->close(conn, NULL)); + if (conn != NULL) + testutil_check(conn->close(conn, NULL)); } +/* + * trace_ops_init -- + * Per thread operation tracing setup. + */ void trace_ops_init(TINFO *tinfo) { diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index 42e6bb8c2ef..45f4f809815 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -56,85 +56,109 @@ track_ts_dots(u_int dot_count) } /* - * track -- + * track_write -- + * Write out a tracking message. + */ +static void +track_write(char *msg, size_t len) +{ + static size_t last_len; /* callers must be single-threaded */ + + if (last_len > len) { + memset(msg + len, ' ', (size_t)(last_len - len)); + msg[last_len] = '\0'; + } + last_len = len; + + if (printf("%s\r", msg) < 0) + testutil_die(EIO, "printf"); + if (fflush(stdout) == EOF) + testutil_die(errno, "fflush"); +} + +/* + * track_ops -- * Show a status line of operations and time stamp progress. */ void -track(const char *tag, uint64_t cnt, TINFO *tinfo) +track_ops(TINFO *tinfo) { - static size_t last_len; static uint64_t last_cur, last_old, last_stable; static u_int cur_dot_cnt, old_dot_cnt, stable_dot_cnt; size_t len; uint64_t cur_ts, old_ts, stable_ts; char msg[128], ts_msg[64]; - if (g.c_quiet || tag == NULL) + if (GV(QUIET)) return; - if (tinfo == NULL && cnt == 0) - testutil_check( - __wt_snprintf_len_set(msg, sizeof(msg), &len, "%4" PRIu32 ": %s", g.run_cnt, tag)); - else if (tinfo == NULL) - testutil_check(__wt_snprintf_len_set( - msg, sizeof(msg), &len, "%4" PRIu32 ": %s: %" PRIu64, g.run_cnt, tag, cnt)); - else { - ts_msg[0] = '\0'; - if (g.c_txn_timestamps) { - /* - * Don't worry about having a completely consistent set of timestamps. - */ - old_ts = g.oldest_timestamp; - stable_ts = g.stable_timestamp; - cur_ts = g.timestamp; - - if (old_ts != last_old) { - ++old_dot_cnt; - last_old = old_ts; - } - if (stable_ts != last_stable) { - ++stable_dot_cnt; - last_stable = stable_ts; - } - if (cur_ts != last_cur) { - ++cur_dot_cnt; - last_cur = cur_ts; - } - - testutil_check(__wt_snprintf(ts_msg, sizeof(ts_msg), - " old%s" - "stb%s%s" - "ts%s%s", - track_ts_dots(old_dot_cnt), track_ts_diff(old_ts, stable_ts), - track_ts_dots(stable_dot_cnt), track_ts_diff(stable_ts, cur_ts), - track_ts_dots(cur_dot_cnt))); + ts_msg[0] = '\0'; + if (g.transaction_timestamps_config) { + /* + * Don't worry about having a completely consistent set of timestamps. + */ + old_ts = g.oldest_timestamp; + stable_ts = g.stable_timestamp; + cur_ts = g.timestamp; + + if (old_ts != last_old) { + ++old_dot_cnt; + last_old = old_ts; } - testutil_check( - __wt_snprintf_len_set(msg, sizeof(msg), &len, - "%4" PRIu32 ": %s: " - "S %" PRIu64 "%s, " - "I %" PRIu64 "%s, " - "U %" PRIu64 "%s, " - "R %" PRIu64 "%s%s", - g.run_cnt, tag, tinfo->search > M(9) ? tinfo->search / M(1) : tinfo->search, - tinfo->search > M(9) ? "M" : "", - tinfo->insert > M(9) ? tinfo->insert / M(1) : tinfo->insert, - tinfo->insert > M(9) ? "M" : "", - tinfo->update > M(9) ? tinfo->update / M(1) : tinfo->update, - tinfo->update > M(9) ? "M" : "", - tinfo->remove > M(9) ? tinfo->remove / M(1) : tinfo->remove, - tinfo->remove > M(9) ? "M" : "", ts_msg)); - } - if (last_len > len) { - memset(msg + len, ' ', (size_t)(last_len - len)); - msg[last_len] = '\0'; + if (stable_ts != last_stable) { + ++stable_dot_cnt; + last_stable = stable_ts; + } + if (cur_ts != last_cur) { + ++cur_dot_cnt; + last_cur = cur_ts; + } + + testutil_check(__wt_snprintf(ts_msg, sizeof(ts_msg), + " old%s" + "stb%s%s" + "ts%s%s", + track_ts_dots(old_dot_cnt), track_ts_diff(old_ts, stable_ts), + track_ts_dots(stable_dot_cnt), track_ts_diff(stable_ts, cur_ts), + track_ts_dots(cur_dot_cnt))); } - last_len = len; + testutil_check(__wt_snprintf_len_set(msg, sizeof(msg), &len, + "ops: " + "S %" PRIu64 + "%s, " + "I %" PRIu64 + "%s, " + "U %" PRIu64 + "%s, " + "R %" PRIu64 "%s%s", + tinfo->search > M(9) ? tinfo->search / M(1) : tinfo->search, tinfo->search > M(9) ? "M" : "", + tinfo->insert > M(9) ? tinfo->insert / M(1) : tinfo->insert, tinfo->insert > M(9) ? "M" : "", + tinfo->update > M(9) ? tinfo->update / M(1) : tinfo->update, tinfo->update > M(9) ? "M" : "", + tinfo->remove > M(9) ? tinfo->remove / M(1) : tinfo->remove, tinfo->remove > M(9) ? "M" : "", + ts_msg)); + + track_write(msg, len); +} - if (printf("%s\r", msg) < 0) - testutil_die(EIO, "printf"); - if (fflush(stdout) == EOF) - testutil_die(errno, "fflush"); +/* + * track -- + * Show general operation progress. + */ +void +track(const char *tag, uint64_t cnt) +{ + size_t len; + char msg[128]; + + if (GV(QUIET)) + return; + + if (cnt == 0) + testutil_check(__wt_snprintf_len_set(msg, sizeof(msg), &len, "%s", tag)); + else + testutil_check(__wt_snprintf_len_set(msg, sizeof(msg), &len, "%s: %" PRIu64, tag, cnt)); + + track_write(msg, len); } /* @@ -162,12 +186,6 @@ path_setup(const char *home) g.home_key = dmalloc(len); testutil_check(__wt_snprintf(g.home_key, len, "%s/%s", g.home, name)); - /* RNG log file. */ - name = "CONFIG.rand"; - len = strlen(g.home) + strlen(name) + 2; - g.home_rand = dmalloc(len); - testutil_check(__wt_snprintf(g.home_rand, len, "%s/%s", g.home, name)); - /* History store dump file. */ name = "FAIL.HSdump"; len = strlen(g.home) + strlen(name) + 2; @@ -195,7 +213,7 @@ bool fp_readv(FILE *fp, char *name, uint32_t *vp) { u_long ulv; - char *endptr, buf[100]; + char *endptr, buf[64]; if (fgets(buf, sizeof(buf), fp) == NULL) testutil_die(errno, "%s: read-value error", name); @@ -239,7 +257,7 @@ timestamp_parse(const char *p, uint64_t *tsp) } /* - * timestamp_stable -- + * timestamp_init -- * Set the stable timestamp on open. */ void @@ -317,7 +335,8 @@ timestamp(void *arg) WT_CONNECTION *conn; WT_SESSION *session; - (void)(arg); + (void)arg; /* Unused argument */ + conn = g.wts_conn; /* Locks need session */ @@ -390,7 +409,7 @@ lock_init(WT_SESSION *session, RWLOCK *lock) { testutil_assert(lock->lock_type == LOCK_NONE); - if (g.c_wt_mutex) { + if (GV(WIREDTIGER_RWLOCK)) { testutil_check(__wt_rwlock_init((WT_SESSION_IMPL *)session, &lock->l.wt)); lock->lock_type = LOCK_WT; } else { @@ -408,10 +427,9 @@ lock_destroy(WT_SESSION *session, RWLOCK *lock) { testutil_assert(LOCK_INITIALIZED(lock)); - if (lock->lock_type == LOCK_WT) { + if (lock->lock_type == LOCK_WT) __wt_rwlock_destroy((WT_SESSION_IMPL *)session, &lock->l.wt); - } else { + else testutil_check(pthread_rwlock_destroy(&lock->l.pthread)); - } lock->lock_type = LOCK_NONE; } diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index 6bb12e063fe..5dc7847fa53 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -28,74 +28,27 @@ #include "format.h" -/* - * Home directory initialize command: create the directory if it doesn't exist, else remove - * everything except the RNG log file. - * - * Redirect the "cd" command to /dev/null so chatty cd implementations don't add the new working - * directory to our output. - */ -#define FORMAT_HOME_INIT_CMD \ - "test -e %s || mkdir %s; " \ - "cd %s > /dev/null && rm -rf `ls | sed /CONFIG.rand/d`" - -/* - * compressor -- - * Configure compression. - */ -static const char * -compressor(uint32_t compress_flag) -{ - const char *p; - - p = "unrecognized compressor flag"; - switch (compress_flag) { - case COMPRESS_NONE: - p = "none"; - break; - case COMPRESS_LZ4: - p = "lz4"; - break; - case COMPRESS_SNAPPY: - p = "snappy"; - break; - case COMPRESS_ZLIB: - p = "zlib"; - break; - case COMPRESS_ZSTD: - p = "zstd"; - break; - default: - testutil_die(EINVAL, "illegal compression flag: %#" PRIx32, compress_flag); - /* NOTREACHED */ - } - return (p); -} +static void create_object(TABLE *, void *); /* * encryptor -- * Configure encryption. */ static const char * -encryptor(uint32_t encrypt_flag) +encryptor(void) { + char *s; const char *p; - p = "unrecognized encryptor flag"; - switch (encrypt_flag) { - case ENCRYPT_NONE: + s = GVS(DISK_ENCRYPTION); + if (strcmp(s, "none") == 0) p = "none"; - break; - case ENCRYPT_ROTN_7: + else if (strcmp(s, "rotn-7") == 0) p = "rotn,keyid=7"; - break; - case ENCRYPT_SODIUM: + else if (strcmp(s, "sodium") == 0) p = "sodium,secretkey=" SODIUM_TESTKEY; - break; - default: - testutil_die(EINVAL, "illegal encryption flag: %#" PRIx32, encrypt_flag); - /* NOTREACHED */ - } + else + testutil_die(EINVAL, "illegal encryption configuration: %s", s); return (p); } @@ -106,34 +59,35 @@ encryptor(uint32_t encrypt_flag) * This must set any secretkey. When keyids are in use it can return NULL. */ static const char * -encryptor_at_open(uint32_t encrypt_flag) +encryptor_at_open(void) { + char *s; const char *p; - p = NULL; - switch (encrypt_flag) { - case ENCRYPT_NONE: - break; - case ENCRYPT_ROTN_7: - break; - case ENCRYPT_SODIUM: + s = GVS(DISK_ENCRYPTION); + if (strcmp(s, "none") == 0) + p = NULL; + else if (strcmp(s, "rotn-7") == 0) + p = NULL; + else if (strcmp(s, "sodium") == 0) p = "sodium,secretkey=" SODIUM_TESTKEY; - break; - default: - testutil_die(EINVAL, "illegal encryption flag: %#" PRIx32, encrypt_flag); - /* NOTREACHED */ - } + else + testutil_die(EINVAL, "illegal encryption configuration: %s", s); return (p); } +/* + * handle_message -- + * Event handler for verbose and error messages. + */ static int handle_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message) { WT_DECL_RET; int nw; - (void)(handler); - (void)(session); + (void)handler; + (void)session; /* * WiredTiger logs a verbose message when the read timestamp is set to a value older than the @@ -149,14 +103,26 @@ handle_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *messa return (nw < 0 ? EIO : (ret == EOF ? errno : 0)); } +/* + * handle_progress -- + * Event handler for progress messages. + */ static int handle_progress( WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *operation, uint64_t progress) { - (void)(handler); - (void)(session); + char buf[256]; - track(operation, progress, NULL); + (void)handler; + (void)session; + + if (session->app_private == NULL) + track(operation, progress); + else { + testutil_check( + __wt_snprintf(buf, sizeof(buf), "%s %s", (char *)session->app_private, operation)); + track(buf, progress); + } return (0); } @@ -182,39 +148,39 @@ static void configure_timing_stress(char *p, size_t max) { CONFIG_APPEND(p, ",timing_stress_for_test=["); - if (g.c_timing_stress_aggressive_sweep) + if (GV(STRESS_AGGRESSIVE_SWEEP)) CONFIG_APPEND(p, ",aggressive_sweep"); - if (g.c_timing_stress_checkpoint) + if (GV(STRESS_CHECKPOINT)) CONFIG_APPEND(p, ",checkpoint_slow"); - if (g.c_timing_stress_checkpoint_prepare) + if (GV(STRESS_CHECKPOINT_PREPARE)) CONFIG_APPEND(p, ",prepare_checkpoint_delay"); - if (g.c_timing_stress_checkpoint_reserved_txnid_delay) + if (GV(STRESS_CHECKPOINT_RESERVED_TXNID_DELAY)) CONFIG_APPEND(p, ",checkpoint_reserved_txnid_delay"); - if (g.c_timing_stress_failpoint_hs_delete_key_from_ts) + if (GV(STRESS_FAILPOINT_HS_DELETE_KEY_FROM_TS)) CONFIG_APPEND(p, ",failpoint_history_store_delete_key_from_ts"); - if (g.c_timing_stress_failpoint_hs_insert_1) + if (GV(STRESS_FAILPOINT_HS_INSERT_1)) CONFIG_APPEND(p, ",failpoint_history_store_insert_1"); - if (g.c_timing_stress_failpoint_hs_insert_2) + if (GV(STRESS_FAILPOINT_HS_INSERT_2)) CONFIG_APPEND(p, ",failpoint_history_store_insert_2"); - if (g.c_timing_stress_hs_checkpoint_delay) + if (GV(STRESS_HS_CHECKPOINT_DELAY)) CONFIG_APPEND(p, ",history_store_checkpoint_delay"); - if (g.c_timing_stress_hs_search) + if (GV(STRESS_HS_SEARCH)) CONFIG_APPEND(p, ",history_store_search"); - if (g.c_timing_stress_hs_sweep) + if (GV(STRESS_HS_SWEEP)) CONFIG_APPEND(p, ",history_store_sweep_race"); - if (g.c_timing_stress_split_1) + if (GV(STRESS_SPLIT_1)) CONFIG_APPEND(p, ",split_1"); - if (g.c_timing_stress_split_2) + if (GV(STRESS_SPLIT_2)) CONFIG_APPEND(p, ",split_2"); - if (g.c_timing_stress_split_3) + if (GV(STRESS_SPLIT_3)) CONFIG_APPEND(p, ",split_3"); - if (g.c_timing_stress_split_4) + if (GV(STRESS_SPLIT_4)) CONFIG_APPEND(p, ",split_4"); - if (g.c_timing_stress_split_5) + if (GV(STRESS_SPLIT_5)) CONFIG_APPEND(p, ",split_5"); - if (g.c_timing_stress_split_6) + if (GV(STRESS_SPLIT_6)) CONFIG_APPEND(p, ",split_6"); - if (g.c_timing_stress_split_7) + if (GV(STRESS_SPLIT_7)) CONFIG_APPEND(p, ",split_7"); CONFIG_APPEND(p, "]"); } @@ -228,7 +194,7 @@ create_database(const char *home, WT_CONNECTION **connp) { WT_CONNECTION *conn; size_t max; - char config[8 * 1024], *p; + char config[8 * 1024], *p, *s; const char *enc; p = config; @@ -241,72 +207,85 @@ create_database(const char *home, WT_CONNECTION **connp) ",checkpoint_sync=false" ",error_prefix=\"%s\"" ",operation_timeout_ms=2000", - g.c_cache, progname); + GV(CACHE), progname); /* In-memory configuration. */ - if (g.c_in_memory != 0) + if (GV(RUNS_IN_MEMORY) != 0) CONFIG_APPEND(p, ",in_memory=1"); + /* FIXME WT-8314: configuring a block cache corrupts tables. */ +#if 0 + /* Block cache configuration. */ + if (GV(BLOCK_CACHE) != 0) + CONFIG_APPEND(p, + ",block_cache=(enabled=true,type=\"dram\"" + ",cache_on_checkpoint=%s" + ",cache_on_writes=%s" + ",size=%" PRIu32 "MB)", + GV(BLOCK_CACHE_CACHE_ON_CHECKPOINT) == 0 ? "false" : "true", + GV(BLOCK_CACHE_CACHE_ON_WRITES) == 0 ? "false" : "true", GV(BLOCK_CACHE_SIZE)); +#endif + /* LSM configuration. */ - if (DATASOURCE("lsm")) - CONFIG_APPEND(p, ",lsm_manager=(worker_thread_max=%" PRIu32 "),", g.c_lsm_worker_threads); + if (g.lsm_config) + CONFIG_APPEND(p, ",lsm_manager=(worker_thread_max=%" PRIu32 "),", GV(LSM_WORKER_THREADS)); - if (DATASOURCE("lsm") || g.c_cache < 20) + if (g.lsm_config || GV(CACHE) < 20) CONFIG_APPEND(p, ",eviction_dirty_trigger=95"); - /* Eviction worker configuration. */ - if (g.c_evict_max != 0) - CONFIG_APPEND(p, ",eviction=(threads_max=%" PRIu32 ")", g.c_evict_max); + /* Eviction configuration. */ + if (GV(CACHE_EVICT_MAX) != 0) + CONFIG_APPEND(p, ",eviction=(threads_max=%" PRIu32 ")", GV(CACHE_EVICT_MAX)); /* Logging configuration. */ - if (g.c_logging) + if (GV(LOGGING)) { + s = GVS(LOGGING_COMPRESSION); CONFIG_APPEND(p, ",log=(enabled=true,archive=%d,prealloc=%d,file_max=%" PRIu32 ",compressor=\"%s\")", - g.c_logging_archive ? 1 : 0, g.c_logging_prealloc ? 1 : 0, KILOBYTE(g.c_logging_file_max), - compressor(g.c_logging_compression_flag)); + GV(LOGGING_ARCHIVE) ? 1 : 0, GV(LOGGING_PREALLOC) ? 1 : 0, KILOBYTE(GV(LOGGING_FILE_MAX)), + s == NULL ? "none" : s); + } /* Encryption. */ - if (g.c_encryption) { - enc = encryptor(g.c_encryption_flag); - if (enc != NULL) - CONFIG_APPEND(p, ",encryption=(name=%s)", enc); - } + enc = encryptor(); + if (enc != NULL) + CONFIG_APPEND(p, ",encryption=(name=%s)", enc); /* Miscellaneous. */ #ifdef HAVE_POSIX_MEMALIGN CONFIG_APPEND(p, ",buffer_alignment=512"); #endif - if (g.c_mmap) + if (GV(DISK_MMAP)) CONFIG_APPEND(p, ",mmap=1"); - if (g.c_mmap_all) + if (GV(DISK_MMAP_ALL)) CONFIG_APPEND(p, ",mmap_all=1"); - if (g.c_direct_io) + if (GV(DISK_DIRECT_IO)) CONFIG_APPEND(p, ",direct_io=(data)"); - if (g.c_data_extend) + if (GV(DISK_DATA_EXTEND)) CONFIG_APPEND(p, ",file_extend=(data=8MB)"); /* * Run the statistics server and/or maintain statistics in the engine. Sometimes specify a set * of sources just to exercise that code. */ - if (g.c_statistics_server) { - if (mmrand(NULL, 0, 5) == 1 && memcmp(g.uri, "file:", strlen("file:")) == 0) + if (GV(STATISTICS_SERVER)) { + if (mmrand(NULL, 0, 20) == 1) CONFIG_APPEND( p, ",statistics=(fast),statistics_log=(json,on_close,wait=5,sources=(\"file:\"))"); else CONFIG_APPEND(p, ",statistics=(fast),statistics_log=(json,on_close,wait=5)"); } else - CONFIG_APPEND(p, ",statistics=(%s)", g.c_statistics ? "fast" : "none"); + CONFIG_APPEND(p, ",statistics=(%s)", GV(STATISTICS) ? "fast" : "none"); /* Optional timing stress. */ configure_timing_stress(p, max); /* Extensions. */ CONFIG_APPEND(p, ",extensions=[\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],", - g.c_reverse ? REVERSE_PATH : "", access(LZ4_PATH, R_OK) == 0 ? LZ4_PATH : "", + REVERSE_PATH, access(LZ4_PATH, R_OK) == 0 ? LZ4_PATH : "", access(ROTN_PATH, R_OK) == 0 ? ROTN_PATH : "", access(SNAPPY_PATH, R_OK) == 0 ? SNAPPY_PATH : "", access(ZLIB_PATH, R_OK) == 0 ? ZLIB_PATH : "", access(ZSTD_PATH, R_OK) == 0 ? ZSTD_PATH : "", @@ -316,8 +295,9 @@ create_database(const char *home, WT_CONNECTION **connp) * Put configuration file configuration options second to last. Put command line configuration * options at the end. Do this so they override the standard configuration. */ - if (g.c_config_open != NULL) - CONFIG_APPEND(p, ",%s", g.c_config_open); + s = GVS(WIREDTIGER_CONFIG); + if (s != NULL) + CONFIG_APPEND(p, ",%s", s); if (g.config_open != NULL) CONFIG_APPEND(p, ",%s", g.config_open); @@ -334,102 +314,96 @@ create_database(const char *home, WT_CONNECTION **connp) * Create the database object. */ static void -create_object(WT_CONNECTION *conn) +create_object(TABLE *table, void *arg) { + WT_CONNECTION *conn; WT_SESSION *session; size_t max; uint32_t maxintlkey, maxleafkey, maxleafvalue; - char config[4096], *p; + char config[4096], *p, *s; + conn = (WT_CONNECTION *)arg; p = config; max = sizeof(config); +/* The page must be a multiple of the allocation size, and 512 always works. */ +#define BLOCK_ALLOCATION_SIZE 512 CONFIG_APPEND(p, "key_format=%s,allocation_size=%d,%s,internal_page_max=%" PRIu32 ",leaf_page_max=%" PRIu32 ",memory_page_max=%" PRIu32, - (g.type == ROW) ? "u" : "r", BLOCK_ALLOCATION_SIZE, - g.c_firstfit ? "block_allocation=first" : "", g.intl_page_max, g.leaf_page_max, - MEGABYTE(g.c_memory_page_max)); + (table->type == ROW) ? "u" : "r", BLOCK_ALLOCATION_SIZE, + TV(DISK_FIRSTFIT) ? "block_allocation=first" : "", table->max_intl_page, table->max_leaf_page, + table->max_mem_page); /* * Configure the maximum key/value sizes, but leave it as the default if we come up with * something crazy. */ - maxintlkey = mmrand(NULL, g.intl_page_max / 50, g.intl_page_max / 40); + maxintlkey = mmrand(NULL, table->max_intl_page / 50, table->max_intl_page / 40); if (maxintlkey > 20) CONFIG_APPEND(p, ",internal_key_max=%" PRIu32, maxintlkey); - maxleafkey = mmrand(NULL, g.leaf_page_max / 50, g.leaf_page_max / 40); + maxleafkey = mmrand(NULL, table->max_leaf_page / 50, table->max_leaf_page / 40); if (maxleafkey > 20) CONFIG_APPEND(p, ",leaf_key_max=%" PRIu32, maxleafkey); - maxleafvalue = mmrand(NULL, g.leaf_page_max * 10, g.leaf_page_max / 40); + maxleafvalue = mmrand(NULL, table->max_leaf_page * 10, table->max_leaf_page / 40); if (maxleafvalue > 40 && maxleafvalue < 100 * 1024) CONFIG_APPEND(p, ",leaf_value_max=%" PRIu32, maxleafvalue); - switch (g.type) { + switch (table->type) { case FIX: - CONFIG_APPEND(p, ",value_format=%" PRIu32 "t", g.c_bitcnt); + CONFIG_APPEND(p, ",value_format=%" PRIu32 "t", TV(BTREE_BITCNT)); break; case ROW: CONFIG_APPEND(p, ",prefix_compression=%s,prefix_compression_min=%" PRIu32, - g.c_prefix_compression == 0 ? "false" : "true", g.c_prefix_compression_min); - if (g.c_reverse) + TV(BTREE_PREFIX_COMPRESSION) == 0 ? "false" : "true", TV(BTREE_PREFIX_COMPRESSION_MIN)); + if (TV(BTREE_REVERSE)) CONFIG_APPEND(p, ",collator=reverse"); /* FALLTHROUGH */ case VAR: - if (g.c_huffman_value) + if (TV(BTREE_HUFFMAN_VALUE)) CONFIG_APPEND(p, ",huffman_value=english"); - if (g.c_dictionary) + if (TV(BTREE_DICTIONARY)) CONFIG_APPEND(p, ",dictionary=%" PRIu32, mmrand(NULL, 123, 517)); break; } /* Configure checksums. */ - switch (g.c_checksum_flag) { - case CHECKSUM_OFF: - CONFIG_APPEND(p, ",checksum=\"off\""); - break; - case CHECKSUM_ON: - CONFIG_APPEND(p, ",checksum=\"on\""); - break; - case CHECKSUM_UNCOMPRESSED: - CONFIG_APPEND(p, ",checksum=\"uncompressed\""); - break; - case CHECKSUM_UNENCRYPTED: - CONFIG_APPEND(p, ",checksum=\"unencrypted\""); - break; - } + if ((s = TVS(DISK_CHECKSUM)) != NULL) + CONFIG_APPEND(p, ",checksum=\"%s\"", s); /* Configure compression. */ - if (g.c_compression_flag != COMPRESS_NONE) - CONFIG_APPEND(p, ",block_compressor=\"%s\"", compressor(g.c_compression_flag)); + if ((s = TVS(BTREE_COMPRESSION)) != NULL) + CONFIG_APPEND(p, ",block_compressor=\"%s\"", s); /* Configure Btree. */ - CONFIG_APPEND(p, ",internal_key_truncate=%s", g.c_internal_key_truncation ? "true" : "false"); - CONFIG_APPEND(p, ",split_pct=%" PRIu32, g.c_split_pct); + CONFIG_APPEND( + p, ",internal_key_truncate=%s", TV(BTREE_INTERNAL_KEY_TRUNCATION) ? "true" : "false"); + CONFIG_APPEND(p, ",split_pct=%" PRIu32, TV(BTREE_SPLIT_PCT)); /* Assertions: assertions slow down the code for additional diagnostic checking. */ - if (g.c_assert_read_timestamp) - CONFIG_APPEND(p, ",assert=(read_timestamp=%s)", g.c_txn_timestamps ? "always" : "never"); - if (g.c_assert_write_timestamp) + if (GV(ASSERT_READ_TIMESTAMP)) + CONFIG_APPEND( + p, ",assert=(read_timestamp=%s)", g.transaction_timestamps_config ? "always" : "never"); + if (GV(ASSERT_WRITE_TIMESTAMP)) CONFIG_APPEND(p, ",assert=(write_timestamp=on),write_timestamp_usage=%s", - g.c_txn_timestamps ? "key_consistent" : "never"); + g.transaction_timestamps_config ? "key_consistent" : "never"); /* Configure LSM. */ - if (DATASOURCE("lsm")) { + if (DATASOURCE(table, "lsm")) { CONFIG_APPEND(p, ",type=lsm,lsm=("); - CONFIG_APPEND(p, "auto_throttle=%s,", g.c_auto_throttle ? "true" : "false"); - CONFIG_APPEND(p, "chunk_size=%" PRIu32 "MB,", g.c_chunk_size); + CONFIG_APPEND(p, "auto_throttle=%s,", TV(LSM_AUTO_THROTTLE) ? "true" : "false"); + CONFIG_APPEND(p, "chunk_size=%" PRIu32 "MB,", TV(LSM_CHUNK_SIZE)); /* * We can't set bloom_oldest without bloom, and we want to test with Bloom filters on most * of the time anyway. */ - if (g.c_bloom_oldest) - g.c_bloom = 1; - CONFIG_APPEND(p, "bloom=%s,", g.c_bloom ? "true" : "false"); - CONFIG_APPEND(p, "bloom_bit_count=%" PRIu32 ",", g.c_bloom_bit_count); - CONFIG_APPEND(p, "bloom_hash_count=%" PRIu32 ",", g.c_bloom_hash_count); - CONFIG_APPEND(p, "bloom_oldest=%s,", g.c_bloom_oldest ? "true" : "false"); - CONFIG_APPEND(p, "merge_max=%" PRIu32 ",", g.c_merge_max); + if (TV(LSM_BLOOM_OLDEST)) + TV(LSM_BLOOM) = 1; + CONFIG_APPEND(p, "bloom=%s,", TV(LSM_BLOOM) ? "true" : "false"); + CONFIG_APPEND(p, "bloom_bit_count=%" PRIu32 ",", TV(LSM_BLOOM_BIT_COUNT)); + CONFIG_APPEND(p, "bloom_hash_count=%" PRIu32 ",", TV(LSM_BLOOM_HASH_COUNT)); + CONFIG_APPEND(p, "bloom_oldest=%s,", TV(LSM_BLOOM_OLDEST) ? "true" : "false"); + CONFIG_APPEND(p, "merge_max=%" PRIu32 ",", TV(LSM_MERGE_MAX)); CONFIG_APPEND(p, ",)"); } @@ -440,35 +414,41 @@ create_object(WT_CONNECTION *conn) * Create the underlying store. */ testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_checkfmt(session->create(session, g.uri, config), "%s", g.uri); + testutil_checkfmt(session->create(session, table->uri, config), "%s", table->uri); testutil_check(session->close(session, NULL)); } /* - * wts_create -- - * Create the database home and objects. + * wts_create_home -- + * Remove and re-create the directory. */ void -wts_create(const char *home) +wts_create_home(void) +{ + char buf[MAX_FORMAT_PATH * 2]; + + testutil_check(__wt_snprintf(buf, sizeof(buf), "rm -rf %s && mkdir %s", g.home, g.home)); + testutil_checkfmt(system(buf), "database home creation (\"%s\") failed", buf); +} + +/* + * wts_create_database -- + * Create the database. + */ +void +wts_create_database(void) { WT_CONNECTION *conn; - WT_DECL_RET; - size_t len; - char *cmd; - - len = strlen(g.home) * 3 + strlen(FORMAT_HOME_INIT_CMD) + 1; - cmd = dmalloc(len); - testutil_check(__wt_snprintf(cmd, len, FORMAT_HOME_INIT_CMD, g.home, g.home, g.home)); - if ((ret = system(cmd)) != 0) - testutil_die(ret, "home initialization (\"%s\") failed", cmd); - free(cmd); - - create_database(home, &conn); - create_object(conn); - if (g.c_in_memory != 0) - g.wts_conn_inmemory = conn; + + create_database(g.home, &conn); + + g.wts_conn = conn; + tables_apply(create_object, g.wts_conn); + if (GV(RUNS_IN_MEMORY) != 0) + g.wts_conn_inmemory = g.wts_conn; else testutil_check(conn->close(conn, NULL)); + g.wts_conn = NULL; } /* @@ -491,7 +471,7 @@ wts_open(const char *home, WT_CONNECTION **connp, WT_SESSION **sessionp, bool al config[0] = '\0'; /* Configuration settings that are not persistent between open calls. */ - enc = encryptor_at_open(g.c_encryption_flag); + enc = encryptor_at_open(); if (enc != NULL) CONFIG_APPEND(p, ",encryption=(name=%s)", enc); @@ -501,11 +481,11 @@ wts_open(const char *home, WT_CONNECTION **connp, WT_SESSION **sessionp, bool al configure_timing_stress(p, max); /* If in-memory, there's only a single, shared WT_CONNECTION handle. */ - if (g.c_in_memory != 0) + if (GV(RUNS_IN_MEMORY) != 0) conn = g.wts_conn_inmemory; else { #if WIREDTIGER_VERSION_MAJOR >= 10 - if (g.c_verify && allow_verify) + if (GV(OPS_VERIFY) && allow_verify) CONFIG_APPEND(p, ",verify_metadata=true"); #else WT_UNUSED(allow_verify); @@ -517,6 +497,10 @@ wts_open(const char *home, WT_CONNECTION **connp, WT_SESSION **sessionp, bool al *connp = conn; } +/* + * wts_close -- + * Close the open database. + */ void wts_close(WT_CONNECTION **connp, WT_SESSION **sessionp) { @@ -538,34 +522,74 @@ wts_close(WT_CONNECTION **connp, WT_SESSION **sessionp) if (g.backward_compatible) testutil_check(conn->reconfigure(conn, "compatibility=(release=3.3)")); - testutil_check(conn->close(conn, g.c_leak_memory ? "leak_memory" : NULL)); + testutil_check(conn->close(conn, GV(WIREDTIGER_LEAK_MEMORY) ? "leak_memory" : NULL)); } +/* + * wts_verify -- + * Verify a table. + */ void -wts_verify(WT_CONNECTION *conn, const char *tag) +wts_verify(TABLE *table, void *arg) { + WT_CONNECTION *conn; WT_DECL_RET; WT_SESSION *session; - if (g.c_verify == 0) - return; - - track("verify", 0ULL, NULL); + conn = (WT_CONNECTION *)arg; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - trace_msg("%s", "=============== verify start"); + if (GV(OPS_VERIFY) == 0) + return; /* * Verify can return EBUSY if the handle isn't available. Don't yield and retry, in the case of * LSM, the handle may not be available for a long time. */ - ret = session->verify(session, g.uri, "strict"); - testutil_assertfmt(ret == 0 || ret == EBUSY, "session.verify: %s: %s", g.uri, tag); - - trace_msg("%s", "=============== verify stop"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + session->app_private = table->track_prefix; + ret = session->verify(session, table->uri, "strict"); + testutil_assert(ret == 0 || ret == EBUSY); testutil_check(session->close(session, NULL)); } +struct stats_args { + FILE *fp; + WT_SESSION *session; +}; + +/* + * stats_data_source -- + * Dump each data source's statistics. + */ +static void +stats_data_source(TABLE *table, void *arg) +{ + struct stats_args *args; + FILE *fp; + WT_CURSOR *cursor; + WT_DECL_RET; + WT_SESSION *session; + uint64_t v; + char buf[1024]; + const char *desc, *pval; + + args = arg; + fp = args->fp; + session = args->session; + + fprintf(fp, "\n\n====== Data source statistics: %s\n", table->uri); + + testutil_check(__wt_snprintf(buf, sizeof(buf), "statistics:%s", table->uri)); + testutil_check(session->open_cursor(session, buf, NULL, NULL, &cursor)); + while ( + (ret = cursor->next(cursor)) == 0 && (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) + if (fprintf(fp, "%s=%s\n", desc, pval) < 0) + testutil_die(errno, "fprintf"); + + testutil_assert(ret == WT_NOTFOUND); + testutil_check(cursor->close(cursor)); +} + /* * wts_stats -- * Dump the run's statistics. @@ -573,22 +597,21 @@ wts_verify(WT_CONNECTION *conn, const char *tag) void wts_stats(void) { + struct stats_args args; FILE *fp; WT_CONNECTION *conn; WT_CURSOR *cursor; WT_DECL_RET; WT_SESSION *session; - size_t len; uint64_t v; - char *stat_name; const char *desc, *pval; /* Ignore statistics if they're not configured. */ - if (g.c_statistics == 0) + if (GV(STATISTICS) == 0) return; conn = g.wts_conn; - track("stat", 0ULL, NULL); + track("stat", 0ULL); testutil_check(conn->open_session(conn, NULL, NULL, &session)); @@ -603,27 +626,13 @@ wts_stats(void) (ret = cursor->next(cursor)) == 0 && (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) if (fprintf(fp, "%s=%s\n", desc, pval) < 0) testutil_die(errno, "fprintf"); - - if (ret != WT_NOTFOUND) - testutil_die(ret, "cursor.next"); + testutil_assert(ret == WT_NOTFOUND); testutil_check(cursor->close(cursor)); /* Data source statistics. */ - fprintf(fp, "\n\n====== Data source statistics:\n"); - len = strlen("statistics:") + strlen(g.uri) + 1; - stat_name = dmalloc(len); - testutil_check(__wt_snprintf(stat_name, len, "statistics:%s", g.uri)); - testutil_check(session->open_cursor(session, stat_name, NULL, NULL, &cursor)); - free(stat_name); - - while ( - (ret = cursor->next(cursor)) == 0 && (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) - if (fprintf(fp, "%s=%s\n", desc, pval) < 0) - testutil_die(errno, "fprintf"); - - if (ret != WT_NOTFOUND) - testutil_die(ret, "cursor.next"); - testutil_check(cursor->close(cursor)); + args.fp = fp; + args.session = session; + tables_apply(stats_data_source, &args); fclose_and_clear(&fp); diff --git a/src/third_party/wiredtiger/test/utility/misc.c b/src/third_party/wiredtiger/test/utility/misc.c index 4826a0b02ff..4d2d989a997 100644 --- a/src/third_party/wiredtiger/test/utility/misc.c +++ b/src/third_party/wiredtiger/test/utility/misc.c @@ -69,7 +69,7 @@ testutil_die(int e, const char *fmt, ...) fprintf(stderr, "\n"); fprintf(stderr, "%s: process aborting\n", progname); - abort(); + __wt_abort(NULL); } /* diff --git a/src/third_party/wiredtiger/test/utility/test_util.h b/src/third_party/wiredtiger/test/utility/test_util.h index 54444684ddf..31283084400 100644 --- a/src/third_party/wiredtiger/test/utility/test_util.h +++ b/src/third_party/wiredtiger/test/utility/test_util.h @@ -245,7 +245,8 @@ extern void (*custom_die)(void); #ifdef _WIN32 __declspec(noreturn) #endif - void testutil_die(int, const char *, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); + void testutil_die(int, const char *, ...) WT_GCC_FUNC_ATTRIBUTE((cold)) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); void *dcalloc(size_t, size_t); void *dmalloc(size_t); |