diff options
Diffstat (limited to 'src/third_party/wiredtiger/test')
| -rw-r--r-- | src/third_party/wiredtiger/test/format/backup.c | 6 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/test/format/bulk.c | 2 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/test/format/checkpoint.c | 6 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/test/format/config.c | 70 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/test/format/config.h | 4 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/test/format/format.h | 31 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/test/format/format.i | 48 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/test/format/ops.c | 26 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/test/format/t.c | 8 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/test/format/util.c | 56 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/test/multiversion/wt_multiversion.sh | 40 | ||||
| -rw-r--r-- | src/third_party/wiredtiger/test/packing/intpack-test3.c | 4 |
12 files changed, 237 insertions, 64 deletions
diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c index 52a79777ca4..5ad1cfe65dc 100644 --- a/src/third_party/wiredtiger/test/format/backup.c +++ b/src/third_party/wiredtiger/test/format/backup.c @@ -379,9 +379,9 @@ backup(void *arg) * with named checkpoints. Wait for the checkpoint to complete, otherwise backups might be * starved out. */ - testutil_check(pthread_rwlock_wrlock(&g.backup_lock)); + lock_writelock(session, &g.backup_lock); if (g.workers_finished) { - testutil_check(pthread_rwlock_unlock(&g.backup_lock)); + lock_writeunlock(session, &g.backup_lock); break; } @@ -471,7 +471,7 @@ backup(void *arg) testutil_check(session->truncate(session, "log:", backup_cursor, NULL, NULL)); testutil_check(backup_cursor->close(backup_cursor)); - testutil_check(pthread_rwlock_unlock(&g.backup_lock)); + lock_writeunlock(session, &g.backup_lock); active_files_sort(active_now); active_files_remove_missing(active_prev, active_now); active_prev = active_now; diff --git a/src/third_party/wiredtiger/test/format/bulk.c b/src/third_party/wiredtiger/test/format/bulk.c index a1d9256e707..0f41a311e43 100644 --- a/src/third_party/wiredtiger/test/format/bulk.c +++ b/src/third_party/wiredtiger/test/format/bulk.c @@ -59,7 +59,7 @@ bulk_commit_transaction(WT_SESSION *session) testutil_check(session->commit_transaction(session, buf)); /* Update the oldest timestamp, otherwise updates are pinned in memory. */ - timestamp_once(); + timestamp_once(session); } /* diff --git a/src/third_party/wiredtiger/test/format/checkpoint.c b/src/third_party/wiredtiger/test/format/checkpoint.c index 0542bb8e54b..36e70ae3125 100644 --- a/src/third_party/wiredtiger/test/format/checkpoint.c +++ b/src/third_party/wiredtiger/test/format/checkpoint.c @@ -85,7 +85,7 @@ checkpoint(void *arg) * few names to test multiple named snapshots in * the system. */ - ret = pthread_rwlock_trywrlock(&g.backup_lock); + ret = lock_try_writelock(session, &g.backup_lock); if (ret == 0) { backup_locked = true; testutil_check(__wt_snprintf( @@ -98,7 +98,7 @@ checkpoint(void *arg) /* * 5% drop all named snapshots. */ - ret = pthread_rwlock_trywrlock(&g.backup_lock); + ret = lock_try_writelock(session, &g.backup_lock); if (ret == 0) { backup_locked = true; ckpt_config = "drop=(all)"; @@ -110,7 +110,7 @@ checkpoint(void *arg) testutil_check(session->checkpoint(session, ckpt_config)); if (backup_locked) - testutil_check(pthread_rwlock_unlock(&g.backup_lock)); + lock_writeunlock(session, &g.backup_lock); secs = mmrand(NULL, 5, 40); } diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index ae104bae989..54a09229ce4 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -30,7 +30,7 @@ #include "config.h" static void config(void); -static void config_backup(void); +static void config_backup_incr(void); static void config_backward_compatible(void); static void config_cache(void); static void config_checkpoint(void); @@ -197,7 +197,7 @@ config(void) config_transaction(); /* Simple selection. */ - config_backup(); + config_backup_incr(); config_checkpoint(); config_checksum(); config_compression("btree.compression"); @@ -257,39 +257,55 @@ config(void) } /* - * config_backup -- - * Backup configuration. + * config_backup_incr -- + * Incremental backup configuration. */ static void -config_backup(void) +config_backup_incr(void) { - const char *cstr; + /* Incremental backup requires backup. */ + if (g.c_backups == 0) + return; /* - * Choose a type of incremental backup. + * Incremental backup using log files is incompatible with logging archival. Testing log file + * archival doesn't seem as useful as testing backup, let the backup configuration override. */ - if (!config_is_perm("backup.incremental")) { - cstr = "backup.incremental=off"; - switch (mmrand(NULL, 1, 10)) { - case 1: /* 30% full backup only */ - case 2: - case 3: - break; - case 4: /* 40% block based incremental */ - case 5: - case 6: - case 7: - cstr = "backup.incremental=block"; - break; - case 8: - case 9: - case 10: /* 30% log based incremental */ - if (!g.c_logging_archive) - cstr = "backup.incremental=log"; - break; + if (config_is_perm("backup.incremental")) { + if (g.c_backup_incr_flag == INCREMENTAL_LOG) { + if (g.c_logging_archive && config_is_perm("logging.archive")) + testutil_die(EINVAL, "backup.incremental=log is incompatible with logging.archive"); + if (g.c_logging_archive) + config_single("logging.archive=0", false); } + return; + } - config_single(cstr, false); + /* + * Choose a type of incremental backup, where the log archival setting can eliminate incremental + * backup based on log files. + */ + switch (mmrand(NULL, 1, 10)) { + case 1: /* 30% full backup only */ + case 2: + case 3: + config_single("backup.incremental=off", false); + break; + case 4: /* 30% log based incremental */ + case 5: + case 6: + if (!g.c_logging_archive || !config_is_perm("logging.archive")) { + if (g.c_logging_archive) + config_single("logging.archive=0", false); + config_single("backup.incremental=log", false); + } + /* FALLTHROUGH */ + case 7: /* 40% block based incremental */ + case 8: + case 9: + case 10: + config_single("backup.incremental=block", false); + break; } } diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index 59dc7cfd86c..04da641386b 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -336,6 +336,10 @@ static CONFIG c[] = { {"wiredtiger.config", "configuration string used to wiredtiger_open", C_IGNORE | C_STRING, 0, 0, 0, NULL, &g.c_config_open}, + /* 80% */ + {"wiredtiger.rwlock", "if wiredtiger read/write mutexes should be used", C_BOOL, 80, 0, 0, + &g.c_wt_mutex, NULL}, + {"wiredtiger.leak_memory", "if memory should be leaked on close", C_BOOL, 0, 0, 0, &g.c_leak_memory, NULL}, diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index b96b858f0f8..6bc213a65ef 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -60,6 +60,20 @@ #define MAX_MODIFY_ENTRIES 5 /* maximum change vectors */ +/* + * Abstract lock that lets us use either pthread reader-writer locks or WiredTiger's own (likely + * faster) implementation. + */ +typedef struct { + union { + WT_RWLOCK wt; + pthread_rwlock_t pthread; + } l; + enum { LOCK_NONE = 0, LOCK_WT, LOCK_PTHREAD } lock_type; +} RWLOCK; + +#define LOCK_INITIALIZED(lock) ((lock)->lock_type != LOCK_NONE) + typedef struct { WT_CONNECTION *wts_conn; WT_EXTENSION_API *wt_api; @@ -92,8 +106,8 @@ typedef struct { bool logging; /* log operations */ FILE *logfp; /* log file */ - pthread_rwlock_t backup_lock; /* Backup running */ - uint64_t backup_id; /* Block incremental id */ + RWLOCK backup_lock; /* Backup running */ + uint64_t backup_id; /* Block incremental id */ WT_RAND_STATE rnd; /* Global RNG state */ @@ -104,13 +118,17 @@ typedef struct { * We get the last committed timestamp periodically in order to update the oldest timestamp, * that requires locking out transactional ops that set a timestamp. */ - pthread_rwlock_t ts_lock; + RWLOCK ts_lock; uint64_t timestamp; /* Counter for timestamps */ uint64_t truncate_cnt; /* Counter for truncation */ - pthread_rwlock_t death_lock; /* Single-thread failure */ + /* + * Single-thread failure. Always use pthread lock rather than WT lock in case WT library is + * misbehaving. + */ + pthread_rwlock_t death_lock; uint32_t c_abort; /* Config values */ uint32_t c_alter; @@ -204,6 +222,7 @@ typedef struct { uint32_t c_value_min; uint32_t c_verify; uint32_t c_write_pct; + uint32_t c_wt_mutex; #define FIX 1 #define ROW 2 @@ -351,6 +370,8 @@ void key_gen_common(WT_ITEM *, uint64_t, const char *); void key_gen_init(WT_ITEM *); void key_gen_teardown(WT_ITEM *); void key_init(void); +void lock_destroy(WT_SESSION *, RWLOCK *); +void lock_init(WT_SESSION *, RWLOCK *); void operations(u_int, bool); WT_THREAD_RET random_kv(void *); void path_setup(const char *); @@ -364,7 +385,7 @@ int snap_repeat_txn(WT_CURSOR *, TINFO *); void snap_repeat_update(TINFO *, bool); void snap_track(TINFO *, thread_op); WT_THREAD_RET timestamp(void *); -void timestamp_once(void); +void timestamp_once(WT_SESSION *); void track(const char *, uint64_t, TINFO *); void val_gen(WT_RAND_STATE *, WT_ITEM *, uint64_t); void val_gen_init(WT_ITEM *); diff --git a/src/third_party/wiredtiger/test/format/format.i b/src/third_party/wiredtiger/test/format/format.i index 00099e1c4cf..661dd096ae9 100644 --- a/src/third_party/wiredtiger/test/format/format.i +++ b/src/third_party/wiredtiger/test/format/format.i @@ -162,3 +162,51 @@ key_gen_insert(WT_RAND_STATE *rnd, WT_ITEM *key, uint64_t keyno) key_gen_common(key, keyno, suffix[mmrand(rnd, 0, 14)]); } + +/* + * lock_try_writelock + * Try to get exclusive lock. Fail immediately if not available. + */ +static inline int +lock_try_writelock(WT_SESSION *session, RWLOCK *lock) +{ + testutil_assert(LOCK_INITIALIZED(lock)); + + if (lock->lock_type == LOCK_WT) { + return (__wt_try_writelock((WT_SESSION_IMPL *)session, &lock->l.wt)); + } else { + return (pthread_rwlock_trywrlock(&lock->l.pthread)); + } +} + +/* + * lock_writelock -- + * Wait to get exclusive lock. + */ +static inline void +lock_writelock(WT_SESSION *session, RWLOCK *lock) +{ + testutil_assert(LOCK_INITIALIZED(lock)); + + if (lock->lock_type == LOCK_WT) { + __wt_writelock((WT_SESSION_IMPL *)session, &lock->l.wt); + } else { + testutil_check(pthread_rwlock_wrlock(&lock->l.pthread)); + } +} + +/* + * lock_writeunlock -- + * Release an exclusive lock. + */ +static inline void +lock_writeunlock(WT_SESSION *session, RWLOCK *lock) +{ + testutil_assert(LOCK_INITIALIZED(lock)); + + if (lock->lock_type == LOCK_WT) { + __wt_writeunlock((WT_SESSION_IMPL *)session, &lock->l.wt); + } else { + testutil_check(pthread_rwlock_unlock(&lock->l.pthread)); + } +} diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 0d679eb4cc3..6a668fa4f45 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -158,11 +158,13 @@ operations(u_int ops_seconds, bool lastrun) if (!SINGLETHREADED) g.rand_log_stop = true; - /* Logging requires a session. */ - if (g.logging) - testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); logop(session, "%s", "=============== thread ops start"); + /* Initialize locks to single-thread backups, failures, and timestamp updates. */ + lock_init(session, &g.backup_lock); + lock_init(session, &g.ts_lock); + /* * Create the per-thread structures and start the worker threads. Allocate the thread structures * separately to minimize false sharing. @@ -295,9 +297,11 @@ operations(u_int ops_seconds, bool lastrun) testutil_check(__wt_thread_join(NULL, ×tamp_tid)); g.workers_finished = false; + lock_destroy(session, &g.backup_lock); + lock_destroy(session, &g.ts_lock); + logop(session, "%s", "=============== thread ops stop"); - if (g.logging) - testutil_check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); for (i = 0; i < g.c_threads; ++i) { tinfo = tinfo_list[i]; @@ -372,13 +376,13 @@ begin_transaction_ts(TINFO *tinfo, u_int *iso_configp) * * Lock out the oldest timestamp update. */ - testutil_check(pthread_rwlock_wrlock(&g.ts_lock)); + lock_writelock(session, &g.ts_lock); ts = __wt_atomic_addv64(&g.timestamp, 1); testutil_check(__wt_snprintf(buf, sizeof(buf), "read_timestamp=%" PRIx64, ts)); testutil_check(session->timestamp_transaction(session, buf)); - testutil_check(pthread_rwlock_unlock(&g.ts_lock)); + lock_writeunlock(session, &g.ts_lock); snap_init(tinfo, ts, false); logop(session, "begin snapshot read-ts=%" PRIu64 " (not repeatable)", ts); @@ -443,7 +447,7 @@ commit_transaction(TINFO *tinfo, bool prepared) ts = 0; /* -Wconditional-uninitialized */ if (g.c_txn_timestamps) { /* Lock out the oldest timestamp update. */ - testutil_check(pthread_rwlock_wrlock(&g.ts_lock)); + lock_writelock(session, &g.ts_lock); ts = __wt_atomic_addv64(&g.timestamp, 1); testutil_check(__wt_snprintf(buf, sizeof(buf), "commit_timestamp=%" PRIx64, ts)); @@ -454,7 +458,7 @@ commit_transaction(TINFO *tinfo, bool prepared) testutil_check(session->timestamp_transaction(session, buf)); } - testutil_check(pthread_rwlock_unlock(&g.ts_lock)); + lock_writeunlock(session, &g.ts_lock); } testutil_check(session->commit_transaction(session, NULL)); @@ -509,7 +513,7 @@ prepare_transaction(TINFO *tinfo) * * Lock out the oldest timestamp update. */ - testutil_check(pthread_rwlock_wrlock(&g.ts_lock)); + lock_writelock(session, &g.ts_lock); ts = __wt_atomic_addv64(&g.timestamp, 1); testutil_check(__wt_snprintf(buf, sizeof(buf), "prepare_timestamp=%" PRIx64, ts)); @@ -517,7 +521,7 @@ prepare_transaction(TINFO *tinfo) logop(session, "prepare ts=%" PRIu64, ts); - testutil_check(pthread_rwlock_unlock(&g.ts_lock)); + lock_writeunlock(session, &g.ts_lock); return (ret); } diff --git a/src/third_party/wiredtiger/test/format/t.c b/src/third_party/wiredtiger/test/format/t.c index 6d897131f28..a2fcf405cf9 100644 --- a/src/third_party/wiredtiger/test/format/t.c +++ b/src/third_party/wiredtiger/test/format/t.c @@ -118,10 +118,8 @@ format_process_env(void) (void)signal(SIGTERM, signal_handler); #endif - /* Initialize locks to single-thread backups, failures, and timestamp updates. */ - testutil_check(pthread_rwlock_init(&g.backup_lock, NULL)); + /* Initialize lock to ensure single threading during failure handling */ testutil_check(pthread_rwlock_init(&g.death_lock, NULL)); - testutil_check(pthread_rwlock_init(&g.ts_lock, NULL)); #if 0 /* Configure the GNU malloc for debugging. */ @@ -328,10 +326,6 @@ main(int argc, char *argv[]) config_print(false); - testutil_check(pthread_rwlock_destroy(&g.backup_lock)); - testutil_check(pthread_rwlock_destroy(&g.death_lock)); - testutil_check(pthread_rwlock_destroy(&g.ts_lock)); - config_clear(); printf("%s: successful run completed\n", progname); diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index e88a9a0aa7e..7aba99c20de 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -233,7 +233,7 @@ fclose_and_clear(FILE **fpp) * Update the timestamp once. */ void -timestamp_once(void) +timestamp_once(WT_SESSION *session) { static const char *oldest_timestamp_str = "oldest_timestamp="; WT_CONNECTION *conn; @@ -246,16 +246,20 @@ timestamp_once(void) /* * Lock out transaction timestamp operations. The lock acts as a barrier ensuring we've checked - * if the workers have finished, we don't want that line reordered. + * if the workers have finished, we don't want that line reordered. We can also be called from + * places, such as bulk load, where we are single-threaded and the locks haven't been + * initialized. */ - testutil_check(pthread_rwlock_wrlock(&g.ts_lock)); + if (LOCK_INITIALIZED(&g.ts_lock)) + lock_writelock(session, &g.ts_lock); ret = conn->query_timestamp(conn, buf + strlen(oldest_timestamp_str), "get=all_durable"); testutil_assert(ret == 0 || ret == WT_NOTFOUND); if (ret == 0) testutil_check(conn->set_timestamp(conn, buf)); - testutil_check(pthread_rwlock_unlock(&g.ts_lock)); + if (LOCK_INITIALIZED(&g.ts_lock)) + lock_writeunlock(session, &g.ts_lock); } /* @@ -265,9 +269,15 @@ timestamp_once(void) WT_THREAD_RET timestamp(void *arg) { + WT_CONNECTION *conn; + WT_SESSION *session; bool done; (void)(arg); + conn = g.wts_conn; + + /* Locks need session */ + testutil_check(conn->open_session(conn, NULL, NULL, &session)); /* Update the oldest timestamp at least once every 15 seconds. */ done = false; @@ -281,10 +291,11 @@ timestamp(void *arg) else random_sleep(&g.rnd, 15); - timestamp_once(); + timestamp_once(session); } while (!done); + testutil_check(session->close(session, NULL)); return (WT_THREAD_RET_VALUE); } @@ -334,3 +345,38 @@ alter(void *arg) testutil_check(session->close(session, NULL)); return (WT_THREAD_RET_VALUE); } + +/* + * lock_init -- + * Initialize abstract lock that can use either pthread of wt reader-writer locks. + */ +void +lock_init(WT_SESSION *session, RWLOCK *lock) +{ + testutil_assert(lock->lock_type == LOCK_NONE); + + if (g.c_wt_mutex) { + testutil_check(__wt_rwlock_init((WT_SESSION_IMPL *)session, &lock->l.wt)); + lock->lock_type = LOCK_WT; + } else { + testutil_check(pthread_rwlock_init(&lock->l.pthread, NULL)); + lock->lock_type = LOCK_PTHREAD; + } +} + +/* + * lock_destroy -- + * Destroy abstract lock. + */ +void +lock_destroy(WT_SESSION *session, RWLOCK *lock) +{ + testutil_assert(LOCK_INITIALIZED(lock)); + + if (lock->lock_type == LOCK_WT) { + __wt_rwlock_destroy((WT_SESSION_IMPL *)session, &lock->l.wt); + } else { + testutil_check(pthread_rwlock_destroy(&lock->l.pthread)); + } + lock->lock_type = LOCK_NONE; +} diff --git a/src/third_party/wiredtiger/test/multiversion/wt_multiversion.sh b/src/third_party/wiredtiger/test/multiversion/wt_multiversion.sh new file mode 100644 index 00000000000..f3528b5d24e --- /dev/null +++ b/src/third_party/wiredtiger/test/multiversion/wt_multiversion.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +last_stable=4.2 +last_stable_dir=wiredtiger_${last_stable}/ +last_stable_branch=mongodb-${last_stable} + +function setup_last_stable { + git clone git@github.com:wiredtiger/wiredtiger.git ${last_stable_dir} + cd ${last_stable_dir}/build_posix/ || exit + git checkout $last_stable_branch || exit 1 + bash reconf + ../configure --enable-python --enable-diagnostic + make -j 10 + # Back to multiversion/ in "latest" repo. + cd ../../ || exit +} + +function run_check { + echo + "$@" + "$@" || exit 1 +} + +# Clone and build v4.2 if it doesn't already exist. +if [ ! -d $last_stable_dir ]; then + setup_last_stable +fi + +latest_workgen=../../bench/workgen/runner/multiversion.py +last_stable_workgen=${last_stable_dir}/bench/workgen/runner/multiversion.py + +# Copy the workload into the v4.2 tree. +cp $latest_workgen $last_stable_workgen + +run_check $latest_workgen --release 4.4 +run_check $latest_workgen --keep --release 4.4 +run_check $last_stable_workgen --keep --release 4.2 +run_check $latest_workgen --keep --release 4.4 + +echo Success. +exit 0 diff --git a/src/third_party/wiredtiger/test/packing/intpack-test3.c b/src/third_party/wiredtiger/test/packing/intpack-test3.c index 8bd8cc8a8c9..43cb8834997 100644 --- a/src/third_party/wiredtiger/test/packing/intpack-test3.c +++ b/src/third_party/wiredtiger/test/packing/intpack-test3.c @@ -110,8 +110,8 @@ main(void) int64_t i; /* - * Test all values in a range, to ensure pack/unpack of small numbers - * (which most actively use different numbers of bits) works. + * Test all values in a range, to ensure pack/unpack of small numbers (which most actively use + * different numbers of bits) works. */ test_spread(0, 100000, 100000); test_spread(INT16_MAX, 1025, 1025); |