diff options
Diffstat (limited to 'test')
46 files changed, 2038 insertions, 985 deletions
diff --git a/test/bloom/Makefile.am b/test/bloom/Makefile.am index 86d87c70071..0592cec7e42 100644 --- a/test/bloom/Makefile.am +++ b/test/bloom/Makefile.am @@ -11,4 +11,4 @@ TESTS = $(noinst_PROGRAMS) LOG_COMPILER = $(TEST_WRAPPER) clean-local: - rm -rf WiredTiger* *.core __* + rm -rf WiredTiger* *.core diff --git a/test/bloom/test_bloom.c b/test/bloom/test_bloom.c index f95bc7faaf9..6955813dc68 100644 --- a/test/bloom/test_bloom.c +++ b/test/bloom/test_bloom.c @@ -189,9 +189,7 @@ run(void) * ensure the value doesn't overlap with existing values. */ item.size = g.c_key_max + 10; - item.data = calloc(item.size, 1); - if (item.data == NULL) - testutil_die(ENOMEM, "value buffer malloc"); + item.data = dcalloc(item.size, 1); memset((void *)item.data, 'a', item.size); for (i = 0, fp = 0; i < g.c_ops; i++) { ((uint8_t *)item.data)[i % item.size] = @@ -232,14 +230,10 @@ populate_entries(void) srand(g.c_srand); - entries = calloc(g.c_ops, sizeof(uint8_t *)); - if (entries == NULL) - testutil_die(ENOMEM, "key buffer malloc"); + entries = dcalloc(g.c_ops, sizeof(uint8_t *)); for (i = 0; i < g.c_ops; i++) { - entries[i] = calloc(g.c_key_max, sizeof(uint8_t)); - if (entries[i] == NULL) - testutil_die(ENOMEM, "key buffer malloc 2"); + entries[i] = dcalloc(g.c_key_max, sizeof(uint8_t)); for (j = 0; j < g.c_key_max; j++) entries[i][j] = 'a' + ((uint8_t)rand() % 26); } diff --git a/test/checkpoint/test_checkpoint.c b/test/checkpoint/test_checkpoint.c index c5524b3c63e..307cfd914bd 100644 --- a/test/checkpoint/test_checkpoint.c +++ b/test/checkpoint/test_checkpoint.c @@ -61,8 +61,7 @@ main(int argc, char *argv[]) working_dir = NULL; ttype = MIX; g.checkpoint_name = "WiredTigerCheckpoint"; - if ((g.home = malloc(512)) == NULL) - testutil_die(ENOMEM, "Unable to allocate memory"); + g.home = dmalloc(512); g.nkeys = 10000; g.nops = 100000; g.ntables = 3; diff --git a/test/checkpoint/test_checkpoint.h b/test/checkpoint/test_checkpoint.h index 09edaeb84bc..10e21289dd3 100644 --- a/test/checkpoint/test_checkpoint.h +++ b/test/checkpoint/test_checkpoint.h @@ -26,19 +26,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include <sys/types.h> -#include <sys/time.h> +#include "test_util.i" -#include <errno.h> -#include <inttypes.h> -#include <pthread.h> #include <signal.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> - -#include "test_util.i" #define URI_BASE "table:__wt" /* File name */ diff --git a/test/cursor_order/Makefile.am b/test/cursor_order/Makefile.am index c0c0ed639bf..8afb8f122d8 100644 --- a/test/cursor_order/Makefile.am +++ b/test/cursor_order/Makefile.am @@ -10,4 +10,4 @@ cursor_order_LDFLAGS = -static TESTS = $(noinst_PROGRAMS) clean-local: - rm -rf WiredTiger* wt.* *.core __stats + rm -rf WT_TEST *.core diff --git a/test/cursor_order/cursor_order_ops.c b/test/cursor_order/cursor_order_ops.c index d44505ab2f3..a2185dd123f 100644 --- a/test/cursor_order/cursor_order_ops.c +++ b/test/cursor_order/cursor_order_ops.c @@ -59,22 +59,16 @@ ops_start(SHARED_CONFIG *cfg) total_nops = 0; /* Create per-thread structures. */ - if ((run_info = calloc( - (size_t)(cfg->reverse_scanners + cfg->append_inserters), - sizeof(*run_info))) == NULL) - testutil_die(errno, "calloc"); - - if ((tids = calloc( - (size_t)(cfg->reverse_scanners + cfg->append_inserters), - sizeof(*tids))) == NULL) - testutil_die(errno, "calloc"); + run_info = dcalloc((size_t) + (cfg->reverse_scanners + cfg->append_inserters), sizeof(*run_info)); + tids = dcalloc((size_t) + (cfg->reverse_scanners + cfg->append_inserters), sizeof(*tids)); /* Create the files and load the initial records. */ for (i = 0; i < cfg->append_inserters; ++i) { run_info[i].cfg = cfg; if (i == 0 || cfg->multiple_files) { - if ((run_info[i].name = malloc(64)) == NULL) - testutil_die(errno, "malloc"); + run_info[i].name = dmalloc(64); snprintf(run_info[i].name, 64, FNAME, (int)i); /* Vary by orders of magnitude */ @@ -96,8 +90,7 @@ ops_start(SHARED_CONFIG *cfg) offset = i + cfg->append_inserters; run_info[offset].cfg = cfg; if (cfg->multiple_files) { - if ((run_info[offset].name = malloc(64)) == NULL) - testutil_die(errno, "malloc"); + run_info[offset].name = dmalloc(64); /* Have reverse scans read from tables with writes. */ name_index = i % cfg->append_inserters; snprintf( diff --git a/test/fops/fops.c b/test/fops/fops.c index 3333ff16858..3c4de161423 100644 --- a/test/fops/fops.c +++ b/test/fops/fops.c @@ -59,10 +59,8 @@ fop_start(u_int nthreads) tids = NULL; /* Silence GCC 4.1 warning. */ /* Create statistics and thread structures. */ - if ((run_stats = calloc( - (size_t)(nthreads), sizeof(*run_stats))) == NULL || - (tids = calloc((size_t)(nthreads), sizeof(*tids))) == NULL) - testutil_die(errno, "calloc"); + run_stats = dcalloc((size_t)(nthreads), sizeof(*run_stats)); + tids = dcalloc((size_t)(nthreads), sizeof(*tids)); (void)gettimeofday(&start, NULL); diff --git a/test/fops/thread.h b/test/fops/thread.h index f9707c14590..630c2061285 100644 --- a/test/fops/thread.h +++ b/test/fops/thread.h @@ -26,25 +26,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include <sys/types.h> -#ifndef _WIN32 -#include <sys/time.h> -#endif +#include "test_util.i" -#include <errno.h> -#include <inttypes.h> -#ifndef _WIN32 -#include <pthread.h> -#endif #include <signal.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#ifndef _WIN32 -#include <unistd.h> -#endif - -#include "test_util.i" extern WT_CONNECTION *conn; /* WiredTiger connection */ diff --git a/test/format/backup.c b/test/format/backup.c index 2b1463bd0e3..69fdf771de9 100644 --- a/test/format/backup.c +++ b/test/format/backup.c @@ -38,7 +38,7 @@ check_copy(void) WT_CONNECTION *conn; WT_SESSION *session; - wts_open(g.home_backup, 0, &conn); + wts_open(g.home_backup, false, &conn); testutil_checkfmt( conn->open_session(conn, NULL, NULL, &session), @@ -53,27 +53,30 @@ check_copy(void) /* * copy_file -- - * Copy a single file into the backup directory. + * Copy a single file into the backup directories. */ static void -copy_file(const char *name) +copy_file(WT_SESSION *session, const char *name) { size_t len; - char *cmd; - - len = strlen(g.home) + strlen(g.home_backup) + strlen(name) * 2 + 20; - cmd = dmalloc(len); - (void)snprintf(cmd, len, - "cp %s/%s %s/%s", g.home, name, g.home_backup, name); - testutil_checkfmt(system(cmd), "backup copy: %s", cmd); - free(cmd); - - len = strlen(g.home) + strlen(g.home_backup2) + strlen(name) * 2 + 20; - cmd = dmalloc(len); - (void)snprintf(cmd, len, - "cp %s/%s %s/%s", g.home, name, g.home_backup2, name); - testutil_checkfmt(system(cmd), "backup copy: %s", cmd); - free(cmd); + char *first, *second; + + len = strlen("BACKUP") + strlen(name) + 10; + first = dmalloc(len); + (void)snprintf(first, len, "BACKUP/%s", name); + testutil_check(__wt_copy_and_sync(session, name, first)); + + /* + * Save another copy of the original file to make debugging recovery + * errors easier. + */ + len = strlen("BACKUP_COPY") + strlen(name) + 10; + second = dmalloc(len); + (void)snprintf(second, len, "BACKUP_COPY/%s", name); + testutil_check(__wt_copy_and_sync(session, first, second)); + + free(first); + free(second); } /* @@ -85,10 +88,11 @@ backup(void *arg) { WT_CONNECTION *conn; WT_CURSOR *backup_cursor; + WT_DECL_RET; WT_SESSION *session; - u_int period; - int ret; - const char *key; + u_int incremental, period; + bool full; + const char *config, *key; (void)(arg); @@ -102,48 +106,86 @@ backup(void *arg) testutil_check(conn->open_session(conn, NULL, NULL, &session)); /* - * Perform a backup at somewhere under 10 seconds (so we get at - * least one done), and then at 45 second intervals. + * Perform a full backup at somewhere under 10 seconds (that way there's + * at least one), then at larger intervals, optionally do incremental + * backups between full backups. */ - for (period = mmrand(NULL, 1, 10);; period = 45) { + incremental = 0; + for (period = mmrand(NULL, 1, 10);; period = mmrand(NULL, 20, 45)) { /* Sleep for short periods so we don't make the run wait. */ while (period > 0 && !g.workers_finished) { --period; sleep(1); } - if (g.workers_finished) - break; - /* Lock out named checkpoints */ + /* + * We can't drop named checkpoints while there's a backup in + * progress, serialize backups with named checkpoints. Wait + * for the checkpoint to complete, otherwise backups might be + * starved out. + */ testutil_check(pthread_rwlock_wrlock(&g.backup_lock)); + if (g.workers_finished) { + testutil_check(pthread_rwlock_unlock(&g.backup_lock)); + break; + } - /* Re-create the backup directory. */ - testutil_checkfmt( - system(g.home_backup_init), - "%s", "backup directory creation failed"); + if (incremental) { + config = "target=(\"log:\")"; + full = false; + } else { + /* Re-create the backup directory. */ + testutil_checkfmt( + system(g.home_backup_init), + "%s", "backup directory creation failed"); + + config = NULL; + full = true; + } /* - * open_cursor can return EBUSY if a metadata operation is - * currently happening - retry in that case. + * open_cursor can return EBUSY if concurrent with a metadata + * operation, retry in that case. */ - while ((ret = session->open_cursor(session, - "backup:", NULL, NULL, &backup_cursor)) == EBUSY) - sleep(1); + while ((ret = session->open_cursor( + session, "backup:", NULL, config, &backup_cursor)) == EBUSY) + __wt_yield(); if (ret != 0) testutil_die(ret, "session.open_cursor: backup"); while ((ret = backup_cursor->next(backup_cursor)) == 0) { testutil_check( backup_cursor->get_key(backup_cursor, &key)); - copy_file(key); + copy_file(session, key); } + if (ret != WT_NOTFOUND) + testutil_die(ret, "backup-cursor"); + + /* After an incremental backup, truncate the log files. */ + if (incremental) + testutil_check(session->truncate( + session, "log:", backup_cursor, NULL, NULL)); testutil_check(backup_cursor->close(backup_cursor)); testutil_check(pthread_rwlock_unlock(&g.backup_lock)); - check_copy(); + /* + * If automatic log archival isn't configured, optionally do + * incremental backups after each full backup. If we're not + * doing any more incrementals, verify the backup (we can't + * verify intermediate states, once we perform recovery on the + * backup database, we can't do any more incremental backups). + */ + if (full) + incremental = + g.c_logging_archive ? 1 : mmrand(NULL, 1, 5); + if (--incremental == 0) + check_copy(); } + if (incremental != 0) + check_copy(); + testutil_check(session->close(session, NULL)); return (NULL); diff --git a/test/format/bdb.c b/test/format/bdb.c index 823fc8ff888..48229cfd5e7 100644 --- a/test/format/bdb.c +++ b/test/format/bdb.c @@ -30,7 +30,7 @@ #include "format.h" static DBT key, value; -static uint8_t *keybuf; +static WT_ITEM keyitem; static int bdb_compare_reverse(DB *dbp, const DBT *k1, const DBT *k2 @@ -78,7 +78,7 @@ bdb_open(void) assert(db->cursor(db, NULL, &dbc, 0) == 0); g.dbc = dbc; - key_gen_setup(&keybuf); + key_gen_setup(&keyitem); } void @@ -95,8 +95,7 @@ bdb_close(void) assert(db->close(db, 0) == 0); assert(dbenv->close(dbenv, 0) == 0); - free(keybuf); - keybuf = NULL; + free(keyitem.mem); } void @@ -144,12 +143,11 @@ void bdb_read(uint64_t keyno, void *valuep, size_t *valuesizep, int *notfoundp) { DBC *dbc = g.dbc; - size_t size; int ret; - key_gen(keybuf, &size, keyno); - key.data = keybuf; - key.size = (uint32_t)size; + key_gen(&keyitem, keyno); + key.data = (void *)keyitem.data; + key.size = keyitem.size; *notfoundp = 0; if ((ret = dbc->get(dbc, &key, &value, DB_SET)) != 0) { @@ -165,7 +163,7 @@ bdb_read(uint64_t keyno, void *valuep, size_t *valuesizep, int *notfoundp) void bdb_update(const void *arg_key, size_t arg_key_size, - const void *arg_value, size_t arg_value_size, int *notfoundp) + const void *arg_value, size_t arg_value_size) { DBC *dbc = g.dbc; int ret; @@ -175,15 +173,10 @@ bdb_update(const void *arg_key, size_t arg_key_size, value.data = (void *)arg_value; value.size = (uint32_t)arg_value_size; - *notfoundp = 0; - if ((ret = dbc->put(dbc, &key, &value, DB_KEYFIRST)) != 0) { - if (ret != DB_NOTFOUND) { - testutil_die(ret, "dbc.put: DB_KEYFIRST: {%.*s}{%.*s}", - (int)key.size, (char *)key.data, - (int)value.size, (char *)value.data); - } - *notfoundp = 1; - } + if ((ret = dbc->put(dbc, &key, &value, DB_KEYFIRST)) != 0) + testutil_die(ret, "dbc.put: DB_KEYFIRST: {%.*s}{%.*s}", + (int)key.size, (char *)key.data, + (int)value.size, (char *)value.data); } void @@ -193,9 +186,9 @@ bdb_remove(uint64_t keyno, int *notfoundp) size_t size; int ret; - key_gen(keybuf, &size, keyno); - key.data = keybuf; - key.size = (uint32_t)size; + key_gen(&keyitem, keyno); + key.data = (void *)keyitem.data; + key.size = keyitem.size; bdb_read(keyno, &value.data, &size, notfoundp); value.size = (uint32_t)size; diff --git a/test/format/bulk.c b/test/format/bulk.c index 64b005d294f..dab23bed404 100644 --- a/test/format/bulk.c +++ b/test/format/bulk.c @@ -33,13 +33,12 @@ wts_load(void) { WT_CONNECTION *conn; WT_CURSOR *cursor; + WT_DECL_RET; WT_ITEM key, value; WT_SESSION *session; - uint8_t *keybuf, *valbuf; bool is_bulk; conn = g.wts_conn; - keybuf = valbuf = NULL; testutil_check(conn->open_session(conn, NULL, NULL, &session)); @@ -63,8 +62,8 @@ wts_load(void) is_bulk ? "bulk,append" : NULL, &cursor)); /* Set up the key/value buffers. */ - key_gen_setup(&keybuf); - val_gen_setup(NULL, &valbuf); + key_gen_setup(&key); + val_gen_setup(NULL, &value); for (;;) { if (++g.key_cnt > g.c_rows) { @@ -73,13 +72,11 @@ wts_load(void) } /* Report on progress every 100 inserts. */ - if (g.key_cnt % 100 == 0) + if (g.key_cnt % 1000 == 0) track("bulk load", g.key_cnt, NULL); - key_gen(keybuf, &key.size, (uint64_t)g.key_cnt); - key.data = keybuf; - val_gen(NULL, valbuf, &value.size, (uint64_t)g.key_cnt); - value.data = valbuf; + key_gen(&key, g.key_cnt); + val_gen(NULL, &value, g.key_cnt); switch (g.type) { case FIX: @@ -88,7 +85,7 @@ wts_load(void) cursor->set_value(cursor, *(uint8_t *)value.data); if (g.logging == LOG_OPS) (void)g.wt_api->msg_printf(g.wt_api, session, - "%-10s %" PRIu32 " {0x%02" PRIx8 "}", + "%-10s %" PRIu64 " {0x%02" PRIx8 "}", "bulk V", g.key_cnt, ((uint8_t *)value.data)[0]); break; @@ -98,7 +95,7 @@ wts_load(void) cursor->set_value(cursor, &value); if (g.logging == LOG_OPS) (void)g.wt_api->msg_printf(g.wt_api, session, - "%-10s %" PRIu32 " {%.*s}", "bulk V", + "%-10s %" PRIu64 " {%.*s}", "bulk V", g.key_cnt, (int)value.size, (char *)value.data); break; @@ -106,18 +103,40 @@ wts_load(void) cursor->set_key(cursor, &key); if (g.logging == LOG_OPS) (void)g.wt_api->msg_printf(g.wt_api, session, - "%-10s %" PRIu32 " {%.*s}", "bulk K", + "%-10s %" PRIu64 " {%.*s}", "bulk K", g.key_cnt, (int)key.size, (char *)key.data); cursor->set_value(cursor, &value); if (g.logging == LOG_OPS) (void)g.wt_api->msg_printf(g.wt_api, session, - "%-10s %" PRIu32 " {%.*s}", "bulk V", + "%-10s %" PRIu64 " {%.*s}", "bulk V", g.key_cnt, (int)value.size, (char *)value.data); break; } - testutil_check(cursor->insert(cursor)); + /* + * We don't want to size the cache to ensure the initial data + * set can load in the in-memory case, guaranteeing the load + * succeeds probably means future updates are also guaranteed + * to succeed, which isn't what we want. If we run out of space + * in the initial load, reset the row counter and continue. + * + * Decrease inserts, they can't be successful if we're at the + * cache limit, and increase the delete percentage to get some + * extra space once the run starts. + */ + if ((ret = cursor->insert(cursor)) != 0) { + if (ret != WT_CACHE_FULL) + testutil_die(ret, "cursor.insert"); + g.rows = --g.key_cnt; + g.c_rows = (uint32_t)g.key_cnt; + + if (g.c_insert_pct > 5) + g.c_insert_pct = 5; + if (g.c_delete_pct < 20) + g.c_delete_pct += 20; + break; + } #ifdef HAVE_BERKELEY_DB if (SINGLETHREADED) @@ -133,6 +152,6 @@ wts_load(void) testutil_check(session->close(session, NULL)); - free(keybuf); - free(valbuf); + free(key.mem); + free(value.mem); } diff --git a/test/format/compact.c b/test/format/compact.c index a75ee4f2adf..240e5553697 100644 --- a/test/format/compact.c +++ b/test/format/compact.c @@ -36,9 +36,9 @@ void * compact(void *arg) { WT_CONNECTION *conn; + WT_DECL_RET; WT_SESSION *session; u_int period; - int ret; (void)(arg); diff --git a/test/format/config.c b/test/format/config.c index 042316d8344..1b09916bd88 100644 --- a/test/format/config.c +++ b/test/format/config.c @@ -35,6 +35,7 @@ static void config_encryption(void); static const char *config_file_type(u_int); static CONFIG *config_find(const char *, size_t); static void config_in_memory(void); +static void config_in_memory_check(void); static int config_is_perm(const char *); static void config_isolation(void); static void config_lrt(void); @@ -43,6 +44,7 @@ static void config_map_compression(const char *, u_int *); static void config_map_encryption(const char *, u_int *); static void config_map_file_type(const char *, u_int *); static void config_map_isolation(const char *, u_int *); +static void config_reset(void); /* * config_setup -- @@ -54,14 +56,10 @@ config_setup(void) CONFIG *cp; /* Clear any temporary values. */ - config_clear(); + config_reset(); - /* - * Periodically, run in-memory; don't do it on the first run, all our - * smoke tests would hit it. - */ - if (!config_is_perm("in_memory") && g.run_cnt % 20 == 19) - g.c_in_memory = 1; + /* Periodically run in-memory. */ + config_in_memory(); /* * Choose a data source type and a file type: they're interrelated (LSM @@ -145,7 +143,7 @@ config_setup(void) /* Some data-sources don't support user-specified collations. */ if (DATASOURCE("helium") || DATASOURCE("kvsbdb")) - g.c_reverse = 0; + config_single("reverse=off", 0); /* * Periodically, run single-threaded so we can compare the results to @@ -159,7 +157,6 @@ config_setup(void) config_compression("compression"); config_compression("logging_compression"); config_encryption(); - config_in_memory(); config_isolation(); config_lrt(); @@ -169,7 +166,7 @@ config_setup(void) * Don't do it on the first run, all our smoke tests would hit it. */ if (!g.replay && g.run_cnt % 10 == 9 && !config_is_perm("delete_pct")) - g.c_delete_pct = 0; + config_single("delete_pct=0", 0); /* * If this is an LSM run, set the cache size and crank up the insert @@ -187,9 +184,12 @@ config_setup(void) if (!config_is_perm("cache") && g.c_cache < g.c_threads) g.c_cache = g.c_threads; + /* Give in-memory configuration a final review. */ + config_in_memory_check(); + /* Make the default maximum-run length 20 minutes. */ if (!config_is_perm("timer")) - g.c_timer = 20; + config_single("timer=20", 0); /* * Key/value minimum/maximum are related, correct unless specified by @@ -329,43 +329,89 @@ config_encryption(void) /* * config_in_memory -- - * In-memory configuration. + * Periodically set up an in-memory configuration. */ static void config_in_memory(void) { + /* + * Configure in-memory before configuring anything else, in-memory has + * many related requirements. Don't configure in-memory if there's any + * incompatible configurations, so we don't have to configure in-memory + * every time we configure something like LSM, that's too painful. + */ + if (config_is_perm("backups")) + return; + if (config_is_perm("checkpoints")) + return; + if (config_is_perm("compression")) + return; + if (config_is_perm("data_source") && DATASOURCE("lsm")) + return; + if (config_is_perm("logging")) + return; + if (config_is_perm("rebalance")) + return; + if (config_is_perm("salvage")) + return; + if (config_is_perm("verify")) + return; + + if (!config_is_perm("in_memory") && mmrand(NULL, 1, 20) == 1) + g.c_in_memory = 1; +} + +/* + * config_in_memory_check -- + * In-memory configuration review. + */ +static void +config_in_memory_check(void) +{ + uint32_t cache; + if (g.c_in_memory == 0) return; /* Turn off a lot of stuff. */ if (!config_is_perm("backups")) - g.c_backups = 0; + config_single("backups=off", 0); if (!config_is_perm("checkpoints")) - g.c_checkpoints = 0; - if (!config_is_perm("compression")) { - g.c_compression = dstrdup("none"); - g.c_compression_flag = COMPRESS_NONE; - } + config_single("checkpoints=off", 0); + if (!config_is_perm("compression")) + config_single("compression=none", 0); if (!config_is_perm("logging")) - g.c_logging = 0; + config_single("logging=off", 0); if (!config_is_perm("rebalance")) - g.c_rebalance = 0; + config_single("rebalance=off", 0); if (!config_is_perm("salvage")) - g.c_salvage = 0; + config_single("salvage=off", 0); if (!config_is_perm("verify")) - g.c_verify = 0; + config_single("verify=off", 0); /* - * Ensure there is 250MB of cache per thread; keep keys/values small, - * overflow items aren't an issue for in-memory configurations and it - * keeps us from overflowing the cache. + * Keep keys/values small, overflow items aren't an issue for in-memory + * configurations and it keeps us from overflowing the cache. */ - if (!config_is_perm("cache")) - g.c_cache = g.c_threads * 250; if (!config_is_perm("key_max")) - g.c_value_max = 64; + config_single("key_max=32", 0); if (!config_is_perm("value_max")) - g.c_value_max = 128; + config_single("value_max=80", 0); + + /* + * Size the cache relative to the initial data set, use 2x the base + * size as a minimum. + */ + if (!config_is_perm("cache")) { + cache = g.c_value_max; + if (g.type == ROW) + cache += g.c_key_max; + cache *= g.c_rows; + cache *= 2; + cache /= WT_MEGABYTE; + if (g.c_cache < cache) + g.c_cache = cache; + } } /* @@ -413,11 +459,11 @@ config_lrt(void) * stores. */ if (g.type == FIX) { - if (g.c_long_running_txn && config_is_perm("long_running_txn")) + if (config_is_perm("long_running_txn")) testutil_die(EINVAL, "long_running_txn not supported with fixed-length " "column store"); - g.c_long_running_txn = 0; + config_single("long_running_txn=off", 0); } } @@ -503,18 +549,36 @@ config_file(const char *name) /* * config_clear -- - * Clear per-run values. + * Clear all configuration values. */ void config_clear(void) { CONFIG *cp; - /* Clear configuration data. */ + /* Clear all allocated configuration data. */ + for (cp = c; cp->name != NULL; ++cp) + if (cp->vstr != NULL) { + free((void *)*cp->vstr); + *cp->vstr = NULL; + } + free(g.uri); + g.uri = NULL; +} + +/* + * config_reset -- + * Clear per-run configuration values. + */ +static void +config_reset(void) +{ + CONFIG *cp; + + /* Clear temporary allocated configuration data. */ for (cp = c; cp->name != NULL; ++cp) { F_CLR(cp, C_TEMP); - if (!F_ISSET(cp, C_PERM) && - F_ISSET(cp, C_STRING) && cp->vstr != NULL) { + if (!F_ISSET(cp, C_PERM) && cp->vstr != NULL) { free((void *)*cp->vstr); *cp->vstr = NULL; } @@ -531,7 +595,7 @@ void config_single(const char *s, int perm) { CONFIG *cp; - uint32_t v; + long v; char *p; const char *ep; @@ -557,43 +621,59 @@ config_single(const char *s, int perm) exit(EXIT_FAILURE); } + /* + * Free the previous setting if a configuration has been + * passed in twice. + */ + if (*cp->vstr != NULL) { + free(*cp->vstr); + *cp->vstr = NULL; + } + if (strncmp(s, "checksum", strlen("checksum")) == 0) { config_map_checksum(ep, &g.c_checksum_flag); - *cp->vstr = strdup(ep); + *cp->vstr = dstrdup(ep); } else if (strncmp( s, "compression", strlen("compression")) == 0) { config_map_compression(ep, &g.c_compression_flag); - *cp->vstr = strdup(ep); + *cp->vstr = dstrdup(ep); } else if (strncmp( s, "encryption", strlen("encryption")) == 0) { config_map_encryption(ep, &g.c_encryption_flag); - *cp->vstr = strdup(ep); + *cp->vstr = dstrdup(ep); } else if (strncmp(s, "isolation", strlen("isolation")) == 0) { config_map_isolation(ep, &g.c_isolation_flag); - *cp->vstr = strdup(ep); + *cp->vstr = dstrdup(ep); } else if (strncmp(s, "file_type", strlen("file_type")) == 0) { config_map_file_type(ep, &g.type); - *cp->vstr = strdup(config_file_type(g.type)); + *cp->vstr = dstrdup(config_file_type(g.type)); } else if (strncmp(s, "logging_compression", strlen("logging_compression")) == 0) { config_map_compression(ep, &g.c_logging_compression_flag); - *cp->vstr = strdup(ep); + *cp->vstr = dstrdup(ep); } else { free((void *)*cp->vstr); - *cp->vstr = strdup(ep); + *cp->vstr = dstrdup(ep); } - if (*cp->vstr == NULL) - testutil_die(errno, "malloc"); return; } - v = (uint32_t)strtoul(ep, &p, 10); - if (*p != '\0') { - fprintf(stderr, "%s: %s: illegal numeric value\n", - g.progname, s); - exit(EXIT_FAILURE); + v = -1; + if (F_ISSET(cp, C_BOOL)) { + if (strncmp(ep, "off", strlen("off")) == 0) + v = 0; + else if (strncmp(ep, "on", strlen("on")) == 0) + v = 1; + } + if (v == -1) { + v = strtol(ep, &p, 10); + if (*p != '\0') { + fprintf(stderr, "%s: %s: illegal numeric value\n", + g.progname, s); + exit(EXIT_FAILURE); + } } if (F_ISSET(cp, C_BOOL)) { if (v != 0 && v != 1) { @@ -607,7 +687,7 @@ config_single(const char *s, int perm) g.progname, s, cp->min, cp->maxset); exit(EXIT_FAILURE); } - *cp->v = v; + *cp->v = (uint32_t)v; } /* diff --git a/test/format/config.h b/test/format/config.h index a17614bc044..16fffb6fafe 100644 --- a/test/format/config.h +++ b/test/format/config.h @@ -294,6 +294,10 @@ static CONFIG c[] = { "maximum time to run in minutes (default 20 minutes)", C_IGNORE, 0, UINT_MAX, UINT_MAX, &g.c_timer, NULL }, + { "transaction-frequency", + "percent operations done inside an explicit transaction", + 0x0, 1, 100, 100, &g.c_txn_freq, NULL }, + { "value_max", "maximum size of values", 0x0, 32, 4096, MEGABYTE(10), &g.c_value_max, NULL }, diff --git a/test/format/format.h b/test/format/format.h index a129c5395fd..beaabe7e83c 100644 --- a/test/format/format.h +++ b/test/format/format.h @@ -26,33 +26,10 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include <sys/stat.h> -#ifndef _WIN32 -#include <sys/time.h> -#endif -#include <sys/types.h> - -#include <assert.h> -#include <ctype.h> -#include <errno.h> -#include <fcntl.h> -#include <inttypes.h> -#include <limits.h> -#ifndef _WIN32 -#include <pthread.h> -#endif -#include <signal.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#ifndef _WIN32 -#include <unistd.h> -#endif -#include <time.h> - #include "test_util.i" #ifdef BDB +#include <assert.h> #include <db.h> #endif @@ -109,7 +86,6 @@ typedef struct { char *home; /* Home directory */ char *home_backup; /* Hot-backup directory */ - char *home_backup2; /* Saved Hot-backup directory */ char *home_backup_init; /* Initialize backup command */ char *home_bdb; /* BDB directory */ char *home_config; /* Run CONFIG file path */ @@ -145,7 +121,8 @@ typedef struct { int replay; /* Replaying a run. */ int workers_finished; /* Operations completed */ - pthread_rwlock_t backup_lock; /* Hot backup running */ + pthread_rwlock_t backup_lock; /* Backup running */ + pthread_rwlock_t checkpoint_lock; /* Checkpoint running */ WT_RAND_STATE rnd; /* Global RNG state */ @@ -224,6 +201,7 @@ typedef struct { uint32_t c_statistics_server; uint32_t c_threads; uint32_t c_timer; + uint32_t c_txn_freq; uint32_t c_value_max; uint32_t c_value_min; uint32_t c_verify; @@ -297,7 +275,7 @@ void bdb_np(int, void *, size_t *, void *, size_t *, int *); void bdb_open(void); void bdb_read(uint64_t, void *, size_t *, int *); void bdb_remove(uint64_t, int *); -void bdb_update(const void *, size_t, const void *, size_t, int *); +void bdb_update(const void *, size_t, const void *, size_t); #endif void *backup(void *); @@ -308,25 +286,23 @@ void config_file(const char *); void config_print(int); void config_setup(void); void config_single(const char *, int); -void *dmalloc(size_t); -char *dstrdup(const char *); void fclose_and_clear(FILE **); -void key_gen(uint8_t *, size_t *, uint64_t); -void key_gen_insert(WT_RAND_STATE *, uint8_t *, size_t *, uint64_t); -void key_gen_setup(uint8_t **); +void key_gen(WT_ITEM *, uint64_t); +void key_gen_insert(WT_RAND_STATE *, WT_ITEM *, uint64_t); +void key_gen_setup(WT_ITEM *); void key_len_setup(void); void *lrt(void *); void path_setup(const char *); -int read_row(WT_CURSOR *, WT_ITEM *, uint64_t, int); +int read_row(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t); uint32_t rng(WT_RAND_STATE *); void track(const char *, uint64_t, TINFO *); -void val_gen(WT_RAND_STATE *, uint8_t *, size_t *, uint64_t); -void val_gen_setup(WT_RAND_STATE *, uint8_t **); +void val_gen(WT_RAND_STATE *, WT_ITEM *, uint64_t); +void val_gen_setup(WT_RAND_STATE *, WT_ITEM *); void wts_close(void); -void wts_create(void); void wts_dump(const char *, int); +void wts_init(void); void wts_load(void); -void wts_open(const char *, int, WT_CONNECTION **); +void wts_open(const char *, bool, WT_CONNECTION **); void wts_ops(int); void wts_read_scan(void); void wts_rebalance(void); diff --git a/test/format/lrt.c b/test/format/lrt.c index 451d2f4fa3c..937525522fa 100644 --- a/test/format/lrt.c +++ b/test/format/lrt.c @@ -43,17 +43,15 @@ lrt(void *arg) uint64_t keyno, saved_keyno; u_int period; int pinned, ret; - uint8_t bitfield, *keybuf; + uint8_t bitfield; void *buf; (void)(arg); /* Unused parameter */ saved_keyno = 0; /* [-Werror=maybe-uninitialized] */ - key_gen_setup(&keybuf); - memset(&key, 0, sizeof(key)); - key.data = keybuf; - memset(&value, 0, sizeof(value)); + key_gen_setup(&key); + val_gen_setup(NULL, &value); buf = NULL; buf_len = buf_size = 0; @@ -67,8 +65,8 @@ lrt(void *arg) for (pinned = 0;;) { if (pinned) { /* Re-read the record at the end of the table. */ - while ((ret = read_row(cursor, - &key, saved_keyno, 1)) == WT_ROLLBACK) + while ((ret = read_row( + cursor, &key, &value, saved_keyno)) == WT_ROLLBACK) ; if (ret != 0) testutil_die(ret, @@ -112,7 +110,7 @@ lrt(void *arg) (u_int)(g.key_cnt - g.key_cnt / 10), (u_int)g.key_cnt); while ((ret = read_row(cursor, - &key, saved_keyno, 1)) == WT_ROLLBACK) + &key, &value, saved_keyno)) == WT_ROLLBACK) ; } while (ret == WT_NOTFOUND); if (ret != 0) @@ -129,9 +127,8 @@ lrt(void *arg) if (ret != 0) testutil_die(ret, "cursor.get_value: %" PRIu64, saved_keyno); - if (buf_len < value.size && - (buf = realloc(buf, buf_len = value.size)) == NULL) - testutil_die(errno, "malloc"); + if (buf_len < value.size) + buf = drealloc(buf, buf_len = value.size); memcpy(buf, value.data, buf_size = value.size); /* @@ -142,7 +139,7 @@ lrt(void *arg) do { keyno = mmrand(NULL, 1, (u_int)g.key_cnt / 5); while ((ret = read_row(cursor, - &key, keyno, 1)) == WT_ROLLBACK) + &key, &value, keyno)) == WT_ROLLBACK) ; } while (ret == WT_NOTFOUND); if (ret != 0) @@ -165,7 +162,8 @@ lrt(void *arg) testutil_check(session->close(session, NULL)); - free(keybuf); + free(key.mem); + free(value.mem); free(buf); return (NULL); diff --git a/test/format/ops.c b/test/format/ops.c index 5d66f4d5391..9275d7f3856 100644 --- a/test/format/ops.c +++ b/test/format/ops.c @@ -28,14 +28,14 @@ #include "format.h" -static int col_insert(TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t *); -static int col_remove(WT_CURSOR *, WT_ITEM *, uint64_t, int *); -static int col_update(TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t); -static int nextprev(WT_CURSOR *, int, int *); +static int col_insert(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t *); +static int col_remove(WT_CURSOR *, WT_ITEM *, uint64_t); +static int col_update(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t); +static int nextprev(WT_CURSOR *, int); static void *ops(void *); -static int row_insert(TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t); -static int row_remove(WT_CURSOR *, WT_ITEM *, uint64_t, int *); -static int row_update(TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t); +static int row_insert(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t); +static int row_remove(WT_CURSOR *, WT_ITEM *, uint64_t); +static int row_update(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t); static void table_append_init(void); #ifdef HAVE_BERKELEY_DB @@ -103,8 +103,7 @@ wts_ops(int lastrun) } /* Create thread structure; start the worker threads. */ - if ((tinfo = calloc((size_t)g.c_threads, sizeof(*tinfo))) == NULL) - testutil_die(errno, "calloc"); + tinfo = dcalloc((size_t)g.c_threads, sizeof(*tinfo)); for (i = 0; i < g.c_threads; ++i) { tinfo[i].id = (int)i + 1; tinfo[i].state = TINFO_RUNNING; @@ -184,6 +183,7 @@ wts_ops(int lastrun) (void)pthread_join(compact_tid, NULL); if (!SINGLETHREADED && g.c_long_running_txn) (void)pthread_join(lrt_tid, NULL); + g.workers_finished = 0; if (g.logging != 0) { (void)g.wt_api->msg_printf(g.wt_api, session, @@ -193,57 +193,229 @@ wts_ops(int lastrun) } /* - * ops_session_config -- - * Return the current session configuration. + * isolation_config -- + * Return an isolation configuration. */ -static const char * -ops_session_config(WT_RAND_STATE *rnd) +static inline const char * +isolation_config(WT_RAND_STATE *rnd, bool *iso_snapshotp) { u_int v; - /* - * The only current session configuration is the isolation level. - */ if ((v = g.c_isolation_flag) == ISOLATION_RANDOM) v = mmrand(rnd, 2, 4); switch (v) { case ISOLATION_READ_UNCOMMITTED: + *iso_snapshotp = false; return ("isolation=read-uncommitted"); case ISOLATION_READ_COMMITTED: + *iso_snapshotp = false; return ("isolation=read-committed"); case ISOLATION_SNAPSHOT: default: + *iso_snapshotp = true; return ("isolation=snapshot"); } } +typedef struct { + uint64_t keyno; /* Row number */ + + void *kdata; /* If an insert, the generated key */ + size_t ksize; + size_t kmemsize; + + void *vdata; /* If not a delete, the value */ + size_t vsize; + size_t vmemsize; + + bool deleted; /* Delete operation */ + bool insert; /* Insert operation */ +} SNAP_OPS; + +/* + * snap_track -- + * Add a single snapshot isolation returned value to the list. + */ +static void +snap_track(SNAP_OPS *snap, uint64_t keyno, WT_ITEM *key, WT_ITEM *value) +{ + snap->keyno = keyno; + if (key == NULL) + snap->insert = false; + else { + snap->insert = true; + + if (snap->kmemsize < key->size) { + snap->kdata = drealloc(snap->kdata, key->size); + snap->kmemsize = key->size; + } + memcpy(snap->kdata, key->data, snap->ksize = key->size); + } + if (value == NULL) + snap->deleted = true; + else { + snap->deleted = false; + if (snap->vmemsize < value->size) { + snap->vdata = drealloc(snap->vdata, value->size); + snap->vmemsize = value->size; + } + memcpy(snap->vdata, value->data, snap->vsize = value->size); + } +} + +/* + * snap_check -- + * Check snapshot isolation operations are repeatable. + */ +static int +snap_check(WT_CURSOR *cursor, + SNAP_OPS *start, SNAP_OPS *stop, WT_ITEM *key, WT_ITEM *value) +{ + WT_DECL_RET; + SNAP_OPS *p; + uint8_t bitfield; + + for (; start < stop; ++start) { + /* Check for subsequent changes to this record. */ + for (p = start + 1; p < stop && p->keyno != start->keyno; ++p) + ; + if (p != stop) + continue; + + /* + * Retrieve the key/value pair by key. Row-store inserts have a + * unique generated key we saved, else generate the key from the + * key number. + */ + if (start->insert == 0) { + switch (g.type) { + case FIX: + case VAR: + cursor->set_key(cursor, start->keyno); + break; + case ROW: + key_gen(key, start->keyno); + cursor->set_key(cursor, key); + break; + } + } else { + key->data = start->kdata; + key->size = start->ksize; + cursor->set_key(cursor, key); + } + if ((ret = cursor->search(cursor)) == 0) { + if (g.type == FIX) { + testutil_check( + cursor->get_value(cursor, &bitfield)); + *(uint8_t *)(value->data) = bitfield; + value->size = 1; + } else + testutil_check( + cursor->get_value(cursor, value)); + } else + if (ret != WT_NOTFOUND) + return (ret); + + /* Check for simple matches. */ + if (ret == 0 && !start->deleted && + value->size == start->vsize && + memcmp(value->data, start->vdata, value->size) == 0) + continue; + if (ret == WT_NOTFOUND && start->deleted) + continue; + + /* + * In fixed length stores, zero values at the end of the key + * space are returned as not-found, and not-found row reads + * are saved as zero values. Map back-and-forth for simplicity. + */ + if (g.type == FIX) { + if (ret == WT_NOTFOUND && + start->vsize == 1 && *(uint8_t *)start->vdata == 0) + continue; + if (start->deleted && + value->size == 1 && *(uint8_t *)value->data == 0) + continue; + } + + /* Things went pear-shaped. */ + switch (g.type) { + case FIX: + testutil_die(ret, + "snap_check: %" PRIu64 " search: " + "expected {0x%02x}, found {0x%02x}", + start->keyno, + start->deleted ? 0 : *(uint8_t *)start->vdata, + ret == WT_NOTFOUND ? 0 : *(uint8_t *)value->data); + /* NOTREACHED */ + case ROW: + testutil_die(ret, + "snap_check: %.*s search: " + "expected {%.*s}, found {%.*s}", + (int)key->size, key->data, + start->deleted ? + (int)strlen("deleted") : (int)start->vsize, + start->deleted ? "deleted" : start->vdata, + ret == WT_NOTFOUND ? + (int)strlen("deleted") : (int)value->size, + ret == WT_NOTFOUND ? "deleted" : value->data); + /* NOTREACHED */ + case VAR: + testutil_die(ret, + "snap_check: %" PRIu64 " search: " + "expected {%.*s}, found {%.*s}", + start->keyno, + start->deleted ? + (int)strlen("deleted") : (int)start->vsize, + start->deleted ? "deleted" : start->vdata, + ret == WT_NOTFOUND ? + (int)strlen("deleted") : (int)value->size, + ret == WT_NOTFOUND ? "deleted" : value->data); + /* NOTREACHED */ + } + } + return (0); +} + +/* + * ops -- + * Per-thread operations. + */ static void * ops(void *arg) { + SNAP_OPS *snap, snap_list[64]; TINFO *tinfo; WT_CONNECTION *conn; WT_CURSOR *cursor, *cursor_insert; + WT_DECL_RET; + WT_ITEM *key, _key, *value, _value; WT_SESSION *session; - WT_ITEM key, value; uint64_t keyno, ckpt_op, reset_op, session_op; - uint32_t op; - uint8_t *keybuf, *valbuf; - u_int np; - int ckpt_available, dir, insert, intxn, notfound, readonly; + uint32_t op, rnd; + u_int i; + int dir; char *ckpt_config, ckpt_name[64]; + bool ckpt_available, intxn, iso_snapshot, positioned, readonly; tinfo = arg; conn = g.wts_conn; - keybuf = valbuf = NULL; - readonly = 0; /* -Wconditional-uninitialized */ + readonly = false; /* -Wconditional-uninitialized */ + + /* Initialize tracking of snapshot isolation transaction returns. */ + snap = NULL; + iso_snapshot = false; + memset(snap_list, 0, sizeof(snap_list)); /* Initialize the per-thread random number generator. */ __wt_random_init(&tinfo->rnd); /* Set up the default key and value buffers. */ - key_gen_setup(&keybuf); - val_gen_setup(&tinfo->rnd, &valbuf); + key = &_key; + key_gen_setup(key); + value = &_value; + val_gen_setup(&tinfo->rnd, value); /* Set the first operation where we'll create sessions and cursors. */ session_op = 0; @@ -252,12 +424,12 @@ ops(void *arg) /* Set the first operation where we'll perform checkpoint operations. */ ckpt_op = g.c_checkpoints ? mmrand(&tinfo->rnd, 100, 10000) : 0; - ckpt_available = 0; + ckpt_available = false; /* Set the first operation where we'll reset the session. */ reset_op = mmrand(&tinfo->rnd, 100, 10000); - for (intxn = 0; !tinfo->quit; ++tinfo->ops) { + for (intxn = false; !tinfo->quit; ++tinfo->ops) { /* * We can't checkpoint or swap sessions/cursors while in a * transaction, resolve any running transaction. @@ -267,7 +439,7 @@ ops(void *arg) testutil_check( session->commit_transaction(session, NULL)); ++tinfo->commit; - intxn = 0; + intxn = false; } /* Open up a new session and cursors. */ @@ -276,8 +448,8 @@ ops(void *arg) if (session != NULL) testutil_check(session->close(session, NULL)); - testutil_check(conn->open_session(conn, NULL, - ops_session_config(&tinfo->rnd), &session)); + testutil_check( + conn->open_session(conn, NULL, NULL, &session)); /* * 10% of the time, perform some read-only operations @@ -299,7 +471,7 @@ ops(void *arg) session_op += 250; /* Checkpoints are read-only. */ - readonly = 1; + readonly = true; } else { /* * Open two cursors: one for overwriting and one @@ -325,21 +497,32 @@ ops(void *arg) session_op += mmrand(&tinfo->rnd, 100, 5000); /* Updates supported. */ - readonly = 0; + readonly = false; } } /* Checkpoint the database. */ if (tinfo->ops == ckpt_op && g.c_checkpoints) { /* - * LSM and data-sources don't support named checkpoints, + * Checkpoints are single-threaded inside WiredTiger, + * skip our checkpoint if another thread is already + * doing one. + */ + ret = pthread_rwlock_trywrlock(&g.checkpoint_lock); + if (ret == EBUSY) + goto skip_checkpoint; + testutil_check(ret); + + /* + * LSM and data-sources don't support named checkpoints * and we can't drop a named checkpoint while there's a - * cursor open on it, otherwise 20% of the time name the - * checkpoint. + * backup in progress, otherwise name the checkpoint 5% + * of the time. */ - if (DATASOURCE("helium") || DATASOURCE("kvsbdb") || - DATASOURCE("lsm") || - readonly || mmrand(&tinfo->rnd, 1, 5) == 1) + if (mmrand(&tinfo->rnd, 1, 20) != 1 || + DATASOURCE("helium") || + DATASOURCE("kvsbdb") || DATASOURCE("lsm") || + pthread_rwlock_trywrlock(&g.backup_lock) == EBUSY) ckpt_config = NULL; else { (void)snprintf(ckpt_name, sizeof(ckpt_name), @@ -347,11 +530,6 @@ ops(void *arg) ckpt_config = ckpt_name; } - /* Named checkpoints lock out backups */ - if (ckpt_config != NULL) - testutil_check( - pthread_rwlock_wrlock(&g.backup_lock)); - testutil_checkfmt( session->checkpoint(session, ckpt_config), "%s", ckpt_config == NULL ? "" : ckpt_config); @@ -359,6 +537,8 @@ ops(void *arg) if (ckpt_config != NULL) testutil_check( pthread_rwlock_unlock(&g.backup_lock)); + testutil_check( + pthread_rwlock_unlock(&g.checkpoint_lock)); /* Rephrase the checkpoint name for cursor open. */ if (ckpt_config == NULL) @@ -367,9 +547,9 @@ ops(void *arg) else (void)snprintf(ckpt_name, sizeof(ckpt_name), "checkpoint=thread-%d", tinfo->id); - ckpt_available = 1; + ckpt_available = true; - /* Pick the next checkpoint operation. */ +skip_checkpoint: /* Pick the next checkpoint operation. */ ckpt_op += mmrand(&tinfo->rnd, 5000, 20000); } @@ -386,21 +566,24 @@ ops(void *arg) } /* - * If we're not single-threaded and we're not in a transaction, - * start a transaction 20% of the time. + * If we're not single-threaded and not in a transaction, choose + * an isolation level and start a transaction some percentage of + * the time. */ if (!SINGLETHREADED && - !intxn && mmrand(&tinfo->rnd, 1, 10) >= 8) { + !intxn && mmrand(&tinfo->rnd, 1, 100) >= g.c_txn_freq) { + testutil_check( + session->reconfigure(session, + isolation_config(&tinfo->rnd, &iso_snapshot))); testutil_check( session->begin_transaction(session, NULL)); - intxn = 1; - } - insert = notfound = 0; + snap = iso_snapshot ? snap_list : NULL; + intxn = true; + } keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows); - key.data = keybuf; - value.data = valbuf; + positioned = false; /* * Perform some number of operations: the percentage of deletes, @@ -414,27 +597,30 @@ ops(void *arg) ++tinfo->remove; switch (g.type) { case ROW: - /* - * If deleting a non-existent record, the cursor - * won't be positioned, and so can't do a next. - */ - if (row_remove(cursor, &key, keyno, ¬found)) - goto deadlock; + ret = row_remove(cursor, key, keyno); break; case FIX: case VAR: - if (col_remove(cursor, &key, keyno, ¬found)) - goto deadlock; + ret = col_remove(cursor, key, keyno); break; } + if (ret == 0) { + positioned = true; + if (snap != NULL && (size_t) + (snap - snap_list) < WT_ELEMENTS(snap_list)) + snap_track(snap++, keyno, NULL, NULL); + } else { + positioned = false; + if (ret == WT_ROLLBACK && intxn) + goto deadlock; + } } else if (op < g.c_delete_pct + g.c_insert_pct) { ++tinfo->insert; switch (g.type) { case ROW: - if (row_insert( - tinfo, cursor, &key, &value, keyno)) - goto deadlock; - insert = 1; + key_gen_insert(&tinfo->rnd, key, keyno); + val_gen(&tinfo->rnd, value, keyno); + ret = row_insert(cursor, key, value, keyno); break; case FIX: case VAR: @@ -447,37 +633,60 @@ ops(void *arg) goto skip_insert; /* Insert, then reset the insert cursor. */ - if (col_insert(tinfo, - cursor_insert, &key, &value, &keyno)) - goto deadlock; + val_gen(&tinfo->rnd, value, g.rows + 1); + ret = col_insert( + cursor_insert, key, value, &keyno); testutil_check( cursor_insert->reset(cursor_insert)); - - insert = 1; break; } + positioned = false; + if (ret == 0) { + if (snap != NULL && (size_t) + (snap - snap_list) < WT_ELEMENTS(snap_list)) + snap_track(snap++, keyno, + g.type == ROW ? key : NULL, value); + } else + if (ret == WT_ROLLBACK && intxn) + goto deadlock; } else if ( op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) { ++tinfo->update; switch (g.type) { case ROW: - if (row_update( - tinfo, cursor, &key, &value, keyno)) - goto deadlock; + key_gen(key, keyno); + val_gen(&tinfo->rnd, value, keyno); + ret = row_update(cursor, key, value, keyno); break; case FIX: case VAR: -skip_insert: if (col_update(tinfo, - cursor, &key, &value, keyno)) - goto deadlock; +skip_insert: val_gen(&tinfo->rnd, value, keyno); + ret = col_update(cursor, key, value, keyno); break; } + if (ret == 0) { + positioned = true; + if (snap != NULL && (size_t) + (snap - snap_list) < WT_ELEMENTS(snap_list)) + snap_track(snap++, keyno, NULL, value); + } else { + positioned = false; + if (ret == WT_ROLLBACK && intxn) + goto deadlock; + } } else { ++tinfo->search; - if (read_row(cursor, &key, keyno, 0)) - if (intxn) + ret = read_row(cursor, key, value, keyno); + if (ret == 0) { + positioned = true; + if (snap != NULL && (size_t) + (snap - snap_list) < WT_ELEMENTS(snap_list)) + snap_track(snap++, keyno, NULL, value); + } else { + positioned = false; + if (ret == WT_ROLLBACK && intxn) goto deadlock; - continue; + } } /* @@ -485,55 +694,64 @@ skip_insert: if (col_update(tinfo, * insert, do a small number of next/prev cursor operations in * a random direction. */ - if (!insert) { + if (positioned) { dir = (int)mmrand(&tinfo->rnd, 0, 1); - for (np = 0; np < mmrand(&tinfo->rnd, 1, 100); ++np) { - if (notfound) - break; - if (nextprev(cursor, dir, ¬found)) + for (i = 0; i < mmrand(&tinfo->rnd, 1, 100); ++i) { + if ((ret = nextprev(cursor, dir)) == 0) + continue; + if (ret == WT_ROLLBACK && intxn) goto deadlock; + break; } } - /* Read to confirm the operation. */ - ++tinfo->search; - if (read_row(cursor, &key, keyno, 0)) - goto deadlock; - /* Reset the cursor: there is no reason to keep pages pinned. */ testutil_check(cursor->reset(cursor)); /* - * If we're in the transaction, commit 40% of the time and + * If we're in a transaction, commit 40% of the time and * rollback 10% of the time. */ - if (intxn) - switch (mmrand(&tinfo->rnd, 1, 10)) { - case 1: case 2: case 3: case 4: /* 40% */ - testutil_check(session->commit_transaction( - session, NULL)); - ++tinfo->commit; - intxn = 0; - break; - case 5: /* 10% */ - if (0) { -deadlock: ++tinfo->deadlock; - } - testutil_check(session->rollback_transaction( - session, NULL)); - ++tinfo->rollback; - intxn = 0; - break; - default: - break; + if (!intxn || (rnd = mmrand(&tinfo->rnd, 1, 10)) > 5) + continue; + + /* + * Ending the transaction. If in snapshot isolation, repeat the + * operations and confirm they're unchanged. + */ + if (snap != NULL && (ret = snap_check( + cursor, snap_list, snap, key, value)) == WT_ROLLBACK) + goto deadlock; + + switch (rnd) { + case 1: case 2: case 3: case 4: /* 40% */ + testutil_check( + session->commit_transaction(session, NULL)); + ++tinfo->commit; + break; + case 5: /* 10% */ + if (0) { +deadlock: ++tinfo->deadlock; } + testutil_check( + session->rollback_transaction(session, NULL)); + ++tinfo->rollback; + break; + } + + intxn = false; + snap = NULL; } if (session != NULL) testutil_check(session->close(session, NULL)); - free(keybuf); - free(valbuf); + for (i = 0; i < WT_ELEMENTS(snap_list); ++i) { + free(snap_list[i].kdata); + free(snap_list[i].vdata); + } + free(key->mem); + free(value->mem); tinfo->state = TINFO_COMPLETE; return (NULL); @@ -548,40 +766,47 @@ wts_read_scan(void) { WT_CONNECTION *conn; WT_CURSOR *cursor; - WT_ITEM key; + WT_DECL_RET; + WT_ITEM key, value; WT_SESSION *session; - uint64_t cnt, last_cnt; - uint8_t *keybuf; + uint64_t keyno, last_keyno; conn = g.wts_conn; - /* Set up the default key buffer. */ - key_gen_setup(&keybuf); + /* Set up the default key/value buffers. */ + key_gen_setup(&key); + val_gen_setup(NULL, &value); /* Open a session and cursor pair. */ - testutil_check(conn->open_session( - conn, NULL, ops_session_config(NULL), &session)); - testutil_check(session->open_cursor( - session, g.uri, NULL, NULL, &cursor)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check( + session->open_cursor(session, g.uri, NULL, NULL, &cursor)); /* Check a random subset of the records using the key. */ - for (last_cnt = cnt = 0; cnt < g.key_cnt;) { - cnt += mmrand(NULL, 1, 17); - if (cnt > g.rows) - cnt = g.rows; - if (cnt - last_cnt > 1000) { - track("read row scan", cnt, NULL); - last_cnt = cnt; + for (last_keyno = keyno = 0; keyno < g.key_cnt;) { + keyno += mmrand(NULL, 1, 17); + if (keyno > g.rows) + keyno = g.rows; + if (keyno - last_keyno > 1000) { + track("read row scan", keyno, NULL); + last_keyno = keyno; } - key.data = keybuf; - testutil_checkfmt( - read_row(cursor, &key, cnt, 0), "%s", "read_scan"); + switch (ret = read_row(cursor, &key, &value, keyno)) { + case 0: + case WT_NOTFOUND: + case WT_ROLLBACK: + break; + default: + testutil_die( + ret, "wts_read_scan: read row %" PRIu64, keyno); + } } testutil_check(session->close(session, NULL)); - free(keybuf); + free(key.mem); + free(value.mem); } /* @@ -589,10 +814,9 @@ wts_read_scan(void) * Read and verify a single element in a row- or column-store file. */ int -read_row(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int notfound_err) +read_row(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) { static int sn = 0; - WT_ITEM value; WT_SESSION *session; int exact, ret; uint8_t bitfield; @@ -611,7 +835,7 @@ read_row(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int notfound_err) cursor->set_key(cursor, keyno); break; case ROW: - key_gen((uint8_t *)key->data, &key->size, keyno); + key_gen(key, keyno); cursor->set_key(cursor, key); break; } @@ -628,37 +852,33 @@ read_row(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int notfound_err) switch (ret) { case 0: if (g.type == FIX) { - ret = cursor->get_value(cursor, &bitfield); - value.data = &bitfield; - value.size = 1; + testutil_check(cursor->get_value(cursor, &bitfield)); + *(uint8_t *)(value->data) = bitfield; + value->size = 1; } else - ret = cursor->get_value(cursor, &value); + testutil_check(cursor->get_value(cursor, value)); break; - case WT_ROLLBACK: - return (WT_ROLLBACK); case WT_NOTFOUND: - if (notfound_err) - return (WT_NOTFOUND); + /* + * In fixed length stores, zero values at the end of the key + * space are returned as not found. Treat this the same as + * a zero value in the key space, to match BDB's behavior. + */ + if (g.type == FIX) { + *(uint8_t *)(value->data) = 0; + value->size = 1; + ret = 0; + } break; + case WT_ROLLBACK: + return (WT_ROLLBACK); default: testutil_die(ret, "read_row: read row %" PRIu64, keyno); } #ifdef HAVE_BERKELEY_DB if (!SINGLETHREADED) - return (0); - - /* - * In fixed length stores, zero values at the end of the key space are - * returned as not found. Treat this the same as a zero value in the - * key space, to match BDB's behavior. - */ - if (ret == WT_NOTFOUND && g.type == FIX) { - bitfield = 0; - value.data = &bitfield; - value.size = 1; - ret = 0; - } + return (ret); /* Retrieve the BDB value. */ { @@ -669,20 +889,20 @@ read_row(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int notfound_err) /* Check for not-found status. */ if (notfound_chk("read_row", ret, notfound, keyno)) - return (0); + return (ret); /* Compare the two. */ - if (value.size != bdb_value.size || - memcmp(value.data, bdb_value.data, value.size) != 0) { + if (value->size != bdb_value.size || + memcmp(value->data, bdb_value.data, value->size) != 0) { fprintf(stderr, "read_row: value mismatch %" PRIu64 ":\n", keyno); print_item("bdb", &bdb_value); - print_item(" wt", &value); + print_item(" wt", value); testutil_die(0, NULL); } } #endif - return (0); + return (ret); } /* @@ -690,21 +910,19 @@ read_row(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int notfound_err) * Read and verify the next/prev element in a row- or column-store file. */ static int -nextprev(WT_CURSOR *cursor, int next, int *notfoundp) +nextprev(WT_CURSOR *cursor, int next) { + WT_DECL_RET; WT_ITEM key, value; uint64_t keyno; - int ret; uint8_t bitfield; const char *which; + keyno = 0; which = next ? "next" : "prev"; - keyno = 0; - ret = next ? cursor->next(cursor) : cursor->prev(cursor); - if (ret == WT_ROLLBACK) - return (WT_ROLLBACK); - if (ret == 0) + switch (ret = (next ? cursor->next(cursor) : cursor->prev(cursor))) { + case 0: switch (g.type) { case FIX: if ((ret = cursor->get_key(cursor, &keyno)) == 0 && @@ -722,13 +940,20 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp) ret = cursor->get_value(cursor, &value); break; } - if (ret != 0 && ret != WT_NOTFOUND) + if (ret != 0) + testutil_die(ret, "nextprev: get_key/get_value"); + break; + case WT_NOTFOUND: + break; + case WT_ROLLBACK: + return (WT_ROLLBACK); + default: testutil_die(ret, "%s", which); - *notfoundp = (ret == WT_NOTFOUND); + } #ifdef HAVE_BERKELEY_DB if (!SINGLETHREADED) - return (0); + return (ret); { WT_ITEM bdb_key, bdb_value; @@ -743,7 +968,7 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp) &bdb_value.data, &bdb_value.size, ¬found); if (notfound_chk( next ? "nextprev(next)" : "nextprev(prev)", ret, notfound, keyno)) - return (0); + return (ret); /* Compare the two. */ if (g.type == ROW) { @@ -794,7 +1019,7 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp) } } #endif - return (0); + return (ret); } /* @@ -802,43 +1027,38 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp) * Update a row in a row-store file. */ static int -row_update(TINFO *tinfo, - WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) +row_update(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) { + WT_DECL_RET; WT_SESSION *session; - int ret; session = cursor->session; - key_gen((uint8_t *)key->data, &key->size, keyno); - val_gen(&tinfo->rnd, (uint8_t *)value->data, &value->size, keyno); - /* Log the operation */ if (g.logging == LOG_OPS) (void)g.wt_api->msg_printf(g.wt_api, session, - "%-10s{%.*s}\n%-10s{%.*s}", - "putK", (int)key->size, (char *)key->data, - "putV", (int)value->size, (char *)value->data); + "%-10s{%.*s}, {%.*s}", + "put", + (int)key->size, key->data, (int)value->size, value->data); cursor->set_key(cursor, key); cursor->set_value(cursor, value); - ret = cursor->update(cursor); - if (ret == WT_ROLLBACK) + switch (ret = cursor->update(cursor)) { + case 0: + break; + case WT_CACHE_FULL: + case WT_ROLLBACK: return (WT_ROLLBACK); - if (ret != 0 && ret != WT_NOTFOUND) + default: testutil_die(ret, "row_update: update row %" PRIu64 " by key", keyno); + } #ifdef HAVE_BERKELEY_DB if (!SINGLETHREADED) return (0); - { - int notfound; - - bdb_update(key->data, key->size, value->data, value->size, ¬found); - (void)notfound_chk("row_update", ret, notfound, keyno); - } + bdb_update(key->data, key->size, value->data, value->size); #endif return (0); } @@ -848,16 +1068,13 @@ row_update(TINFO *tinfo, * Update a row in a column-store file. */ static int -col_update(TINFO *tinfo, - WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) +col_update(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) { + WT_DECL_RET; WT_SESSION *session; - int ret; session = cursor->session; - val_gen(&tinfo->rnd, (uint8_t *)value->data, &value->size, keyno); - /* Log the operation */ if (g.logging == LOG_OPS) { if (g.type == FIX) @@ -877,23 +1094,22 @@ col_update(TINFO *tinfo, cursor->set_value(cursor, *(uint8_t *)value->data); else cursor->set_value(cursor, value); - ret = cursor->update(cursor); - if (ret == WT_ROLLBACK) + switch (ret = cursor->update(cursor)) { + case 0: + break; + case WT_CACHE_FULL: + case WT_ROLLBACK: return (WT_ROLLBACK); - if (ret != 0 && ret != WT_NOTFOUND) + default: testutil_die(ret, "col_update: %" PRIu64, keyno); + } #ifdef HAVE_BERKELEY_DB if (!SINGLETHREADED) return (0); - { - int notfound; - - key_gen((uint8_t *)key->data, &key->size, keyno); - bdb_update(key->data, key->size, value->data, value->size, ¬found); - (void)notfound_chk("col_update", ret, notfound, keyno); - } + key_gen(key, keyno); + bdb_update(key->data, key->size, value->data, value->size); #else (void)key; /* [-Wunused-variable] */ #endif @@ -912,8 +1128,7 @@ table_append_init(void) g.append_cnt = 0; free(g.append); - if ((g.append = calloc(g.append_max, sizeof(uint64_t))) == NULL) - testutil_die(errno, "calloc"); + g.append = dcalloc(g.append_max, sizeof(uint64_t)); } /* @@ -1005,43 +1220,38 @@ table_append(uint64_t keyno) * Insert a row in a row-store file. */ static int -row_insert(TINFO *tinfo, - WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) +row_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) { + WT_DECL_RET; WT_SESSION *session; - int ret; session = cursor->session; - key_gen_insert(&tinfo->rnd, (uint8_t *)key->data, &key->size, keyno); - val_gen(&tinfo->rnd, (uint8_t *)value->data, &value->size, keyno); - /* Log the operation */ if (g.logging == LOG_OPS) (void)g.wt_api->msg_printf(g.wt_api, session, - "%-10s{%.*s}\n%-10s{%.*s}", - "insertK", (int)key->size, (char *)key->data, - "insertV", (int)value->size, (char *)value->data); + "%-10s{%.*s}, {%.*s}", + "insert", + (int)key->size, key->data, (int)value->size, value->data); cursor->set_key(cursor, key); cursor->set_value(cursor, value); - ret = cursor->insert(cursor); - if (ret == WT_ROLLBACK) + switch (ret = cursor->insert(cursor)) { + case 0: + break; + case WT_CACHE_FULL: + case WT_ROLLBACK: return (WT_ROLLBACK); - if (ret != 0 && ret != WT_NOTFOUND) + default: testutil_die(ret, "row_insert: insert row %" PRIu64 " by key", keyno); + } #ifdef HAVE_BERKELEY_DB if (!SINGLETHREADED) return (0); - { - int notfound; - - bdb_update(key->data, key->size, value->data, value->size, ¬found); - (void)notfound_chk("row_insert", ret, notfound, keyno); - } + bdb_update(key->data, key->size, value->data, value->size); #endif return (0); } @@ -1051,24 +1261,25 @@ row_insert(TINFO *tinfo, * Insert an element in a column-store file. */ static int -col_insert(TINFO *tinfo, - WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t *keynop) +col_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t *keynop) { + WT_DECL_RET; WT_SESSION *session; uint64_t keyno; - int ret; session = cursor->session; - val_gen(&tinfo->rnd, (uint8_t *)value->data, &value->size, g.rows + 1); - if (g.type == FIX) cursor->set_value(cursor, *(uint8_t *)value->data); else cursor->set_value(cursor, value); - if ((ret = cursor->insert(cursor)) != 0) { - if (ret == WT_ROLLBACK) - return (WT_ROLLBACK); + switch (ret = cursor->insert(cursor)) { + case 0: + break; + case WT_CACHE_FULL: + case WT_ROLLBACK: + return (WT_ROLLBACK); + default: testutil_die(ret, "cursor.insert"); } testutil_check(cursor->get_key(cursor, &keyno)); @@ -1093,12 +1304,8 @@ col_insert(TINFO *tinfo, if (!SINGLETHREADED) return (0); - { - int notfound; - - key_gen((uint8_t *)key->data, &key->size, keyno); - bdb_update(key->data, key->size, value->data, value->size, ¬found); - } + key_gen(key, keyno); + bdb_update(key->data, key->size, value->data, value->size); #else (void)key; /* [-Wunused-variable] */ #endif @@ -1110,14 +1317,14 @@ col_insert(TINFO *tinfo, * Remove an row from a row-store file. */ static int -row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp) +row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno) { + WT_DECL_RET; WT_SESSION *session; - int ret; session = cursor->session; - key_gen((uint8_t *)key->data, &key->size, keyno); + key_gen(key, keyno); /* Log the operation */ if (g.logging == LOG_OPS) @@ -1128,16 +1335,20 @@ row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp) /* We use the cursor in overwrite mode, check for existence. */ if ((ret = cursor->search(cursor)) == 0) ret = cursor->remove(cursor); - if (ret == WT_ROLLBACK) + switch (ret) { + case 0: + case WT_NOTFOUND: + break; + case WT_ROLLBACK: return (WT_ROLLBACK); - if (ret != 0 && ret != WT_NOTFOUND) + default: testutil_die(ret, "row_remove: remove %" PRIu64 " by key", keyno); - *notfoundp = (ret == WT_NOTFOUND); + } #ifdef HAVE_BERKELEY_DB if (!SINGLETHREADED) - return (0); + return (ret); { int notfound; @@ -1148,7 +1359,7 @@ row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp) #else (void)key; /* [-Wunused-variable] */ #endif - return (0); + return (ret); } /* @@ -1156,10 +1367,10 @@ row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp) * Remove a row from a column-store file. */ static int -col_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp) +col_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno) { + WT_DECL_RET; WT_SESSION *session; - int ret; session = cursor->session; @@ -1172,35 +1383,38 @@ col_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp) /* We use the cursor in overwrite mode, check for existence. */ if ((ret = cursor->search(cursor)) == 0) ret = cursor->remove(cursor); - if (ret == WT_ROLLBACK) + switch (ret) { + case 0: + case WT_NOTFOUND: + break; + case WT_ROLLBACK: return (WT_ROLLBACK); - if (ret != 0 && ret != WT_NOTFOUND) + default: testutil_die(ret, "col_remove: remove %" PRIu64 " by key", keyno); - *notfoundp = (ret == WT_NOTFOUND); + } #ifdef HAVE_BERKELEY_DB if (!SINGLETHREADED) - return (0); - - { - int notfound; + return (ret); /* * Deleting a fixed-length item is the same as setting the bits to 0; * do the same thing for the BDB store. */ if (g.type == FIX) { - key_gen((uint8_t *)key->data, &key->size, keyno); - bdb_update(key->data, key->size, "\0", 1, ¬found); - } else + key_gen(key, keyno); + bdb_update(key->data, key->size, "\0", 1); + } else { + int notfound; + bdb_remove(keyno, ¬found); - (void)notfound_chk("col_remove", ret, notfound, keyno); + (void)notfound_chk("col_remove", ret, notfound, keyno); } #else (void)key; /* [-Wunused-variable] */ #endif - return (0); + return (ret); } #ifdef HAVE_BERKELEY_DB diff --git a/test/format/salvage.c b/test/format/salvage.c index 526e1563390..8274c556364 100644 --- a/test/format/salvage.c +++ b/test/format/salvage.c @@ -36,8 +36,8 @@ static void salvage(void) { WT_CONNECTION *conn; + WT_DECL_RET; WT_SESSION *session; - int ret; conn = g.wts_conn; track("salvage", 0ULL, NULL); @@ -141,7 +141,7 @@ found: if (fstat(fd, &sb) == -1) void wts_salvage(void) { - int ret; + WT_DECL_RET; /* Some data-sources don't support salvage. */ if (DATASOURCE("helium") || DATASOURCE("kvsbdb")) @@ -158,7 +158,7 @@ wts_salvage(void) testutil_die(ret, "salvage copy step failed"); /* Salvage, then verify. */ - wts_open(g.home, 1, &g.wts_conn); + wts_open(g.home, true, &g.wts_conn); salvage(); wts_verify("post-salvage verify"); wts_close(); @@ -174,7 +174,7 @@ wts_salvage(void) /* Corrupt the file randomly, salvage, then verify. */ if (corrupt()) { - wts_open(g.home, 1, &g.wts_conn); + wts_open(g.home, true, &g.wts_conn); salvage(); wts_verify("post-corrupt-salvage verify"); wts_close(); diff --git a/test/format/t.c b/test/format/t.c index 28c22e23cb8..085163befe2 100644 --- a/test/format/t.c +++ b/test/format/t.c @@ -181,6 +181,7 @@ main(int argc, char *argv[]) */ testutil_check(pthread_rwlock_init(&g.append_lock, NULL)); testutil_check(pthread_rwlock_init(&g.backup_lock, NULL)); + testutil_check(pthread_rwlock_init(&g.checkpoint_lock, NULL)); testutil_check(pthread_rwlock_init(&g.death_lock, NULL)); printf("%s: process %" PRIdMAX "\n", g.progname, (intmax_t)getpid()); @@ -198,8 +199,8 @@ main(int argc, char *argv[]) if (SINGLETHREADED) bdb_open(); /* Initial file config */ #endif - wts_open(g.home, 1, &g.wts_conn); - wts_create(); + wts_open(g.home, true, &g.wts_conn); + wts_init(); wts_load(); /* Load initial records */ wts_verify("post-bulk verify"); /* Verify */ @@ -275,6 +276,8 @@ main(int argc, char *argv[]) testutil_check(pthread_rwlock_destroy(&g.append_lock)); testutil_check(pthread_rwlock_destroy(&g.backup_lock)); + testutil_check(pthread_rwlock_destroy(&g.checkpoint_lock)); + testutil_check(pthread_rwlock_destroy(&g.death_lock)); config_clear(); @@ -288,7 +291,7 @@ main(int argc, char *argv[]) static void startup(void) { - int ret; + WT_DECL_RET; /* Flush/close any logging information. */ fclose_and_clear(&g.logfp); diff --git a/test/format/util.c b/test/format/util.c index 2e4c869366c..f2b4d18029e 100644 --- a/test/format/util.c +++ b/test/format/util.c @@ -32,56 +32,11 @@ #define MAX(a, b) (((a) > (b)) ? (a) : (b)) #endif -/* - * dmalloc -- - * Call malloc, dying on failure. - */ -void * -dmalloc(size_t len) -{ - void *p; - - if ((p = malloc(len)) == NULL) - testutil_die(errno, "malloc"); - return (p); -} - -/* - * dstrdup -- - * Call strdup, dying on failure. - */ -char * -dstrdup(const char *str) -{ - char *p; - - if ((p = strdup(str)) == NULL) - testutil_die(errno, "strdup"); - return (p); -} - -static inline uint32_t -kv_len(WT_RAND_STATE *rnd, uint64_t keyno, uint32_t min, uint32_t max) -{ - /* - * Focus on relatively small key/value items, admitting the possibility - * of larger items. Pick a size close to the minimum most of the time, - * only create a larger item 1 in 20 times, and a really big item 1 in - * 1000 times. (Configuration can force large key/value minimum sizes, - * where every key/value item is an overflow.) - */ - if (keyno % 1000 == 0 && max < KILOBYTE(80)) { - min = KILOBYTE(80); - max = KILOBYTE(100); - } else if (keyno % 20 != 0 && max > min + 20) - max = min + 20; - return (mmrand(rnd, min, max)); -} - void key_len_setup(void) { size_t i; + uint32_t max; /* * The key is a variable length item with a leading 10-digit value. @@ -91,72 +46,113 @@ key_len_setup(void) * the pre-loaded lengths. * * Fill in the random key lengths. + * + * Focus on relatively small items, admitting the possibility of larger + * items. Pick a size close to the minimum most of the time, only create + * a larger item 1 in 20 times. */ - for (i = 0; i < sizeof(g.key_rand_len) / sizeof(g.key_rand_len[0]); ++i) - g.key_rand_len[i] = - kv_len(NULL, (uint64_t)i, g.c_key_min, g.c_key_max); + for (i = 0; + i < sizeof(g.key_rand_len) / sizeof(g.key_rand_len[0]); ++i) { + max = g.c_key_max; + if (i % 20 != 0 && max > g.c_key_min + 20) + max = g.c_key_min + 20; + g.key_rand_len[i] = mmrand(NULL, g.c_key_min, max); + } } void -key_gen_setup(uint8_t **keyp) +key_gen_setup(WT_ITEM *key) { - uint8_t *key; size_t i, len; - - *keyp = NULL; + char *p; len = MAX(KILOBYTE(100), g.c_key_max); - key = dmalloc(len); + p = dmalloc(len); for (i = 0; i < len; ++i) - key[i] = (uint8_t)("abcdefghijklmnopqrstuvwxyz"[i % 26]); - *keyp = key; + p[i] = "abcdefghijklmnopqrstuvwxyz"[i % 26]; + + key->mem = p; + key->memsize = len; + key->data = key->mem; + key->size = 0; } static void -key_gen_common(uint8_t *key, size_t *sizep, uint64_t keyno, int suffix) +key_gen_common(WT_ITEM *key, uint64_t keyno, int suffix) { int len; + char *p; + + p = key->mem; /* * The key always starts with a 10-digit string (the specified cnt) * followed by two digits, a random number between 1 and 15 if it's * an insert, otherwise 00. */ - len = sprintf((char *)key, "%010" PRIu64 ".%02d", keyno, suffix); + len = sprintf(p, "%010" PRIu64 ".%02d", keyno, suffix); /* - * In a column-store, the key is only used for BDB, and so it doesn't - * need a random length. + * In a column-store, the key is only used for Berkeley DB inserts, + * and so it doesn't need a random length. */ if (g.type == ROW) { - key[len] = '/'; - len = (int)g.key_rand_len[keyno % - (sizeof(g.key_rand_len) / sizeof(g.key_rand_len[0]))]; + p[len] = '/'; + + /* + * Because we're doing table lookup for key sizes, we weren't + * able to set really big keys sizes in the table, the table + * isn't big enough to keep our hash from selecting too many + * big keys and blowing out the cache. Handle that here, use a + * really big key 1 in 2500 times. + */ + len = keyno % 2500 == 0 && g.c_key_max < KILOBYTE(80) ? + KILOBYTE(80) : + (int)g.key_rand_len[keyno % WT_ELEMENTS(g.key_rand_len)]; } - *sizep = (size_t)len; + + key->data = key->mem; + key->size = (size_t)len; } void -key_gen(uint8_t *key, size_t *sizep, uint64_t keyno) +key_gen(WT_ITEM *key, uint64_t keyno) { - key_gen_common(key, sizep, keyno, 0); + key_gen_common(key, keyno, 0); } void -key_gen_insert(WT_RAND_STATE *rnd, uint8_t *key, size_t *sizep, uint64_t keyno) +key_gen_insert(WT_RAND_STATE *rnd, WT_ITEM *key, uint64_t keyno) { - key_gen_common(key, sizep, keyno, (int)mmrand(rnd, 1, 15)); + key_gen_common(key, keyno, (int)mmrand(rnd, 1, 15)); } static uint32_t val_dup_data_len; /* Length of duplicate data items */ +static inline uint32_t +value_len(WT_RAND_STATE *rnd, uint64_t keyno, uint32_t min, uint32_t max) +{ + /* + * Focus on relatively small items, admitting the possibility of larger + * items. Pick a size close to the minimum most of the time, only create + * a larger item 1 in 20 times, and a really big item 1 in somewhere + * around 2500 items. + */ + if (keyno % 2500 == 0 && max < KILOBYTE(80)) { + min = KILOBYTE(80); + max = KILOBYTE(100); + } else if (keyno % 20 != 0 && max > min + 20) + max = min + 20; + return (mmrand(rnd, min, max)); +} + void -val_gen_setup(WT_RAND_STATE *rnd, uint8_t **valp) +val_gen_setup(WT_RAND_STATE *rnd, WT_ITEM *value) { - uint8_t *val; size_t i, len; + char *p; - *valp = NULL; + memset(value, 0, sizeof(WT_ITEM)); /* * Set initial buffer contents to recognizable text. @@ -166,35 +162,43 @@ val_gen_setup(WT_RAND_STATE *rnd, uint8_t **valp) * data for column-store run-length encoded files. */ len = MAX(KILOBYTE(100), g.c_value_max) + 20; - val = dmalloc(len); + p = dmalloc(len); for (i = 0; i < len; ++i) - val[i] = (uint8_t)("ABCDEFGHIJKLMNOPQRSTUVWXYZ"[i % 26]); + p[i] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"[i % 26]; - *valp = val; + value->mem = p; + value->memsize = len; + value->data = value->mem; + value->size = 0; - val_dup_data_len = kv_len(rnd, + val_dup_data_len = value_len(rnd, (uint64_t)mmrand(rnd, 1, 20), g.c_value_min, g.c_value_max); } void -val_gen(WT_RAND_STATE *rnd, uint8_t *val, size_t *sizep, uint64_t keyno) +val_gen(WT_RAND_STATE *rnd, WT_ITEM *value, uint64_t keyno) { + char *p; + + p = value->mem; + value->data = value->mem; + /* * Fixed-length records: take the low N bits from the last digit of * the record number. */ if (g.type == FIX) { switch (g.c_bitcnt) { - case 8: val[0] = (uint8_t)mmrand(rnd, 1, 0xff); break; - case 7: val[0] = (uint8_t)mmrand(rnd, 1, 0x7f); break; - case 6: val[0] = (uint8_t)mmrand(rnd, 1, 0x3f); break; - case 5: val[0] = (uint8_t)mmrand(rnd, 1, 0x1f); break; - case 4: val[0] = (uint8_t)mmrand(rnd, 1, 0x0f); break; - case 3: val[0] = (uint8_t)mmrand(rnd, 1, 0x07); break; - case 2: val[0] = (uint8_t)mmrand(rnd, 1, 0x03); break; - case 1: val[0] = 1; break; + case 8: p[0] = (char)mmrand(rnd, 1, 0xff); break; + case 7: p[0] = (char)mmrand(rnd, 1, 0x7f); break; + case 6: p[0] = (char)mmrand(rnd, 1, 0x3f); break; + case 5: p[0] = (char)mmrand(rnd, 1, 0x1f); break; + case 4: p[0] = (char)mmrand(rnd, 1, 0x0f); break; + case 3: p[0] = (char)mmrand(rnd, 1, 0x07); break; + case 2: p[0] = (char)mmrand(rnd, 1, 0x03); break; + case 1: p[0] = 1; break; } - *sizep = 1; + value->size = 1; return; } @@ -203,8 +207,8 @@ val_gen(WT_RAND_STATE *rnd, uint8_t *val, size_t *sizep, uint64_t keyno) * test that by inserting a zero-length data item every so often. */ if (keyno % 63 == 0) { - val[0] = '\0'; - *sizep = 0; + p[0] = '\0'; + value->size = 0; return; } @@ -219,13 +223,14 @@ val_gen(WT_RAND_STATE *rnd, uint8_t *val, size_t *sizep, uint64_t keyno) if ((g.type == ROW || g.type == VAR) && g.c_repeat_data_pct != 0 && mmrand(rnd, 1, 100) < g.c_repeat_data_pct) { - (void)strcpy((char *)val, "DUPLICATEV"); - val[10] = '/'; - *sizep = val_dup_data_len; + (void)strcpy(p, "DUPLICATEV"); + p[10] = '/'; + value->size = val_dup_data_len; } else { - (void)sprintf((char *)val, "%010" PRIu64, keyno); - val[10] = '/'; - *sizep = kv_len(rnd, keyno, g.c_value_min, g.c_value_max); + (void)sprintf(p, "%010" PRIu64, keyno); + p[10] = '/'; + value->size = + value_len(rnd, keyno, g.c_value_min, g.c_value_max); } } @@ -305,15 +310,6 @@ path_setup(const char *home) g.home_stats = dmalloc(len); snprintf(g.home_stats, len, "%s/%s", g.home, "stats"); - /* Backup directory. */ - len = strlen(g.home) + strlen("BACKUP") + 2; - g.home_backup = dmalloc(len); - snprintf(g.home_backup, len, "%s/%s", g.home, "BACKUP"); - - len = strlen(g.home) + strlen("BACKUP2") + 2; - g.home_backup2 = dmalloc(len); - snprintf(g.home_backup2, len, "%s/%s", g.home, "BACKUP2"); - /* BDB directory. */ len = strlen(g.home) + strlen("bdb") + 2; g.home_bdb = dmalloc(len); @@ -341,18 +337,27 @@ path_setup(const char *home) g.home_init = dmalloc(len); snprintf(g.home_init, len, CMD, g.home, g.home, g.home); - /* Backup directory initialize command, remove and re-create it. */ + /* Primary backup directory. */ + len = strlen(g.home) + strlen("BACKUP") + 2; + g.home_backup = dmalloc(len); + snprintf(g.home_backup, len, "%s/%s", g.home, "BACKUP"); + + /* + * Backup directory initialize command, remove and re-create the primary + * backup directory, plus a copy we maintain for recovery testing. + */ #undef CMD #ifdef _WIN32 -#define CMD "del /s /q >:nul && mkdir %s %s" +#define CMD "del %s/%s %s/%s /s /q >:nul && mkdir %s/%s %s/%s" #else -#define CMD "rm -rf %s %s && mkdir %s %s" +#define CMD "rm -rf %s/%s %s/%s && mkdir %s/%s %s/%s" #endif - len = strlen(g.home_backup) * 2 + - strlen(g.home_backup2) * 2 + strlen(CMD) + 1; + len = strlen(g.home) * 4 + + strlen("BACKUP") * 2 + strlen("BACKUP_COPY") * 2 + strlen(CMD) + 1; g.home_backup_init = dmalloc(len); - snprintf(g.home_backup_init, len, CMD, g.home_backup, g.home_backup2, - g.home_backup, g.home_backup2); + snprintf(g.home_backup_init, len, CMD, + g.home, "BACKUP", g.home, "BACKUP_COPY", + g.home, "BACKUP", g.home, "BACKUP_COPY"); /* * Salvage command, save the interesting files so we can replay the diff --git a/test/format/wts.c b/test/format/wts.c index 81e484296e2..2ee01aa75b5 100644 --- a/test/format/wts.c +++ b/test/format/wts.c @@ -126,10 +126,10 @@ static WT_EVENT_HANDLER event_handler = { * Open a connection to a WiredTiger database. */ void -wts_open(const char *home, int set_api, WT_CONNECTION **connp) +wts_open(const char *home, bool set_api, WT_CONNECTION **connp) { WT_CONNECTION *conn; - int ret; + WT_DECL_RET; char *config, *end, *p, helium_config[1024]; *connp = NULL; @@ -138,10 +138,11 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) end = config + sizeof(g.wiredtiger_open_config); p += snprintf(p, REMAIN(p, end), - "create,checkpoint_sync=false,cache_size=%" PRIu32 "MB", - g.c_cache); - - p += snprintf(p, REMAIN(p, end), ",error_prefix=\"%s\"", g.progname); + "create=true," + "cache_size=%" PRIu32 "MB," + "checkpoint_sync=false," + "error_prefix=\"%s\"", + g.c_cache, g.progname); /* In-memory configuration. */ if (g.c_in_memory != 0) @@ -273,8 +274,13 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) void wts_reopen(void) { + WT_CONNECTION *conn; + testutil_checkfmt(wiredtiger_open(g.home, &event_handler, - g.wiredtiger_open_config, &g.wts_conn), "%s", g.home); + g.wiredtiger_open_config, &conn), "%s", g.home); + + g.wt_api = conn->get_extension_api(conn); + g.wts_conn = conn; } /* @@ -282,7 +288,7 @@ wts_reopen(void) * Create the underlying store. */ void -wts_create(void) +wts_init(void) { WT_CONNECTION *conn; WT_SESSION *session; @@ -497,8 +503,8 @@ void wts_verify(const char *tag) { WT_CONNECTION *conn; + WT_DECL_RET; WT_SESSION *session; - int ret; if (g.c_verify == 0) return; @@ -531,12 +537,12 @@ wts_stats(void) { WT_CONNECTION *conn; WT_CURSOR *cursor; + WT_DECL_RET; WT_SESSION *session; FILE *fp; char *stat_name; const char *pval, *desc; uint64_t v; - int ret; /* Ignore statistics if they're not configured. */ if (g.c_statistics == 0) diff --git a/test/huge/Makefile.am b/test/huge/Makefile.am index bc76bdc0f3c..151d3a40dd4 100644 --- a/test/huge/Makefile.am +++ b/test/huge/Makefile.am @@ -10,4 +10,4 @@ t_LDFLAGS = -static TESTS = smoke.sh clean-local: - rm -rf WiredTiger* *.core __* + rm -rf WT_TEST *.core diff --git a/test/huge/huge.c b/test/huge/huge.c index ad19035ff99..e7bfd08882f 100644 --- a/test/huge/huge.c +++ b/test/huge/huge.c @@ -26,14 +26,6 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include <errno.h> -#include <stdlib.h> -#include <stdio.h> -#include <string.h> -#ifndef _WIN32 -#include <unistd.h> -#endif - #include "test_util.i" static char home[512]; /* Program working dir */ @@ -205,8 +197,7 @@ main(int argc, char *argv[]) /* Allocate a buffer to use. */ len = small ? ((size_t)SMALL_MAX) : ((size_t)4 * GIGABYTE); - if ((big = malloc(len)) == NULL) - testutil_die(errno, ""); + big = dmalloc(len); memset(big, 'a', len); /* Make sure the configurations all work. */ diff --git a/test/manydbs/Makefile.am b/test/manydbs/Makefile.am index 53559b25243..d347868aa4f 100644 --- a/test/manydbs/Makefile.am +++ b/test/manydbs/Makefile.am @@ -10,4 +10,4 @@ t_LDFLAGS = -static TESTS = smoke.sh clean-local: - rm -rf WiredTiger* *.core __* + rm -rf WT_TEST *.core diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 1d3412a7b06..4ab455f3620 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -26,22 +26,10 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include <sys/wait.h> -#include <errno.h> -#include <signal.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#ifndef _WIN32 -#include <unistd.h> -#endif - -#include <wiredtiger.h> - #include "test_util.i" #define HOME_SIZE 512 -#define HOME_BASE "WT_HOME" +#define HOME_BASE "WT_TEST" static char home[HOME_SIZE]; /* Base home directory */ static char hometmp[HOME_SIZE]; /* Each conn home directory */ static const char *progname; /* Program name */ @@ -172,17 +160,10 @@ main(int argc, char *argv[]) * Allocate arrays for connection handles, sessions, statistics * cursors and, if needed, data cursors. */ - if ((connections = calloc( - (size_t)dbs, sizeof(WT_CONNECTION *))) == NULL) - testutil_die(ENOMEM, "connection array malloc"); - if ((sessions = calloc( - (size_t)dbs, sizeof(WT_SESSION *))) == NULL) - testutil_die(ENOMEM, "session array malloc"); - if ((cond_reset_orig = calloc((size_t)dbs, sizeof(uint64_t))) == NULL) - testutil_die(ENOMEM, "orig stat malloc"); - if (!idle && ((cursors = calloc( - (size_t)dbs, sizeof(WT_CURSOR *))) == NULL)) - testutil_die(ENOMEM, "cursor array malloc"); + connections = dcalloc((size_t)dbs, sizeof(WT_CONNECTION *)); + sessions = dcalloc((size_t)dbs, sizeof(WT_SESSION *)); + cond_reset_orig = dcalloc((size_t)dbs, sizeof(uint64_t)); + cursors = idle ? NULL : dcalloc((size_t)dbs, sizeof(WT_CURSOR *)); memset(cmd, 0, sizeof(cmd)); /* * Set up all the directory names. @@ -257,8 +238,7 @@ main(int argc, char *argv[]) free(connections); free(sessions); free(cond_reset_orig); - if (!idle) - free(cursors); + free(cursors); return (EXIT_SUCCESS); } diff --git a/test/packing/Makefile.am b/test/packing/Makefile.am index a9e7e16e5c2..0e7c8cc8b2e 100644 --- a/test/packing/Makefile.am +++ b/test/packing/Makefile.am @@ -1,4 +1,5 @@ -AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/test/utility noinst_PROGRAMS = intpack-test intpack-test2 intpack-test3 packing-test LDADD = $(top_builddir)/libwiredtiger.la diff --git a/test/packing/intpack-test.c b/test/packing/intpack-test.c index 08cc3807725..6412ed296aa 100644 --- a/test/packing/intpack-test.c +++ b/test/packing/intpack-test.c @@ -26,9 +26,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "wt_internal.h" /* For __wt_XXX */ +#include "test_util.i" -#include <assert.h> +void (*custom_die)(void) = NULL; int main(void) @@ -47,9 +47,10 @@ main(void) #if 1 p = buf; - assert(__wt_vpack_uint(&p, sizeof(buf), r) == 0); + testutil_check(__wt_vpack_uint(&p, sizeof(buf), r)); cp = buf; - assert(__wt_vunpack_uint(&cp, sizeof(buf), &r2) == 0); + testutil_check( + __wt_vunpack_uint(&cp, sizeof(buf), &r2)); #else /* * Note: use memmove for comparison because GCC does diff --git a/test/packing/intpack-test2.c b/test/packing/intpack-test2.c index 7555d2724e7..e9443ad7ed1 100644 --- a/test/packing/intpack-test2.c +++ b/test/packing/intpack-test2.c @@ -26,9 +26,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "wt_internal.h" /* For __wt_XXX */ +#include "test_util.i" -#include <assert.h> +void (*custom_die)(void) = NULL; int main(void) @@ -38,14 +38,15 @@ main(void) for (i = 1; i < 1LL << 60; i <<= 1) { end = buf; - assert(__wt_vpack_uint(&end, sizeof(buf), (uint64_t)i) == 0); + testutil_check( + __wt_vpack_uint(&end, sizeof(buf), (uint64_t)i)); printf("%" PRId64 " ", i); for (p = buf; p < end; p++) printf("%02x", *p); printf("\n"); end = buf; - assert(__wt_vpack_int(&end, sizeof(buf), -i) == 0); + testutil_check(__wt_vpack_int(&end, sizeof(buf), -i)); printf("%" PRId64 " ", -i); for (p = buf; p < end; p++) printf("%02x", *p); diff --git a/test/packing/intpack-test3.c b/test/packing/intpack-test3.c index 2ebc01f9e2e..328b45d1bf7 100644 --- a/test/packing/intpack-test3.c +++ b/test/packing/intpack-test3.c @@ -26,9 +26,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "wt_internal.h" /* For __wt_XXX */ +#include "test_util.i" -#include <assert.h> +void (*custom_die)(void) = NULL; void test_value(int64_t); void test_spread(int64_t, int64_t, int64_t); @@ -43,11 +43,12 @@ test_value(int64_t val) size_t used_len; sinput = val; + soutput = 0; /* Make GCC happy. */ p = buf; - assert(__wt_vpack_int(&p, sizeof(buf), sinput) == 0); + testutil_check(__wt_vpack_int(&p, sizeof(buf), sinput)); used_len = (size_t)(p - buf); cp = buf; - assert(__wt_vunpack_int(&cp, used_len, &soutput) == 0); + testutil_check(__wt_vunpack_int(&cp, used_len, &soutput)); /* Ensure we got the correct value back */ if (sinput != soutput) { fprintf(stderr, "mismatch %" PRIu64 ", %" PRIu64 "\n", @@ -69,10 +70,9 @@ test_value(int64_t val) uinput = (uint64_t)val; p = buf; - assert(__wt_vpack_uint(&p, sizeof(buf), uinput) == 0); + testutil_check(__wt_vpack_uint(&p, sizeof(buf), uinput)); cp = buf; - assert(__wt_vunpack_uint( - &cp, sizeof(buf), &uoutput) == 0); + testutil_check(__wt_vunpack_uint(&cp, sizeof(buf), &uoutput)); /* Ensure we got the correct value back */ if (sinput != soutput) { fprintf(stderr, "mismatch %" PRIu64 ", %" PRIu64 "\n", diff --git a/test/packing/packing-test.c b/test/packing/packing-test.c index 9b7105d7d4a..706eeb0935c 100644 --- a/test/packing/packing-test.c +++ b/test/packing/packing-test.c @@ -26,9 +26,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "wt_internal.h" /* For __wt_XXX */ +#include "test_util.i" -#include <assert.h> +void (*custom_die)(void) = NULL; static void check(const char *fmt, ...) @@ -40,13 +40,15 @@ check(const char *fmt, ...) len = 0; /* -Werror=maybe-uninitialized */ va_start(ap, fmt); - assert(__wt_struct_sizev(NULL, &len, fmt, ap) == 0); + testutil_check(__wt_struct_sizev(NULL, &len, fmt, ap)); va_end(ap); - assert(len > 0 && len < sizeof(buf)); + if (len < 1 || len >= sizeof(buf)) + testutil_die(EINVAL, + "Unexpected length from __wt_struct_sizev"); va_start(ap, fmt); - assert(__wt_struct_packv(NULL, buf, sizeof(buf), fmt, ap) == 0); + testutil_check(__wt_struct_packv(NULL, buf, sizeof(buf), fmt, ap)); va_end(ap); printf("%s ", fmt); diff --git a/test/readonly/Makefile.am b/test/readonly/Makefile.am index 3abcd2386a1..8028e2ab845 100644 --- a/test/readonly/Makefile.am +++ b/test/readonly/Makefile.am @@ -10,4 +10,4 @@ t_LDFLAGS = -static TESTS = smoke.sh clean-local: - rm -rf WT_RD* WiredTiger* *.core __* + rm -rf WT_RD* *.core diff --git a/test/readonly/readonly.c b/test/readonly/readonly.c index 41400da2605..a35e7ee23fc 100644 --- a/test/readonly/readonly.c +++ b/test/readonly/readonly.c @@ -26,20 +26,10 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include <sys/wait.h> -#include <errno.h> -#include <signal.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#ifndef _WIN32 -#include <unistd.h> -#endif - -#include <wiredtiger.h> - #include "test_util.i" +#include <sys/wait.h> + #define HOME_SIZE 512 static char home[HOME_SIZE]; /* Program working dir lock file */ #define HOME_WR_SUFFIX ".WRNOLOCK" /* Writable dir copy no lock file */ diff --git a/test/recovery/Makefile.am b/test/recovery/Makefile.am index 35f8dd15823..6865d5edf3e 100644 --- a/test/recovery/Makefile.am +++ b/test/recovery/Makefile.am @@ -15,4 +15,4 @@ TESTS = $(noinst_PROGRAMS) LOG_COMPILER = $(TEST_WRAPPER) clean-local: - rm -rf WT_TEST* *.core __* + rm -rf WT_TEST.* *.core diff --git a/test/salvage/Makefile.am b/test/salvage/Makefile.am index 3e686dd2951..0fd46aefcb1 100644 --- a/test/salvage/Makefile.am +++ b/test/salvage/Makefile.am @@ -11,4 +11,4 @@ TESTS = $(noinst_PROGRAMS) LOG_COMPILER = $(TEST_WRAPPER) clean-local: - rm -rf WiredTiger* *.core __* + rm -rf WiredTiger* __slvg* *.core diff --git a/test/salvage/salvage.c b/test/salvage/salvage.c index a1517d70787..f264be99e2b 100644 --- a/test/salvage/salvage.c +++ b/test/salvage/salvage.c @@ -159,7 +159,7 @@ int usage(void) { (void)fprintf(stderr, - "usage: %s [-v] [-r run] [-t fix|rle|var|row]\n", progname); + "usage: %s [-v] [-r run] [-t fix|var|row]\n", progname); return (EXIT_FAILURE); } @@ -170,7 +170,7 @@ run(int r) printf("\t%s: run %d\n", __wt_page_type_string(page_type), r); - CHECK(system("rm -f WiredTiger* __slvg.* __schema.*") == 0); + CHECK(system("rm -f WiredTiger* __slvg.*") == 0); CHECK((res_fp = fopen(RSLT, "w")) != NULL); /* diff --git a/test/suite/test_backup05.py b/test/suite/test_backup05.py index 991a9f71b19..fbe219d8de8 100644 --- a/test/suite/test_backup05.py +++ b/test/suite/test_backup05.py @@ -37,10 +37,12 @@ import fnmatch, os, shutil, time from suite_subprocess import suite_subprocess from wtscenario import multiply_scenarios, number_scenarios, prune_scenarios from helper import copy_wiredtiger_home -import wttest +import wiredtiger, wttest class test_backup05(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:test_backup05' + emptyuri = 'table:test_empty05' + newuri = 'table:test_new05' create_params = 'key_format=i,value_format=i' freq = 5 @@ -51,12 +53,35 @@ class test_backup05(wttest.WiredTigerTestCase, suite_subprocess): # With the connection still open, copy files to new directory. # Half the time use an unaligned copy. - aligned = (i % (self.freq * 2) != 0) or os.name == "nt" + even = i % (self.freq * 2) == 0 + aligned = even or os.name == "nt" copy_wiredtiger_home(olddir, newdir, aligned) + # Half the time try to rename a table and the other half try + # to remove a table. They should fail. + if not even: + self.assertRaises(wiredtiger.WiredTigerError, + lambda: self.session.rename( + self.emptyuri, self.newuri, None)) + else: + self.assertRaises(wiredtiger.WiredTigerError, + lambda: self.session.drop(self.emptyuri, None)) + # Now simulate fsyncUnlock by closing the backup cursor. cbkup.close() + # Once the backup cursor is closed we should be able to perform + # schema operations. Test that and then reset the files to their + # expected initial names. + if not even: + self.session.rename(self.emptyuri, self.newuri, None) + self.session.drop(self.newuri, None) + self.session.create(self.emptyuri, self.create_params) + else: + self.session.drop(self.emptyuri, None) + self.session.create(self.emptyuri, self.create_params) + + # Open the new directory and verify conn = self.setUpConnectionOpen(newdir) session = self.setUpSessionOpen(conn) @@ -77,6 +102,10 @@ class test_backup05(wttest.WiredTigerTestCase, suite_subprocess): # # If the metadata isn't flushed, eventually the metadata we copy will # be sufficiently out-of-sync with the data file that it won't verify. + + self.session.create(self.emptyuri, self.create_params) + self.reopen_conn() + self.session.create(self.uri, self.create_params) for i in range(100): c = self.session.open_cursor(self.uri) @@ -88,7 +117,7 @@ class test_backup05(wttest.WiredTigerTestCase, suite_subprocess): self.session.verify(self.uri) def test_backup(self): - with self.expectedStdoutPattern('Recreating metadata'): + with self.expectedStdoutPattern('recreating metadata'): self.backup() if __name__ == '__main__': diff --git a/test/suite/test_join01.py b/test/suite/test_join01.py index 4aa2bc6e269..f8d96a2718a 100644 --- a/test/suite/test_join01.py +++ b/test/suite/test_join01.py @@ -35,10 +35,44 @@ from wtscenario import check_scenarios, multiply_scenarios, number_scenarios class test_join01(wttest.WiredTigerTestCase): nentries = 100 - scenarios = [ + type_scen = [ ('table', dict(ref='table')), ('index', dict(ref='index')) ] + bloom0_scen = [ + ('bloom0=0', dict(joincfg0='')), + ('bloom0=1000', dict(joincfg0=',strategy=bloom,count=1000')), + ('bloom0=10000', dict(joincfg0=',strategy=bloom,count=10000')), + ] + bloom1_scen = [ + ('bloom1=0', dict(joincfg1='')), + ('bloom1=1000', dict(joincfg1=',strategy=bloom,count=1000')), + ('bloom1=10000', dict(joincfg1=',strategy=bloom,count=10000')), + ] + projection_scen = [ + ('no-projection', dict(do_proj=False)), + ('projection', dict(do_proj=True)) + ] + nested_scen = [ + ('simple', dict(do_nested=False)), + ('nested', dict(do_nested=True)) + ] + stats_scen = [ + ('no-stats', dict(do_stats=False)), + ('stats', dict(do_stats=True)) + ] + order_scen = [ + ('order=0', dict(join_order=0)), + ('order=1', dict(join_order=1)), + ('order=2', dict(join_order=2)), + ('order=3', dict(join_order=3)), + ] + scenarios = number_scenarios(multiply_scenarios('.', type_scen, + bloom0_scen, bloom1_scen, + projection_scen, + nested_scen, stats_scen, + order_scen)) + # We need statistics for these tests. conn_config = 'statistics=(all)' @@ -52,9 +86,29 @@ class test_join01(wttest.WiredTigerTestCase): return [s, rs, sort3] # Common function for testing iteration of join cursors - def iter_common(self, jc, do_proj): + def iter_common(self, jc, do_proj, do_nested, join_order): # See comments in join_common() - expect = [73, 82, 62, 83, 92] + # The order that the results are seen depends on + # the ordering of the joins. Specifically, the first + # join drives the order that results are seen. + if do_nested: + if join_order == 0: + expect = [73, 82, 83, 92] + elif join_order == 1: + expect = [73, 82, 83, 92] + elif join_order == 2: + expect = [82, 92, 73, 83] + elif join_order == 3: + expect = [92, 73, 82, 83] + else: + if join_order == 0: + expect = [73, 82, 62, 83, 92] + elif join_order == 1: + expect = [62, 73, 82, 83, 92] + elif join_order == 2: + expect = [62, 82, 92, 73, 83] + elif join_order == 3: + expect = [73, 82, 62, 83, 92] while jc.next() == 0: [k] = jc.get_keys() i = k - 1 @@ -64,7 +118,9 @@ class test_join01(wttest.WiredTigerTestCase): [v0,v1,v2] = jc.get_values() self.assertEquals(self.gen_values(i), [v0,v1,v2]) if len(expect) == 0 or i != expect[0]: - self.tty(' result ' + str(i) + ' is not in: ' + str(expect)) + self.tty('ERROR: ' + str(i) + ' is not next in: ' + + str(expect)) + self.tty('JOIN ORDER=' + str(join_order) + ', NESTED=' + str(do_nested)) self.assertTrue(i == expect[0]) expect.remove(i) self.assertEquals(0, len(expect)) @@ -81,6 +137,8 @@ class test_join01(wttest.WiredTigerTestCase): 'join: index:join01:index2: ' + statdesc ] if self.ref == 'index': expectstats.append('join: index:join01:index0: ' + statdesc) + elif self.do_proj: + expectstats.append('join: table:join01(v2,v1,v0): ' + statdesc) else: expectstats.append('join: table:join01: ' + statdesc) self.check_stats(statcur, expectstats) @@ -118,11 +176,46 @@ class test_join01(wttest.WiredTigerTestCase): self.assertTrue(len(expectstats) == 0, 'missing expected values in stats: ' + str(expectstats)) + def session_record_join(self, jc, refc, config, order, joins): + joins.append([order, [jc, refc, config]]) + + def session_play_one_join(self, firsturi, jc, refc, config): + if refc.uri == firsturi and config != None: + config = config.replace('strategy=bloom','') + #self.tty('->join(jc, uri="' + refc.uri + + # '", config="' + str(config) + '"') + self.session.join(jc, refc, config) + + def session_play_joins(self, joins, join_order): + #self.tty('->') + firsturi = None + for [i, joinargs] in joins: + if i >= join_order: + if firsturi == None: + firsturi = joinargs[1].uri + self.session_play_one_join(firsturi, *joinargs) + for [i, joinargs] in joins: + if i < join_order: + if firsturi == None: + firsturi = joinargs[1].uri + self.session_play_one_join(firsturi, *joinargs) + # Common function for testing the most basic functionality # of joins - def join_common(self, joincfg0, joincfg1, do_proj, do_stats): + def test_join(self): + joincfg0 = self.joincfg0 + joincfg1 = self.joincfg1 + do_proj = self.do_proj + do_nested = self.do_nested + do_stats = self.do_stats + join_order = self.join_order #self.tty('join_common(' + joincfg0 + ',' + joincfg1 + ',' + - # str(do_proj) + ')') + # str(do_proj) + ',' + str(do_nested) + ',' + + # str(do_stats) + ',' + str(join_order) + ')') + + closeme = [] + joins = [] # cursors to be joined + self.session.create('table:join01', 'key_format=r' + ',value_format=SSi,columns=(k,v0,v1,v2)') self.session.create('index:join01:index0','columns=(v0)') @@ -143,7 +236,7 @@ class test_join01(wttest.WiredTigerTestCase): # We join on index2 first, not using bloom indices. # This defines the order that items are returned. - # index2 is sorts multiples of 3 first (see gen_values()) + # index2 sorts multiples of 3 first (see gen_values()) # and by using 'gt' and key 99, we'll skip multiples of 3, # and examine primary keys 2,5,8,...,95,98,1,4,7,...,94,97. jc = self.session.open_cursor('join:table:join01' + proj_suffix, @@ -152,7 +245,7 @@ class test_join01(wttest.WiredTigerTestCase): c2 = self.session.open_cursor('index:join01:index2(v1)', None, None) c2.set_key(99) # skips all entries w/ primary key divisible by three self.assertEquals(0, c2.search()) - self.session.join(jc, c2, 'compare=gt') + self.session_record_join(jc, c2, 'compare=gt', 0, joins) # Then select all the numbers 0-99 whose string representation # sort >= '60'. @@ -163,285 +256,87 @@ class test_join01(wttest.WiredTigerTestCase): c0 = self.session.open_cursor('table:join01', None, None) c0.set_key(60) self.assertEquals(0, c0.search()) - self.session.join(jc, c0, 'compare=ge' + joincfg0) + self.session_record_join(jc, c0, 'compare=ge' + joincfg0, 1, joins) # Then select all numbers whose reverse string representation # is in '20' < x < '40'. c1a = self.session.open_cursor('index:join01:index1(v1)', None, None) c1a.set_key('21') self.assertEquals(0, c1a.search()) - self.session.join(jc, c1a, 'compare=gt' + joincfg1) + self.session_record_join(jc, c1a, 'compare=gt' + joincfg1, 2, joins) c1b = self.session.open_cursor('index:join01:index1(v1)', None, None) c1b.set_key('41') self.assertEquals(0, c1b.search()) - self.session.join(jc, c1b, 'compare=lt' + joincfg1) + self.session_record_join(jc, c1b, 'compare=lt' + joincfg1, 2, joins) # Numbers that satisfy these 3 conditions (with ordering implied by c2): # [73, 82, 62, 83, 92]. # # After iterating, we should be able to reset and iterate again. + if do_nested: + # To test nesting, we create two new levels of conditions: + # + # x == 72 or x == 73 or x == 82 or x == 83 or + # (x >= 90 and x <= 99) + # + # that will get AND-ed into our existing join. The expected + # result is [73, 82, 83, 92]. + # + # We don't specify the projection here, it should be picked up + # from the 'enclosing' join. + nest1 = self.session.open_cursor('join:table:join01', None, None) + nest2 = self.session.open_cursor('join:table:join01', None, None) + + nc = self.session.open_cursor('index:join01:index0', None, None) + nc.set_key('90') + self.assertEquals(0, nc.search()) + self.session.join(nest2, nc, 'compare=ge') # joincfg left out + closeme.append(nc) + + nc = self.session.open_cursor('index:join01:index0', None, None) + nc.set_key('99') + self.assertEquals(0, nc.search()) + self.session.join(nest2, nc, 'compare=le') + closeme.append(nc) + + self.session.join(nest1, nest2, "operation=or") + + for val in [ '72', '73', '82', '83' ]: + nc = self.session.open_cursor('index:join01:index0', None, None) + nc.set_key(val) + self.assertEquals(0, nc.search()) + self.session.join(nest1, nc, 'compare=eq,operation=or' + + joincfg0) + closeme.append(nc) + self.session_record_join(jc, nest1, None, 3, joins) + + self.session_play_joins(joins, join_order) + self.iter_common(jc, do_proj, do_nested, join_order) if do_stats: self.stats(jc, 0) - self.iter_common(jc, do_proj) + jc.reset() + self.iter_common(jc, do_proj, do_nested, join_order) if do_stats: self.stats(jc, 1) jc.reset() - self.iter_common(jc, do_proj) + self.iter_common(jc, do_proj, do_nested, join_order) if do_stats: self.stats(jc, 2) jc.reset() - self.iter_common(jc, do_proj) + self.iter_common(jc, do_proj, do_nested, join_order) jc.close() c2.close() c1a.close() c1b.close() c0.close() + if do_nested: + nest1.close() + nest2.close() + for c in closeme: + c.close() self.session.drop('table:join01') - # Test joins with basic functionality - def test_join(self): - bloomcfg1000 = ',strategy=bloom,count=1000' - bloomcfg10000 = ',strategy=bloom,count=10000' - for cfga in [ '', bloomcfg1000, bloomcfg10000 ]: - for cfgb in [ '', bloomcfg1000, bloomcfg10000 ]: - for do_proj in [ False, True ]: - #self.tty('cfga=' + cfga + - # ', cfgb=' + cfgb + - # ', doproj=' + str(do_proj)) - self.join_common(cfga, cfgb, do_proj, False) - - def test_join_errors(self): - self.session.create('table:join01', 'key_format=r,value_format=SS' - ',columns=(k,v0,v1)') - self.session.create('table:join01B', 'key_format=r,value_format=SS' - ',columns=(k,v0,v1)') - self.session.create('index:join01:index0','columns=(v0)') - self.session.create('index:join01:index1','columns=(v1)') - self.session.create('index:join01B:index0','columns=(v0)') - jc = self.session.open_cursor('join:table:join01', None, None) - tc = self.session.open_cursor('table:join01', None, None) - fc = self.session.open_cursor('file:join01.wt', None, None) - ic0 = self.session.open_cursor('index:join01:index0', None, None) - ic0again = self.session.open_cursor('index:join01:index0', None, None) - ic1 = self.session.open_cursor('index:join01:index1', None, None) - icB = self.session.open_cursor('index:join01B:index0', None, None) - tcB = self.session.open_cursor('table:join01B', None, None) - - tc.set_key(1) - tc.set_value('val1', 'val1') - tc.insert() - tcB.set_key(1) - tcB.set_value('val1', 'val1') - tcB.insert() - fc.next() - - # Joining using a non join-cursor - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(tc, ic0, 'compare=ge'), - '/not a join cursor/') - # Joining a table cursor, not index - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, fc, 'compare=ge'), - '/not an index or table cursor/') - # Joining a non positioned cursor - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic0, 'compare=ge'), - '/requires reference cursor be positioned/') - ic0.set_key('val1') - # Joining a non positioned cursor (no search or next has been done) - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic0, 'compare=ge'), - '/requires reference cursor be positioned/') - ic0.set_key('valXX') - self.assertEqual(ic0.search(), wiredtiger.WT_NOTFOUND) - # Joining a non positioned cursor after failed search - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic0, 'compare=ge'), - '/requires reference cursor be positioned/') - - # position the cursors now - ic0.set_key('val1') - ic0.search() - ic0again.next() - icB.next() - - # Joining non matching index - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, icB, 'compare=ge'), - '/table for join cursor does not match/') - - # The cursor must be positioned - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic1, 'compare=ge'), - '/requires reference cursor be positioned/') - ic1.next() - - # The first cursor joined cannot be bloom - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic1, - 'compare=ge,strategy=bloom,count=1000'), - '/first joined cursor cannot specify strategy=bloom/') - - # This succeeds. - self.session.join(jc, ic1, 'compare=ge'), - - # With bloom filters, a count is required - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic0, 'compare=ge,strategy=bloom'), - '/count must be nonzero/') - - # This succeeds. - self.session.join(jc, ic0, 'compare=ge,strategy=bloom,count=1000'), - - bloom_config = ',strategy=bloom,count=1000' - # Cannot use the same index cursor - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic0, - 'compare=le' + bloom_config), - '/index cursor already used in a join/') - - # When joining with the same index, need compatible compares - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic0again, 'compare=ge' + bloom_config), - '/join has overlapping ranges/') - - # Another incompatible compare - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic0again, 'compare=gt' + bloom_config), - '/join has overlapping ranges/') - - # Compare is compatible, but bloom args need to match - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic0again, 'compare=le'), - '/join has incompatible strategy/') - - # Counts need to match for bloom filters - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.join(jc, ic0again, 'compare=le,strategy=bloom,' - 'count=100'), '/count.* does not match previous count/') - - # This succeeds - self.session.join(jc, ic0again, 'compare=le,strategy=bloom,count=1000') - - # Need to do initial next() before getting key/values - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: jc.get_keys(), - '/join cursor must be advanced with next/') - - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: jc.get_values(), - '/join cursor must be advanced with next/') - - # Operations on the joined cursor are frozen until the join is closed. - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: ic0.next(), - '/index cursor is being used in a join/') - - # Operations on the joined cursor are frozen until the join is closed. - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: ic0.prev(), - '/index cursor is being used in a join/') - - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: ic0.reset(), - '/index cursor is being used in a join/') - - # Only a small number of operations allowed on a join cursor - msg = "/Unsupported cursor/" - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: jc.search(), msg) - - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: jc.prev(), msg) - - self.assertEquals(jc.next(), 0) - self.assertEquals(jc.next(), wiredtiger.WT_NOTFOUND) - - # Only after the join cursor is closed can we use the index cursor - # normally - jc.close() - self.assertEquals(ic0.next(), wiredtiger.WT_NOTFOUND) - self.assertEquals(ic0.prev(), 0) - - # common code for making sure that cursors can be - # implicitly closed, no matter the order they are created - def cursor_close_common(self, joinfirst): - self.session.create('table:join01', 'key_format=r' + - ',value_format=SS,columns=(k,v0,v1)') - self.session.create('index:join01:index0','columns=(v0)') - self.session.create('index:join01:index1','columns=(v1)') - c = self.session.open_cursor('table:join01', None, None) - for i in range(0, self.nentries): - c.set_key(*self.gen_key(i)) - c.set_value(*self.gen_values(i)) - c.insert() - c.close() - - if joinfirst: - jc = self.session.open_cursor('join:table:join01', None, None) - c0 = self.session.open_cursor('index:join01:index0', None, None) - c1 = self.session.open_cursor('index:join01:index1', None, None) - c0.next() # index cursors must be positioned - c1.next() - if not joinfirst: - jc = self.session.open_cursor('join:table:join01', None, None) - self.session.join(jc, c0, 'compare=ge') - self.session.join(jc, c1, 'compare=ge') - self.session.close() - self.session = None - - def test_cursor_close1(self): - self.cursor_close_common(True) - - def test_cursor_close2(self): - self.cursor_close_common(False) - - # test statistics using the framework set up for this test - def test_stats(self): - bloomcfg1000 = ',strategy=bloom,count=1000' - bloomcfg10 = ',strategy=bloom,count=10' - self.join_common(bloomcfg1000, bloomcfg1000, False, True) - - # Intentially run with an underconfigured Bloom filter, - # statistics should pick up some false positives. - self.join_common(bloomcfg10, bloomcfg10, False, True) - - # test statistics with a simple one index join cursor - def test_simple_stats(self): - self.session.create("table:join01b", - "key_format=i,value_format=i,columns=(k,v)") - self.session.create("index:join01b:index", "columns=(v)") - - cursor = self.session.open_cursor("table:join01b", None, None) - cursor[1] = 11 - cursor[2] = 12 - cursor[3] = 13 - cursor.close() - - cursor = self.session.open_cursor("index:join01b:index", None, None) - cursor.set_key(11) - cursor.search() - - jcursor = self.session.open_cursor("join:table:join01b", None, None) - self.session.join(jcursor, cursor, "compare=gt") - - while jcursor.next() == 0: - [k] = jcursor.get_keys() - [v] = jcursor.get_values() - - statcur = self.session.open_cursor("statistics:join", jcursor, None) - found = False - while statcur.next() == 0: - [desc, pvalue, value] = statcur.get_values() - #self.tty(str(desc) + "=" + str(pvalue)) - found = True - self.assertEquals(found, True) - - jcursor.close() - cursor.close() - - if __name__ == '__main__': wttest.run() diff --git a/test/suite/test_join07.py b/test/suite/test_join07.py new file mode 100644 index 00000000000..36e91361329 --- /dev/null +++ b/test/suite/test_join07.py @@ -0,0 +1,548 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os, re, run +import wiredtiger, wttest, suite_random +from wtscenario import check_scenarios, multiply_scenarios, number_scenarios + +class ParseException(Exception): + def __init__(self, msg): + super(ParseException, self).__init__(msg) + +class Token: + UNKNOWN = '<unknown>' + NUMBER = 'Number' + STRING = 'String' + COLUMN = 'Column' + LPAREN = '(' + RPAREN = ')' + LBRACKET = '{' + RBRACKET = '}' + COMMA = ',' + OR = '||' + AND = '&&' + LT = '<' + GT = '>' + LE = '<=' + GE = '>=' + EQ = '==' + ATTRIBUTE = 'Attribute' # bracketed key value pair + + COMPARE_OPS = [LT, GT, LE, GE, EQ] + COMPARATORS = [NUMBER, STRING] + + def __init__(self, kind, tokenizer): + self.kind = kind + self.pos = tokenizer.off + tokenizer.pos + self.n = 0 + self.s = '' + self.index = '' + self.attr_key = '' + self.attr_value = '' + self.groups = None + + def __str__(self): + return '<Token ' + self.kind + ' at char ' + str(self.pos) + '>' + +class Tokenizer: + def __init__(self, s): + self.off = 0 + self.s = s + '?' # add a char that won't match anything + self.pos = 0 + self.end = len(s) + self.re_num = re.compile(r"(\d+)") + self.re_quote1 = re.compile(r"'([^']*)'") + self.re_quote2 = re.compile(r"\"([^\"]*)\"") + self.re_attr = re.compile(r"\[(\w+)=(\w+)\]") + self.pushed = None + + def newToken(self, kind, sz): + t = Token(kind, self) + self.pos += sz + return t + + def error(self, s): + raise ParseException(str(self.pos) + ': ' + s) + + def matched(self, kind, repat): + pos = self.pos + match = re.match(repat, self.s[pos:]) + if not match: + end = pos + 10 + if end > self.end: + end = self.end + self.error('matching ' + kind + ' at "' + + self.s[pos:end] + '..."') + t = self.newToken(kind, match.end()) + t.groups = match.groups() + t.s = self.s[pos:pos + match.end()] + return t + + def available(self): + if self.pushed == None: + self.pushback(self.token()) + return (self.pushed != None) + + def pushback(self, token): + if self.pushed != None: + raise AssertionError('pushback more than once') + self.pushed = token + + def peek(self): + token = self.token() + self.pushback(token) + return token + + def scan(self): + while self.pos < self.end and self.s[self.pos].isspace(): + self.pos += 1 + return '' if self.pos >= self.end else self.s[self.pos] + + def token(self): + if self.pushed != None: + ret = self.pushed + self.pushed = None + return ret + c = self.scan() + if self.pos >= self.end: + return None + lookahead = '' if self.pos + 1 >= self.end else self.s[self.pos+1] + #self.tty("Tokenizer.token char=" + c + ", lookahead=" + lookahead) + if c == "'": + t = self.matched(Token.STRING, self.re_quote1) + t.s = t.groups[0] + return t + if c == '"': + t = self.matched(Token.STRING, self.re_quote2) + t.s = t.groups[0] + return t + if c in "{}(),": + return self.newToken(c, 1) + if c == "|": + if lookahead != "|": + self.error('matching OR') + return self.newToken(Token.OR, 2) + if c == "&": + if lookahead != "&": + self.error('matching AND') + return self.newToken(Token.AND, 2) + if c in "0123456789": + t = self.matched(Token.NUMBER, self.re_num) + t.s = t.groups[0] + t.n = int(t.s) + return t + if c in "ABCDEFGHIJ": + t = self.newToken(Token.COLUMN, 1) + t.s = c + return t + if c == '<': + if lookahead == '=': + return self.newToken(Token.LE, 2) + else: + return self.newToken(Token.LT, 1) + if c == '>': + if lookahead == '=': + return self.newToken(Token.GE, 2) + else: + return self.newToken(Token.GT, 1) + if c in "=": + if lookahead != "=": + self.error('matching EQ') + return self.newToken(Token.EQ, 2) + if c in "[": + t = self.matched(Token.ATTRIBUTE, self.re_attr) + t.attr_key = t.groups[0] + t.attr_value = t.groups[1] + return t + return None + + def tty(self, s): + wttest.WiredTigerTestCase.tty(s) + +# test_join07.py +# Join interpreter +class test_join07(wttest.WiredTigerTestCase): + reverseop = { '==' : '==', '<=' : '>=', '<' : '>', '>=' : '<=', '>' : '<' } + compareop = { '==' : 'eq', '<=' : 'le', '<' : 'lt', '>=' : 'ge', + '>' : 'gt' } + columnmult = { 'A' : 1, 'B' : 2, 'C' : 3, 'D' : 4, 'E' : 5, + 'F' : 6, 'G' : 7, 'H' : 8, 'I' : 9, 'J' : 10 } + + extractscen = [ + ('extractor', dict(extractor=True)), + ('noextractor', dict(extractor=False)) + ] + + scenarios = number_scenarios(extractscen) + + # Return the wiredtiger_open extension argument for a shared library. + def extensionArg(self, exts): + extfiles = [] + for ext in exts: + (dirname, name, libname) = ext + if name != None and name != 'none': + testdir = os.path.dirname(__file__) + extdir = os.path.join(run.wt_builddir, 'ext', dirname) + extfile = os.path.join( + extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') + if not os.path.exists(extfile): + self.skipTest('extension "' + extfile + '" not built') + if not extfile in extfiles: + extfiles.append(extfile) + if len(extfiles) == 0: + return '' + else: + return ',extensions=["' + '","'.join(extfiles) + '"]' + + # Override WiredTigerTestCase, we have extensions. + def setUpConnectionOpen(self, dir): + extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor')]) + connarg = 'create,error_prefix="{0}: ",{1}'.format( + self.shortid(), extarg) + conn = self.wiredtiger_open(dir, connarg) + self.pr(`conn`) + return conn + + def expect(self, token, expected): + if token == None or token.kind not in expected: + self.err(token, 'expected one of: ' + str(expected)) + return token + + def err(self, token, msg): + self.assertTrue(False, 'ERROR at token ' + str(token) + ': ' + msg) + + def gen_key(self, i): + if self.keyformat == 'S': + return [ 'key%06d' % i ] # zero pad so it sorts expectedly + else: + return [ i ] + + def gen_values(self, i): + s = "" + ret = [] + for x in range(1, 11): + v = (i * x) % self.N + if x <= 5: + ret.append(v) + else: + ret.append(str(v)) + if s != "": + s += "," + s += str(v) + ret.insert(0, s) + return ret + + def iterate(self, jc, mbr): + mbr = set(mbr) # we need a mutable set + gotkeys = [] + #self.tty('iteration expects ' + str(len(mbr)) + + # ' entries: ' + str(mbr)) + while jc.next() == 0: + [k] = jc.get_keys() + values = jc.get_values() + if self.keyformat == 'S': + i = int(str(k[3:])) + else: + i = k + #self.tty('GOT key=' + str(k) + ', values=' + str(values)) + + # Duplicates may be returned when the disjunctions are used, + # so we ignore them. + if not i in gotkeys: + self.assertEquals(self.gen_values(i), values) + if not i in mbr: + self.tty('ERROR: result ' + str(i) + ' is not in: ' + + str(mbr)) + self.assertTrue(i in mbr) + mbr.remove(i) + gotkeys.append(i) + self.assertEquals(0, len(mbr)) + + def token_literal(self, token): + if token.kind == Token.STRING: + return token.s + elif token.kind == Token.NUMBER: + return token.n + + def idx_sim(self, x, mult, isstr): + if isstr: + return str(int(x) * mult % self.N) + else: + return (x * mult % self.N) + + def mkmbr(self, expr): + return frozenset([x for x in self.allN if expr(x)]) + + def join_one_side(self, jc, coltok, littok, optok, conjunction, + isright, mbr): + idxname = 'index:join07:' + coltok.s + cursor = self.session.open_cursor(idxname, None, None) + jc.cursors.append(cursor) + literal = self.token_literal(littok) + cursor.set_key(literal) + searchret = cursor.search() + if searchret != 0: + self.tty('ERROR: cannot find value ' + str(literal) + + ' in ' + idxname) + self.assertEquals(0, searchret) + op = optok.kind + if not isright: + op = self.reverseop[op] + mult = self.columnmult[coltok.s] + config = 'compare=' + self.compareop[op] + ',operation=' + \ + ('and' if conjunction else 'or') + if hasattr(coltok, 'bloom'): + config += ',strategy=bloom,count=' + str(coltok.bloom) + #self.tty('join(jc, cursor=' + str(literal) + ', ' + config) + self.session.join(jc, cursor, config) + isstr = type(literal) is str + if op == '==': + tmbr = self.mkmbr(lambda x: self.idx_sim(x, mult, isstr) == literal) + elif op == '<=': + tmbr = self.mkmbr(lambda x: self.idx_sim(x, mult, isstr) <= literal) + elif op == '<': + tmbr = self.mkmbr(lambda x: self.idx_sim(x, mult, isstr) < literal) + elif op == '>=': + tmbr = self.mkmbr(lambda x: self.idx_sim(x, mult, isstr) >= literal) + elif op == '>': + tmbr = self.mkmbr(lambda x: self.idx_sim(x, mult, isstr) > literal) + if conjunction: + mbr = mbr.intersection(tmbr) + else: + mbr = mbr.union(tmbr) + return mbr + + def parse_join(self, jc, tokenizer, conjunction, mbr): + left = None + right = None + leftop = None + rightop = None + col = None + token = tokenizer.token() + if token.kind == Token.LPAREN: + subjc = self.session.open_cursor('join:table:join07', None, None) + jc.cursors.append(subjc) + submbr = self.parse_junction(subjc, tokenizer) + config = 'operation=' + ('and' if conjunction else 'or') + self.session.join(jc, subjc, config) + if conjunction: + mbr = mbr.intersection(submbr) + else: + mbr = mbr.union(submbr) + return mbr + if token.kind in Token.COMPARATORS: + left = token + leftop = self.expect(tokenizer.token(), Token.COMPARE_OPS) + token = tokenizer.token() + col = self.expect(token, [Token.COLUMN]) + token = tokenizer.token() + if token.kind in Token.ATTRIBUTE: + tokenizer.pushback(token) + self.parse_column_attributes(tokenizer, col) + token = tokenizer.token() + if token.kind in Token.COMPARE_OPS: + rightop = token + right = self.expect(tokenizer.token(), Token.COMPARATORS) + token = tokenizer.token() + tokenizer.pushback(token) + + # Now we have everything we need to do a join. + if left != None: + mbr = self.join_one_side(jc, col, left, leftop, conjunction, + False, mbr) + if right != None: + mbr = self.join_one_side(jc, col, right, rightop, conjunction, + True, mbr) + return mbr + + # Parse a set of joins, grouped by && or || + def parse_junction(self, jc, tokenizer): + jc.cursors = [] + + # Take a peek at the tokenizer's stream to see if we + # have a conjunction or disjunction + token = tokenizer.peek() + s = tokenizer.s[token.pos:] + (andpos, orpos) = self.find_nonparen(s, ['&', '|']) + if orpos >= 0 and (andpos < 0 or orpos < andpos): + conjunction = False + mbr = frozenset() + else: + conjunction = True + mbr = frozenset(self.allN) + + while tokenizer.available(): + mbr = self.parse_join(jc, tokenizer, conjunction, mbr) + token = tokenizer.token() + if token != None: + if token.kind == Token.OR: + self.assertTrue(not conjunction) + elif token.kind == Token.AND: + self.assertTrue(conjunction) + elif token.kind == Token.RPAREN: + break + else: + self.err(token, 'unexpected token') + return mbr + + def parse_attributes(self, tokenizer): + attributes = [] + token = tokenizer.token() + while token != None and token.kind == Token.ATTRIBUTE: + attributes.append(token) + token = tokenizer.token() + tokenizer.pushback(token) + return attributes + + # Find a set of chars that aren't within parentheses. + # For this simple language, we don't allow parentheses in quoted literals. + def find_nonparen(self, s, matchlist): + pos = 0 + end = len(s) + nmatch = len(matchlist) + nfound = 0 + result = [-1 for i in range(0, nmatch)] + parennest = 0 + while pos < end and nfound < nmatch: + c = s[pos] + if c == '(': + parennest += 1 + elif c == ')': + parennest -= 1 + if parennest < 0: + break + elif parennest == 0 and c in matchlist: + m = matchlist.index(c) + if result[m] < 0: + result[m] = pos + nfound += 1 + pos += 1 + return result + + def parse_toplevel(self, jc, tokenizer): + return self.parse_junction(jc, tokenizer) + + def parse_toplevel_attributes(self, tokenizer): + for attrtoken in self.parse_attributes(tokenizer): + key = attrtoken.attr_key + value = attrtoken.attr_value + #self.tty('ATTR:' + str([key,value])) + if key == 'N': + self.N = int(value) + elif key == 'key': + self.keyformat = value + else: + tokenizer.error('bad attribute key: ' + str(key)) + + def parse_column_attributes(self, tokenizer, c): + for attrtoken in self.parse_attributes(tokenizer): + key = attrtoken.attr_key + value = attrtoken.attr_value + #self.tty('ATTR:' + str([key,value])) + if key == 'bloom': + c.bloom = int(value) + else: + tokenizer.error('bad column attribute key: ' + str(key)) + + def close_cursors(self, jc): + jc.close() + for c in jc.cursors: + if c.uri[0:5] == 'join:': + self.close_cursors(c) + else: + c.close() + + def interpret(self, s): + #self.tty('INTERPRET: ' + s) + self.N = 1000 + self.keyformat = "r" + self.keycols = 'k' + + # Grab attributes before creating anything, as some attributes + # may override needed parameters. + tokenizer = Tokenizer(s) + self.parse_toplevel_attributes(tokenizer) + self.allN = range(1, self.N + 1) + + self.session.create('table:join07', 'key_format=' + self.keyformat + + ',value_format=SiiiiiSSSSS,' + + 'columns=(' + self.keycols + + ',S,A,B,C,D,E,F,G,H,I,J)') + mdfieldnum = 0 + mdformat = 'i' + mdconfig = '' + for colname in [ 'A','B','C','D','E','F','G','H','I','J' ]: + if self.extractor: + if colname == 'F': + mdformat = 'S' + mdconfig = 'app_metadata={"format" : "%s","field" : "%d"}' % \ + (mdformat, mdfieldnum) + config = 'extractor=csv,key_format=%s' % mdformat + mdfieldnum += 1 + else: + config = 'columns=(%s)' % colname + self.session.create('index:join07:%s' % colname, + '%s,%s' % (config, mdconfig)) + c = self.session.open_cursor('table:join07', None, None) + for i in self.allN: + c.set_key(*self.gen_key(i)) + c.set_value(*self.gen_values(i)) + c.insert() + c.close() + + jc = self.session.open_cursor('join:table:join07', None, None) + mbr = self.parse_toplevel(jc, tokenizer) + self.iterate(jc, mbr) + + self.close_cursors(jc) + self.session.drop('table:join07') + + def test_join_string(self): + self.interpret("[N=1000][key=r] 7 < A <= 500 && B < 150 && C > 17") + self.interpret("[N=1001][key=r] 7 < A <= 500 && B < 150 && F > '234'") + self.interpret("[N=10000][key=r] 7 < A <= 500 && B < 150 && " + + "(F > '234' || G < '100')") + self.interpret("[N=7919][key=r](7 < A <= 9)&&(F > '234')") + self.interpret("[N=1000][key=S](A>=0 && A<0)||(A>999)") + self.interpret("[N=2000][key=S](A>=0 && A<0)||(A>1999)") + self.interpret("(7<A<=10 && B < 150)||(B>998)") + self.interpret("(7<A<=10 && B < 150)||(J=='990')") + clause1 = "(7 < A <= 500 && B < 150)" + clause2 = "(F > '234' || G < '100')" + self.interpret("[N=1000][key=r]" + clause1 + "&&" + clause2) + self.interpret("(7<A<=10)||(B>994||C<12)") + self.interpret("(7<A<=10 && B < 150)||(B>996||C<6)") + self.interpret("[N=1000][key=r]" + clause2 + "||" + clause1) + self.interpret("[N=1000][key=r]" + clause1 + "||" + clause2) + self.interpret("[N=1000][key=S]" + clause2 + "&&" + clause1) + clause1 = "(7 < A <= 500 && B[bloom=300] < 150)" + clause2 = "(F[bloom=500] > '234' || G[bloom=20] < '100')" + self.interpret("[N=1000][key=S]" + clause1 + "&&" + clause2) + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_join08.py b/test/suite/test_join08.py new file mode 100644 index 00000000000..6d674ab8193 --- /dev/null +++ b/test/suite/test_join08.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import check_scenarios, multiply_scenarios, number_scenarios + +# test_join08.py +# Test join error paths +class test_join08(wttest.WiredTigerTestCase): + nentries = 100 + + # We need statistics for these tests. + conn_config = 'statistics=(all)' + + def gen_key(self, i): + return [ i + 1 ] + + def gen_values(self, i): + s = str(i) + rs = s[::-1] + sort3 = (self.nentries * (i % 3)) + i # multiples of 3 sort first + return [s, rs, sort3] + + def test_join_errors(self): + self.session.create('table:join08', 'key_format=r,value_format=SS' + ',columns=(k,v0,v1)') + self.session.create('table:join08B', 'key_format=r,value_format=SS' + ',columns=(k,v0,v1)') + self.session.create('index:join08:index0','columns=(v0)') + self.session.create('index:join08:index1','columns=(v1)') + self.session.create('index:join08B:index0','columns=(v0)') + jc = self.session.open_cursor('join:table:join08', None, None) + tc = self.session.open_cursor('table:join08', None, None) + fc = self.session.open_cursor('file:join08.wt', None, None) + ic0 = self.session.open_cursor('index:join08:index0', None, None) + ic0again = self.session.open_cursor('index:join08:index0', None, None) + ic1 = self.session.open_cursor('index:join08:index1', None, None) + icB = self.session.open_cursor('index:join08B:index0', None, None) + tcB = self.session.open_cursor('table:join08B', None, None) + + tc.set_key(1) + tc.set_value('val1', 'val1') + tc.insert() + tcB.set_key(1) + tcB.set_value('val1', 'val1') + tcB.insert() + fc.next() + + # Joining using a non join-cursor + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.join(tc, ic0, 'compare=ge'), + '/not a join cursor/') + # Joining a table cursor, not index + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.join(jc, fc, 'compare=ge'), + '/must be an index, table or join cursor/') + # Joining a non positioned cursor + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.join(jc, ic0, 'compare=ge'), + '/requires reference cursor be positioned/') + ic0.set_key('val1') + # Joining a non positioned cursor (no search or next has been done) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.join(jc, ic0, 'compare=ge'), + '/requires reference cursor be positioned/') + ic0.set_key('valXX') + self.assertEqual(ic0.search(), wiredtiger.WT_NOTFOUND) + # Joining a non positioned cursor after failed search + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.join(jc, ic0, 'compare=ge'), + '/requires reference cursor be positioned/') + + # position the cursors now + ic0.set_key('val1') + ic0.search() + ic0again.next() + icB.next() + + # Joining non matching index + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.join(jc, icB, 'compare=ge'), + '/table for join cursor does not match/') + + # The cursor must be positioned + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.join(jc, ic1, 'compare=ge'), + '/requires reference cursor be positioned/') + ic1.next() + + # This succeeds. + self.session.join(jc, ic1, 'compare=ge'), + + # With bloom filters, a count is required + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.join(jc, ic0, 'compare=ge,strategy=bloom'), + '/count must be nonzero/') + + # This succeeds. + self.session.join(jc, ic0, 'compare=ge,strategy=bloom,count=1000'), + + bloom_config = ',strategy=bloom,count=1000' + # Cannot use the same index cursor + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.join(jc, ic0, + 'compare=le' + bloom_config), + '/cursor already used in a join/') + + # When joining with the same index, need compatible compares + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.join(jc, ic0again, 'compare=ge' + bloom_config), + '/join has overlapping ranges/') + + # Another incompatible compare + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.join(jc, ic0again, 'compare=gt' + bloom_config), + '/join has overlapping ranges/') + + # Compare is compatible, but bloom args need to match + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.join(jc, ic0again, 'compare=le'), + '/join has incompatible strategy/') + + # Counts need to match for bloom filters + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.join(jc, ic0again, 'compare=le,strategy=bloom,' + 'count=100'), '/count.* does not match previous count/') + + # This succeeds + self.session.join(jc, ic0again, 'compare=le,strategy=bloom,count=1000') + + # Need to do initial next() before getting key/values + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: jc.get_keys(), + '/join cursor must be advanced with next/') + + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: jc.get_values(), + '/join cursor must be advanced with next/') + + # Operations on the joined cursor are frozen until the join is closed. + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: ic0.next(), + '/cursor is being used in a join/') + + # Operations on the joined cursor are frozen until the join is closed. + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: ic0.prev(), + '/cursor is being used in a join/') + + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: ic0.reset(), + '/cursor is being used in a join/') + + # Only a small number of operations allowed on a join cursor + msg = "/Unsupported cursor/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: jc.search(), msg) + + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: jc.prev(), msg) + + self.assertEquals(jc.next(), 0) + self.assertEquals(jc.next(), wiredtiger.WT_NOTFOUND) + + # Only after the join cursor is closed can we use the index cursor + # normally + jc.close() + self.assertEquals(ic0.next(), wiredtiger.WT_NOTFOUND) + self.assertEquals(ic0.prev(), 0) + + # common code for making sure that cursors can be + # implicitly closed, no matter the order they are created + def cursor_close_common(self, joinfirst): + self.session.create('table:join08', 'key_format=r' + + ',value_format=SS,columns=(k,v0,v1)') + self.session.create('index:join08:index0','columns=(v0)') + self.session.create('index:join08:index1','columns=(v1)') + c = self.session.open_cursor('table:join08', None, None) + for i in range(0, self.nentries): + c.set_key(*self.gen_key(i)) + c.set_value(*self.gen_values(i)) + c.insert() + c.close() + + if joinfirst: + jc = self.session.open_cursor('join:table:join08', None, None) + c0 = self.session.open_cursor('index:join08:index0', None, None) + c1 = self.session.open_cursor('index:join08:index1', None, None) + c0.next() # index cursors must be positioned + c1.next() + if not joinfirst: + jc = self.session.open_cursor('join:table:join08', None, None) + self.session.join(jc, c0, 'compare=ge') + self.session.join(jc, c1, 'compare=ge') + self.session.close() + self.session = None + + def test_cursor_close1(self): + self.cursor_close_common(True) + + def test_cursor_close2(self): + self.cursor_close_common(False) + + # test statistics with a simple one index join cursor + def test_simple_stats(self): + self.session.create("table:join01b", + "key_format=i,value_format=i,columns=(k,v)") + self.session.create("index:join01b:index", "columns=(v)") + + cursor = self.session.open_cursor("table:join01b", None, None) + cursor[1] = 11 + cursor[2] = 12 + cursor[3] = 13 + cursor.close() + + cursor = self.session.open_cursor("index:join01b:index", None, None) + cursor.set_key(11) + cursor.search() + + jcursor = self.session.open_cursor("join:table:join01b", None, None) + self.session.join(jcursor, cursor, "compare=gt") + + while jcursor.next() == 0: + [k] = jcursor.get_keys() + [v] = jcursor.get_values() + + statcur = self.session.open_cursor("statistics:join", jcursor, None) + found = False + while statcur.next() == 0: + [desc, pvalue, value] = statcur.get_values() + #self.tty(str(desc) + "=" + str(pvalue)) + found = True + self.assertEquals(found, True) + + jcursor.close() + cursor.close() + + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_reconfig02.py b/test/suite/test_reconfig02.py index aee8ee4458b..85a9ceb2a34 100644 --- a/test/suite/test_reconfig02.py +++ b/test/suite/test_reconfig02.py @@ -74,9 +74,15 @@ class test_reconfig02(wttest.WiredTigerTestCase): # Now turn on pre-allocation. Sleep to give the worker thread # a chance to run and verify pre-allocated log files exist. + # + # Potentially loop a few times in case it is a very slow system. self.conn.reconfigure("log=(prealloc=true)") - time.sleep(2) - prep_logs = fnmatch.filter(os.listdir('.'), "*Prep*") + for x in xrange(0, 20): + time.sleep(1) + prep_logs = fnmatch.filter(os.listdir('.'), "*Prep*") + if len(prep_logs) != 0: + break + self.assertNotEqual(0, len(prep_logs)) # Logging starts on, but archive is off. Verify it is off. diff --git a/test/suite/test_stat05.py b/test/suite/test_stat05.py index 6a93ec2c84d..9bcedd65089 100644 --- a/test/suite/test_stat05.py +++ b/test/suite/test_stat05.py @@ -37,9 +37,13 @@ from helper import complex_value_populate, key_populate, value_populate # Statistics cursor using size only class test_stat_cursor_config(wttest.WiredTigerTestCase): pfx = 'test_stat_cursor_size' + conn_config = 'statistics=(fast)' + uri = [ ('file', dict(uri='file:' + pfx, pop=simple_populate, cfg='')), ('table', dict(uri='table:' + pfx, pop=simple_populate, cfg='')), + ('inmem', dict(uri='table:' + pfx, pop=simple_populate, cfg='', + conn_config='in_memory,statistics=(fast)')), ('table-lsm', dict(uri='table:' + pfx, pop=simple_populate, cfg=',type=lsm,lsm=(chunk_size=1MB,merge_min=2)')), ('complex', dict(uri='table:' + pfx, pop=complex_populate, cfg='')), @@ -49,7 +53,6 @@ class test_stat_cursor_config(wttest.WiredTigerTestCase): ] scenarios = number_scenarios(uri) - conn_config = 'statistics=(fast)' def openAndWalkStatCursor(self): c = self.session.open_cursor( diff --git a/test/suite/test_txn04.py b/test/suite/test_txn04.py index bbd6ce8c4e2..9d9d2db62c6 100644 --- a/test/suite/test_txn04.py +++ b/test/suite/test_txn04.py @@ -193,7 +193,7 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): self.hot_backup(self.uri, committed) def test_ops(self): - with self.expectedStdoutPattern('Recreating metadata'): + with self.expectedStdoutPattern('recreating metadata'): self.ops() if __name__ == '__main__': diff --git a/test/thread/Makefile.am b/test/thread/Makefile.am index a58f019b513..ead783185f8 100644 --- a/test/thread/Makefile.am +++ b/test/thread/Makefile.am @@ -9,4 +9,4 @@ t_LDFLAGS = -static TESTS = smoke.sh clean-local: - rm -rf WiredTiger* wt.* *.core __stats + rm -rf WT_TEST __stats *.core diff --git a/test/thread/rw.c b/test/thread/rw.c index 913fa6e6c25..10f13b9eb04 100644 --- a/test/thread/rw.c +++ b/test/thread/rw.c @@ -59,16 +59,13 @@ rw_start(u_int readers, u_int writers) total_nops = 0; /* Create per-thread structures. */ - if ((run_info = calloc( - (size_t)(readers + writers), sizeof(*run_info))) == NULL || - (tids = calloc((size_t)(readers + writers), sizeof(*tids))) == NULL) - testutil_die(errno, "calloc"); + run_info = dcalloc((size_t)(readers + writers), sizeof(*run_info)); + tids = dcalloc((size_t)(readers + writers), sizeof(*tids)); /* Create the files and load the initial records. */ for (i = 0; i < writers; ++i) { if (i == 0 || multiple_files) { - if ((run_info[i].name = malloc(64)) == NULL) - testutil_die(errno, "malloc"); + run_info[i].name = dmalloc(64); snprintf(run_info[i].name, 64, FNAME, i); /* Vary by orders of magnitude */ @@ -88,8 +85,7 @@ rw_start(u_int readers, u_int writers) for (i = 0; i < readers; ++i) { offset = i + writers; if (multiple_files) { - if ((run_info[offset].name = malloc(64)) == NULL) - testutil_die(errno, "malloc"); + run_info[offset].name = dmalloc(64); /* Have readers read from tables with writes. */ name_index = i % writers; snprintf( diff --git a/test/thread/thread.h b/test/thread/thread.h index 36cdbebd210..d5f0f42ea35 100644 --- a/test/thread/thread.h +++ b/test/thread/thread.h @@ -26,19 +26,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include <sys/types.h> -#include <sys/time.h> +#include "test_util.i" -#include <errno.h> -#include <inttypes.h> -#include <pthread.h> #include <signal.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> - -#include "test_util.i" #define FNAME "file:wt.%03d" /* File name */ #define FNAME_STAT "__stats" /* File name for statistics */ diff --git a/test/utility/test_util.i b/test/utility/test_util.i index 43982d9e4a1..833eddd87aa 100644 --- a/test/utility/test_util.i +++ b/test/utility/test_util.i @@ -64,9 +64,11 @@ testutil_die(int e, const char *fmt, ...) if (custom_die != NULL) (*custom_die)(); - va_start(ap, fmt); - vfprintf(stderr, fmt, ap); - va_end(ap); + if (fmt != NULL) { + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + } if (e != 0) fprintf(stderr, ": %s", wiredtiger_strerror(e)); fprintf(stderr, "\n"); @@ -161,3 +163,58 @@ testutil_make_work_dir(char *dir) testutil_die(ret, "%s", buf); free(buf); } + +/* + * dcalloc -- + * Call calloc, dying on failure. + */ +static inline void * +dcalloc(size_t number, size_t size) +{ + void *p; + + if ((p = calloc(number, size)) != NULL) + return (p); + testutil_die(errno, "calloc: %" WT_SIZET_FMT "B", number * size); +} + +/* + * dmalloc -- + * Call malloc, dying on failure. + */ +static inline void * +dmalloc(size_t len) +{ + void *p; + + if ((p = malloc(len)) != NULL) + return (p); + testutil_die(errno, "malloc: %" WT_SIZET_FMT "B", len); +} + +/* + * drealloc -- + * Call realloc, dying on failure. + */ +static inline void * +drealloc(void *p, size_t len) +{ + void *t; + if ((t = realloc(p, len)) != NULL) + return (t); + testutil_die(errno, "realloc: %" WT_SIZET_FMT "B", len); +} + +/* + * dstrdup -- + * Call strdup, dying on failure. + */ +static inline void * +dstrdup(const void *str) +{ + char *p; + + if ((p = strdup(str)) != NULL) + return (p); + testutil_die(errno, "strdup"); +} |