diff options
Diffstat (limited to 'test')
114 files changed, 3962 insertions, 1193 deletions
diff --git a/test/bloom/test_bloom.c b/test/bloom/test_bloom.c index 67249ff887e..b6299bbbadc 100644 --- a/test/bloom/test_bloom.c +++ b/test/bloom/test_bloom.c @@ -29,8 +29,6 @@ #include "test_util.h" static struct { - char *progname; /* Program name */ - WT_CONNECTION *wt_conn; /* WT_CONNECTION handle */ WT_SESSION *wt_session; /* WT_SESSION handle */ @@ -61,10 +59,7 @@ main(int argc, char *argv[]) { int ch; - if ((g.progname = strrchr(argv[0], DIR_DELIM)) == NULL) - g.progname = argv[0]; - else - ++g.progname; + (void)testutil_set_progname(argv); /* Set default configuration values. */ g.c_cache = 10; @@ -75,7 +70,7 @@ main(int argc, char *argv[]) g.c_srand = 3233456; /* Set values from the command line. */ - while ((ch = __wt_getopt(g.progname, argc, argv, "c:f:k:o:s:")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "c:f:k:o:s:")) != EOF) switch (ch) { case 'c': /* Cache size */ g.c_cache = (u_int)atoi(__wt_optarg); @@ -126,9 +121,9 @@ setup(void) * Open configuration -- put command line configuration options at the * end so they can override "standard" configuration. */ - snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "create,error_prefix=\"%s\",cache_size=%" PRIu32 "MB,%s", - g.progname, g.c_cache, g.config_open == NULL ? "" : g.config_open); + progname, g.c_cache, g.config_open == NULL ? "" : g.config_open)); testutil_check(wiredtiger_open(NULL, NULL, config, &conn)); @@ -246,7 +241,7 @@ populate_entries(void) void usage(void) { - fprintf(stderr, "usage: %s [-cfkos]\n", g.progname); + fprintf(stderr, "usage: %s [-cfkos]\n", progname); fprintf(stderr, "%s", "\t-c cache size\n" "\t-f number of bits per item\n" diff --git a/test/checkpoint/checkpointer.c b/test/checkpoint/checkpointer.c index ef49a9492ce..84d2765843a 100644 --- a/test/checkpoint/checkpointer.c +++ b/test/checkpoint/checkpointer.c @@ -74,7 +74,7 @@ checkpointer(void *arg) WT_UNUSED(arg); - __wt_thread_id(tid, sizeof(tid)); + testutil_check(__wt_thread_id(tid, sizeof(tid))); printf("checkpointer thread starting: tid: %s\n", tid); (void)real_checkpointer(); @@ -107,8 +107,9 @@ real_checkpointer(void) "WiredTigerCheckpoint", strlen("WiredTigerCheckpoint")) == 0) checkpoint_config = NULL; else { + testutil_check(__wt_snprintf( + _buf, sizeof(_buf), "name=%s", g.checkpoint_name)); checkpoint_config = _buf; - snprintf(checkpoint_config, 128, "name=%s", g.checkpoint_name); } while (g.running) { /* Execute a checkpoint */ @@ -147,7 +148,8 @@ verify_checkpoint(WT_SESSION *session) ret = t_ret = 0; key_count = 0; - snprintf(ckpt, 128, "checkpoint=%s", g.checkpoint_name); + testutil_check(__wt_snprintf( + ckpt, sizeof(ckpt), "checkpoint=%s", g.checkpoint_name)); cursors = calloc((size_t)g.ntables, sizeof(*cursors)); if (cursors == NULL) return (log_print_err("verify_checkpoint", ENOMEM, 1)); @@ -159,7 +161,8 @@ verify_checkpoint(WT_SESSION *session) */ if (g.cookies[i].type == LSM) continue; - snprintf(next_uri, 128, "table:__wt%04d", i); + testutil_check(__wt_snprintf( + next_uri, sizeof(next_uri), "table:__wt%04d", i)); if ((ret = session->open_cursor( session, next_uri, NULL, ckpt, &cursors[i])) != 0) { (void)log_print_err( @@ -296,7 +299,8 @@ diagnose_key_error( session = cursor1->session; key1_orig = key2_orig = 0; - snprintf(ckpt, 128, "checkpoint=%s", g.checkpoint_name); + testutil_check(__wt_snprintf( + ckpt, sizeof(ckpt), "checkpoint=%s", g.checkpoint_name)); /* Save the failed keys. */ if (cursor1->get_key(cursor1, &key1_orig) != 0 || @@ -338,7 +342,8 @@ diagnose_key_error( * Now try opening new cursors on the checkpoints and see if we * get the same missing key via searching. */ - snprintf(next_uri, 128, "table:__wt%04d", index1); + testutil_check(__wt_snprintf( + next_uri, sizeof(next_uri), "table:__wt%04d", index1)); if (session->open_cursor(session, next_uri, NULL, ckpt, &c) != 0) return (1); c->set_key(c, key1_orig); @@ -350,7 +355,8 @@ diagnose_key_error( if (c->close(c) != 0) return (1); - snprintf(next_uri, 128, "table:__wt%04d", index2); + testutil_check(__wt_snprintf( + next_uri, sizeof(next_uri), "table:__wt%04d", index2)); if (session->open_cursor(session, next_uri, NULL, ckpt, &c) != 0) return (1); c->set_key(c, key1_orig); @@ -367,7 +373,8 @@ live_check: * Now try opening cursors on the live checkpoint to see if we get the * same missing key via searching. */ - snprintf(next_uri, 128, "table:__wt%04d", index1); + testutil_check(__wt_snprintf( + next_uri, sizeof(next_uri), "table:__wt%04d", index1)); if (session->open_cursor(session, next_uri, NULL, NULL, &c) != 0) return (1); c->set_key(c, key1_orig); @@ -376,7 +383,8 @@ live_check: if (c->close(c) != 0) return (1); - snprintf(next_uri, 128, "table:__wt%04d", index2); + testutil_check(__wt_snprintf( + next_uri, sizeof(next_uri), "table:__wt%04d", index2)); if (session->open_cursor(session, next_uri, NULL, NULL, &c) != 0) return (1); c->set_key(c, key2_orig); diff --git a/test/checkpoint/smoke.sh b/test/checkpoint/smoke.sh index 123d4e00df5..39b1f428c2c 100755 --- a/test/checkpoint/smoke.sh +++ b/test/checkpoint/smoke.sh @@ -6,8 +6,8 @@ set -e echo "checkpoint: 3 mixed tables" $TEST_WRAPPER ./t -T 3 -t m -# We are done if short tests are requested -test -z "$TESTUTIL_DISABLE_LONG_TESTS" || exit 0 +# We are done unless long tests are enabled. +test "$TESTUTIL_ENABLE_LONG_TESTS" = "1" || exit 0 echo "checkpoint: 6 column-store tables" $TEST_WRAPPER ./t -T 6 -t c diff --git a/test/checkpoint/test_checkpoint.c b/test/checkpoint/test_checkpoint.c index 4998019ad8e..e7e1a0b81a5 100644 --- a/test/checkpoint/test_checkpoint.c +++ b/test/checkpoint/test_checkpoint.c @@ -50,10 +50,7 @@ main(int argc, char *argv[]) char *working_dir; const char *config_open; - if ((g.progname = strrchr(argv[0], DIR_DELIM)) == NULL) - g.progname = argv[0]; - else - ++g.progname; + (void)testutil_set_progname(argv); config_open = NULL; ret = 0; @@ -68,7 +65,7 @@ main(int argc, char *argv[]) runs = 1; while ((ch = __wt_getopt( - g.progname, argc, argv, "c:C:h:k:l:n:r:t:T:W:")) != EOF) + progname, argc, argv, "c:C:h:k:l:n:r:t:T:W:")) != EOF) switch (ch) { case 'c': g.checkpoint_name = __wt_optarg; @@ -132,7 +129,7 @@ main(int argc, char *argv[]) testutil_work_dir_from_path(g.home, 512, working_dir); - printf("%s: process %" PRIu64 "\n", g.progname, (uint64_t)getpid()); + printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); for (cnt = 1; (runs == 0 || cnt <= runs) && g.status == 0; ++cnt) { printf(" %d: %d workers, %d tables\n", cnt, g.nworkers, g.ntables); @@ -202,11 +199,11 @@ wt_connect(const char *config_open) testutil_make_work_dir(g.home); - snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "create,statistics=(fast),error_prefix=\"%s\",cache_size=1GB%s%s", - g.progname, + progname, config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open); + config_open == NULL ? "" : config_open)); if ((ret = wiredtiger_open( g.home, &event_handler, config, &g.conn)) != 0) @@ -297,10 +294,10 @@ log_print_err(const char *m, int e, int fatal) g.running = 0; g.status = e; } - fprintf(stderr, "%s: %s: %s\n", g.progname, m, wiredtiger_strerror(e)); + fprintf(stderr, "%s: %s: %s\n", progname, m, wiredtiger_strerror(e)); if (g.logfp != NULL) fprintf(g.logfp, "%s: %s: %s\n", - g.progname, m, wiredtiger_strerror(e)); + progname, m, wiredtiger_strerror(e)); return (e); } @@ -333,7 +330,7 @@ usage(void) "usage: %s " "[-S] [-C wiredtiger-config] [-k keys] [-l log]\n\t" "[-n ops] [-c checkpoint] [-r runs] [-t f|r|v] [-W workers]\n", - g.progname); + progname); fprintf(stderr, "%s", "\t-C specify wiredtiger_open configuration arguments\n" "\t-c checkpoint name to used named checkpoints\n" diff --git a/test/checkpoint/test_checkpoint.h b/test/checkpoint/test_checkpoint.h index 0d0d02447d5..347bd2c6e89 100644 --- a/test/checkpoint/test_checkpoint.h +++ b/test/checkpoint/test_checkpoint.h @@ -58,7 +58,6 @@ typedef struct { u_int nkeys; /* Keys to load */ u_int nops; /* Operations per thread */ FILE *logfp; /* Message log file. */ - char *progname; /* Program name */ int nworkers; /* Number workers configured */ int ntables; /* Number tables configured */ int ntables_created; /* Number tables opened */ diff --git a/test/checkpoint/workers.c b/test/checkpoint/workers.c index e4fe7bd1b29..82d1b8685c4 100644 --- a/test/checkpoint/workers.c +++ b/test/checkpoint/workers.c @@ -39,14 +39,12 @@ static int create_table(WT_SESSION *session, COOKIE *cookie) { int ret; - char *p, *end, config[128]; + char config[128]; - p = config; - end = config + sizeof(config); - p += snprintf(p, (size_t)(end - p), - "key_format=%s,value_format=S", cookie->type == COL ? "r" : "q"); - if (cookie->type == LSM) - (void)snprintf(p, (size_t)(end - p), ",type=lsm"); + testutil_check(__wt_snprintf(config, sizeof(config), + "key_format=%s,value_format=S,%s", + cookie->type == COL ? "r" : "q", + cookie->type == LSM ? ",type=lsm" : "")); if ((ret = session->create(session, cookie->uri, config)) != 0) if (ret != EEXIST) @@ -88,8 +86,9 @@ start_workers(table_type type) (table_type)((i % MAX_TABLE_TYPE) + 1); else g.cookies[i].type = type; - (void)snprintf(g.cookies[i].uri, 128, - "%s%04d", URI_BASE, g.cookies[i].id); + testutil_check(__wt_snprintf( + g.cookies[i].uri, sizeof(g.cookies[i].uri), + "%s%04d", URI_BASE, g.cookies[i].id)); /* Should probably be atomic to avoid races. */ if ((ret = create_table(session, &g.cookies[i])) != 0) @@ -132,7 +131,8 @@ worker_op(WT_CURSOR *cursor, uint64_t keyno, u_int new_val) char valuebuf[64]; cursor->set_key(cursor, keyno); - (void)snprintf(valuebuf, sizeof(valuebuf), "%037u", new_val); + testutil_check(__wt_snprintf( + valuebuf, sizeof(valuebuf), "%037u", new_val)); cursor->set_value(cursor, valuebuf); if ((ret = cursor->insert(cursor)) != 0) { if (ret == WT_ROLLBACK) @@ -153,7 +153,7 @@ worker(void *arg) WT_UNUSED(arg); - __wt_thread_id(tid, sizeof(tid)); + testutil_check(__wt_thread_id(tid, sizeof(tid))); printf("worker thread starting: tid: %s\n", tid); (void)real_worker(); diff --git a/test/csuite/Makefile.am b/test/csuite/Makefile.am index a96492c1e71..10ab890f2f5 100644 --- a/test/csuite/Makefile.am +++ b/test/csuite/Makefile.am @@ -4,8 +4,13 @@ LDADD = $(top_builddir)/test/utility/libtest_util.la \ $(top_builddir)/libwiredtiger.la AM_LDFLAGS = -static +noinst_PROGRAMS= + +test_scope_SOURCES = scope/main.c +noinst_PROGRAMS += test_scope + test_wt1965_col_efficiency_SOURCES = wt1965_col_efficiency/main.c -noinst_PROGRAMS = test_wt1965_col_efficiency +noinst_PROGRAMS += test_wt1965_col_efficiency test_wt2403_lsm_workload_SOURCES = wt2403_lsm_workload/main.c noinst_PROGRAMS += test_wt2403_lsm_workload @@ -37,9 +42,21 @@ noinst_PROGRAMS += test_wt2834_join_bloom_fix test_wt2853_perf_SOURCES = wt2853_perf/main.c noinst_PROGRAMS += test_wt2853_perf +test_wt2909_checkpoint_integrity_SOURCES = wt2909_checkpoint_integrity/main.c +noinst_PROGRAMS += test_wt2909_checkpoint_integrity + test_wt2999_join_extractor_SOURCES = wt2999_join_extractor/main.c noinst_PROGRAMS += test_wt2999_join_extractor +test_wt3120_filesys_SOURCES = wt3120_filesys/main.c +noinst_PROGRAMS += test_wt3120_filesys + +test_wt3135_search_near_collator_SOURCES = wt3135_search_near_collator/main.c +noinst_PROGRAMS += test_wt3135_search_near_collator + +test_wt3184_dup_index_collator_SOURCES = wt3184_dup_index_collator/main.c +noinst_PROGRAMS += test_wt3184_dup_index_collator + # Run this during a "make check" smoke test. TESTS = $(noinst_PROGRAMS) LOG_COMPILER = $(TEST_WRAPPER) diff --git a/test/csuite/scope/main.c b/test/csuite/scope/main.c new file mode 100644 index 00000000000..15dabd97c40 --- /dev/null +++ b/test/csuite/scope/main.c @@ -0,0 +1,288 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +#define KEY "key" +#define VALUE "value" + +static int ignore_errors; + +static int +handle_error(WT_EVENT_HANDLER *handler, + WT_SESSION *session, int error, const char *message) +{ + (void)(handler); + + /* Skip the error messages we're expecting to see. */ + if (ignore_errors > 0 && + (strstr(message, "requires key be set") != NULL || + strstr(message, "requires value be set") != NULL)) { + --ignore_errors; + return (0); + } + + (void)fprintf(stderr, "%s: %s\n", + message, session->strerror(session, error)); + return (0); +} + +static WT_EVENT_HANDLER event_handler = { + handle_error, + NULL, + NULL, + NULL +}; + +static void +cursor_scope_ops(WT_SESSION *session, const char *uri) +{ + struct { + const char *op; + enum { INSERT, SEARCH, SEARCH_NEAR, + REMOVE, REMOVE_POS, RESERVE, UPDATE } func; + const char *config; + } *op, ops[] = { + /* + * The ops order is fixed and shouldn't change, that is, insert + * has to happen first so search, update and remove operations + * are possible, and remove has to be last. + */ + { "insert", INSERT, NULL, }, + { "search", SEARCH, NULL, }, + { "search", SEARCH_NEAR, NULL, }, +#if 0 + { "reserve", RESERVE, NULL, }, +#endif + { "update", UPDATE, NULL, }, + { "remove", REMOVE, NULL, }, + { "remove", REMOVE_POS, NULL, }, + { NULL, INSERT, NULL } + }; + WT_CURSOR *cursor; + uint64_t keyr; + const char *key, *value; + char keybuf[100], valuebuf[100]; + int exact; + bool recno; + + /* Reserve requires a running transaction. */ + testutil_check(session->begin_transaction(session, NULL)); + + cursor = NULL; + for (op = ops; op->op != NULL; op++) { + key = value = NULL; + + /* Open a cursor. */ + if (cursor != NULL) + testutil_check(cursor->close(cursor)); + testutil_check(session->open_cursor( + session, uri, NULL, op->config, &cursor)); + recno = strcmp(cursor->key_format, "r") == 0; + + /* + * Set up application buffers so we can detect overwrites + * or failure to copy application information into library + * memory. + */ + if (recno) + cursor->set_key(cursor, (uint64_t)1); + else { + strcpy(keybuf, KEY); + cursor->set_key(cursor, keybuf); + } + strcpy(valuebuf, VALUE); + cursor->set_value(cursor, valuebuf); + + /* + * The application must keep key and value memory valid until + * the next operation that positions the cursor, modifies the + * data, or resets or closes the cursor. + * + * Modifying either the key or value buffers is not permitted. + */ + switch (op->func) { + case INSERT: + testutil_check(cursor->insert(cursor)); + break; + case SEARCH: + testutil_check(cursor->search(cursor)); + break; + case SEARCH_NEAR: + testutil_check(cursor->search_near(cursor, &exact)); + break; + case REMOVE_POS: + /* + * Remove has two modes, one where the remove is based + * on a cursor position, the other where it's based on + * a set key. The results are different, so test them + * separately. + */ + testutil_check(cursor->search(cursor)); + /* FALLTHROUGH */ + case REMOVE: + testutil_check(cursor->remove(cursor)); + break; + case RESERVE: +#if 0 + testutil_check(cursor->reserve(cursor)); +#endif + break; + case UPDATE: + testutil_check(cursor->update(cursor)); + break; + } + + /* + * The cursor should no longer reference application memory, + * and application buffers can be safely overwritten. + */ + memset(keybuf, 'K', sizeof(keybuf)); + memset(valuebuf, 'V', sizeof(valuebuf)); + + /* + * Check that get_key/get_value behave as expected after the + * operation. + */ + switch (op->func) { + case INSERT: + case REMOVE: + /* + * Insert and remove configured with a search key do + * not position the cursor and have no key or value. + * + * There should be two error messages, ignore them. + */ + ignore_errors = 2; + if (recno) + testutil_assert( + cursor->get_key(cursor, &keyr) != 0); + else + testutil_assert( + cursor->get_key(cursor, &key) != 0); + testutil_assert(cursor->get_value(cursor, &value) != 0); + testutil_assert(ignore_errors == 0); + break; + case REMOVE_POS: + /* + * Remove configured with a cursor position has a key, + * but no value. + * + * There should be one error message, ignore it. + */ + if (recno) { + testutil_assert( + cursor->get_key(cursor, &keyr) == 0); + testutil_assert(keyr == 1); + } else { + testutil_assert( + cursor->get_key(cursor, &key) == 0); + testutil_assert(key != keybuf); + testutil_assert(strcmp(key, KEY) == 0); + } + ignore_errors = 1; + testutil_assert(cursor->get_value(cursor, &value) != 0); + testutil_assert(ignore_errors == 0); + break; + case RESERVE: + case SEARCH: + case SEARCH_NEAR: + case UPDATE: + /* + * Reserve, search, search-near and update position the + * cursor and have both a key and value. + * + * Any key/value should not reference application + * memory. + */ + if (recno) { + testutil_assert( + cursor->get_key(cursor, &keyr) == 0); + testutil_assert(keyr == 1); + } else { + testutil_assert( + cursor->get_key(cursor, &key) == 0); + testutil_assert(key != keybuf); + testutil_assert(strcmp(key, KEY) == 0); + } + testutil_assert(cursor->get_value(cursor, &value) == 0); + testutil_assert(value != valuebuf); + testutil_assert(strcmp(value, VALUE) == 0); + break; + } + + /* + * We have more than one remove operation, add the key back + * in. + */ + if (op->func == REMOVE || op->func == REMOVE_POS) { + if (recno) + cursor->set_key(cursor, (uint64_t)1); + else { + cursor->set_key(cursor, KEY); + } + cursor->set_value(cursor, VALUE); + testutil_check(cursor->insert(cursor)); + } + } +} + +static void +run(WT_CONNECTION *conn, const char *uri, const char *config) +{ + WT_SESSION *session; + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->create(session, uri, config)); + cursor_scope_ops(session, uri); + testutil_check(session->close(session, NULL)); +} + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check( + wiredtiger_open(opts->home, &event_handler, "create", &opts->conn)); + + run(opts->conn, "file:file.SS", "key_format=S,value_format=S"); + run(opts->conn, "file:file.rS", "key_format=r,value_format=S"); + run(opts->conn, "lsm:lsm.SS", "key_format=S,value_format=S"); + run(opts->conn, "lsm:lsm.rS", "key_format=r,value_format=S"); + run(opts->conn, "table:table.SS", "key_format=S,value_format=S"); + run(opts->conn, "table:table.rS", "key_format=r,value_format=S"); + + testutil_cleanup(opts); + + return (EXIT_SUCCESS); +} diff --git a/test/csuite/wt1965_col_efficiency/main.c b/test/csuite/wt1965_col_efficiency/main.c index a7235d81b31..e5b73d5e642 100644 --- a/test/csuite/wt1965_col_efficiency/main.c +++ b/test/csuite/wt1965_col_efficiency/main.c @@ -132,7 +132,8 @@ main(int argc, char *argv[]) testutil_check(opts->conn->open_session( opts->conn, NULL, NULL, &session)); - sprintf(table_format, "key_format=r,value_format="); + testutil_check(__wt_snprintf( + table_format, sizeof(table_format), "key_format=r,value_format=")); for (i = 0; i < NR_FIELDS; i++) strcat(table_format, "Q"); diff --git a/test/csuite/wt2246_col_append/main.c b/test/csuite/wt2246_col_append/main.c index 4b352b26051..9876582fffa 100644 --- a/test/csuite/wt2246_col_append/main.c +++ b/test/csuite/wt2246_col_append/main.c @@ -68,8 +68,8 @@ page_init(uint64_t n) else { if (recno % 3 == 0) ++vrecno; - snprintf(buf, - sizeof(buf), "%" PRIu64 " VALUE ------", vrecno); + testutil_check(__wt_snprintf(buf, + sizeof(buf), "%" PRIu64 " VALUE ------", vrecno)); cursor->set_value(cursor, buf); } testutil_check(cursor->insert(cursor)); @@ -101,9 +101,10 @@ main(int argc, char *argv[]) uint64_t i, id; char buf[100]; + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + opts = &_opts; - if (testutil_disable_long_tests()) - return (0); memset(opts, 0, sizeof(*opts)); opts->table_type = TABLE_ROW; opts->n_append_threads = N_APPEND_THREADS; @@ -111,19 +112,19 @@ main(int argc, char *argv[]) testutil_check(testutil_parse_opts(argc, argv, opts)); testutil_make_work_dir(opts->home); - snprintf(buf, sizeof(buf), + testutil_check(__wt_snprintf(buf, sizeof(buf), "create," "cache_size=%s," "eviction=(threads_max=5)," "statistics=(fast)", - opts->table_type == TABLE_FIX ? "500MB" : "2GB"); + opts->table_type == TABLE_FIX ? "500MB" : "2GB")); testutil_check(wiredtiger_open(opts->home, NULL, buf, &opts->conn)); testutil_check( opts->conn->open_session(opts->conn, NULL, NULL, &session)); - snprintf(buf, sizeof(buf), + testutil_check(__wt_snprintf(buf, sizeof(buf), "key_format=r,value_format=%s," "allocation_size=4K,leaf_page_max=64K", - opts->table_type == TABLE_FIX ? "8t" : "S"); + opts->table_type == TABLE_FIX ? "8t" : "S")); testutil_check(session->create(session, opts->uri, buf)); testutil_check(session->close(session, NULL)); diff --git a/test/csuite/wt2323_join_visibility/main.c b/test/csuite/wt2323_join_visibility/main.c index 239a3f300d0..617490fec4d 100644 --- a/test/csuite/wt2323_join_visibility/main.c +++ b/test/csuite/wt2323_join_visibility/main.c @@ -92,10 +92,11 @@ main(int argc, char *argv[]) TEST_OPTS *opts, _opts; const char *tablename; + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + opts = &_opts; sharedopts = &_sharedopts; - if (testutil_disable_long_tests()) - return (0); memset(opts, 0, sizeof(*opts)); memset(sharedopts, 0, sizeof(*sharedopts)); @@ -105,14 +106,18 @@ main(int argc, char *argv[]) tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); tablename++; - snprintf(sharedopts->posturi, sizeof(sharedopts->posturi), - "index:%s:post", tablename); - snprintf(sharedopts->baluri, sizeof(sharedopts->baluri), - "index:%s:bal", tablename); - snprintf(sharedopts->flaguri, sizeof(sharedopts->flaguri), - "index:%s:flag", tablename); - snprintf(sharedopts->joinuri, sizeof(sharedopts->joinuri), - "join:%s", opts->uri); + testutil_check(__wt_snprintf( + sharedopts->posturi, sizeof(sharedopts->posturi), + "index:%s:post", tablename)); + testutil_check(__wt_snprintf( + sharedopts->baluri, sizeof(sharedopts->baluri), + "index:%s:bal", tablename)); + testutil_check(__wt_snprintf( + sharedopts->flaguri, sizeof(sharedopts->flaguri), + "index:%s:flag", tablename)); + testutil_check(__wt_snprintf( + sharedopts->joinuri, sizeof(sharedopts->joinuri), + "join:%s", opts->uri)); testutil_check(wiredtiger_open(opts->home, NULL, "create,cache_size=1G", &opts->conn)); @@ -349,19 +354,21 @@ static void *thread_join(void *arg) balcur->set_key(balcur, 0); testutil_check(balcur->search(balcur)); if (sharedopts->bloom) - sprintf(cfg, "compare=lt,strategy=bloom,count=%d", - N_RECORDS); + testutil_check(__wt_snprintf(cfg, sizeof(cfg), + "compare=lt,strategy=bloom,count=%d", N_RECORDS)); else - sprintf(cfg, "compare=lt"); + testutil_check(__wt_snprintf( + cfg, sizeof(cfg), "compare=lt")); testutil_check(session->join(session, joincur, balcur, cfg)); flagcur->set_key(flagcur, 0); testutil_check(flagcur->search(flagcur)); if (sharedopts->bloom) - sprintf(cfg, "compare=eq,strategy=bloom,count=%d", - N_RECORDS); + testutil_check(__wt_snprintf(cfg, sizeof(cfg), + "compare=eq,strategy=bloom,count=%d", N_RECORDS)); else - sprintf(cfg, "compare=eq"); + testutil_check(__wt_snprintf( + cfg, sizeof(cfg), "compare=eq")); testutil_check(session->join(session, joincur, flagcur, cfg)); /* Expect no values returned */ diff --git a/test/csuite/wt2447_join_main_table/main.c b/test/csuite/wt2447_join_main_table/main.c index 1368e7c8c09..656cea04145 100644 --- a/test/csuite/wt2447_join_main_table/main.c +++ b/test/csuite/wt2447_join_main_table/main.c @@ -102,9 +102,12 @@ main(int argc, char *argv[]) tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); tablename++; - snprintf(index1uri, sizeof(index1uri), "index:%s:index1", tablename); - snprintf(index2uri, sizeof(index2uri), "index:%s:index2", tablename); - snprintf(joinuri, sizeof(joinuri), "join:%s", opts->uri); + testutil_check(__wt_snprintf( + index1uri, sizeof(index1uri), "index:%s:index1", tablename)); + testutil_check(__wt_snprintf( + index2uri, sizeof(index2uri), "index:%s:index2", tablename)); + testutil_check(__wt_snprintf( + joinuri, sizeof(joinuri), "join:%s", opts->uri)); testutil_check(wiredtiger_open(opts->home, NULL, "statistics=(all),create", &opts->conn)); @@ -150,7 +153,8 @@ main(int argc, char *argv[]) cursor2->set_key(cursor2, half + 1); testutil_check(cursor2->search(cursor2)); - sprintf(bloom_cfg, "compare=lt,strategy=bloom,count=%d", half); + testutil_check(__wt_snprintf(bloom_cfg, sizeof(bloom_cfg), + "compare=lt,strategy=bloom,count=%d", half)); testutil_check(session->open_cursor(session, joinuri, NULL, NULL, &jcursor)); diff --git a/test/csuite/wt2535_insert_race/main.c b/test/csuite/wt2535_insert_race/main.c index ae18760a829..ba17d485e07 100644 --- a/test/csuite/wt2535_insert_race/main.c +++ b/test/csuite/wt2535_insert_race/main.c @@ -49,9 +49,10 @@ main(int argc, char *argv[]) uint64_t current_value; int i; + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + opts = &_opts; - if (testutil_disable_long_tests()) - return (0); memset(opts, 0, sizeof(*opts)); opts->nthreads = 10; opts->nrecords = 1000; diff --git a/test/csuite/wt2592_join_schema/main.c b/test/csuite/wt2592_join_schema/main.c index 0ec1c765d99..be3eff6136c 100644 --- a/test/csuite/wt2592_join_schema/main.c +++ b/test/csuite/wt2592_join_schema/main.c @@ -82,9 +82,12 @@ main(int argc, char *argv[]) tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); tablename++; - snprintf(countryuri, sizeof(countryuri), "index:%s:country", tablename); - snprintf(yearuri, sizeof(yearuri), "index:%s:year", tablename); - snprintf(joinuri, sizeof(joinuri), "join:%s", opts->uri); + testutil_check(__wt_snprintf( + countryuri, sizeof(countryuri), "index:%s:country", tablename)); + testutil_check(__wt_snprintf( + yearuri, sizeof(yearuri), "index:%s:year", tablename)); + testutil_check(__wt_snprintf( + joinuri, sizeof(joinuri), "join:%s", opts->uri)); testutil_check(wiredtiger_open(opts->home, NULL, "create,cache_size=200M", &opts->conn)); diff --git a/test/csuite/wt2834_join_bloom_fix/main.c b/test/csuite/wt2834_join_bloom_fix/main.c index 7c80496f1b6..e128df29f41 100644 --- a/test/csuite/wt2834_join_bloom_fix/main.c +++ b/test/csuite/wt2834_join_bloom_fix/main.c @@ -59,11 +59,11 @@ main(int argc, char *argv[]) char flaguri[256]; char joinuri[256]; + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + opts = &_opts; - if (testutil_disable_long_tests()) - return (0); memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); testutil_make_work_dir(opts->home); @@ -83,10 +83,14 @@ main(int argc, char *argv[]) tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); tablename++; - snprintf(posturi, sizeof(posturi), "index:%s:post", tablename); - snprintf(balanceuri, sizeof(balanceuri), "index:%s:balance", tablename); - snprintf(flaguri, sizeof(flaguri), "index:%s:flag", tablename); - snprintf(joinuri, sizeof(joinuri), "join:%s", opts->uri); + testutil_check(__wt_snprintf( + posturi, sizeof(posturi), "index:%s:post", tablename)); + testutil_check(__wt_snprintf( + balanceuri, sizeof(balanceuri), "index:%s:balance", tablename)); + testutil_check(__wt_snprintf( + flaguri, sizeof(flaguri), "index:%s:flag", tablename)); + testutil_check(__wt_snprintf( + joinuri, sizeof(joinuri), "join:%s", opts->uri)); testutil_check(session->create(session, posturi, "columns=(post)")); testutil_check(session->create(session, balanceuri, @@ -126,14 +130,14 @@ main(int argc, char *argv[]) balancecur->set_key(balancecur, 0); testutil_check(balancecur->search(balancecur)); - sprintf(cfg, "compare=lt,strategy=bloom,count=%d", - N_RECORDS / 100); + testutil_check(__wt_snprintf(cfg, sizeof(cfg), + "compare=lt,strategy=bloom,count=%d", N_RECORDS / 100)); testutil_check(session->join(session, joincur, balancecur, cfg)); flagcur->set_key(flagcur, 0); testutil_check(flagcur->search(flagcur)); - sprintf(cfg, "compare=eq,strategy=bloom,count=%d", - N_RECORDS / 100); + testutil_check(__wt_snprintf(cfg, sizeof(cfg), + "compare=eq,strategy=bloom,count=%d", N_RECORDS / 100)); testutil_check(session->join(session, joincur, flagcur, cfg)); /* Expect no values returned */ diff --git a/test/csuite/wt2853_perf/main.c b/test/csuite/wt2853_perf/main.c index 6cec9634cd1..46ba71372e5 100644 --- a/test/csuite/wt2853_perf/main.c +++ b/test/csuite/wt2853_perf/main.c @@ -82,11 +82,11 @@ main(int argc, char *argv[]) int i, nfail; const char *tablename; + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + opts = &_opts; sharedopts = &_sharedopts; - - if (testutil_disable_long_tests()) - return (0); memset(opts, 0, sizeof(*opts)); memset(sharedopts, 0, sizeof(*sharedopts)); memset(insert_args, 0, sizeof(insert_args)); @@ -114,12 +114,15 @@ main(int argc, char *argv[]) tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); tablename++; - snprintf(sharedopts->posturi, sizeof(sharedopts->posturi), - "index:%s:post", tablename); - snprintf(sharedopts->baluri, sizeof(sharedopts->baluri), - "index:%s:bal", tablename); - snprintf(sharedopts->flaguri, sizeof(sharedopts->flaguri), - "index:%s:flag", tablename); + testutil_check(__wt_snprintf( + sharedopts->posturi, sizeof(sharedopts->posturi), + "index:%s:post", tablename)); + testutil_check(__wt_snprintf( + sharedopts->baluri, sizeof(sharedopts->baluri), + "index:%s:bal", tablename)); + testutil_check(__wt_snprintf( + sharedopts->flaguri, sizeof(sharedopts->flaguri), + "index:%s:flag", tablename)); testutil_check(session->create(session, sharedopts->posturi, "columns=(post)")); diff --git a/test/csuite/wt2909_checkpoint_integrity/main.c b/test/csuite/wt2909_checkpoint_integrity/main.c new file mode 100644 index 00000000000..ce7bd72fa3f --- /dev/null +++ b/test/csuite/wt2909_checkpoint_integrity/main.c @@ -0,0 +1,669 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +#include <sys/time.h> +#include <sys/resource.h> +#include <sys/wait.h> + +/* + * JIRA ticket reference: WT-2909 + * Test case description: + * + * This test attempts to check the integrity of checkpoints by injecting + * failures (by means of a custom file system) and then trying to recover. To + * insulate the top level program from various crashes that may occur when + * injecting failures, the "populate" code runs in another process, and is + * expected to sometimes fail. Then the top level program runs recovery (with + * the normal file system) and checks the results. Any failure at the top level + * indicates a checkpoint integrity problem. + * + * Each subtest uses the same kind of schema and data, the only variance is + * when the faults are injected. At the moment, this test only injects during + * checkpoints, and only injects write failures. It varies in the number of + * successful writes that occur before an injected failure (during a checkpoint + * operation), this can be indicated with "-o N". When N is not specified, the + * test attempts to find the optimal range of N for testing. Clearly when N is + * large, then the checkpoint may be successfully written, and the data + * represented by the checkpoint will be fully present. When N is small, + * nothing of interest is written and no data is present. To find the sweet + * spot where interesting failures occur, the test does a binary search to find + * the approximate N that divides the "small" and "large" cases. This is not + * strictly deterministic, a given N may give different results on different + * runs. But approximate optimal N can be determined, allowing a series of + * additional tests clustered around this N. + * + * The data is stored in two tables, one having indices. Both tables have + * the same keys and are updated with the same key in a single transaction. + * + * Failure mode: + * If one table is out of step with the other, that is detected as a failure at + * the top level. If an index is missing values (or has extra values), that is + * likewise a failure at the top level. If the tables or the home directory + * cannot be opened, that is a top level error. The tables must be present + * as an initial checkpoint is done without any injected fault. + */ + +/* + * This program does not run on Windows. The non-portable aspects at minimum + * are fork/exec the use of environment variables (used by fail_fs), and file + * name and build locations of dynamically loaded libraries. + */ +#define BIG_SIZE (1024 * 10) +#define BIG_CONTENTS "<Big String Contents>" +#define MAX_ARGS 20 +#define MAX_OP_RANGE 1000 +#define STDERR_FILE "stderr.txt" +#define STDOUT_FILE "stdout.txt" +#define TESTS_PER_OP_VALUE 3 +#define VERBOSE_PRINT 10000 + +static int check_results(TEST_OPTS *, uint64_t *); +static void check_values(WT_CURSOR *, int, int, int, char *); +static int create_big_string(char **); +static void cursor_count_items(WT_CURSOR *, uint64_t *); +static void disable_failures(void); +static void enable_failures(uint64_t, uint64_t); +static void generate_key(uint64_t, int *); +static void generate_value(uint32_t, uint64_t, char *, int *, int *, int *, + char **); +static void run_check_subtest(TEST_OPTS *, const char *, uint64_t, bool, + uint64_t *); +static void run_check_subtest_range(TEST_OPTS *, const char *, bool); +static int run_process(TEST_OPTS *, const char *, char *[], int *); +static void subtest_main(int, char *[], bool); +static void subtest_populate(TEST_OPTS *, bool); + +extern int __wt_optind; + +#define WT_FAIL_FS_LIB "../../ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so" + +/* + * check_results -- + * Check all the tables and verify the results. + */ +static int +check_results(TEST_OPTS *opts, uint64_t *foundp) +{ + WT_CURSOR *maincur, *maincur2, *v0cur, *v1cur, *v2cur; + WT_SESSION *session; + uint64_t count, idxcount, nrecords; + uint32_t rndint; + int key, key_got, ret, v0, v1, v2; + char *bigref, *big; + + testutil_check(create_big_string(&bigref)); + nrecords = opts->nrecords; + testutil_check(wiredtiger_open(opts->home, NULL, + "create,log=(enabled)", &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(session->open_cursor(session, "table:subtest", NULL, + NULL, &maincur)); + testutil_check(session->open_cursor(session, "table:subtest2", NULL, + NULL, &maincur2)); + testutil_check(session->open_cursor(session, "index:subtest:v0", NULL, + NULL, &v0cur)); + testutil_check(session->open_cursor(session, "index:subtest:v1", NULL, + NULL, &v1cur)); + testutil_check(session->open_cursor(session, "index:subtest:v2", NULL, + NULL, &v2cur)); + + count = 0; + while ((ret = maincur->next(maincur)) == 0) { + testutil_check(maincur2->next(maincur2)); + testutil_check(maincur2->get_key(maincur2, &key_got)); + testutil_check(maincur2->get_value(maincur2, &rndint)); + + generate_key(count, &key); + generate_value(rndint, count, bigref, &v0, &v1, &v2, &big); + testutil_assert(key == key_got); + + /* Check the key/values in main table. */ + testutil_check(maincur->get_key(maincur, &key_got)); + testutil_assert(key == key_got); + check_values(maincur, v0, v1, v2, big); + + /* Check the values in the indices. */ + v0cur->set_key(v0cur, v0); + testutil_check(v0cur->search(v0cur)); + check_values(v0cur, v0, v1, v2, big); + v1cur->set_key(v1cur, v1); + testutil_check(v1cur->search(v1cur)); + check_values(v1cur, v0, v1, v2, big); + v2cur->set_key(v2cur, v2); + testutil_check(v2cur->search(v2cur)); + check_values(v2cur, v0, v1, v2, big); + + count++; + if (count % VERBOSE_PRINT == 0 && opts->verbose) + printf("checked %" PRIu64 "/%" PRIu64 "\n", count, + nrecords); + } + if (count % VERBOSE_PRINT != 0 && opts->verbose) + printf("checked %" PRIu64 "/%" PRIu64 "\n", count, nrecords); + + /* + * Always expect at least one entry, as populate does a + * checkpoint after the first insert. + */ + testutil_assert(count > 0); + testutil_assert(ret == WT_NOTFOUND); + testutil_assert(maincur2->next(maincur2) == WT_NOTFOUND); + cursor_count_items(v0cur, &idxcount); + testutil_assert(count == idxcount); + cursor_count_items(v1cur, &idxcount); + testutil_assert(count == idxcount); + cursor_count_items(v2cur, &idxcount); + testutil_assert(count == idxcount); + + testutil_check(opts->conn->close(opts->conn, NULL)); + opts->conn = NULL; + + free(bigref); + *foundp = count; + return (0); +} + +/* + * check_values -- + * Check that the values in the cursor match the given values. + */ +static void +check_values(WT_CURSOR *cursor, int v0, int v1, int v2, char *big) +{ + int v0_got, v1_got, v2_got; + char *big_got; + + testutil_check(cursor->get_value(cursor, &v0_got, &v1_got, &v2_got, + &big_got)); + testutil_assert(v0 == v0_got); + testutil_assert(v1 == v1_got); + testutil_assert(v2 == v2_got); + testutil_assert(strcmp(big, big_got) == 0); +} + +/* + * create_big_string -- + * Create and fill the "reference" big array. + */ +static int create_big_string(char **bigp) +{ + size_t i, mod; + char *big; + + if ((big = malloc(BIG_SIZE + 1)) == NULL) + return (ENOMEM); + mod = strlen(BIG_CONTENTS); + for (i = 0; i < BIG_SIZE; i++) { + big[i] = BIG_CONTENTS[i % mod]; + } + big[BIG_SIZE] = '\0'; + *bigp = big; + return (0); +} + +/* + * cursor_count_items -- + * Count the number of items in the table by traversing + * through the cursor. + */ +static void +cursor_count_items(WT_CURSOR *cursor, uint64_t *countp) +{ + int ret; + + *countp = 0; + + testutil_check(cursor->reset(cursor)); + while ((ret = cursor->next(cursor)) == 0) + (*countp)++; + testutil_assert(ret == WT_NOTFOUND); +} + +/* + * disable_failures -- + * Disable failures in the fail file system. + */ +static void +disable_failures(void) +{ + testutil_check(setenv("WT_FAIL_FS_ENABLE", "0", 1)); +} + +/* + * enable_failures -- + * Enable failures in the fail file system. + */ +static void +enable_failures(uint64_t allow_writes, uint64_t allow_reads) +{ + char value[100]; + + testutil_check(setenv("WT_FAIL_FS_ENABLE", "1", 1)); + testutil_check(__wt_snprintf( + value, sizeof(value), "%" PRIu64, allow_writes)); + testutil_check(setenv("WT_FAIL_FS_WRITE_ALLOW", value, 1)); + testutil_check(__wt_snprintf( + value, sizeof(value), "%" PRIu64, allow_reads)); + testutil_check(setenv("WT_FAIL_FS_READ_ALLOW", value, 1)); +} + +/* + * generate_key -- + * Generate a key used by the "subtest" and "subtest2" tables. + */ +static void +generate_key(uint64_t i, int *keyp) +{ + *keyp = (int)i; +} + +/* + * generate_value -- + * Generate values for the "subtest" table. + */ +static void +generate_value(uint32_t rndint, uint64_t i, char *bigref, + int *v0p, int *v1p, int *v2p, char **bigp) +{ + *v0p = (int)(i * 7); + *v1p = (int)(i * 10007); + *v2p = (int)(i * 100000007); + *bigp = &bigref[rndint % BIG_SIZE]; +} + +/* + * run_check_subtest -- + * Run the subtest with the given parameters and check the results. + */ +static void +run_check_subtest(TEST_OPTS *opts, const char *debugger, uint64_t nops, + bool close_test, uint64_t *nresultsp) +{ + int estatus, narg; + char rarg[20], sarg[20], *subtest_args[MAX_ARGS]; + + narg = 0; + if (debugger != NULL) { + subtest_args[narg++] = (char *)debugger; + subtest_args[narg++] = (char *)"--"; + } + + subtest_args[narg++] = (char *)opts->progname; + /* "subtest" must appear before arguments */ + if (close_test) + subtest_args[narg++] = (char *)"subtest_close"; + else + subtest_args[narg++] = (char *)"subtest"; + subtest_args[narg++] = (char *)"-h"; + subtest_args[narg++] = opts->home; + subtest_args[narg++] = (char *)"-v"; /* subtest is always verbose */ + subtest_args[narg++] = (char *)"-p"; + subtest_args[narg++] = (char *)"-o"; + testutil_check(__wt_snprintf(sarg, sizeof(sarg), "%" PRIu64, nops)); + subtest_args[narg++] = sarg; /* number of operations */ + subtest_args[narg++] = (char *)"-n"; + testutil_check(__wt_snprintf( + rarg, sizeof(rarg), "%" PRIu64, opts->nrecords)); + subtest_args[narg++] = rarg; /* number of records */ + subtest_args[narg++] = NULL; + testutil_assert(narg <= MAX_ARGS); + if (opts->verbose) + printf("running a separate process with %" PRIu64 + " operations until fail...\n", nops); + testutil_clean_work_dir(opts->home); + testutil_check(run_process( + opts, debugger != NULL ? debugger : opts->progname, + subtest_args, &estatus)); + if (opts->verbose) + printf("process exited %d\n", estatus); + + /* + * Verify results in parent process. + */ + testutil_check(check_results(opts, nresultsp)); +} + +/* + * run_check_subtest_range -- + * + * Run successive tests via binary search that determines the approximate + * crossover point between when data is recoverable or not. Once that is + * determined, run the subtest in a range near that crossover point. + * + * The theory is that running at the crossover point will tend to trigger + * "interesting" failures at the borderline when the checkpoint is about to, + * or has, succeeded. If any of those failures creates a WT home directory + * that cannot be recovered, the top level test will fail. + */ +static void +run_check_subtest_range(TEST_OPTS *opts, const char *debugger, bool close_test) +{ + uint64_t cutoff, high, low, mid, nops, nresults; + int i; + bool got_failure, got_success; + + if (opts->verbose) + printf("Determining best range of operations until failure, " + "with close_test %s.\n", + (close_test ? "enabled" : "disabled")); + + run_check_subtest(opts, debugger, 1, close_test, &cutoff); + low = 0; + high = MAX_OP_RANGE; + mid = (low + high) / 2; + while (mid != low) { + run_check_subtest(opts, debugger, mid, close_test, + &nresults); + if (nresults > cutoff) + high = mid; + else + low = mid; + mid = (low + high) / 2; + } + /* + * mid is the number of ops that is the crossover point. + * Run some tests near that point to try to trigger weird + * failures. If mid is too low or too high, it indicates + * there is a fundamental problem with the test. + */ + testutil_assert(mid > 1 && mid < MAX_OP_RANGE - 1); + if (opts->verbose) + printf("Retesting around %" PRIu64 " operations.\n", + mid); + + got_failure = false; + got_success = false; + for (nops = mid - 10; nops < mid + 10; nops++) { + for (i = 0; i < TESTS_PER_OP_VALUE; i++) { + run_check_subtest(opts, debugger, nops, + close_test, &nresults); + if (nresults > cutoff) + got_failure = true; + else + got_success = true; + } + } + /* + * Check that it really ran with a crossover point. + */ + testutil_assert(got_failure); + testutil_assert(got_success); +} + +/* + * run_process -- + * Run a program with arguments, wait until it completes. + */ +static int +run_process(TEST_OPTS *opts, const char *prog, char *argv[], int *status) +{ + int pid; + char **arg; + + if (opts->verbose) { + printf("running: "); + for (arg = argv; *arg != NULL; arg++) + printf("%s ", *arg); + printf("\n"); + } + if ((pid = fork()) == 0) { + (void)execv(prog, argv); + testutil_die(errno, "%s", prog); + } else if (pid < 0) + return (errno); + + (void)waitpid(pid, status, 0); + return (0); +} + +/* + * subtest_main -- + * The main program for the subtest + */ +static void +subtest_main(int argc, char *argv[], bool close_test) +{ + TEST_OPTS *opts, _opts; + WT_SESSION *session; + char config[1024], filename[1024]; + struct rlimit rlim; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + memset(&rlim, 0, sizeof(rlim)); + + /* No core files during fault injection tests. */ + testutil_check(setrlimit(RLIMIT_CORE, &rlim)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + /* Redirect stderr, stdout. */ + testutil_check(__wt_snprintf( + filename, sizeof(filename), "%s/%s", opts->home, STDERR_FILE)); + testutil_assert(freopen(filename, "a", stderr) != NULL); + testutil_check(__wt_snprintf( + filename, sizeof(filename), "%s/%s", opts->home, STDOUT_FILE)); + testutil_assert(freopen(filename, "a", stdout) != NULL); + testutil_check(__wt_snprintf(config, sizeof(config), + "create,cache_size=250M,log=(enabled)," + "transaction_sync=(enabled,method=none),extensions=(" + WT_FAIL_FS_LIB + "=(early_load,config={environment=true,verbose=true})]")); + + testutil_check(wiredtiger_open(opts->home, NULL, config, &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(session->create(session, "table:subtest", + "key_format=i,value_format=iiiS," + "columns=(id,v0,v1,v2,big)")); + + testutil_check(session->create(session, "table:subtest2", + "key_format=i,value_format=i")); + + testutil_check(session->create(session, "index:subtest:v0", + "columns=(v0)")); + testutil_check(session->create(session, "index:subtest:v1", + "columns=(v1)")); + testutil_check(session->create(session, "index:subtest:v2", + "columns=(v2)")); + + testutil_check(session->close(session, NULL)); + + subtest_populate(opts, close_test); + + testutil_cleanup(opts); +} + +/* + * This macro is used as a substitute for testutil_check, except that it is + * aware of when a failure may be expected due to the effects of the fail_fs. + * This macro is used only in subtest_populate(), it uses local variables. + */ +#define CHECK(expr) { \ + int _ret; \ + _ret = expr; \ + if (_ret != 0) { \ + if (!failmode || \ + (_ret != WT_RUN_RECOVERY && _ret != EIO)) { \ + fprintf(stderr, " BAD RETURN %d for \"%s\"\n", \ + _ret, #expr); \ + testutil_check(_ret); \ + } else \ + failed = true; \ + } \ +} + +/* + * subtest_populate -- + * Populate the tables. + */ +static void +subtest_populate(TEST_OPTS *opts, bool close_test) +{ + WT_CURSOR *maincur, *maincur2; + WT_RAND_STATE rnd; + WT_SESSION *session; + uint64_t i, nrecords; + uint32_t rndint; + int key, v0, v1, v2; + char *big, *bigref; + bool failed, failmode; + + failmode = failed = false; + __wt_random_init_seed(NULL, &rnd); + CHECK(create_big_string(&bigref)); + nrecords = opts->nrecords; + + CHECK(opts->conn->open_session( + opts->conn, NULL, NULL, &session)); + + CHECK(session->open_cursor(session, "table:subtest", NULL, + NULL, &maincur)); + + CHECK(session->open_cursor(session, "table:subtest2", NULL, + NULL, &maincur2)); + + for (i = 0; i < nrecords && !failed; i++) { + rndint = __wt_random(&rnd); + generate_key(i, &key); + generate_value(rndint, i, bigref, &v0, &v1, &v2, &big); + CHECK(session->begin_transaction(session, NULL)); + maincur->set_key(maincur, key); + maincur->set_value(maincur, v0, v1, v2, big); + CHECK(maincur->insert(maincur)); + + maincur2->set_key(maincur2, key); + maincur2->set_value(maincur2, rndint); + CHECK(maincur2->insert(maincur2)); + CHECK(session->commit_transaction(session, NULL)); + + if (i == 0) + /* + * Force an initial checkpoint, that helps to + * distinguish a clear failure from just not running + * long enough. + */ + CHECK(session->checkpoint(session, NULL)); + + if ((i + 1) % VERBOSE_PRINT == 0 && opts->verbose) + printf(" %" PRIu64 "/%" PRIu64 "\n", + (i + 1), nrecords); + /* Attempt to isolate the failures to checkpointing. */ + if (i == (nrecords/100)) { + enable_failures(opts->nops, 1000000); + failmode = true; /* CHECK should expect failures. */ + CHECK(session->checkpoint(session, NULL)); + failmode = false; + disable_failures(); + if (failed && opts->verbose) + printf("checkpoint failed (expected).\n"); + } + } + + /* + * Closing handles after an extreme fail is likely to cause + * cascading failures (or crashes), so recommended practice is + * to immediately exit. We're interested in testing both with + * and without the recommended practice. + */ + if (failed) { + if (!close_test) { + fprintf(stderr, "exit early.\n"); + exit(0); + } else + fprintf(stderr, "closing after failure.\n"); + } + + free(bigref); + CHECK(maincur->close(maincur)); + CHECK(maincur2->close(maincur2)); + CHECK(session->close(session, NULL)); +} + +/* + * main -- + * The main program for the test. When invoked with "subtest" + * argument, run the subtest. Otherwise, run a separate process + * for each needed subtest, and check the results. + */ +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + uint64_t nresults; + const char *debugger; + + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + debugger = NULL; + + testutil_check(testutil_parse_opts(argc, argv, opts)); + argc -= __wt_optind; + argv += __wt_optind; + if (opts->nrecords == 0) + opts->nrecords = 50000; + + while (argc > 0) { + if (strcmp(argv[0], "subtest") == 0) { + subtest_main(argc, argv, false); + return (0); + } else if (strcmp(argv[0], "subtest_close") == 0) { + subtest_main(argc, argv, true); + return (0); + } else if (strcmp(argv[0], "gdb") == 0) + debugger = "/usr/bin/gdb"; + else + testutil_assert(false); + argc--; + argv++; + } + if (opts->verbose) { + printf("Number of operations until failure: %" PRIu64 + " (change with -o N)\n", opts->nops); + printf("Number of records: %" PRIu64 + " (change with -n N)\n", opts->nrecords); + } + if (opts->nops == 0) { + run_check_subtest_range(opts, debugger, false); + run_check_subtest_range(opts, debugger, true); + } else + run_check_subtest(opts, debugger, opts->nops, + opts->nrecords, &nresults); + + testutil_clean_work_dir(opts->home); + testutil_cleanup(opts); + + return (0); +} diff --git a/test/csuite/wt3120_filesys/main.c b/test/csuite/wt3120_filesys/main.c new file mode 100644 index 00000000000..2fae85017d4 --- /dev/null +++ b/test/csuite/wt3120_filesys/main.c @@ -0,0 +1,99 @@ +/*- + * Public Domain 2014-2017 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * JIRA ticket reference: WT-3120 + * Test case description: A simple file system extension built into + * a shared library. + * Failure mode: Loading the file system and closing the connection + * is enough to evoke the failure. This test does slightly more + * than that. + */ + +#define WT_FAIL_FS_LIB "../../ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so" + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + WT_CURSOR *cursor; + WT_SESSION *session; + char *kstr, *vstr; + char buf[1024]; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(__wt_snprintf(buf, sizeof(buf), + "create,extensions=(" WT_FAIL_FS_LIB "=(early_load=true))")); + testutil_check(wiredtiger_open(opts->home, NULL, buf, &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(session->create(session, opts->uri, + "key_format=S,value_format=S")); + + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, + &cursor)); + cursor->set_key(cursor, "a"); + cursor->set_value(cursor, "0"); + testutil_check(cursor->insert(cursor)); + cursor->set_key(cursor, "b"); + cursor->set_value(cursor, "1"); + testutil_check(cursor->insert(cursor)); + testutil_check(cursor->close(cursor)); + testutil_check(session->close(session, NULL)); + + /* Force to disk and re-open. */ + testutil_check(opts->conn->close(opts->conn, NULL)); + testutil_check(wiredtiger_open(opts->home, NULL, NULL, &opts->conn)); + + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, + &cursor)); + testutil_check(cursor->next(cursor)); + testutil_check(cursor->get_key(cursor, &kstr)); + testutil_check(cursor->get_value(cursor, &vstr)); + testutil_assert(strcmp(kstr, "a") == 0); + testutil_assert(strcmp(vstr, "0") == 0); + testutil_check(cursor->next(cursor)); + testutil_check(cursor->get_key(cursor, &kstr)); + testutil_check(cursor->get_value(cursor, &vstr)); + testutil_assert(strcmp(kstr, "b") == 0); + testutil_assert(strcmp(vstr, "1") == 0); + testutil_assert(cursor->next(cursor) == WT_NOTFOUND); + testutil_check(cursor->close(cursor)); + testutil_check(session->close(session, NULL)); + printf("Success\n"); + + testutil_cleanup(opts); + return (EXIT_SUCCESS); +} diff --git a/test/csuite/wt3135_search_near_collator/main.c b/test/csuite/wt3135_search_near_collator/main.c new file mode 100644 index 00000000000..8783034a7d8 --- /dev/null +++ b/test/csuite/wt3135_search_near_collator/main.c @@ -0,0 +1,360 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * JIRA ticket reference: WT-3135 + * Test case description: Each set of data is ordered and contains + * five elements (0-4). We insert elements 1 and 3, and then do + * search_near and search for each element. For each set of data, we perform + * these tests first using a custom collator, and second using a custom collator + * and extractor. In each case there are index keys having variable length. + * Failure mode: In the reported test case, the custom compare routine is + * given a truncated key to compare, and the unpack functions return errors + * because the truncation appeared in the middle of a key. + */ + +#define TEST_ENTRY_COUNT 5 +typedef const char *TEST_SET[TEST_ENTRY_COUNT]; +static TEST_SET test_sets[] = { + { "0", "01", "012", "0123", "01234" }, + { "A", "B", "C", "D", "E" }, + { "5", "54", "543", "5432", "54321" }, + { "54321", "5433", "544", "55", "6" } +}; +#define TEST_SET_COUNT (sizeof(test_sets) / sizeof(test_sets[0])) + +static bool +item_str_equal(WT_ITEM *item, const char *str) +{ + return (item->size == strlen(str) + 1 && strncmp((char *)item->data, + str, item->size) == 0); +} + +static int +compare_int(int64_t a, int64_t b) +{ + return (a < b ? -1 : (a > b ? 1 : 0)); +} + +static int +index_compare_primary(WT_PACK_STREAM *s1, WT_PACK_STREAM *s2, int *cmp) +{ + int64_t pkey1, pkey2; + int rc1, rc2; + + rc1 = wiredtiger_unpack_int(s1, &pkey1); + rc2 = wiredtiger_unpack_int(s2, &pkey2); + + if (rc1 == 0 && rc2 == 0) + *cmp = compare_int(pkey1, pkey2); + else if (rc1 != 0 && rc2 != 0) + *cmp = 0; + else if (rc1 != 0) + *cmp = -1; + else + *cmp = 1; + return (0); +} + +static int +index_compare_S(WT_COLLATOR *collator, WT_SESSION *session, + const WT_ITEM *key1, const WT_ITEM *key2, int *cmp) +{ + WT_PACK_STREAM *s1, *s2; + const char *skey1, *skey2; + + (void)collator; + + testutil_check(wiredtiger_unpack_start(session, "Si", key1->data, + key1->size, &s1)); + testutil_check(wiredtiger_unpack_start(session, "Si", key2->data, + key2->size, &s2)); + + testutil_check(wiredtiger_unpack_str(s1, &skey1)); + testutil_check(wiredtiger_unpack_str(s2, &skey2)); + + if ((*cmp = strcmp(skey1, skey2)) == 0) + testutil_check(index_compare_primary(s1, s2, cmp)); + + testutil_check(wiredtiger_pack_close(s1, NULL)); + testutil_check(wiredtiger_pack_close(s2, NULL)); + + return (0); +} + +static int +index_compare_u(WT_COLLATOR *collator, WT_SESSION *session, + const WT_ITEM *key1, const WT_ITEM *key2, int *cmp) +{ + WT_ITEM skey1, skey2; + WT_PACK_STREAM *s1, *s2; + + (void)collator; + + testutil_check(wiredtiger_unpack_start(session, "ui", key1->data, + key1->size, &s1)); + testutil_check(wiredtiger_unpack_start(session, "ui", key2->data, + key2->size, &s2)); + + testutil_check(wiredtiger_unpack_item(s1, &skey1)); + testutil_check(wiredtiger_unpack_item(s2, &skey2)); + + if ((*cmp = strcmp(skey1.data, skey2.data)) == 0) + testutil_check(index_compare_primary(s1, s2, cmp)); + + testutil_check(wiredtiger_pack_close(s1, NULL)); + testutil_check(wiredtiger_pack_close(s2, NULL)); + + return (0); +} + +static int +index_extractor_u(WT_EXTRACTOR *extractor, WT_SESSION *session, + const WT_ITEM *key, const WT_ITEM *value, WT_CURSOR *result_cursor) +{ + (void)extractor; + (void)session; + (void)key; + + result_cursor->set_key(result_cursor, value); + return result_cursor->insert(result_cursor); +} + +static WT_COLLATOR collator_S = { index_compare_S, NULL, NULL }; +static WT_COLLATOR collator_u = { index_compare_u, NULL, NULL }; +static WT_EXTRACTOR extractor_u = { index_extractor_u, NULL, NULL }; + +/* + * Check search() and search_near() using the test string indicated + * by test_index. + */ +static void +search_using_str(WT_CURSOR *cursor, TEST_SET test_set, int test_index) +{ + int exact, ret; + const char *result; + const char *str_01, *str_0123, *test_str; + + testutil_assert(test_index >= 0 && test_index <= 4); + str_01 = test_set[1]; + str_0123 = test_set[3]; + test_str = test_set[test_index]; + + cursor->set_key(cursor, test_str); + testutil_check(cursor->search_near(cursor, &exact)); + testutil_check(cursor->get_key(cursor, &result)); + + if (test_index == 0) + testutil_assert(strcmp(result, str_01) == 0 && exact > 0); + else if (test_index == 1) + testutil_assert(strcmp(result, str_01) == 0 && exact == 0); + else if (test_index == 2) + testutil_assert((strcmp(result, str_0123) == 0 && exact > 0) || + (strcmp(result, str_01) == 0 && exact < 0)); + else if (test_index == 3) + testutil_assert(strcmp(result, str_0123) == 0 && exact == 0); + else if (test_index == 4) + testutil_assert(strcmp(result, str_0123) == 0 && exact < 0); + + cursor->set_key(cursor, test_str); + ret = cursor->search(cursor); + + if (test_index == 0 || test_index == 2 || test_index == 4) + testutil_assert(ret == WT_NOTFOUND); + else if (test_index == 1 || test_index == 3) + testutil_assert(ret == 0); +} + +/* + * Check search() and search_near() using the test string indicated + * by test_index against a table containing a variable sized item. + */ +static void +search_using_item(WT_CURSOR *cursor, TEST_SET test_set, int test_index) +{ + WT_ITEM item; + size_t testlen; + int exact, ret; + const char *str_01, *str_0123, *test_str; + + testutil_assert(test_index >= 0 && test_index <= 4); + str_01 = test_set[1]; + str_0123 = test_set[3]; + test_str = test_set[test_index]; + + testlen = strlen(test_str) + 1; + item.data = test_str; + item.size = testlen; + cursor->set_key(cursor, &item); + testutil_check(cursor->search_near(cursor, &exact)); + testutil_check(cursor->get_key(cursor, &item)); + + if (test_index == 0) + testutil_assert(item_str_equal(&item, str_01) && exact > 0); + else if (test_index == 1) + testutil_assert(item_str_equal(&item, str_01) && exact == 0); + else if (test_index == 2) + testutil_assert((item_str_equal(&item, str_0123) && exact > 0) + || (item_str_equal(&item, str_01) && exact < 0)); + else if (test_index == 3) + testutil_assert(item_str_equal(&item, str_0123) && exact == 0); + else if (test_index == 4) + testutil_assert(item_str_equal(&item, str_0123) && exact < 0); + + item.data = test_str; + item.size = testlen; + cursor->set_key(cursor, &item); + ret = cursor->search(cursor); + + if (test_index == 0 || test_index == 2 || test_index == 4) + testutil_assert(ret == WT_NOTFOUND); + else if (test_index == 1 || test_index == 3) + testutil_assert(ret == 0); +} + +/* + * For each set of data, perform tests. + */ +static void +test_one_set(WT_SESSION *session, TEST_SET set) +{ + WT_CURSOR *cursor; + WT_ITEM item; + int32_t i; + + /* + * Part 1: Using a custom collator, insert some elements + * and verify results from search_near. + */ + + testutil_check(session->create(session, + "table:main", "key_format=i,value_format=S,columns=(k,v)")); + testutil_check(session->create(session, + "index:main:def_collator", "columns=(v)")); + testutil_check(session->create(session, + "index:main:custom_collator", + "columns=(v),collator=collator_S")); + + /* Insert only elements #1 and #3. */ + testutil_check(session->open_cursor(session, + "table:main", NULL, NULL, &cursor)); + cursor->set_key(cursor, 0); + cursor->set_value(cursor, set[1]); + testutil_check(cursor->insert(cursor)); + cursor->set_key(cursor, 1); + cursor->set_value(cursor, set[3]); + testutil_check(cursor->insert(cursor)); + testutil_check(cursor->close(cursor)); + + /* Check all elements in def_collator index. */ + testutil_check(session->open_cursor(session, + "index:main:def_collator", NULL, NULL, &cursor)); + for (i = 0; i < (int32_t)TEST_ENTRY_COUNT; i++) + search_using_str(cursor, set, i); + testutil_check(cursor->close(cursor)); + + /* Check all elements in custom_collator index */ + testutil_check(session->open_cursor(session, + "index:main:custom_collator", NULL, NULL, &cursor)); + for (i = 0; i < (int32_t)TEST_ENTRY_COUNT; i++) + search_using_str(cursor, set, i); + testutil_check(cursor->close(cursor)); + + /* + * Part 2: perform the same checks using a custom collator and + * extractor. + */ + testutil_check(session->create(session, + "table:main2", "key_format=i,value_format=u,columns=(k,v)")); + + testutil_check(session->create(session, "index:main2:idx_w_coll", + "key_format=u,collator=collator_u,extractor=extractor_u")); + + testutil_check(session->open_cursor(session, + "table:main2", NULL, NULL, &cursor)); + + memset(&item, 0, sizeof(item)); + item.size = strlen(set[1]) + 1; + item.data = set[1]; + cursor->set_key(cursor, 1); + cursor->set_value(cursor, &item); + testutil_check(cursor->insert(cursor)); + + item.size = strlen(set[3]) + 1; + item.data = set[3]; + cursor->set_key(cursor, 3); + cursor->set_value(cursor, &item); + testutil_check(cursor->insert(cursor)); + + testutil_check(cursor->close(cursor)); + + testutil_check(session->open_cursor(session, + "index:main2:idx_w_coll", NULL, NULL, &cursor)); + for (i = 0; i < (int32_t)TEST_ENTRY_COUNT; i++) + search_using_item(cursor, set, i); + testutil_check(cursor->close(cursor)); + + testutil_check(session->drop(session, "table:main", NULL)); + testutil_check(session->drop(session, "table:main2", NULL)); +} + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + WT_SESSION *session; + size_t i; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(wiredtiger_open(opts->home, NULL, "create", + &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + /* Add any collators and extractors used by tests */ + testutil_check(opts->conn->add_collator(opts->conn, "collator_S", + &collator_S, NULL)); + testutil_check(opts->conn->add_collator(opts->conn, "collator_u", + &collator_u, NULL)); + testutil_check(opts->conn->add_extractor(opts->conn, "extractor_u", + &extractor_u, NULL)); + + for (i = 0; i < TEST_SET_COUNT; i++) { + printf("test set %" WT_SIZET_FMT "\n", i); + test_one_set(session, test_sets[i]); + } + + testutil_check(session->close(session, NULL)); + testutil_cleanup(opts); + return (EXIT_SUCCESS); +} diff --git a/test/csuite/wt3184_dup_index_collator/main.c b/test/csuite/wt3184_dup_index_collator/main.c new file mode 100644 index 00000000000..c969e7a1d7e --- /dev/null +++ b/test/csuite/wt3184_dup_index_collator/main.c @@ -0,0 +1,168 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * JIRA ticket reference: WT-3184 + * Test case description: Each set of data is ordered and contains + * five elements (0-4). We insert elements 1 and 3, and then do + * search_near and search for each element. For each set of data, we perform + * these tests first using a custom collator, and second using a custom collator + * and extractor. In each case there are index keys having variable length. + * Failure mode: In the reported test case, the custom compare routine is + * given a truncated key to compare, and the unpack functions return errors + * because the truncation appeared in the middle of a key. + */ + +static int +compare_int(int32_t a, int32_t b) +{ + return (a < b ? -1 : (a > b ? 1 : 0)); +} + +static int32_t +item_to_int(WT_ITEM *item) +{ + testutil_assert(item->size == sizeof(int32_t)); + return (*(int32_t *)item->data); +} + +static int +compare_int_items(WT_ITEM *itema, WT_ITEM *itemb) +{ + testutil_assert(itema->size == sizeof(int32_t)); + testutil_assert(itemb->size == sizeof(int32_t)); + return (compare_int(item_to_int(itema), item_to_int(itemb))); +} + +static void +print_int_item(const char *str, const WT_ITEM *item) +{ + if (item->size > 0) { + testutil_assert(item->size == sizeof(int32_t)); + printf("%s%" PRId32, str, *(int32_t *)item->data); + } else + printf("%s<empty>", str); +} + +static int +index_compare(WT_COLLATOR *collator, WT_SESSION *session, + const WT_ITEM *key1, const WT_ITEM *key2, int *cmp) +{ + WT_ITEM ikey1, pkey1, ikey2, pkey2; + + (void)collator; + testutil_check(wiredtiger_struct_unpack(session, + key1->data, key1->size, "uu", &ikey1, &pkey1)); + testutil_check(wiredtiger_struct_unpack(session, + key2->data, key2->size, "uu", &ikey2, &pkey2)); + + print_int_item("index_compare: index key1 = ", &ikey1); + print_int_item(", primary key1 = ", &pkey1); + print_int_item(", index key2 = ", &ikey2); + print_int_item(", primary key2 = ", &pkey2); + printf("\n"); + + if ((*cmp = compare_int_items(&ikey1, &ikey2)) != 0) + return (0); + + if (pkey1.size != 0 && pkey2.size != 0) + *cmp = compare_int_items(&pkey1, &pkey2); + else if (pkey1.size != 0) + *cmp = 1; + else if (pkey2.size != 0) + *cmp = -1; + else + *cmp = 0; + + return (0); +} + +static WT_COLLATOR index_coll = { index_compare, NULL, NULL }; + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + WT_CURSOR *cursor, *cursor1; + WT_ITEM got, k, v; + WT_SESSION *session; + int32_t ki, vi; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(wiredtiger_open(opts->home, NULL, "create", + &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(opts->conn->add_collator(opts->conn, "index_coll", + &index_coll, NULL)); + + testutil_check(session->create(session, + "table:main", "key_format=u,value_format=u,columns=(k,v)")); + testutil_check(session->create(session, + "index:main:index", "columns=(v),collator=index_coll")); + + printf("adding new record\n"); + testutil_check(session->open_cursor(session, "table:main", NULL, NULL, + &cursor)); + + ki = 13; + vi = 17; + + k.data = &ki; k.size = sizeof(ki); + v.data = &vi; v.size = sizeof(vi); + + cursor->set_key(cursor, &k); + cursor->set_value(cursor, &v); + testutil_check(cursor->insert(cursor)); + testutil_check(cursor->close(cursor)); + + printf("positioning index cursor\n"); + + testutil_check(session->open_cursor(session, "index:main:index", NULL, + NULL, &cursor)); + cursor->set_key(cursor, &v); + testutil_check(cursor->search(cursor)); + + printf("duplicating cursor\n"); + testutil_check(session->open_cursor(session, NULL, cursor, NULL, + &cursor1)); + testutil_check(cursor->get_value(cursor, &got)); + testutil_assert(item_to_int(&got) == 17); + testutil_check(cursor1->get_value(cursor1, &got)); + testutil_assert(item_to_int(&got) == 17); + + testutil_check(session->close(session, NULL)); + testutil_cleanup(opts); + return (EXIT_SUCCESS); +} diff --git a/test/cursor_order/cursor_order.c b/test/cursor_order/cursor_order.c index 85b8c68e545..d3c64b54ab5 100644 --- a/test/cursor_order/cursor_order.c +++ b/test/cursor_order/cursor_order.c @@ -29,7 +29,6 @@ #include "cursor_order.h" static char home[512]; /* Program working dir */ -static char *progname; /* Program name */ static FILE *logfp; /* Log file */ static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); @@ -51,10 +50,7 @@ main(int argc, char *argv[]) int ch, cnt, runs; char *config_open, *working_dir; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); cfg = &_cfg; config_open = NULL; @@ -185,19 +181,15 @@ wt_connect(SHARED_CONFIG *cfg, char *config_open) }; int ret; char config[512]; - size_t print_count; testutil_clean_work_dir(home); testutil_make_work_dir(home); - print_count = (size_t)snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "create,statistics=(all),error_prefix=\"%s\",%s%s", progname, config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open); - - if (print_count >= sizeof(config)) - testutil_die(EINVAL, "Config string too long"); + config_open == NULL ? "" : config_open)); if ((ret = wiredtiger_open( home, &event_handler, config, &cfg->conn)) != 0) diff --git a/test/cursor_order/cursor_order_file.c b/test/cursor_order/cursor_order_file.c index 5dc7194b5fb..42d7af54de4 100644 --- a/test/cursor_order/cursor_order_file.c +++ b/test/cursor_order/cursor_order_file.c @@ -34,23 +34,21 @@ file_create(SHARED_CONFIG *cfg, const char *name) WT_CONNECTION *conn; WT_SESSION *session; int ret; - char *p, *end, config[128]; + char config[128]; conn = cfg->conn; if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) testutil_die(ret, "conn.session"); - p = config; - end = config + sizeof(config); - p += snprintf(p, (size_t)(end - p), + testutil_check(__wt_snprintf(config, sizeof(config), "key_format=%s," "internal_page_max=%d," "split_deepen_min_child=200," - "leaf_page_max=%d,", - cfg->ftype == ROW ? "S" : "r", 16 * 1024, 128 * 1024); - if (cfg->ftype == FIX) - (void)snprintf(p, (size_t)(end - p), ",value_format=3t"); + "leaf_page_max=%d," + "%s", + cfg->ftype == ROW ? "S" : "r", 16 * 1024, 128 * 1024, + cfg->ftype == FIX ? ",value_format=3t" : "")); if ((ret = session->create(session, name, config)) != 0) if (ret != EEXIST) @@ -67,9 +65,10 @@ load(SHARED_CONFIG *cfg, const char *name) WT_CURSOR *cursor; WT_ITEM *value, _value; WT_SESSION *session; - char keybuf[64], valuebuf[64]; - int64_t keyno; + size_t len; + uint64_t keyno; int ret; + char keybuf[64], valuebuf[64]; conn = cfg->conn; @@ -83,9 +82,10 @@ load(SHARED_CONFIG *cfg, const char *name) testutil_die(ret, "cursor.open"); value = &_value; - for (keyno = 1; keyno <= (int64_t)cfg->nkeys; ++keyno) { + for (keyno = 1; keyno <= cfg->nkeys; ++keyno) { if (cfg->ftype == ROW) { - snprintf(keybuf, sizeof(keybuf), "%016u", (u_int)keyno); + testutil_check(__wt_snprintf( + keybuf, sizeof(keybuf), "%016" PRIu64, keyno)); cursor->set_key(cursor, keybuf); } else cursor->set_key(cursor, (uint32_t)keyno); @@ -93,8 +93,10 @@ load(SHARED_CONFIG *cfg, const char *name) if (cfg->ftype == FIX) cursor->set_value(cursor, 0x01); else { - value->size = (uint32_t)snprintf( - valuebuf, sizeof(valuebuf), "%37u", (u_int)keyno); + testutil_check(__wt_snprintf_len_set( + valuebuf, sizeof(valuebuf), + &len, "%37" PRIu64, keyno)); + value->size = (uint32_t)len; cursor->set_value(cursor, value); } if ((ret = cursor->insert(cursor)) != 0) diff --git a/test/cursor_order/cursor_order_ops.c b/test/cursor_order/cursor_order_ops.c index 58da49b2991..299f22684c9 100644 --- a/test/cursor_order/cursor_order_ops.c +++ b/test/cursor_order/cursor_order_ops.c @@ -69,7 +69,8 @@ ops_start(SHARED_CONFIG *cfg) run_info[i].cfg = cfg; if (i == 0 || cfg->multiple_files) { run_info[i].name = dmalloc(64); - snprintf(run_info[i].name, 64, FNAME, (int)i); + testutil_check(__wt_snprintf( + run_info[i].name, 64, FNAME, (int)i)); /* Vary by orders of magnitude */ if (cfg->vary_nops) @@ -93,8 +94,8 @@ ops_start(SHARED_CONFIG *cfg) run_info[offset].name = dmalloc(64); /* Have reverse scans read from tables with writes. */ name_index = i % cfg->append_inserters; - snprintf( - run_info[offset].name, 64, FNAME, (int)name_index); + testutil_check(__wt_snprintf( + run_info[offset].name, 64, FNAME, (int)name_index)); /* Vary by orders of magnitude */ if (cfg->vary_nops) @@ -231,7 +232,7 @@ reverse_scan(void *arg) id = (uintmax_t)arg; s = &run_info[id]; cfg = s->cfg; - __wt_thread_id(tid, sizeof(tid)); + testutil_check(__wt_thread_id(tid, sizeof(tid))); __wt_random_init(&s->rnd); printf(" reverse scan thread %2" PRIuMAX @@ -272,6 +273,7 @@ append_insert_op( { WT_ITEM *value, _value; uint64_t keyno; + size_t len; int ret; char keybuf[64], valuebuf[64]; @@ -281,7 +283,8 @@ append_insert_op( keyno = __wt_atomic_add64(&cfg->key_range, 1); if (cfg->ftype == ROW) { - snprintf(keybuf, sizeof(keybuf), "%016u", (u_int)keyno); + testutil_check(__wt_snprintf( + keybuf, sizeof(keybuf), "%016" PRIu64, keyno)); cursor->set_key(cursor, keybuf); } else cursor->set_key(cursor, (uint32_t)keyno); @@ -291,8 +294,9 @@ append_insert_op( if (cfg->ftype == FIX) cursor->set_value(cursor, 0x10); else { - value->size = (uint32_t)snprintf( - valuebuf, sizeof(valuebuf), "XXX %37u", (u_int)keyno); + testutil_check(__wt_snprintf_len_set( + valuebuf, sizeof(valuebuf), &len, "XXX %37" PRIu64, keyno)); + value->size = (uint32_t)len; cursor->set_value(cursor, value); } if ((ret = cursor->insert(cursor)) != 0) @@ -318,7 +322,7 @@ append_insert(void *arg) id = (uintmax_t)arg; s = &run_info[id]; cfg = s->cfg; - __wt_thread_id(tid, sizeof(tid)); + testutil_check(__wt_thread_id(tid, sizeof(tid))); __wt_random_init(&s->rnd); printf("write thread %2" PRIuMAX " starting: tid: %s, file: %s\n", diff --git a/test/fops/file.c b/test/fops/file.c index ea15f1ee80d..d1cd22ab391 100644 --- a/test/fops/file.c +++ b/test/fops/file.c @@ -51,7 +51,7 @@ obj_bulk(void) if ((ret = c->close(c)) != 0) testutil_die(ret, "cursor.close"); } else if (ret != ENOENT && ret != EBUSY && ret != EINVAL) - testutil_die(ret, "session.open_cursor"); + testutil_die(ret, "session.open_cursor bulk"); } if ((ret = session->close(session, NULL)) != 0) testutil_die(ret, "session.close"); @@ -71,7 +71,8 @@ obj_bulk_unique(int force) /* Generate a unique object name. */ if ((ret = pthread_rwlock_wrlock(&single)) != 0) testutil_die(ret, "pthread_rwlock_wrlock single"); - (void)snprintf(new_uri, sizeof(new_uri), "%s.%u", uri, ++uid); + testutil_check(__wt_snprintf( + new_uri, sizeof(new_uri), "%s.%u", uri, ++uid)); if ((ret = pthread_rwlock_unlock(&single)) != 0) testutil_die(ret, "pthread_rwlock_unlock single"); @@ -79,12 +80,17 @@ obj_bulk_unique(int force) testutil_die(ret, "session.create: %s", new_uri); __wt_yield(); - if ((ret = - session->open_cursor(session, new_uri, NULL, "bulk", &c)) != 0) - testutil_die(ret, "session.open_cursor: %s", new_uri); - - if ((ret = c->close(c)) != 0) - testutil_die(ret, "cursor.close"); + /* + * Opening a bulk cursor may have raced with a forced checkpoint + * which created a checkpoint of the empty file, and triggers an EINVAL + */ + if ((ret = session->open_cursor( + session, new_uri, NULL, "bulk", &c)) == 0) { + if ((ret = c->close(c)) != 0) + testutil_die(ret, "cursor.close"); + } else if (ret != EINVAL) + testutil_die(ret, + "session.open_cursor bulk unique: %s, new_uri"); while ((ret = session->drop( session, new_uri, force ? "force" : NULL)) != 0) @@ -147,7 +153,8 @@ obj_create_unique(int force) /* Generate a unique object name. */ if ((ret = pthread_rwlock_wrlock(&single)) != 0) testutil_die(ret, "pthread_rwlock_wrlock single"); - (void)snprintf(new_uri, sizeof(new_uri), "%s.%u", uri, ++uid); + testutil_check(__wt_snprintf( + new_uri, sizeof(new_uri), "%s.%u", uri, ++uid)); if ((ret = pthread_rwlock_unlock(&single)) != 0) testutil_die(ret, "pthread_rwlock_unlock single"); @@ -190,9 +197,13 @@ obj_checkpoint(void) if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) testutil_die(ret, "conn.session"); - /* Force the checkpoint so it has to be taken. */ + /* + * Force the checkpoint so it has to be taken. Forced checkpoints can + * race with other metadata operations and return EBUSY - we'd expect + * applications using forced checkpoints to retry on EBUSY. + */ if ((ret = session->checkpoint(session, "force")) != 0) - if (ret != ENOENT) + if (ret != EBUSY && ret != ENOENT) testutil_die(ret, "session.checkpoint"); if ((ret = session->close(session, NULL)) != 0) diff --git a/test/fops/t.c b/test/fops/t.c index 7b4a7cf8fca..07ac07349e3 100644 --- a/test/fops/t.c +++ b/test/fops/t.c @@ -34,7 +34,6 @@ u_int nops; /* Operations */ const char *uri; /* Object */ const char *config; /* Object config */ -static char *progname; /* Program name */ static FILE *logfp; /* Log file */ static char home[512]; @@ -71,22 +70,15 @@ main(int argc, char *argv[]) int ch, cnt, ret, runs; char *config_open, *working_dir; - working_dir = NULL; - - /* Remove directories */ - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); if ((ret = pthread_rwlock_init(&single, NULL)) != 0) testutil_die(ret, "pthread_rwlock_init: single"); - config_open = NULL; nops = 1000; nthreads = 10; runs = 1; - + config_open = working_dir = NULL; while ((ch = __wt_getopt(progname, argc, argv, "C:h:l:n:r:t:")) != EOF) switch (ch) { case 'C': /* wiredtiger_open config */ @@ -165,11 +157,11 @@ wt_startup(char *config_open) testutil_make_work_dir(home); - snprintf(config_buf, sizeof(config_buf), + testutil_check(__wt_snprintf(config_buf, sizeof(config_buf), "create,error_prefix=\"%s\",cache_size=5MB%s%s", progname, config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open); + config_open == NULL ? "" : config_open)); if ((ret = wiredtiger_open( home, &event_handler, config_buf, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); @@ -225,6 +217,11 @@ handle_message(WT_EVENT_HANDLER *handler, (void)(handler); (void)(session); + /* Ignore messages about failing to create forced checkpoints. */ + if (strstr( + message, "forced or named checkpoint") != NULL) + return (0); + if (logfp != NULL) return (fprintf(logfp, "%s\n", message) < 0 ? -1 : 0); diff --git a/test/format/backup.c b/test/format/backup.c index 69fdf771de9..8aa614fa970 100644 --- a/test/format/backup.c +++ b/test/format/backup.c @@ -63,7 +63,7 @@ copy_file(WT_SESSION *session, const char *name) len = strlen("BACKUP") + strlen(name) + 10; first = dmalloc(len); - (void)snprintf(first, len, "BACKUP/%s", name); + testutil_check(__wt_snprintf(first, len, "BACKUP/%s", name)); testutil_check(__wt_copy_and_sync(session, name, first)); /* @@ -72,7 +72,7 @@ copy_file(WT_SESSION *session, const char *name) */ len = strlen("BACKUP_COPY") + strlen(name) + 10; second = dmalloc(len); - (void)snprintf(second, len, "BACKUP_COPY/%s", name); + testutil_check(__wt_snprintf(second, len, "BACKUP_COPY/%s", name)); testutil_check(__wt_copy_and_sync(session, first, second)); free(first); diff --git a/test/format/config.c b/test/format/config.c index cf922b5db04..22b40f7164d 100644 --- a/test/format/config.c +++ b/test/format/config.c @@ -44,6 +44,7 @@ static void config_map_compression(const char *, u_int *); static void config_map_encryption(const char *, u_int *); static void config_map_file_type(const char *, u_int *); static void config_map_isolation(const char *, u_int *); +static void config_pct(void); static void config_reset(void); /* @@ -62,39 +63,42 @@ config_setup(void) config_in_memory(); /* - * Choose a data source type and a file type: they're interrelated (LSM - * trees are only compatible with row-store) and other items depend on - * them. + * Choose a file format and a data source: they're interrelated (LSM is + * only compatible with row-store) and other items depend on them. */ + if (!config_is_perm("file_type")) { + if (config_is_perm("data_source") && DATASOURCE("lsm")) + config_single("file_type=row", 0); + else + switch (mmrand(NULL, 1, 10)) { + case 1: /* 10% */ + config_single("file_type=fix", 0); + break; + case 2: case 3: case 4: /* 30% */ + config_single("file_type=var", 0); + break; /* 60% */ + case 5: case 6: case 7: case 8: case 9: case 10: + config_single("file_type=row", 0); + break; + } + } + config_map_file_type(g.c_file_type, &g.type); + if (!config_is_perm("data_source")) switch (mmrand(NULL, 1, 3)) { case 1: config_single("data_source=file", 0); break; case 2: - if (!g.c_in_memory) { - config_single("data_source=lsm", 0); - break; - } - /* FALLTHROUGH */ - case 3: config_single("data_source=table", 0); break; - } - - if (!config_is_perm("file_type")) - switch (DATASOURCE("lsm") ? 5 : mmrand(NULL, 1, 10)) { - case 1: - config_single("file_type=fix", 0); - break; - case 2: case 3: case 4: - config_single("file_type=var", 0); - break; - case 5: case 6: case 7: case 8: case 9: case 10: - config_single("file_type=row", 0); + case 3: + if (g.c_in_memory || g.type != ROW) + config_single("data_source=table", 0); + else + config_single("data_source=lsm", 0); break; } - config_map_file_type(g.c_file_type, &g.type); /* * If data_source and file_type were both "permanent", we may still @@ -103,7 +107,7 @@ config_setup(void) if (DATASOURCE("lsm") && g.type != ROW) { fprintf(stderr, "%s: lsm data_source is only compatible with row file_type\n", - g.progname); + progname); exit(EXIT_FAILURE); } @@ -159,31 +163,19 @@ config_setup(void) config_encryption(); config_isolation(); config_lrt(); + config_pct(); /* - * Periodically, set the delete percentage to 0 so salvage gets run, - * as long as the delete percentage isn't nailed down. - * Don't do it on the first run, all our smoke tests would hit it. - */ - if (!g.replay && g.run_cnt % 10 == 9 && !config_is_perm("delete_pct")) - config_single("delete_pct=0", 0); - - /* - * If this is an LSM run, set the cache size and crank up the insert - * percentage. + * If this is an LSM run, ensure cache size sanity. + * Ensure there is at least 1MB of cache per thread. */ - if (DATASOURCE("lsm")) { - if (!config_is_perm("cache")) + if (!config_is_perm("cache")) { + if (DATASOURCE("lsm")) g.c_cache = 30 * g.c_chunk_size; - - if (!config_is_perm("insert_pct")) - g.c_insert_pct = mmrand(NULL, 50, 85); + if (g.c_cache < g.c_threads) + g.c_cache = g.c_threads; } - /* Ensure there is at least 1MB of cache per thread. */ - if (!config_is_perm("cache") && g.c_cache < g.c_threads) - g.c_cache = g.c_threads; - /* Give in-memory configuration a final review. */ config_in_memory_check(); @@ -265,8 +257,8 @@ config_compression(const char *conf_name) */ cstr = "none"; if (strcmp(conf_name, "logging_compression") == 0 && g.c_logging == 0) { - (void)snprintf( - confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr); + testutil_check(__wt_snprintf( + confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr)); config_single(confbuf, 0); return; } @@ -310,7 +302,8 @@ config_compression(const char *conf_name) break; } - (void)snprintf(confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr); + testutil_check(__wt_snprintf( + confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr)); config_single(confbuf, 0); } @@ -473,7 +466,7 @@ config_lrt(void) * stores. */ if (g.type == FIX) { - if (config_is_perm("long_running_txn")) + if (config_is_perm("long_running_txn") && g.c_long_running_txn) testutil_die(EINVAL, "long_running_txn not supported with fixed-length " "column store"); @@ -482,6 +475,83 @@ config_lrt(void) } /* + * config_pct -- + * Configure operation percentages. + */ +static void +config_pct(void) +{ + static struct { + const char *name; /* Operation */ + uint32_t *vp; /* Value store */ + u_int order; /* Order of assignment */ + } list[] = { +#define CONFIG_DELETE_ENTRY 0 + { "delete_pct", &g.c_delete_pct, 0 }, + { "insert_pct", &g.c_insert_pct, 0 }, + { "read_pct", &g.c_read_pct, 0 }, + { "write_pct", &g.c_write_pct, 0 }, + }; + u_int i, max_order, max_slot, n, pct; + + /* + * Walk the list of operations, checking for an illegal configuration + * and creating a random order in the list. + */ + pct = 0; + for (i = 0; i < WT_ELEMENTS(list); ++i) + if (config_is_perm(list[i].name)) + pct += *list[i].vp; + else + list[i].order = mmrand(NULL, 1, 1000); + if (pct > 100) + testutil_die(EINVAL, + "operation percentages total to more than 100%%"); + + /* + * If the delete percentage isn't nailed down, periodically set it to + * 0 so salvage gets run. Don't do it on the first run, all our smoke + * tests would hit it. + */ + if (!config_is_perm("delete_pct") && !g.replay && g.run_cnt % 10 == 9) { + list[CONFIG_DELETE_ENTRY].order = 0; + *list[CONFIG_DELETE_ENTRY].vp = 0; + } + + /* + * Walk the list, allocating random numbers of operations in a random + * order. + * + * If the "order" field is non-zero, we need to create a value for this + * operation. Find the largest order field in the array; if one non-zero + * order field is found, it's the last entry and gets the remainder of + * the operations. + */ + for (pct = 100 - pct;;) { + for (i = n = + max_order = max_slot = 0; i < WT_ELEMENTS(list); ++i) { + if (list[i].order != 0) + ++n; + if (list[i].order > max_order) { + max_order = list[i].order; + max_slot = i; + } + } + if (n == 0) + break; + if (n == 1) { + *list[max_slot].vp = pct; + break; + } + *list[max_slot].vp = mmrand(NULL, 0, pct); + list[max_slot].order = 0; + pct -= *list[max_slot].vp; + } + testutil_assert(g.c_delete_pct + + g.c_insert_pct + g.c_read_pct + g.c_write_pct == 100); +} + +/* * config_error -- * Display configuration information on error. */ @@ -609,13 +679,14 @@ void config_single(const char *s, int perm) { CONFIG *cp; - long v; + long vlong; + uint32_t v; char *p; const char *ep; if ((ep = strchr(s, '=')) == NULL) { fprintf(stderr, - "%s: %s: illegal configuration value\n", g.progname, s); + "%s: %s: illegal configuration value\n", progname, s); exit(EXIT_FAILURE); } @@ -674,34 +745,35 @@ config_single(const char *s, int perm) return; } - v = -1; + vlong = -1; if (F_ISSET(cp, C_BOOL)) { if (strncmp(ep, "off", strlen("off")) == 0) - v = 0; + vlong = 0; else if (strncmp(ep, "on", strlen("on")) == 0) - v = 1; + vlong = 1; } - if (v == -1) { - v = strtol(ep, &p, 10); + if (vlong == -1) { + vlong = strtol(ep, &p, 10); if (*p != '\0') { fprintf(stderr, "%s: %s: illegal numeric value\n", - g.progname, s); + progname, s); exit(EXIT_FAILURE); } } + v = (uint32_t)vlong; if (F_ISSET(cp, C_BOOL)) { if (v != 0 && v != 1) { fprintf(stderr, "%s: %s: value of boolean not 0 or 1\n", - g.progname, s); + progname, s); exit(EXIT_FAILURE); } } else if (v < cp->min || v > cp->maxset) { fprintf(stderr, "%s: %s: value outside min/max values of %" PRIu32 "-%" PRIu32 "\n", - g.progname, s, cp->min, cp->maxset); + progname, s, cp->min, cp->maxset); exit(EXIT_FAILURE); } - *cp->v = (uint32_t)v; + *cp->v = v; } /* @@ -817,7 +889,7 @@ config_find(const char *s, size_t len) return (cp); fprintf(stderr, - "%s: %s: unknown configuration keyword\n", g.progname, s); + "%s: %s: unknown configuration keyword\n", progname, s); config_error(); exit(EXIT_FAILURE); } diff --git a/test/format/config.h b/test/format/config.h index e4f7af2e1b2..b5feb7a5321 100644 --- a/test/format/config.h +++ b/test/format/config.h @@ -131,7 +131,7 @@ static CONFIG c[] = { { "delete_pct", "percent operations that are deletes", - 0x0, 0, 45, 90, &g.c_delete_pct, NULL }, + C_IGNORE, 0, 0, 100, &g.c_delete_pct, NULL }, { "dictionary", "if values are dictionary compressed", /* 20% */ @@ -171,7 +171,7 @@ static CONFIG c[] = { { "insert_pct", "percent operations that are inserts", - 0x0, 0, 45, 90, &g.c_insert_pct, NULL }, + C_IGNORE, 0, 0, 100, &g.c_insert_pct, NULL }, { "internal_key_truncation", "if internal keys are truncated", /* 95% */ @@ -254,6 +254,14 @@ static CONFIG c[] = { "quiet run (same as -q)", C_IGNORE|C_BOOL, 0, 0, 0, &g.c_quiet, NULL }, + { "read_pct", + "percent operations that are reads", + C_IGNORE, 0, 0, 100, &g.c_read_pct, NULL }, + + { "rebalance", + "rebalance testing", /* 100% */ + C_BOOL, 100, 1, 0, &g.c_rebalance, NULL }, + { "repeat_data_pct", "percent duplicate values in row- or var-length column-stores", 0x0, 0, 90, 90, &g.c_repeat_data_pct, NULL }, @@ -270,17 +278,13 @@ static CONFIG c[] = { "the number of runs", C_IGNORE, 0, UINT_MAX, UINT_MAX, &g.c_runs, NULL }, - { "rebalance", - "rebalance testing", /* 100% */ - C_BOOL, 100, 1, 0, &g.c_rebalance, NULL }, - { "salvage", "salvage testing", /* 100% */ C_BOOL, 100, 1, 0, &g.c_salvage, NULL }, { "split_pct", "page split size as a percentage of the maximum page size", - 0x0, 40, 85, 85, &g.c_split_pct, NULL }, + 0x0, 50, 100, 100, &g.c_split_pct, NULL }, { "statistics", "maintain statistics", /* 20% */ @@ -320,7 +324,7 @@ static CONFIG c[] = { { "write_pct", "percent operations that are writes", - 0x0, 0, 90, 90, &g.c_write_pct, NULL }, + C_IGNORE, 0, 0, 100, &g.c_write_pct, NULL }, { NULL, NULL, 0x0, 0, 0, 0, NULL, NULL } }; diff --git a/test/format/format.h b/test/format/format.h index c1f4875dbb2..41cc48c4278 100644 --- a/test/format/format.h +++ b/test/format/format.h @@ -79,8 +79,6 @@ #define FORMAT_OPERATION_REPS 3 /* 3 thread operations sets */ typedef struct { - char *progname; /* Program name */ - char *home; /* Home directory */ char *home_backup; /* Hot-backup directory */ char *home_backup_init; /* Initialize backup command */ @@ -192,6 +190,7 @@ typedef struct { uint32_t c_reverse; uint32_t c_rows; uint32_t c_runs; + uint32_t c_read_pct; uint32_t c_rebalance; uint32_t c_salvage; uint32_t c_split_pct; diff --git a/test/format/ops.c b/test/format/ops.c index 940318c87a9..72e885bd0d6 100644 --- a/test/format/ops.c +++ b/test/format/ops.c @@ -28,14 +28,17 @@ #include "format.h" -static int col_insert(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t *); -static int col_remove(WT_CURSOR *, WT_ITEM *, uint64_t); -static int col_update(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t); +static int col_insert(TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t *); +static int col_remove(WT_CURSOR *, WT_ITEM *, uint64_t, bool); +static int col_update( + TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t, bool); static int nextprev(WT_CURSOR *, int); static void *ops(void *); -static int row_insert(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t); -static int row_remove(WT_CURSOR *, WT_ITEM *, uint64_t); -static int row_update(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t); +static int row_insert( + TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t, bool); +static int row_remove(WT_CURSOR *, WT_ITEM *, uint64_t, bool); +static int row_update( + TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t, bool); static void table_append_init(void); #ifdef HAVE_BERKELEY_DB @@ -243,6 +246,9 @@ typedef struct { bool insert; /* Insert operation */ } SNAP_OPS; +#define SNAP_TRACK \ + (snap != NULL && (size_t)(snap - snap_list) < WT_ELEMENTS(snap_list)) + /* * snap_track -- * Add a single snapshot isolation returned value to the list. @@ -395,15 +401,16 @@ snap_check(WT_CURSOR *cursor, static void * ops(void *arg) { + enum { INSERT, READ, REMOVE, UPDATE } op; SNAP_OPS *snap, snap_list[64]; TINFO *tinfo; WT_CONNECTION *conn; - WT_CURSOR *cursor, *cursor_insert; + WT_CURSOR *cursor; WT_DECL_RET; WT_ITEM *key, _key, *value, _value; WT_SESSION *session; uint64_t keyno, ckpt_op, reset_op, session_op; - uint32_t op, rnd; + uint32_t rnd; u_int i; int dir; char *ckpt_config, ckpt_name[64]; @@ -429,9 +436,9 @@ ops(void *arg) val_gen_setup(&tinfo->rnd, value); /* Set the first operation where we'll create sessions and cursors. */ - session_op = 0; + cursor = NULL; session = NULL; - cursor = cursor_insert = NULL; + session_op = 0; /* Set the first operation where we'll perform checkpoint operations. */ ckpt_op = g.c_checkpoints ? mmrand(&tinfo->rnd, 100, 10000) : 0; @@ -485,24 +492,11 @@ ops(void *arg) readonly = true; } else { /* - * Open two cursors: one for overwriting and one - * for append (if it's a column-store). - * - * The reason is when testing with existing - * records, we don't track if a record was - * deleted or not, which means we must use - * cursor->insert with overwriting configured. - * But, in column-store files where we're - * testing with new, appended records, we don't - * want to have to specify the record number, - * which requires an append configuration. + * Configure "append", in the case of column + * stores, we append when inserting new rows. */ - testutil_check(session->open_cursor(session, - g.uri, NULL, "overwrite", &cursor)); - if (g.type == FIX || g.type == VAR) - testutil_check(session->open_cursor( - session, g.uri, - NULL, "append", &cursor_insert)); + testutil_check(session->open_cursor( + session, g.uri, NULL, "append", &cursor)); /* Pick the next session/cursor close/open. */ session_op += mmrand(&tinfo->rnd, 100, 5000); @@ -536,8 +530,9 @@ ops(void *arg) pthread_rwlock_trywrlock(&g.backup_lock) == EBUSY) ckpt_config = NULL; else { - (void)snprintf(ckpt_name, sizeof(ckpt_name), - "name=thread-%d", tinfo->id); + testutil_check(__wt_snprintf( + ckpt_name, sizeof(ckpt_name), + "name=thread-%d", tinfo->id)); ckpt_config = ckpt_name; } @@ -563,8 +558,9 @@ ops(void *arg) strcpy(ckpt_name, "checkpoint=WiredTigerCheckpoint"); else - (void)snprintf(ckpt_name, sizeof(ckpt_name), - "checkpoint=thread-%d", tinfo->id); + testutil_check(__wt_snprintf( + ckpt_name, sizeof(ckpt_name), + "checkpoint=thread-%d", tinfo->id)); ckpt_available = true; skip_checkpoint: /* Pick the next checkpoint operation. */ @@ -600,111 +596,174 @@ skip_checkpoint: /* Pick the next checkpoint operation. */ intxn = true; } + /* Select a row. */ keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows); positioned = false; + /* Select an operation. */ + op = READ; + if (!readonly) { + i = mmrand(&tinfo->rnd, 1, 100); + if (i < g.c_delete_pct) + op = REMOVE; + else if (i < g.c_delete_pct + g.c_insert_pct) + op = INSERT; + else if (i < + g.c_delete_pct + g.c_insert_pct + g.c_write_pct) + op = UPDATE; + else + op = READ; + } + /* - * Perform some number of operations: the percentage of deletes, - * inserts and writes are specified, reads are the rest. The - * percentages don't have to add up to 100, a high percentage - * of deletes will mean fewer inserts and writes. Modifications - * are always followed by a read to confirm it worked. + * Inserts, removes and updates can be done following a cursor + * set-key, or based on a cursor position taken from a previous + * search. If not already doing a read, position the cursor at + * an existing point in the tree 20% of the time. */ - op = readonly ? UINT32_MAX : mmrand(&tinfo->rnd, 1, 100); - if (op < g.c_delete_pct) { - ++tinfo->remove; + positioned = false; + if (op != READ && mmrand(&tinfo->rnd, 1, 5) == 1) { + ++tinfo->search; + ret = read_row(cursor, key, value, keyno); + if (ret == 0) { + positioned = true; + if (SNAP_TRACK) + snap_track(snap++, keyno, NULL, value); + } else { + positioned = false; + if (ret == WT_ROLLBACK && intxn) + goto deadlock; + testutil_assert(ret == WT_NOTFOUND); + } + } +#if 0 + /* Optionally reserve a row. */ + if (!readonly && intxn && mmrand(&tinfo->rnd, 0, 20) == 1) { switch (g.type) { case ROW: - ret = row_remove(cursor, key, keyno); + ret = + row_reserve(cursor, key, keyno, positioned); break; case FIX: case VAR: - ret = col_remove(cursor, key, keyno); + ret = col_reserve(cursor, keyno, positioned); break; } if (ret == 0) { positioned = true; - if (snap != NULL && (size_t) - (snap - snap_list) < WT_ELEMENTS(snap_list)) - snap_track(snap++, keyno, NULL, NULL); + __wt_yield(); } else { positioned = false; if (ret == WT_ROLLBACK && intxn) goto deadlock; + testutil_assert(ret == WT_NOTFOUND); } - } else if (op < g.c_delete_pct + g.c_insert_pct) { - ++tinfo->insert; + } +#endif + /* Perform the operation. */ + switch (op) { + case INSERT: switch (g.type) { case ROW: - key_gen_insert(&tinfo->rnd, key, keyno); - val_gen(&tinfo->rnd, value, keyno); - ret = row_insert(cursor, key, value, keyno); + ret = row_insert(tinfo, + cursor, key, value, keyno, positioned); break; case FIX: case VAR: /* - * We can only append so many new records, if - * we've reached that limit, update a record - * instead of doing an insert. + * We can only append so many new records, once + * we reach that limit, update a record instead + * of inserting. */ if (g.append_cnt >= g.append_max) - goto skip_insert; + goto update_instead_of_insert; - /* Insert, then reset the insert cursor. */ - val_gen(&tinfo->rnd, value, g.rows + 1); ret = col_insert( - cursor_insert, key, value, &keyno); - testutil_check( - cursor_insert->reset(cursor_insert)); + tinfo, cursor, key, value, &keyno); break; } + + /* Insert never leaves the cursor positioned. */ positioned = false; if (ret == 0) { - if (snap != NULL && (size_t) - (snap - snap_list) < WT_ELEMENTS(snap_list)) + ++tinfo->insert; + if (SNAP_TRACK) snap_track(snap++, keyno, g.type == ROW ? key : NULL, value); - } else + } else { if (ret == WT_ROLLBACK && intxn) goto deadlock; - } else if ( - op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) { - ++tinfo->update; + testutil_assert(ret == 0 || ret == WT_ROLLBACK); + } + break; + case READ: + ++tinfo->search; + ret = read_row(cursor, key, value, keyno); + if (ret == 0) { + positioned = true; + if (SNAP_TRACK) + snap_track(snap++, keyno, NULL, value); + } else { + positioned = false; + if (ret == WT_ROLLBACK && intxn) + goto deadlock; + testutil_assert(ret == WT_NOTFOUND); + } + break; + case REMOVE: switch (g.type) { case ROW: - key_gen(key, keyno); - val_gen(&tinfo->rnd, value, keyno); - ret = row_update(cursor, key, value, keyno); + ret = + row_remove(cursor, key, keyno, positioned); break; case FIX: case VAR: -skip_insert: val_gen(&tinfo->rnd, value, keyno); - ret = col_update(cursor, key, value, keyno); + ret = + col_remove(cursor, key, keyno, positioned); break; } if (ret == 0) { - positioned = true; - if (snap != NULL && (size_t) - (snap - snap_list) < WT_ELEMENTS(snap_list)) - snap_track(snap++, keyno, NULL, value); + ++tinfo->remove; + /* + * Don't set positioned: it's unchanged from the + * previous state, but not necessarily set. + */ + if (SNAP_TRACK) + snap_track(snap++, keyno, NULL, NULL); } else { positioned = false; if (ret == WT_ROLLBACK && intxn) goto deadlock; + testutil_assert(ret == WT_NOTFOUND); + } + break; + case UPDATE: +update_instead_of_insert: + ++tinfo->update; + + /* Update the row. */ + switch (g.type) { + case ROW: + ret = row_update(tinfo, + cursor, key, value, keyno, positioned); + break; + case FIX: + case VAR: + ret = col_update(tinfo, + cursor, key, value, keyno, positioned); + break; } - } else { - ++tinfo->search; - ret = read_row(cursor, key, value, keyno); if (ret == 0) { positioned = true; - if (snap != NULL && (size_t) - (snap - snap_list) < WT_ELEMENTS(snap_list)) + if (SNAP_TRACK) snap_track(snap++, keyno, NULL, value); } else { positioned = false; if (ret == WT_ROLLBACK && intxn) goto deadlock; + testutil_assert(ret == 0 || ret == WT_ROLLBACK); } + break; } /* @@ -727,8 +786,8 @@ skip_insert: val_gen(&tinfo->rnd, value, keyno); testutil_check(cursor->reset(cursor)); /* - * If we're in a transaction, commit 40% of the time and - * rollback 10% of the time. + * Continue if not in a transaction, else add more operations + * to the transaction half the time. */ if (!intxn || (rnd = mmrand(&tinfo->rnd, 1, 10)) > 5) continue; @@ -741,6 +800,10 @@ skip_insert: val_gen(&tinfo->rnd, value, keyno); cursor, snap_list, snap, key, value)) == WT_ROLLBACK) goto deadlock; + /* + * If we're in a transaction, commit 40% of the time and + * rollback 10% of the time. + */ switch (rnd) { case 1: case 2: case 3: case 4: /* 40% */ testutil_check( @@ -1040,27 +1103,94 @@ nextprev(WT_CURSOR *cursor, int next) return (ret); } +#if 0 +/* + * row_reserve -- + * Reserve a row in a row-store file. + */ +static int +row_reserve(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, bool positioned) +{ + WT_DECL_RET; + + if (!positioned) { + key_gen(key, keyno); + cursor->set_key(cursor, key); + } + + if (g.logging == LOG_OPS) + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, + "%-10s{%.*s}", "reserve", (int)key->size, key->data); + + switch (ret = cursor->reserve(cursor)) { + case 0: + break; + case WT_CACHE_FULL: + case WT_ROLLBACK: + return (WT_ROLLBACK); + case WT_NOTFOUND: + return (WT_NOTFOUND); + default: + testutil_die(ret, + "row_reserve: reserve row %" PRIu64 " by key", keyno); + } + return (0); +} + +/* + * col_reserve -- + * Reserve a row in a column-store file. + */ +static int +col_reserve(WT_CURSOR *cursor, uint64_t keyno, bool positioned) +{ + WT_DECL_RET; + + if (!positioned) + cursor->set_key(cursor, keyno); + + if (g.logging == LOG_OPS) + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, + "%-10s%" PRIu64, "reserve", keyno); + + switch (ret = cursor->reserve(cursor)) { + case 0: + break; + case WT_CACHE_FULL: + case WT_ROLLBACK: + return (WT_ROLLBACK); + case WT_NOTFOUND: + return (WT_NOTFOUND); + default: + testutil_die(ret, "col_reserve: %" PRIu64, keyno); + } + return (0); +} +#endif + /* * row_update -- * Update a row in a row-store file. */ static int -row_update(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) +row_update(TINFO *tinfo, WT_CURSOR *cursor, + WT_ITEM *key, WT_ITEM *value, uint64_t keyno, bool positioned) { WT_DECL_RET; - WT_SESSION *session; - session = cursor->session; + if (!positioned) { + key_gen(key, keyno); + cursor->set_key(cursor, key); + } + val_gen(&tinfo->rnd, value, keyno); + cursor->set_value(cursor, value); - /* Log the operation */ if (g.logging == LOG_OPS) - (void)g.wt_api->msg_printf(g.wt_api, session, + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, "%-10s{%.*s}, {%.*s}", "put", (int)key->size, key->data, (int)value->size, value->data); - cursor->set_key(cursor, key); - cursor->set_value(cursor, value); switch (ret = cursor->update(cursor)) { case 0: break; @@ -1086,32 +1216,32 @@ row_update(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) * Update a row in a column-store file. */ static int -col_update(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) +col_update(TINFO *tinfo, WT_CURSOR *cursor, + WT_ITEM *key, WT_ITEM *value, uint64_t keyno, bool positioned) { WT_DECL_RET; - WT_SESSION *session; - session = cursor->session; + if (!positioned) + cursor->set_key(cursor, keyno); + val_gen(&tinfo->rnd, value, keyno); + if (g.type == FIX) + cursor->set_value(cursor, *(uint8_t *)value->data); + else + cursor->set_value(cursor, value); - /* Log the operation */ if (g.logging == LOG_OPS) { if (g.type == FIX) - (void)g.wt_api->msg_printf(g.wt_api, session, + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, "%-10s%" PRIu64 " {0x%02" PRIx8 "}", "update", keyno, ((uint8_t *)value->data)[0]); else - (void)g.wt_api->msg_printf(g.wt_api, session, + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, "%-10s%" PRIu64 " {%.*s}", "update", keyno, (int)value->size, (char *)value->data); } - cursor->set_key(cursor, keyno); - if (g.type == FIX) - cursor->set_value(cursor, *(uint8_t *)value->data); - else - cursor->set_value(cursor, value); switch (ret = cursor->update(cursor)) { case 0: break; @@ -1238,22 +1368,29 @@ table_append(uint64_t keyno) * Insert a row in a row-store file. */ static int -row_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) +row_insert(TINFO *tinfo, WT_CURSOR *cursor, + WT_ITEM *key, WT_ITEM *value, uint64_t keyno, bool positioned) { WT_DECL_RET; - WT_SESSION *session; - session = cursor->session; + /* + * If we positioned the cursor already, it's a test of an update using + * the insert method. Otherwise, generate a unique key and insert. + */ + if (!positioned) { + key_gen_insert(&tinfo->rnd, key, keyno); + cursor->set_key(cursor, key); + } + val_gen(&tinfo->rnd, value, keyno); + cursor->set_value(cursor, value); /* Log the operation */ if (g.logging == LOG_OPS) - (void)g.wt_api->msg_printf(g.wt_api, session, + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, "%-10s{%.*s}, {%.*s}", "insert", (int)key->size, key->data, (int)value->size, value->data); - cursor->set_key(cursor, key); - cursor->set_value(cursor, value); switch (ret = cursor->insert(cursor)) { case 0: break; @@ -1279,14 +1416,13 @@ row_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) * Insert an element in a column-store file. */ static int -col_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t *keynop) +col_insert(TINFO *tinfo, + WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t *keynop) { WT_DECL_RET; - WT_SESSION *session; uint64_t keyno; - session = cursor->session; - + val_gen(&tinfo->rnd, value, g.rows + 1); if (g.type == FIX) cursor->set_value(cursor, *(uint8_t *)value->data); else @@ -1307,12 +1443,12 @@ col_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t *keynop) if (g.logging == LOG_OPS) { if (g.type == FIX) - (void)g.wt_api->msg_printf(g.wt_api, session, + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, "%-10s%" PRIu64 " {0x%02" PRIx8 "}", "insert", keyno, ((uint8_t *)value->data)[0]); else - (void)g.wt_api->msg_printf(g.wt_api, session, + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, "%-10s%" PRIu64 " {%.*s}", "insert", keyno, (int)value->size, (char *)value->data); @@ -1335,21 +1471,19 @@ col_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t *keynop) * Remove an row from a row-store file. */ static int -row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno) +row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, bool positioned) { WT_DECL_RET; - WT_SESSION *session; - session = cursor->session; - - key_gen(key, keyno); + if (!positioned) { + key_gen(key, keyno); + cursor->set_key(cursor, key); + } - /* Log the operation */ if (g.logging == LOG_OPS) - (void)g.wt_api->msg_printf( - g.wt_api, session, "%-10s%" PRIu64, "remove", keyno); + (void)g.wt_api->msg_printf(g.wt_api, + cursor->session, "%-10s%" PRIu64, "remove", keyno); - cursor->set_key(cursor, key); /* We use the cursor in overwrite mode, check for existence. */ if ((ret = cursor->search(cursor)) == 0) ret = cursor->remove(cursor); @@ -1385,19 +1519,17 @@ row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno) * Remove a row from a column-store file. */ static int -col_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno) +col_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, bool positioned) { WT_DECL_RET; - WT_SESSION *session; - session = cursor->session; + if (!positioned) + cursor->set_key(cursor, keyno); - /* Log the operation */ if (g.logging == LOG_OPS) - (void)g.wt_api->msg_printf( - g.wt_api, session, "%-10s%" PRIu64, "remove", keyno); + (void)g.wt_api->msg_printf(g.wt_api, + cursor->session, "%-10s%" PRIu64, "remove", keyno); - cursor->set_key(cursor, keyno); /* We use the cursor in overwrite mode, check for existence. */ if ((ret = cursor->search(cursor)) == 0) ret = cursor->remove(cursor); @@ -1448,7 +1580,7 @@ notfound_chk(const char *f, int wt_ret, int bdb_notfound, uint64_t keyno) return (1); if (bdb_notfound) { - fprintf(stderr, "%s: %s:", g.progname, f); + fprintf(stderr, "%s: %s:", progname, f); if (keyno != 0) fprintf(stderr, " row %" PRIu64 ":", keyno); fprintf(stderr, @@ -1456,7 +1588,7 @@ notfound_chk(const char *f, int wt_ret, int bdb_notfound, uint64_t keyno) testutil_die(0, NULL); } if (wt_ret == WT_NOTFOUND) { - fprintf(stderr, "%s: %s:", g.progname, f); + fprintf(stderr, "%s: %s:", progname, f); if (keyno != 0) fprintf(stderr, " row %" PRIu64 ":", keyno); fprintf(stderr, diff --git a/test/format/rebalance.c b/test/format/rebalance.c index 9849b7df82b..e35c62e7255 100644 --- a/test/format/rebalance.c +++ b/test/format/rebalance.c @@ -41,10 +41,10 @@ wts_rebalance(void) track("rebalance", 0ULL, NULL); /* Dump the current object. */ - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), ".." DIR_DELIM_STR ".." DIR_DELIM_STR "wt" " -h %s dump -f %s/rebalance.orig %s", - g.home, g.home, g.uri); + g.home, g.home, g.uri)); testutil_checkfmt(system(cmd), "command failed: %s", cmd); /* Rebalance, then verify the object. */ @@ -66,21 +66,21 @@ wts_rebalance(void) wts_verify("post-rebalance verify"); wts_close(); - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), ".." DIR_DELIM_STR ".." DIR_DELIM_STR "wt" " -h %s dump -f %s/rebalance.new %s", - g.home, g.home, g.uri); + g.home, g.home, g.uri)); testutil_checkfmt(system(cmd), "command failed: %s", cmd); /* Compare the old/new versions of the object. */ #ifdef _WIN32 - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "fc /b %s\\rebalance.orig %s\\rebalance.new > NUL", - g.home, g.home); + g.home, g.home)); #else - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "cmp %s/rebalance.orig %s/rebalance.new > /dev/null", - g.home, g.home); + g.home, g.home)); #endif testutil_checkfmt(system(cmd), "command failed: %s", cmd); } diff --git a/test/format/salvage.c b/test/format/salvage.c index 69805fb1018..f82dc34dd5f 100644 --- a/test/format/salvage.c +++ b/test/format/salvage.c @@ -70,29 +70,31 @@ corrupt(void) * It's a little tricky: if the data source is a file, we're looking * for "wt", if the data source is a table, we're looking for "wt.wt". */ - (void)snprintf(buf, sizeof(buf), "%s/%s", g.home, WT_NAME); + testutil_check(__wt_snprintf( + buf, sizeof(buf), "%s/%s", g.home, WT_NAME)); if ((fd = open(buf, O_RDWR)) != -1) { #ifdef _WIN32 - (void)snprintf(copycmd, sizeof(copycmd), + testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), "copy %s\\%s %s\\slvg.copy\\%s.corrupted", - g.home, WT_NAME, g.home, WT_NAME); + g.home, WT_NAME, g.home, WT_NAME)); #else - (void)snprintf(copycmd, sizeof(copycmd), + testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), "cp %s/%s %s/slvg.copy/%s.corrupted", - g.home, WT_NAME, g.home, WT_NAME); + g.home, WT_NAME, g.home, WT_NAME)); #endif goto found; } - (void)snprintf(buf, sizeof(buf), "%s/%s.wt", g.home, WT_NAME); + testutil_check(__wt_snprintf( + buf, sizeof(buf), "%s/%s.wt", g.home, WT_NAME)); if ((fd = open(buf, O_RDWR)) != -1) { #ifdef _WIN32 - (void)snprintf(copycmd, sizeof(copycmd), + testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), "copy %s\\%s.wt %s\\slvg.copy\\%s.wt.corrupted", - g.home, WT_NAME, g.home, WT_NAME); + g.home, WT_NAME, g.home, WT_NAME)); #else - (void)snprintf(copycmd, sizeof(copycmd), + testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), "cp %s/%s.wt %s/slvg.copy/%s.wt.corrupted", - g.home, WT_NAME, g.home, WT_NAME); + g.home, WT_NAME, g.home, WT_NAME)); #endif goto found; } @@ -103,7 +105,8 @@ found: if (fstat(fd, &sb) == -1) offset = mmrand(NULL, 0, (u_int)sb.st_size); len = (size_t)(20 + (sb.st_size / 100) * 2); - (void)snprintf(buf, sizeof(buf), "%s/slvg.corrupt", g.home); + testutil_check(__wt_snprintf( + buf, sizeof(buf), "%s/slvg.corrupt", g.home)); if ((fp = fopen(buf, "w")) == NULL) testutil_die(errno, "salvage-corrupt: open: %s", buf); (void)fprintf(fp, diff --git a/test/format/t.c b/test/format/t.c index 7701595776c..c6686ae8b91 100644 --- a/test/format/t.c +++ b/test/format/t.c @@ -49,14 +49,7 @@ main(int argc, char *argv[]) config = NULL; -#ifdef _WIN32 - g.progname = "t_format.exe"; -#else - if ((g.progname = strrchr(argv[0], DIR_DELIM)) == NULL) - g.progname = argv[0]; - else - ++g.progname; -#endif + (void)testutil_set_progname(argv); #if 0 /* Configure the GNU malloc for debugging. */ @@ -74,7 +67,7 @@ main(int argc, char *argv[]) home = NULL; onerun = 0; while ((ch = __wt_getopt( - g.progname, argc, argv, "1C:c:H:h:Llqrt:")) != EOF) + progname, argc, argv, "1C:c:H:h:Llqrt:")) != EOF) switch (ch) { case '1': /* One run */ onerun = 1; @@ -179,7 +172,7 @@ main(int argc, char *argv[]) testutil_check(pthread_rwlock_init(&g.checkpoint_lock, NULL)); testutil_check(pthread_rwlock_init(&g.death_lock, NULL)); - printf("%s: process %" PRIdMAX "\n", g.progname, (intmax_t)getpid()); + printf("%s: process %" PRIdMAX "\n", progname, (intmax_t)getpid()); while (++g.run_cnt <= g.c_runs || g.c_runs == 0 ) { startup(); /* Start a run */ @@ -344,7 +337,7 @@ usage(void) "usage: %s [-1Llqr] [-C wiredtiger-config]\n " "[-c config-file] [-H mount] [-h home] " "[name=value ...]\n", - g.progname); + progname); fprintf(stderr, "%s", "\t-1 run once\n" "\t-C specify wiredtiger_open configuration arguments\n" diff --git a/test/format/util.c b/test/format/util.c index b9788f1ac75..983d03e2525 100644 --- a/test/format/util.c +++ b/test/format/util.c @@ -241,20 +241,23 @@ val_gen(WT_RAND_STATE *rnd, WT_ITEM *value, uint64_t keyno) void track(const char *tag, uint64_t cnt, TINFO *tinfo) { - static int lastlen = 0; - int len; + static size_t lastlen = 0; + size_t len; char msg[128]; if (g.c_quiet || tag == NULL) return; if (tinfo == NULL && cnt == 0) - len = snprintf(msg, sizeof(msg), "%4d: %s", g.run_cnt, tag); + testutil_check(__wt_snprintf_len_set( + msg, sizeof(msg), &len, "%4d: %s", g.run_cnt, tag)); else if (tinfo == NULL) - len = snprintf( - msg, sizeof(msg), "%4d: %s: %" PRIu64, g.run_cnt, tag, cnt); + testutil_check(__wt_snprintf_len_set( + msg, sizeof(msg), &len, + "%4d: %s: %" PRIu64, g.run_cnt, tag, cnt)); else - len = snprintf(msg, sizeof(msg), + testutil_check(__wt_snprintf_len_set( + msg, sizeof(msg), &len, "%4d: %s: " "search %" PRIu64 "%s, " "insert %" PRIu64 "%s, " @@ -268,7 +271,7 @@ track(const char *tag, uint64_t cnt, TINFO *tinfo) tinfo->update > M(9) ? tinfo->update / M(1) : tinfo->update, tinfo->update > M(9) ? "M" : "", tinfo->remove > M(9) ? tinfo->remove / M(1) : tinfo->remove, - tinfo->remove > M(9) ? "M" : ""); + tinfo->remove > M(9) ? "M" : "")); if (lastlen > len) { memset(msg + len, ' ', (size_t)(lastlen - len)); @@ -297,27 +300,30 @@ path_setup(const char *home) /* Log file. */ len = strlen(g.home) + strlen("log") + 2; g.home_log = dmalloc(len); - snprintf(g.home_log, len, "%s/%s", g.home, "log"); + testutil_check(__wt_snprintf(g.home_log, len, "%s/%s", g.home, "log")); /* RNG log file. */ len = strlen(g.home) + strlen("rand") + 2; g.home_rand = dmalloc(len); - snprintf(g.home_rand, len, "%s/%s", g.home, "rand"); + testutil_check(__wt_snprintf( + g.home_rand, len, "%s/%s", g.home, "rand")); /* Run file. */ len = strlen(g.home) + strlen("CONFIG") + 2; g.home_config = dmalloc(len); - snprintf(g.home_config, len, "%s/%s", g.home, "CONFIG"); + testutil_check(__wt_snprintf( + g.home_config, len, "%s/%s", g.home, "CONFIG")); /* Statistics file. */ len = strlen(g.home) + strlen("stats") + 2; g.home_stats = dmalloc(len); - snprintf(g.home_stats, len, "%s/%s", g.home, "stats"); + testutil_check(__wt_snprintf( + g.home_stats, len, "%s/%s", g.home, "stats")); /* BDB directory. */ len = strlen(g.home) + strlen("bdb") + 2; g.home_bdb = dmalloc(len); - snprintf(g.home_bdb, len, "%s/%s", g.home, "bdb"); + testutil_check(__wt_snprintf(g.home_bdb, len, "%s/%s", g.home, "bdb")); /* * Home directory initialize command: create the directory if it doesn't @@ -336,21 +342,23 @@ path_setup(const char *home) "cd %s & mkdir KVS" len = strlen(g.home) * 7 + strlen(CMD) + 1; g.home_init = dmalloc(len); - snprintf(g.home_init, len, CMD, - g.home, g.home, g.home, g.home, g.home, g.home, g.home); + testutil_check(__wt_snprintf(g.home_init, len, CMD, + g.home, g.home, g.home, g.home, g.home, g.home, g.home)); #else #define CMD "test -e %s || mkdir %s; " \ "cd %s > /dev/null && rm -rf `ls | sed /rand/d`; " \ "mkdir KVS" len = strlen(g.home) * 3 + strlen(CMD) + 1; g.home_init = dmalloc(len); - snprintf(g.home_init, len, CMD, g.home, g.home, g.home); + testutil_check(__wt_snprintf( + g.home_init, len, CMD, g.home, g.home, g.home)); #endif /* Primary backup directory. */ len = strlen(g.home) + strlen("BACKUP") + 2; g.home_backup = dmalloc(len); - snprintf(g.home_backup, len, "%s/%s", g.home, "BACKUP"); + testutil_check(__wt_snprintf( + g.home_backup, len, "%s/%s", g.home, "BACKUP")); /* * Backup directory initialize command, remove and re-create the primary @@ -365,9 +373,9 @@ path_setup(const char *home) len = strlen(g.home) * 4 + strlen("BACKUP") * 2 + strlen("BACKUP_COPY") * 2 + strlen(CMD) + 1; g.home_backup_init = dmalloc(len); - snprintf(g.home_backup_init, len, CMD, + testutil_check(__wt_snprintf(g.home_backup_init, len, CMD, g.home, "BACKUP", g.home, "BACKUP_COPY", - g.home, "BACKUP", g.home, "BACKUP_COPY"); + g.home, "BACKUP", g.home, "BACKUP_COPY")); /* * Salvage command, save the interesting files so we can replay the @@ -390,7 +398,7 @@ path_setup(const char *home) #endif len = strlen(g.home) + strlen(CMD) + 1; g.home_salvage_copy = dmalloc(len); - snprintf(g.home_salvage_copy, len, CMD, g.home); + testutil_check(__wt_snprintf(g.home_salvage_copy, len, CMD, g.home)); } /* @@ -489,8 +497,9 @@ alter(void *arg) while (!g.workers_finished) { period = mmrand(NULL, 1, 10); - snprintf(buf, sizeof(buf), - "access_pattern_hint=%s", access_value ? "random" : "none"); + testutil_check(__wt_snprintf(buf, sizeof(buf), + "access_pattern_hint=%s", + access_value ? "random" : "none")); access_value = !access_value; if (session->alter(session, g.uri, buf) != 0) break; diff --git a/test/format/wts.c b/test/format/wts.c index da234ce53c7..6aa4784d1c1 100644 --- a/test/format/wts.c +++ b/test/format/wts.c @@ -120,8 +120,15 @@ static WT_EVENT_HANDLER event_handler = { NULL /* Close handler. */ }; -#undef REMAIN -#define REMAIN(p, end) (size_t)((p) >= (end) ? 0 : (end) - (p)) +#define CONFIG_APPEND(p, ...) do { \ + size_t __len; \ + testutil_check( \ + __wt_snprintf_len_set(p, max, &__len, __VA_ARGS__)); \ + if (__len > max) \ + __len = max; \ + p += __len; \ + max -= __len; \ +} while (0) /* * wts_open -- @@ -132,42 +139,42 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) { WT_CONNECTION *conn; WT_DECL_RET; - char *config, *end, *p, helium_config[1024]; + size_t max; + char *config, *p, helium_config[1024]; *connp = NULL; config = p = g.wiredtiger_open_config; - end = config + sizeof(g.wiredtiger_open_config); + max = sizeof(g.wiredtiger_open_config); - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, "create=true," "cache_size=%" PRIu32 "MB," "checkpoint_sync=false," "error_prefix=\"%s\"", - g.c_cache, g.progname); + g.c_cache, progname); /* In-memory configuration. */ if (g.c_in_memory != 0) - p += snprintf(p, REMAIN(p, end), ",in_memory=1"); + CONFIG_APPEND(p, ",in_memory=1"); /* LSM configuration. */ if (DATASOURCE("lsm")) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",lsm_manager=(worker_thread_max=%" PRIu32 "),", g.c_lsm_worker_threads); - if (DATASOURCE("lsm") || g.c_cache < 20) { - p += snprintf(p, REMAIN(p, end), ",eviction_dirty_trigger=95"); - } + if (DATASOURCE("lsm") || g.c_cache < 20) + CONFIG_APPEND(p, ",eviction_dirty_trigger=95"); /* Eviction worker configuration. */ if (g.c_evict_max != 0) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",eviction=(threads_max=%" PRIu32 ")", g.c_evict_max); /* Logging configuration. */ if (g.c_logging) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",log=(enabled=true,archive=%d,prealloc=%d" ",compressor=\"%s\")", g.c_logging_archive ? 1 : 0, @@ -175,21 +182,21 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) compressor(g.c_logging_compression_flag)); if (g.c_encryption) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",encryption=(name=%s)", encryptor(g.c_encryption_flag)); /* Miscellaneous. */ #ifdef HAVE_POSIX_MEMALIGN - p += snprintf(p, REMAIN(p, end), ",buffer_alignment=512"); + CONFIG_APPEND(p, ",buffer_alignment=512"); #endif - p += snprintf(p, REMAIN(p, end), ",mmap=%d", g.c_mmap ? 1 : 0); + CONFIG_APPEND(p, ",mmap=%d", g.c_mmap ? 1 : 0); if (g.c_direct_io) - p += snprintf(p, REMAIN(p, end), ",direct_io=(data)"); + CONFIG_APPEND(p, ",direct_io=(data)"); if (g.c_data_extend) - p += snprintf(p, REMAIN(p, end), ",file_extend=(data=8MB)"); + CONFIG_APPEND(p, ",file_extend=(data=8MB)"); /* * Run the statistics server and/or maintain statistics in the engine. @@ -198,18 +205,18 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) if (g.c_statistics_server) { if (mmrand(NULL, 0, 5) == 1 && memcmp(g.uri, "file:", strlen("file:")) == 0) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",statistics=(fast)" ",statistics_log=(wait=5,sources=(\"file:\"))"); else - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",statistics=(fast),statistics_log=(wait=5)"); } else - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",statistics=(%s)", g.c_statistics ? "fast" : "none"); /* Extensions. */ - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",extensions=[" "\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],", g.c_reverse ? REVERSE_PATH : "", @@ -227,11 +234,11 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) * override the standard configuration. */ if (g.c_config_open != NULL) - p += snprintf(p, REMAIN(p, end), ",%s", g.c_config_open); + CONFIG_APPEND(p, ",%s", g.c_config_open); if (g.config_open != NULL) - p += snprintf(p, REMAIN(p, end), ",%s", g.config_open); + CONFIG_APPEND(p, ",%s", g.config_open); - if (REMAIN(p, end) == 0) + if (max == 0) testutil_die(ENOMEM, "wiredtiger_open configuration buffer too small"); @@ -259,12 +266,13 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) if (DATASOURCE("helium")) { if (g.helium_mount == NULL) testutil_die(EINVAL, "no Helium mount point specified"); - (void)snprintf(helium_config, sizeof(helium_config), + testutil_check( + __wt_snprintf(helium_config, sizeof(helium_config), "entry=wiredtiger_extension_init,config=[" "helium_verbose=0," "dev1=[helium_devices=\"he://./%s\"," "helium_o_volume_truncate=1]]", - g.helium_mount); + g.helium_mount)); if ((ret = conn->load_extension( conn, HELIUM_PATH, helium_config)) != 0) testutil_die(ret, @@ -299,13 +307,13 @@ wts_init(void) { WT_CONNECTION *conn; WT_SESSION *session; + size_t max; uint32_t maxintlpage, maxintlkey, maxleafpage, maxleafkey, maxleafvalue; - char config[4096], *end, *p; + char config[4096], *p; conn = g.wts_conn; - p = config; - end = config + sizeof(config); + max = sizeof(config); /* * Ensure that we can service at least one operation per-thread @@ -326,7 +334,7 @@ wts_init(void) if (maxleafpage > 512) maxleafpage >>= 1; } - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, "key_format=%s," "allocation_size=512,%s" "internal_page_max=%" PRIu32 ",leaf_page_max=%" PRIu32, @@ -340,43 +348,35 @@ wts_init(void) */ maxintlkey = mmrand(NULL, maxintlpage / 50, maxintlpage / 40); if (maxintlkey > 20) - p += snprintf(p, REMAIN(p, end), - ",internal_key_max=%" PRIu32, maxintlkey); + CONFIG_APPEND(p, ",internal_key_max=%" PRIu32, maxintlkey); maxleafkey = mmrand(NULL, maxleafpage / 50, maxleafpage / 40); if (maxleafkey > 20) - p += snprintf(p, REMAIN(p, end), - ",leaf_key_max=%" PRIu32, maxleafkey); + CONFIG_APPEND(p, ",leaf_key_max=%" PRIu32, maxleafkey); maxleafvalue = mmrand(NULL, maxleafpage * 10, maxleafpage / 40); if (maxleafvalue > 40 && maxleafvalue < 100 * 1024) - p += snprintf(p, REMAIN(p, end), - ",leaf_value_max=%" PRIu32, maxleafvalue); + CONFIG_APPEND(p, ",leaf_value_max=%" PRIu32, maxleafvalue); switch (g.type) { case FIX: - p += snprintf(p, REMAIN(p, end), - ",value_format=%" PRIu32 "t", g.c_bitcnt); + CONFIG_APPEND(p, ",value_format=%" PRIu32 "t", g.c_bitcnt); break; case ROW: if (g.c_huffman_key) - p += snprintf(p, REMAIN(p, end), - ",huffman_key=english"); + CONFIG_APPEND(p, ",huffman_key=english"); if (g.c_prefix_compression) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",prefix_compression_min=%" PRIu32, g.c_prefix_compression_min); else - p += snprintf(p, REMAIN(p, end), - ",prefix_compression=false"); + CONFIG_APPEND(p, ",prefix_compression=false"); if (g.c_reverse) - p += snprintf(p, REMAIN(p, end), - ",collator=reverse"); + CONFIG_APPEND(p, ",collator=reverse"); /* FALLTHROUGH */ case VAR: if (g.c_huffman_value) - p += snprintf(p, REMAIN(p, end), - ",huffman_value=english"); + CONFIG_APPEND(p, ",huffman_value=english"); if (g.c_dictionary) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",dictionary=%" PRIu32, mmrand(NULL, 123, 517)); break; } @@ -384,66 +384,63 @@ wts_init(void) /* Configure checksums. */ switch (g.c_checksum_flag) { case CHECKSUM_OFF: - p += snprintf(p, REMAIN(p, end), ",checksum=\"off\""); + CONFIG_APPEND(p, ",checksum=\"off\""); break; case CHECKSUM_ON: - p += snprintf(p, REMAIN(p, end), ",checksum=\"on\""); + CONFIG_APPEND(p, ",checksum=\"on\""); break; case CHECKSUM_UNCOMPRESSED: - p += snprintf(p, REMAIN(p, end), ",checksum=\"uncompressed\""); + CONFIG_APPEND(p, ",checksum=\"uncompressed\""); break; } /* Configure compression. */ if (g.c_compression_flag != COMPRESS_NONE) - p += snprintf(p, REMAIN(p, end), ",block_compressor=\"%s\"", + CONFIG_APPEND(p, ",block_compressor=\"%s\"", compressor(g.c_compression_flag)); /* Configure Btree internal key truncation. */ - p += snprintf(p, REMAIN(p, end), ",internal_key_truncate=%s", + CONFIG_APPEND(p, ",internal_key_truncate=%s", g.c_internal_key_truncation ? "true" : "false"); /* Configure Btree page key gap. */ - p += snprintf(p, REMAIN(p, end), ",key_gap=%" PRIu32, g.c_key_gap); + CONFIG_APPEND(p, ",key_gap=%" PRIu32, g.c_key_gap); /* Configure Btree split page percentage. */ - p += snprintf(p, REMAIN(p, end), ",split_pct=%" PRIu32, g.c_split_pct); + CONFIG_APPEND(p, ",split_pct=%" PRIu32, g.c_split_pct); /* Configure LSM and data-sources. */ if (DATASOURCE("helium")) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",type=helium,helium_o_compress=%d,helium_o_truncate=1", g.c_compression_flag == COMPRESS_NONE ? 0 : 1); if (DATASOURCE("kvsbdb")) - p += snprintf(p, REMAIN(p, end), ",type=kvsbdb"); + CONFIG_APPEND(p, ",type=kvsbdb"); if (DATASOURCE("lsm")) { - p += snprintf(p, REMAIN(p, end), ",type=lsm,lsm=("); - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",type=lsm,lsm=("); + CONFIG_APPEND(p, "auto_throttle=%s,", g.c_auto_throttle ? "true" : "false"); - p += snprintf(p, REMAIN(p, end), - "chunk_size=%" PRIu32 "MB,", g.c_chunk_size); + CONFIG_APPEND(p, "chunk_size=%" PRIu32 "MB,", g.c_chunk_size); /* * We can't set bloom_oldest without bloom, and we want to test * with Bloom filters on most of the time anyway. */ if (g.c_bloom_oldest) g.c_bloom = 1; - p += snprintf(p, REMAIN(p, end), - "bloom=%s,", g.c_bloom ? "true" : "false"); - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, "bloom=%s,", g.c_bloom ? "true" : "false"); + CONFIG_APPEND(p, "bloom_bit_count=%" PRIu32 ",", g.c_bloom_bit_count); - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, "bloom_hash_count=%" PRIu32 ",", g.c_bloom_hash_count); - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, "bloom_oldest=%s,", g.c_bloom_oldest ? "true" : "false"); - p += snprintf(p, REMAIN(p, end), - "merge_max=%" PRIu32 ",", g.c_merge_max); - p += snprintf(p, REMAIN(p, end), ",)"); + CONFIG_APPEND(p, "merge_max=%" PRIu32 ",", g.c_merge_max); + CONFIG_APPEND(p, ",)"); } - if (REMAIN(p, end) == 0) + if (max == 0) testutil_die(ENOMEM, "WT_SESSION.create configuration buffer too small"); @@ -490,14 +487,14 @@ wts_dump(const char *tag, int dump_bdb) len = strlen(g.home) + strlen(BERKELEY_DB_PATH) + strlen(g.uri) + 100; cmd = dmalloc(len); - (void)snprintf(cmd, len, + testutil_check(__wt_snprintf(cmd, len, "sh s_dumpcmp -h %s %s %s %s %s %s", g.home, dump_bdb ? "-b " : "", dump_bdb ? BERKELEY_DB_PATH : "", g.type == FIX || g.type == VAR ? "-c" : "", g.uri == NULL ? "" : "-n", - g.uri == NULL ? "" : g.uri); + g.uri == NULL ? "" : g.uri)); testutil_checkfmt(system(cmd), "%s: dump comparison failed", tag); free(cmd); @@ -587,7 +584,7 @@ wts_stats(void) fprintf(fp, "\n\n====== Data source statistics:\n"); len = strlen("statistics:") + strlen(g.uri) + 1; stat_name = dmalloc(len); - snprintf(stat_name, len, "statistics:%s", g.uri); + testutil_check(__wt_snprintf(stat_name, len, "statistics:%s", g.uri)); testutil_check(session->open_cursor( session, stat_name, NULL, NULL, &cursor)); free(stat_name); diff --git a/test/huge/huge.c b/test/huge/huge.c index 17e2db353d5..2b0d5f498e3 100644 --- a/test/huge/huge.c +++ b/test/huge/huge.c @@ -29,7 +29,6 @@ #include "test_util.h" static char home[512]; /* Program working dir */ -static const char *progname; /* Program name */ static uint8_t *big; /* Big key/value buffer */ #define GIGABYTE (1073741824) @@ -167,14 +166,10 @@ main(int argc, char *argv[]) int ch, small; char *working_dir; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); small = 0; working_dir = NULL; - while ((ch = __wt_getopt(progname, argc, argv, "h:s")) != EOF) switch (ch) { case 'h': diff --git a/test/java/com/wiredtiger/test/CursorTest03.java b/test/java/com/wiredtiger/test/CursorTest03.java new file mode 100644 index 00000000000..64f33f4d7b6 --- /dev/null +++ b/test/java/com/wiredtiger/test/CursorTest03.java @@ -0,0 +1,175 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +package com.wiredtiger.test; + +import com.wiredtiger.db.Connection; +import com.wiredtiger.db.Cursor; +import com.wiredtiger.db.SearchStatus; +import com.wiredtiger.db.Session; +import com.wiredtiger.db.WiredTigerPackingException; +import com.wiredtiger.db.WiredTigerException; +import com.wiredtiger.db.wiredtiger; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; +import org.junit.Assert; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/* + * Test cases for WT-3238. + * + * Most WiredTiger methods return int, and our SWIG typemaps for Java add + * checking that throws exceptions for non-zero returns. Certain methods + * (Cursor.compare, Cursor.equals) are declared as returning int in Java, + * but should not throw exceptions for normal returns (which may be + * non-zero). + */ +public class CursorTest03 { + Connection conn; + Session s; + static String values[] = { "key0", "key1" }; + + @Test + public void cursor_int_methods() + throws WiredTigerPackingException { + setup(); + + Cursor c1 = s.open_cursor("table:t", null, null); + Cursor c2 = s.open_cursor("table:t", null, null); + for (String s : values) { + c1.putKeyString(s); + c1.putValueString(s); + c1.insert(); + } + c1.reset(); + + // "key1" compared to "key1" + c1.putKeyString(values[1]); + Assert.assertEquals(c1.search_near(), SearchStatus.FOUND); + c2.putKeyString(values[1]); + Assert.assertEquals(c2.search_near(), SearchStatus.FOUND); + Assert.assertEquals(c1.compare(c2), 0); + Assert.assertEquals(c2.compare(c1), 0); + Assert.assertEquals(c1.compare(c1), 0); + Assert.assertEquals(c1.equals(c2), 1); + Assert.assertEquals(c2.equals(c1), 1); + Assert.assertEquals(c1.equals(c1), 1); + + // "key0" compared to "key1" + c1.putKeyString(values[0]); + Assert.assertEquals(c1.search_near(), SearchStatus.FOUND); + Assert.assertEquals(c1.compare(c2), -1); + Assert.assertEquals(c2.compare(c1), 1); + Assert.assertEquals(c1.equals(c2), 0); + Assert.assertEquals(c2.equals(c1), 0); + + c1.close(); + c2.close(); + teardown(); + } + + public void expectException(Cursor c1, Cursor c2) + { + boolean caught = false; + try { + c1.compare(c2); + } + catch (WiredTigerException wte) { + caught = true; + } + Assert.assertTrue(caught); + + caught = false; + try { + c1.equals(c2); + } + catch (WiredTigerException wte) { + caught = true; + } + Assert.assertTrue(caught); + } + + @Test + public void cursor_int_methods_errors() + throws WiredTigerPackingException { + setup(); + + Cursor c1 = s.open_cursor("table:t", null, null); + Cursor c2 = s.open_cursor("table:t", null, null); + Cursor cx = s.open_cursor("table:t2", null, null); + for (String s : values) { + c1.putKeyString(s); + c1.putValueString(s); + c1.insert(); + cx.putKeyString(s); + cx.putValueString(s); + cx.insert(); + } + c1.reset(); + cx.reset(); + + // With both cursors not set, should be an exception. + expectException(c1, c2); + expectException(c1, c2); + + // With any one cursor not set, should be an exception. + c1.putKeyString(values[1]); + Assert.assertEquals(c1.search_near(), SearchStatus.FOUND); + expectException(c1, c2); + expectException(c1, c2); + + // With two cursors from different tables, should be an exception. + cx.putKeyString(values[1]); + Assert.assertEquals(cx.search_near(), SearchStatus.FOUND); + expectException(c1, cx); + expectException(c1, cx); + + c1.close(); + c2.close(); + cx.close(); + teardown(); + } + + private void setup() { + conn = wiredtiger.open("WT_HOME", "create"); + s = conn.open_session(null); + s.create("table:t", "key_format=S,value_format=S"); + s.create("table:t2", "key_format=S,value_format=S"); + } + + private void teardown() { + s.drop("table:t", ""); + s.drop("table:t2", ""); + s.close(""); + conn.close(""); + } + +} + diff --git a/test/java/com/wiredtiger/test/WiredTigerSuite.java b/test/java/com/wiredtiger/test/WiredTigerSuite.java index 5bd98d53fac..9322d30671a 100644 --- a/test/java/com/wiredtiger/test/WiredTigerSuite.java +++ b/test/java/com/wiredtiger/test/WiredTigerSuite.java @@ -38,6 +38,7 @@ import org.junit.runners.Suite; ConfigTest.class, CursorTest.class, CursorTest02.class, + CursorTest03.class, ExceptionTest.class, PackTest.class, PackTest02.class, diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 7e986d47af3..42020d6ce9a 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -32,7 +32,6 @@ #define HOME_BASE "WT_TEST" static char home[HOME_SIZE]; /* Base home directory */ static char hometmp[HOME_SIZE]; /* Each conn home directory */ -static const char *progname; /* Program name */ static const char * const uri = "table:main"; #define WTOPEN_CFG_COMMON \ @@ -129,10 +128,8 @@ main(int argc, char *argv[]) const char *working_dir, *wt_cfg; char cmd[128]; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); + dbs = MAX_DBS; working_dir = HOME_BASE; idle = false; @@ -171,7 +168,8 @@ main(int argc, char *argv[]) testutil_make_work_dir(home); __wt_random_init(&rnd); for (i = 0; i < dbs; ++i) { - snprintf(hometmp, HOME_SIZE, "%s/%s.%d", home, HOME_BASE, i); + testutil_check(__wt_snprintf( + hometmp, HOME_SIZE, "%s/%s.%d", home, HOME_BASE, i)); testutil_make_work_dir(hometmp); /* * Open each database. Rotate different configurations diff --git a/test/mciproject.yml b/test/mciproject.yml index eb74914eb46..6456475aa00 100644 --- a/test/mciproject.yml +++ b/test/mciproject.yml @@ -65,7 +65,7 @@ tasks: ./build_posix/reconf ${configure_env_vars|} ./configure --enable-diagnostic --enable-python --enable-zlib --enable-strict --enable-verbose ${make_command|make} ${smp_command|} 2>&1 - ${make_command|make} VERBOSE=1 check 2>&1 + TESTUTIL_ENABLE_LONG_TESTS=1 ${make_command|make} VERBOSE=1 check 2>&1 fi - command: archive.targz_pack params: diff --git a/test/readonly/readonly.c b/test/readonly/readonly.c index a4b79f5859f..66c7a0ca692 100644 --- a/test/readonly/readonly.c +++ b/test/readonly/readonly.c @@ -39,7 +39,6 @@ static char home_rd[HOME_SIZE + sizeof(HOME_RD_SUFFIX)]; #define HOME_RD2_SUFFIX ".RDNOLOCK" /* Read-only dir no lock file */ static char home_rd2[HOME_SIZE + sizeof(HOME_RD2_SUFFIX)]; -static const char *progname; /* Program name */ static const char *saved_argv0; /* Program command */ static const char * const uri = "table:main"; @@ -172,10 +171,8 @@ main(int argc, char *argv[]) char cmd[512]; uint8_t buf[MAX_VAL]; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); + /* * Needed unaltered for system command later. */ @@ -209,10 +206,12 @@ main(int argc, char *argv[]) * Set up all the directory names. */ testutil_work_dir_from_path(home, sizeof(home), working_dir); - (void)snprintf(home_wr, sizeof(home_wr), "%s%s", home, HOME_WR_SUFFIX); - (void)snprintf(home_rd, sizeof(home_rd), "%s%s", home, HOME_RD_SUFFIX); - (void)snprintf( - home_rd2, sizeof(home_rd2), "%s%s", home, HOME_RD2_SUFFIX); + testutil_check(__wt_snprintf( + home_wr, sizeof(home_wr), "%s%s", home, HOME_WR_SUFFIX)); + testutil_check(__wt_snprintf( + home_rd, sizeof(home_rd), "%s%s", home, HOME_RD_SUFFIX)); + testutil_check(__wt_snprintf( + home_rd2, sizeof(home_rd2), "%s%s", home, HOME_RD2_SUFFIX)); if (!child) { testutil_make_work_dir(home); testutil_make_work_dir(home_wr); @@ -271,22 +270,22 @@ main(int argc, char *argv[]) * Copy the database. Remove any lock file from one copy * and chmod the copies to be read-only permissions. */ - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "cp -rp %s/* %s; rm -f %s/WiredTiger.lock", - home, home_wr, home_wr); + home, home_wr, home_wr)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "cp -rp %s/* %s; chmod 0555 %s; chmod -R 0444 %s/*", - home, home_rd, home_rd, home_rd); + home, home_rd, home_rd, home_rd)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "cp -rp %s/* %s; rm -f %s/WiredTiger.lock; " "chmod 0555 %s; chmod -R 0444 %s/*", - home, home_rd2, home_rd2, home_rd2, home_rd2); + home, home_rd2, home_rd2, home_rd2, home_rd2)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); @@ -330,8 +329,8 @@ main(int argc, char *argv[]) * * The child will exit with success if its test passes. */ - (void)snprintf( - cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); if (WEXITSTATUS(status) != 0) @@ -340,8 +339,8 @@ main(int argc, char *argv[]) /* * Scenario 2. Run child with writable config. */ - (void)snprintf( - cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); if (WEXITSTATUS(status) != 0) @@ -361,8 +360,8 @@ main(int argc, char *argv[]) /* * Scenario 3. Child read-only. */ - (void)snprintf( - cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); if (WEXITSTATUS(status) != 0) @@ -371,8 +370,8 @@ main(int argc, char *argv[]) /* * Scenario 4. Run child with writable config. */ - (void)snprintf( - cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); if (WEXITSTATUS(status) != 0) @@ -393,11 +392,12 @@ main(int argc, char *argv[]) * We need to chmod the read-only databases back so that they can * be removed by scripts. */ - (void)snprintf(cmd, sizeof(cmd), "chmod 0777 %s %s", home_rd, home_rd2); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "chmod 0777 %s %s", home_rd, home_rd2)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); - (void)snprintf(cmd, sizeof(cmd), "chmod -R 0666 %s/* %s/*", - home_rd, home_rd2); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "chmod -R 0666 %s/* %s/*", home_rd, home_rd2)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); printf(" *** Readonly test successful ***\n"); diff --git a/test/recovery/random-abort.c b/test/recovery/random-abort.c index c407361c7eb..febe6530534 100644 --- a/test/recovery/random-abort.c +++ b/test/recovery/random-abort.c @@ -31,9 +31,13 @@ #include <sys/wait.h> #include <signal.h> -static char home[512]; /* Program working dir */ -static const char *progname; /* Program name */ +static char home[1024]; /* Program working dir */ + +/* + * These two names for the URI and file system must be maintained in tandem. + */ static const char * const uri = "table:main"; +static const char * const fs_main = "main.wt"; static bool inmem; #define MAX_TH 12 @@ -90,14 +94,16 @@ thread_run(void *arg) /* * The value is the name of the record file with our id appended. */ - snprintf(buf, sizeof(buf), RECORDS_FILE, td->id); + testutil_check(__wt_snprintf(buf, sizeof(buf), RECORDS_FILE, td->id)); /* * Set up a large value putting our id in it. Write it in there a * bunch of times, but the rest of the buffer can just be zero. */ - snprintf(lgbuf, sizeof(lgbuf), "th-%" PRIu32, td->id); + testutil_check(__wt_snprintf( + lgbuf, sizeof(lgbuf), "th-%" PRIu32, td->id)); for (i = 0; i < 128; i += strlen(lgbuf)) - snprintf(&large[i], lsize - i, "%s", lgbuf); + testutil_check(__wt_snprintf( + &large[i], lsize - i, "%s", lgbuf)); /* * Keep a separate file with the records we wrote for checking. */ @@ -120,7 +126,8 @@ thread_run(void *arg) * Write our portion of the key space until we're killed. */ for (i = td->start; ; ++i) { - snprintf(kname, sizeof(kname), "%" PRIu64, i); + testutil_check(__wt_snprintf( + kname, sizeof(kname), "%" PRIu64, i)); cursor->set_key(cursor, kname); /* * Every 30th record write a very large record that exceeds the @@ -211,6 +218,7 @@ extern char *__wt_optarg; int main(int argc, char *argv[]) { + struct stat sb; FILE *fp; WT_CONNECTION *conn; WT_CURSOR *cursor; @@ -222,12 +230,9 @@ main(int argc, char *argv[]) pid_t pid; bool fatal, rand_th, rand_time, verify_only; const char *working_dir; - char fname[64], kname[64]; + char fname[64], kname[64], statname[1024]; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); inmem = false; nth = MIN_TH; @@ -263,7 +268,7 @@ main(int argc, char *argv[]) if (argc != 0) usage(); - testutil_work_dir_from_path(home, 512, working_dir); + testutil_work_dir_from_path(home, sizeof(home), working_dir); /* * If the user wants to verify they need to tell us how many threads * there were so we can find the old record files. @@ -305,8 +310,16 @@ main(int argc, char *argv[]) /* parent */ /* * Sleep for the configured amount of time before killing - * the child. + * the child. Start the timeout from the time we notice that + * the table has been created. That allows the test to run + * correctly on really slow machines. Verify the process ID + * still exists in case the child aborts for some reason we + * don't stay in this loop forever. */ + testutil_check(__wt_snprintf( + statname, sizeof(statname), "%s/%s", home, fs_main)); + while (stat(statname, &sb) != 0 && kill(pid, 0) == 0) + sleep(1); sleep(timeout); /* @@ -339,12 +352,10 @@ main(int argc, char *argv[]) fatal = false; for (i = 0; i < nth; ++i) { middle = 0; - snprintf(fname, sizeof(fname), RECORDS_FILE, i); - if ((fp = fopen(fname, "r")) == NULL) { - fprintf(stderr, - "Failed to open %s. i %" PRIu32 "\n", fname, i); - testutil_die(errno, "fopen"); - } + testutil_check(__wt_snprintf( + fname, sizeof(fname), RECORDS_FILE, i)); + if ((fp = fopen(fname, "r")) == NULL) + testutil_die(errno, "fopen: %s", fname); /* * For every key in the saved file, verify that the key exists @@ -370,7 +381,8 @@ main(int argc, char *argv[]) fname, key, last_key); break; } - snprintf(kname, sizeof(kname), "%" PRIu64, key); + testutil_check(__wt_snprintf( + kname, sizeof(kname), "%" PRIu64, key)); cursor->set_key(cursor, kname); if ((ret = cursor->search(cursor)) != 0) { if (ret != WT_NOTFOUND) diff --git a/test/recovery/truncated-log.c b/test/recovery/truncated-log.c index c265263d44c..a127d8c1c63 100644 --- a/test/recovery/truncated-log.c +++ b/test/recovery/truncated-log.c @@ -30,13 +30,7 @@ #include <sys/wait.h> -#ifdef _WIN32 -/* snprintf is not supported on <= VS2013 */ -#define snprintf _snprintf -#endif - -static char home[512]; /* Program working dir */ -static const char *progname; /* Program name */ +static char home[1024]; /* Program working dir */ static const char * const uri = "table:main"; #define RECORDS_FILE "records" @@ -138,7 +132,8 @@ usage(void) * Child process creates the database and table, and then writes data into * the table until it is killed by the parent. */ -static void fill_db(void)WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void fill_db(void) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void fill_db(void) { @@ -194,9 +189,9 @@ fill_db(void) max_key = min_key * 2; first = true; for (i = 0; i < max_key; ++i) { - snprintf(k, sizeof(k), "key%03d", (int)i); - snprintf(v, sizeof(v), "value%0*d", - (int)(V_SIZE - strlen("value")), (int)i); + testutil_check(__wt_snprintf(k, sizeof(k), "key%03d", (int)i)); + testutil_check(__wt_snprintf(v, sizeof(v), "value%0*d", + (int)(V_SIZE - (strlen("value") + 1)), (int)i)); cursor->set_key(cursor, k); cursor->set_value(cursor, v); if ((ret = cursor->insert(cursor)) != 0) @@ -271,10 +266,7 @@ main(int argc, char *argv[]) pid_t pid; const char *working_dir; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); working_dir = "WT_TEST.truncated-log"; while ((ch = __wt_getopt(progname, argc, argv, "h:")) != EOF) @@ -290,7 +282,7 @@ main(int argc, char *argv[]) if (argc != 0) usage(); - testutil_work_dir_from_path(home, 512, working_dir); + testutil_work_dir_from_path(home, sizeof(home), working_dir); testutil_make_work_dir(home); /* diff --git a/test/salvage/salvage.c b/test/salvage/salvage.c index b8553bbd72d..83f9c6349bc 100644 --- a/test/salvage/salvage.c +++ b/test/salvage/salvage.c @@ -54,8 +54,6 @@ void run(int); void t(int, u_int, int); int usage(void); -static const char *progname; /* Program name */ - static FILE *res_fp; /* Results file */ static u_int page_type; /* File types */ static int value_unique; /* Values are unique */ @@ -70,10 +68,7 @@ main(int argc, char *argv[]) u_int ptype; int ch, r; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); r = 0; ptype = 0; @@ -445,7 +440,8 @@ run(int r) process(); - snprintf(buf, sizeof(buf), "cmp %s %s > /dev/null", DUMP, RSLT); + testutil_check(__wt_snprintf( + buf, sizeof(buf), "cmp %s %s > /dev/null", DUMP, RSLT)); if (system(buf)) { fprintf(stderr, "check failed, salvage results were incorrect\n"); @@ -490,28 +486,28 @@ build(int ikey, int ivalue, int cnt) switch (page_type) { case WT_PAGE_COL_FIX: - (void)snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "key_format=r,value_format=7t," "allocation_size=%d," "internal_page_max=%d,internal_item_max=%d," "leaf_page_max=%d,leaf_item_max=%d", - PSIZE, PSIZE, OSIZE, PSIZE, OSIZE); + PSIZE, PSIZE, OSIZE, PSIZE, OSIZE)); break; case WT_PAGE_COL_VAR: - (void)snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "key_format=r," "allocation_size=%d," "internal_page_max=%d,internal_item_max=%d," "leaf_page_max=%d,leaf_item_max=%d", - PSIZE, PSIZE, OSIZE, PSIZE, OSIZE); + PSIZE, PSIZE, OSIZE, PSIZE, OSIZE)); break; case WT_PAGE_ROW_LEAF: - (void)snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "key_format=u," "allocation_size=%d," "internal_page_max=%d,internal_item_max=%d," "leaf_page_max=%d,leaf_item_max=%d", - PSIZE, PSIZE, OSIZE, PSIZE, OSIZE); + PSIZE, PSIZE, OSIZE, PSIZE, OSIZE)); break; default: assert(0); @@ -525,7 +521,8 @@ build(int ikey, int ivalue, int cnt) case WT_PAGE_COL_VAR: break; case WT_PAGE_ROW_LEAF: - snprintf(kbuf, sizeof(kbuf), "%010d KEY------", ikey); + testutil_check(__wt_snprintf( + kbuf, sizeof(kbuf), "%010d KEY------", ikey)); key.data = kbuf; key.size = 20; cursor->set_key(cursor, &key); @@ -538,8 +535,8 @@ build(int ikey, int ivalue, int cnt) break; case WT_PAGE_COL_VAR: case WT_PAGE_ROW_LEAF: - snprintf(vbuf, sizeof(vbuf), - "%010d VALUE----", value_unique ? ivalue : 37); + testutil_check(__wt_snprintf(vbuf, sizeof(vbuf), + "%010d VALUE----", value_unique ? ivalue : 37)); value.data = vbuf; value.size = 20; cursor->set_value(cursor, &value); @@ -626,9 +623,9 @@ process(void) /* Salvage. */ config[0] = '\0'; if (verbose) - snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "error_prefix=\"%s\",verbose=[salvage,verify],", - progname); + progname)); strcat(config, "log=(enabled=false),"); CHECK(wiredtiger_open(NULL, NULL, config, &conn) == 0); diff --git a/test/suite/run.py b/test/suite/run.py index ba6d9f78503..97c58bfdccf 100644 --- a/test/suite/run.py +++ b/test/suite/run.py @@ -324,7 +324,8 @@ if __name__ == '__main__': # All global variables should be set before any test classes are loaded. # That way, verbose printing can be done at the class definition level. wttest.WiredTigerTestCase.globalSetup(preserve, timestamp, gdbSub, - verbose, dirarg, longtest) + verbose, wt_builddir, dirarg, + longtest) # Without any tests listed as arguments, do discovery if len(testargs) == 0: diff --git a/test/suite/test_async01.py b/test/suite/test_async01.py index cbb3dad8de6..158c16a9381 100644 --- a/test/suite/test_async01.py +++ b/test/suite/test_async01.py @@ -132,7 +132,7 @@ class test_async01(wttest.WiredTigerTestCase, suite_subprocess): ]) # Enable async for this test. - def conn_config(self, dir): + def conn_config(self): return 'async=(enabled=true,ops_max=%s,' % self.async_ops + \ 'threads=%s)' % self.async_threads diff --git a/test/suite/test_async02.py b/test/suite/test_async02.py index 50652da6dfd..28435fe85b2 100644 --- a/test/suite/test_async02.py +++ b/test/suite/test_async02.py @@ -129,7 +129,7 @@ class test_async02(wttest.WiredTigerTestCase, suite_subprocess): ]) # Enable async for this test. - def conn_config(self, dir): + def conn_config(self): return 'async=(enabled=true,ops_max=%s,' % self.async_ops + \ 'threads=%s)' % self.async_threads diff --git a/test/suite/test_backup03.py b/test/suite/test_backup03.py index 73d05f0b0a1..c1ed3cc9e1a 100644 --- a/test/suite/test_backup03.py +++ b/test/suite/test_backup03.py @@ -74,7 +74,7 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): ('backup_9', dict(big=3,list=[])), # Backup everything ] - scenarios = make_scenarios(list) + scenarios = make_scenarios(list, prune=3, prunelong=1000) # Create a large cache, otherwise this test runs quite slowly. conn_config = 'cache_size=1G' diff --git a/test/suite/test_backup04.py b/test/suite/test_backup04.py index 919649fed57..be52a5e1e97 100644 --- a/test/suite/test_backup04.py +++ b/test/suite/test_backup04.py @@ -60,7 +60,7 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): ]) # Create a large cache, otherwise this test runs quite slowly. - def conn_config(self, dir): + def conn_config(self): return 'cache_size=1G,log=(archive=false,enabled,file_max=%s)' % \ self.logmax diff --git a/test/suite/test_bug011.py b/test/suite/test_bug011.py index 969aaeb5b39..5e0721b93f1 100644 --- a/test/suite/test_bug011.py +++ b/test/suite/test_bug011.py @@ -43,7 +43,7 @@ class test_bug011(wttest.WiredTigerTestCase): nrows = 10000 nops = 10000 # Add connection configuration for this test. - def conn_config(self, dir): + def conn_config(self): return 'cache_size=1GB' @wttest.longtest("Eviction copes with lots of files") diff --git a/test/suite/test_collator.py b/test/suite/test_collator.py index 3fae4ff47cb..7ce135c8976 100644 --- a/test/suite/test_collator.py +++ b/test/suite/test_collator.py @@ -48,34 +48,10 @@ class test_collator(wttest.WiredTigerTestCase): nentries = 100 nindices = 4 - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name, libname) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' - - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor'), - ('collators', 'revint', 'revint_collator')]) - connarg = 'create,error_prefix="{0}: ",{1}'.format( - self.shortid(), extarg) - conn = self.wiredtiger_open(dir, connarg) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('extractors', 'csv') + extlist.extension('collators', 'revint') def create_indices(self): # Create self.nindices index files, each with a column from the CSV diff --git a/test/suite/test_compact02.py b/test/suite/test_compact02.py index 7af76b5fd58..803600eea14 100644 --- a/test/suite/test_compact02.py +++ b/test/suite/test_compact02.py @@ -99,7 +99,7 @@ class test_compact02(wttest.WiredTigerTestCase): def ConnectionOpen(self, cacheSize): self.home = '.' conn_params = 'create,' + \ - cacheSize + ',error_prefix="%s: ",' % self.shortid() + \ + cacheSize + ',error_prefix="%s",' % self.shortid() + \ 'statistics=(all),' + \ 'eviction_dirty_target=99,eviction_dirty_trigger=99' try: diff --git a/test/suite/test_compress01.py b/test/suite/test_compress01.py index 606f7b63235..ef1064d294e 100644 --- a/test/suite/test_compress01.py +++ b/test/suite/test_compress01.py @@ -51,22 +51,10 @@ class test_compress01(wttest.WiredTigerTestCase): nrecords = 10000 bigvalue = "abcdefghij" * 1000 - # Load the compression extension, compression is enabled elsewhere. - def conn_config(self, dir): - return self.extensionArg(self.compress) - - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, name): - if name == None: - return '' - - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext/compressors') - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('compression extension "' + extfile + '" not built') - return ',extensions=["' + extfile + '"]' + # Load the compression extension, skip the test if missing + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('compressors', self.compress) # Create a table, add keys with both big and small values, then verify them. def test_compress(self): diff --git a/test/suite/test_config03.py b/test/suite/test_config03.py index 6699f7d2650..89038d71319 100644 --- a/test/suite/test_config03.py +++ b/test/suite/test_config03.py @@ -71,7 +71,7 @@ class test_config03(test_base03.test_base03): cache_size_scenarios, create_scenarios, error_prefix_scenarios, eviction_target_scenarios, eviction_trigger_scenarios, multiprocess_scenarios, session_max_scenarios, - transactional_scenarios, verbose_scenarios, prune=1000) + transactional_scenarios, verbose_scenarios, prune=100, prunelong=1000) #wttest.WiredTigerTestCase.printVerbose(2, 'test_config03: running ' + \ # str(len(scenarios)) + ' of ' + \ diff --git a/test/suite/test_cursor07.py b/test/suite/test_cursor07.py index d6078183fc1..19db718fd11 100644 --- a/test/suite/test_cursor07.py +++ b/test/suite/test_cursor07.py @@ -49,7 +49,7 @@ class test_cursor07(wttest.WiredTigerTestCase, suite_subprocess): ('reopen', dict(reopen=True)) ]) # Enable logging for this test. - def conn_config(self, dir): + def conn_config(self): return 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ 'transaction_sync="(method=dsync,enabled)"' diff --git a/test/suite/test_cursor08.py b/test/suite/test_cursor08.py index 3f8f50defa7..cc76f528aa9 100644 --- a/test/suite/test_cursor08.py +++ b/test/suite/test_cursor08.py @@ -54,24 +54,14 @@ class test_cursor08(wttest.WiredTigerTestCase, suite_subprocess): ] scenarios = make_scenarios(reopens, compress) # Load the compression extension, and enable it for logging. - def conn_config(self, dir): + def conn_config(self): return 'log=(archive=false,enabled,file_max=%s,' % self.logmax + \ 'compressor=%s),' % self.compress + \ - 'transaction_sync="(method=dsync,enabled)",' + \ - self.extensionArg(self.compress) + 'transaction_sync="(method=dsync,enabled)"' - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, name): - if name == None or name == 'none': - return '' - - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext/compressors') - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('compression extension "' + extfile + '" not built') - return ',extensions=["' + extfile + '"]' + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('compressors', self.compress) def test_log_cursor(self): # print "Creating %s with config '%s'" % (self.uri, self.create_params) diff --git a/test/suite/test_cursor10.py b/test/suite/test_cursor10.py index b3cffeab4e9..6cabfde9f1f 100644 --- a/test/suite/test_cursor10.py +++ b/test/suite/test_cursor10.py @@ -31,11 +31,11 @@ from wtscenario import make_scenarios # test_cursor10.py # Cursors with projections. -class test_cursor04(wttest.WiredTigerTestCase): +class test_cursor10(wttest.WiredTigerTestCase): """ Test cursor search and search_near """ - table_name1 = 'test_cursor04' + table_name1 = 'test_cursor10' nentries = 20 scenarios = make_scenarios([ diff --git a/test/suite/test_cursor11.py b/test/suite/test_cursor11.py new file mode 100644 index 00000000000..e159ec499e6 --- /dev/null +++ b/test/suite/test_cursor11.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet, SimpleIndexDataSet +from wtdataset import SimpleLSMDataSet, ComplexDataSet, ComplexLSMDataSet +from wtscenario import make_scenarios + +# test_cursor11.py +# WT_CURSOR position tests: remove (if not already positioned), and insert +# leave the cursor without position or information. +class test_cursor11(wttest.WiredTigerTestCase): + + keyfmt = [ + ('integer', dict(keyfmt='i')), + ('recno', dict(keyfmt='r')), + ('string', dict(keyfmt='S')), + ] + types = [ + ('file', dict(uri='file', ds=SimpleDataSet)), + ('lsm', dict(uri='lsm', ds=SimpleDataSet)), + ('table-complex', dict(uri='table', ds=ComplexDataSet)), + ('table-complex-lsm', dict(uri='table', ds=ComplexLSMDataSet)), + ('table-index', dict(uri='table', ds=SimpleIndexDataSet)), + ('table-simple', dict(uri='table', ds=SimpleDataSet)), + ('table-simple-lsm', dict(uri='table', ds=SimpleLSMDataSet)), + ] + scenarios = make_scenarios(types, keyfmt) + + def skip(self): + return self.keyfmt == 'r' and \ + (self.ds.is_lsm() or self.uri == 'lsm') + + # Do a remove using the cursor after setting a position, and confirm + # the key and position remain set but no value. + def test_cursor_remove_with_position(self): + if self.skip(): + return + + # Build an object. + uri = self.uri + ':test_cursor11' + ds = self.ds(self, uri, 50, key_format=self.keyfmt) + ds.populate() + s = self.conn.open_session() + c = s.open_cursor(uri, None) + + c.set_key(ds.key(25)) + self.assertEquals(c.search(), 0) + self.assertEquals(c.next(), 0) + self.assertEquals(c.get_key(), ds.key(26)) + c.remove() + self.assertEquals(c.get_key(), ds.key(26)) + msg = '/requires value be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_value, msg) + self.assertEquals(c.next(), 0) + self.assertEquals(c.get_key(), ds.key(27)) + + # Do a remove using the cursor without setting a position, and confirm + # no key, value or position remains. + def test_cursor_remove_without_position(self): + if self.skip(): + return + + # Build an object. + uri = self.uri + ':test_cursor11' + ds = self.ds(self, uri, 50, key_format=self.keyfmt) + ds.populate() + s = self.conn.open_session() + c = s.open_cursor(uri, None) + + c.set_key(ds.key(25)) + c.remove() + msg = '/requires key be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_key, msg) + msg = '/requires value be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_value, msg) + self.assertEquals(c.next(), 0) + self.assertEquals(c.get_key(), ds.key(1)) + + # Do a remove using the key after also setting a position, and confirm + # no key, value or position remains. + def test_cursor_remove_with_key_and_position(self): + if self.skip(): + return + + # Build an object. + uri = self.uri + ':test_cursor11' + ds = self.ds(self, uri, 50, key_format=self.keyfmt) + ds.populate() + s = self.conn.open_session() + c = s.open_cursor(uri, None) + + c.set_key(ds.key(25)) + self.assertEquals(c.search(), 0) + c.set_key(ds.key(25)) + c.remove() + msg = '/requires key be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_key, msg) + msg = '/requires value be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_value, msg) + self.assertEquals(c.next(), 0) + self.assertEquals(c.get_key(), ds.key(1)) + + # Do an insert and confirm no key, value or position remains. + def test_cursor_insert(self): + if self.skip(): + return + + # Build an object. + uri = self.uri + ':test_cursor11' + ds = self.ds(self, uri, 50, key_format=self.keyfmt) + ds.populate() + s = self.conn.open_session() + c = s.open_cursor(uri, None) + + c.set_key(ds.key(25)) + c.set_value(ds.value(300)) + c.insert() + msg = '/requires key be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_key, msg) + msg = '/requires value be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_value, msg) + self.assertEquals(c.next(), 0) + self.assertEquals(c.get_key(), ds.key(1)) + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_cursor_random.py b/test/suite/test_cursor_random.py index 3bda6dc9946..ee0f85a29ee 100644 --- a/test/suite/test_cursor_random.py +++ b/test/suite/test_cursor_random.py @@ -71,6 +71,15 @@ class test_cursor_random(wttest.WiredTigerTestCase): self.assertEquals(cursor.reset(), 0) cursor.close() + # Check that next_random fails with an empty tree, repeatedly. + def test_cursor_random_empty(self): + uri = self.type + self.session.create(uri, 'key_format=S,value_format=S') + cursor = self.session.open_cursor(uri, None, self.config) + for i in range(1,5): + self.assertTrue(cursor.next(), wiredtiger.WT_NOTFOUND) + cursor.close + # Check that next_random works with a single value, repeatedly. def test_cursor_random_single_record(self): uri = self.type @@ -127,6 +136,46 @@ class test_cursor_random(wttest.WiredTigerTestCase): def test_cursor_random_multiple_page_records(self): self.cursor_random_multiple_page_records(0) + # Check that next_random fails in the presence of a set of values, some of + # which are deleted. + def test_cursor_random_deleted_partial(self): + uri = self.type + ds = self.dataset(self, uri, 10000, + config='allocation_size=512,leaf_page_max=512') + ds.populate() + + # Close the connection so everything is forced to disk. + self.reopen_conn() + + start = self.session.open_cursor(uri, None) + start.set_key(ds.key(10)) + end = self.session.open_cursor(uri, None) + end.set_key(ds.key(10000-10)) + self.session.truncate(None, start, end, None) + self.assertEqual(start.close(), 0) + self.assertEqual(end.close(), 0) + + cursor = self.session.open_cursor(uri, None, self.config) + for i in range(1,10): + self.assertEqual(cursor.next(), 0) + + # Check that next_random fails in the presence of a set of values, all of + # which are deleted. + def test_cursor_random_deleted_all(self): + uri = self.type + ds = self.dataset(self, uri, 10000, + config='allocation_size=512,leaf_page_max=512') + ds.populate() + + # Close the connection so everything is forced to disk. + self.reopen_conn() + + self.session.truncate(uri, None, None, None) + + cursor = self.session.open_cursor(uri, None, self.config) + for i in range(1,10): + self.assertTrue(cursor.next(), wiredtiger.WT_NOTFOUND) + # Check that opening a random cursor on column-store returns not-supported. class test_cursor_random_column(wttest.WiredTigerTestCase): scenarios = make_scenarios([ diff --git a/test/suite/test_dump.py b/test/suite/test_dump.py index f6a83c32489..3127c7aef00 100644 --- a/test/suite/test_dump.py +++ b/test/suite/test_dump.py @@ -32,7 +32,7 @@ import wiredtiger, wttest from suite_subprocess import suite_subprocess from wtscenario import make_scenarios from wtdataset import SimpleDataSet, SimpleIndexDataSet, SimpleLSMDataSet, \ - ComplexDataSet, ComplexLSMDataSet + ComplexDataSet, ComplexLSMDataSet, ProjectionDataSet, ProjectionIndexDataSet # test_dump.py # Utilities: wt dump @@ -62,6 +62,10 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): ('table-simple-lsm', dict(uri='table:', dataset=SimpleLSMDataSet)), ('table-complex', dict(uri='table:', dataset=ComplexDataSet)), ('table-complex-lsm', dict(uri='table:', dataset=ComplexLSMDataSet)), + ('table-simple-proj', dict(uri='table:', + dataset=ProjectionDataSet, projection=True)), + ('table-index-proj', dict(uri='table:', + dataset=ProjectionIndexDataSet, projection=True)), ] scenarios = make_scenarios(types, keyfmt, dumpfmt) @@ -158,5 +162,53 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): pop = self.dataset(self, uri2, self.nentries, key_format=self.keyfmt) pop.check() +# test_dump_projection +# Utilities: wt dump +# Test the dump utility with projections +class test_dump_projection(wttest.WiredTigerTestCase, suite_subprocess): + dir = 'dump.dir' # Backup directory name + + name = 'test_dump' + nentries = 2500 + uri = 'table:' + + # Dump, re-load and do a content comparison. + def test_dump(self): + + # Create the object. + uri = self.uri + self.name + pop = ProjectionDataSet(self, uri, self.nentries, key_format='S') + pop.populate() + + # Check some cases with invalid projections. + self.runWt(['dump', '-x', uri + '('], \ + outfilename='bad1.out', errfilename='err1.out', failure=True) + self.check_non_empty_file('err1.out') + self.runWt(['dump', '-x', uri + '(xx)'], \ + outfilename='bad2.out', errfilename='err2.out', failure=True) + self.check_non_empty_file('err2.out') + self.runWt(['dump', '-x', uri + pop.projection[:-1]], \ + outfilename='bad3.out', errfilename='err3.out', failure=True) + self.check_non_empty_file('err3.out') + + # Dump the object with a valid projection. + self.runWt(['dump', '-x', uri + pop.projection], outfilename='dump.out') + + # Re-load the object in a new home. + os.mkdir(self.dir) + self.runWt(['-h', self.dir, 'load', '-f', 'dump.out']) + + # Check the database contents. + self.runWt(['list'], outfilename='list.out') + self.runWt(['-h', self.dir, 'list'], outfilename='list.out.new') + s1 = set(open('list.out').read().split()) + s2 = set(open('list.out.new').read().split()) + self.assertEqual(not s1.symmetric_difference(s2), True) + + # Check the object's contents. + self.reopen_conn(self.dir) + pop_reload = ProjectionDataSet(self, uri, self.nentries, key_format='S') + pop_reload.check() + if __name__ == '__main__': wttest.run() diff --git a/test/suite/test_encrypt01.py b/test/suite/test_encrypt01.py index 746c9d13e96..317bed93246 100644 --- a/test/suite/test_encrypt01.py +++ b/test/suite/test_encrypt01.py @@ -66,41 +66,20 @@ class test_encrypt01(wttest.WiredTigerTestCase): nrecords = 5000 bigvalue = "abcdefghij" * 1001 # len(bigvalue) = 10010 - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('encryptors', self.sys_encrypt) + extlist.extension('encryptors', self.file_encrypt) + extlist.extension('compressors', self.block_compress) + extlist.extension('compressors', self.log_compress) + + def conn_config(self): encarg = 'encryption=(name={0}{1}),'.format( self.sys_encrypt, self.sys_encrypt_args) comparg = '' if self.log_compress != None: comparg='log=(compressor={0}),'.format(self.log_compress) - extarg = self.extensionArg([('encryptors', self.sys_encrypt), - ('encryptors', self.file_encrypt), - ('compressors', self.block_compress), - ('compressors', self.log_compress)]) - conn = self.wiredtiger_open(dir, - 'create,error_prefix="{0}: ",{1}{2}{3}'.format( - self.shortid(), encarg, comparg, extarg)) - self.pr(`conn`) - return conn - - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' + return encarg + comparg # Create a table, add keys with both big and small values, then verify them. def test_encrypt(self): diff --git a/test/suite/test_encrypt02.py b/test/suite/test_encrypt02.py index 648686274c4..d950be067e2 100644 --- a/test/suite/test_encrypt02.py +++ b/test/suite/test_encrypt02.py @@ -39,51 +39,27 @@ from wtscenario import make_scenarios class test_encrypt02(wttest.WiredTigerTestCase, suite_subprocess): uri = 'file:test_encrypt02' encrypt_type = [ - ('noarg', dict( encrypt='rotn', encrypt_args='name=rotn', - secret_arg=None)), - ('keyid', dict( encrypt='rotn', encrypt_args='name=rotn,keyid=11', - secret_arg=None)), - ('pass', dict( encrypt='rotn', encrypt_args='name=rotn', - secret_arg='ABC')), - ('keyid-pass', dict( encrypt='rotn', encrypt_args='name=rotn,keyid=11', - secret_arg='ABC')), + ('noarg', dict( encrypt_args='name=rotn', secret_arg=None)), + ('keyid', dict( encrypt_args='name=rotn,keyid=11', secret_arg=None)), + ('pass', dict( encrypt_args='name=rotn', secret_arg='ABC')), + ('keyid-pass', dict( + encrypt_args='name=rotn,keyid=11', secret_arg='ABC')), ] scenarios = make_scenarios(encrypt_type) + def conn_extensions(self, extlist): + # Load the compression extension, skip the test if missing + extlist.skip_if_missing = True + extlist.extension('encryptors', 'rotn') + nrecords = 5000 bigvalue = "abcdefghij" * 1001 # len(bigvalue) = 10010 - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' - - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): + def conn_config(self): secretarg = '' if self.secret_arg != None: secretarg = ',secretkey=' + self.secret_arg - encarg = 'encryption=({0}{1})'.format(self.encrypt_args, secretarg) - extarg = self.extensionArg([('encryptors', self.encrypt)]) - connarg = 'create,error_prefix="{0}: ",{1},{2}'.format( - self.shortid(), encarg, extarg) - conn = self.wiredtiger_open(dir, connarg) - self.pr(`conn`) - return conn + return 'encryption=({0}{1})'.format(self.encrypt_args, secretarg) # Create a table, add keys with both big and small values, then verify them. def test_pass(self): diff --git a/test/suite/test_encrypt03.py b/test/suite/test_encrypt03.py index cf459190637..302572bd044 100644 --- a/test/suite/test_encrypt03.py +++ b/test/suite/test_encrypt03.py @@ -50,37 +50,14 @@ class test_encrypt03(wttest.WiredTigerTestCase): ] scenarios = make_scenarios(types, encrypt) - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - encarg = 'encryption=(name={0}{1}),'.format( - self.sys_encrypt, self.sys_encrypt_args) - extarg = self.extensionArg([('encryptors', self.sys_encrypt), - ('encryptors', self.file_encrypt)]) - self.pr('encarg = ' + encarg + ' extarg = ' + extarg) - conn = self.wiredtiger_open(dir, - 'create,error_prefix="{0}: ",{1}{2}'.format( - self.shortid(), encarg, extarg)) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('encryptors', self.sys_encrypt) + extlist.extension('encryptors', self.file_encrypt) - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' + def conn_config(self): + return 'encryption=(name={0}{1}),'.format( + self.sys_encrypt, self.sys_encrypt_args) # Create a table with encryption values that are in error. def test_encrypt(self): diff --git a/test/suite/test_encrypt04.py b/test/suite/test_encrypt04.py index a244cf97961..19c0b85d427 100644 --- a/test/suite/test_encrypt04.py +++ b/test/suite/test_encrypt04.py @@ -77,9 +77,16 @@ class test_encrypt04(wttest.WiredTigerTestCase, suite_subprocess): wttest.WiredTigerTestCase.__init__(self, *args, **kwargs) self.part = 1 + def conn_extensions(self, extlist): + extarg = None + if self.expect_forceerror: + extarg='(config=\"rotn_force_error=true\")' + extlist.skip_if_missing = True + extlist.extension('encryptors', self.name, extarg) + # Override WiredTigerTestCase, we have extensions. def setUpConnectionOpen(self, dir): - forceerror = None + self.expect_forceerror = False if self.part == 1: self.name = self.name1 self.keyid = self.keyid1 @@ -93,21 +100,20 @@ class test_encrypt04(wttest.WiredTigerTestCase, suite_subprocess): self.fileinclear = self.fileinclear2 if \ hasattr(self, 'fileinclear2') else False if hasattr(self, 'forceerror1') and hasattr(self, 'forceerror2'): - forceerror = "rotn_force_error=true" - self.expect_forceerror = forceerror != None + self.expect_forceerror = True self.got_forceerror = False encarg = 'encryption=(name={0},keyid={1},secretkey={2}),'.format( self.name, self.keyid, self.secretkey) - # If forceerror is set for this test, add a config arg to - # the extension string. That signals rotn to return a (-1000) - # error code, which we'll detect here. - extarg = self.extensionArg([('encryptors', self.name, forceerror)]) + # If forceerror is set for this test, conn_extensions adds a + # config arg to the extension string. That signals rotn to + # return a (-1000) error code, which we'll detect here. + extarg = self.extensionsConfig() self.pr('encarg = ' + encarg + ' extarg = ' + extarg) completed = False try: conn = self.wiredtiger_open(dir, - 'create,error_prefix="{0}: ",{1}{2}'.format( + 'create,error_prefix="{0}",{1}{2}'.format( self.shortid(), encarg, extarg)) except (BaseException) as err: # Capture the recognizable error created by rotn @@ -135,29 +141,6 @@ class test_encrypt04(wttest.WiredTigerTestCase, suite_subprocess): self.assertEqual(cursor.search(), 0) self.assertEquals(cursor.get_value(), val) - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name, extarg) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - extfile = '"' + extfile + '"' - if not extfile in extfiles: - s = extfile - if extarg != None: - s += "=(config=\"" + extarg + "\")" - extfiles.append(s) - if len(extfiles) == 0: - return '' - else: - return ',extensions=[' + ','.join(extfiles) + ']' - # Evaluate expression, which either must succeed (if expect_okay) # or must fail (if !expect_okay). def check_okay(self, expect_okay, expr): diff --git a/test/suite/test_encrypt05.py b/test/suite/test_encrypt05.py index 19a3522b3d5..d8862321821 100644 --- a/test/suite/test_encrypt05.py +++ b/test/suite/test_encrypt05.py @@ -49,41 +49,20 @@ class test_encrypt05(wttest.WiredTigerTestCase): nrecords = 500 bigvalue = 'a' * 500 # we use values that will definitely give compression - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('encryptors', self.sys_encrypt) + extlist.extension('encryptors', self.file_encrypt) + extlist.extension('compressors', self.block_compress) + extlist.extension('compressors', self.log_compress) + + def conn_config(self): encarg = 'encryption=(name={0}{1}),'.format( self.sys_encrypt, self.sys_encrypt_args) comparg = '' if self.log_compress != None: comparg='log=(compressor={0}),'.format(self.log_compress) - extarg = self.extensionArg([('encryptors', self.sys_encrypt), - ('encryptors', self.file_encrypt), - ('compressors', self.block_compress), - ('compressors', self.log_compress)]) - conn = self.wiredtiger_open(dir, - 'create,error_prefix="{0}: ",{1}{2}{3}'.format( - self.shortid(), encarg, comparg, extarg)) - self.pr(`conn`) - return conn - - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' + return encarg + comparg def getvalue(self, r, n): if n < len(self.bigvalue): diff --git a/test/suite/test_encrypt06.py b/test/suite/test_encrypt06.py index 893c4ba3095..72718e53b2b 100644 --- a/test/suite/test_encrypt06.py +++ b/test/suite/test_encrypt06.py @@ -89,38 +89,15 @@ class test_encrypt06(wttest.WiredTigerTestCase): scenarios = make_scenarios(encrypt, storagetype) nrecords = 1000 - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - encarg = 'encryption=(name={0}{1}),'.format( + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('encryptors', self.sys_encrypt) + extlist.extension('encryptors', self.file0_encrypt) + extlist.extension('encryptors', self.file1_encrypt) + + def conn_config(self): + return 'encryption=(name={0}{1}),'.format( self.sys_encrypt, self.sys_encrypt_args) - comparg = '' - extarg = self.extensionArg([('encryptors', self.sys_encrypt), - ('encryptors', self.file0_encrypt), - ('encryptors', self.file1_encrypt)]) - self.open_params = 'create,error_prefix="{0}: ",{1}{2}{3}'.format( - self.shortid(), encarg, comparg, extarg) - conn = self.wiredtiger_open(dir, self.open_params) - self.pr(`conn`) - return conn - - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' def encrypt_file_params(self, name, args): if name == None: diff --git a/test/suite/test_encrypt07.py b/test/suite/test_encrypt07.py index 97ab1987d4f..81c9f1a49ea 100644 --- a/test/suite/test_encrypt07.py +++ b/test/suite/test_encrypt07.py @@ -44,35 +44,14 @@ class test_encrypt07(test_salvage.test_salvage): nrecords = 5000 bigvalue = "abcdefghij" * 1007 # len(bigvalue) = 10070 - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - encarg = 'encryption=(name={0}{1}),'.format( - self.sys_encrypt, self.sys_encrypt_args) - extarg = self.extensionArg([('encryptors', self.sys_encrypt)]) - conn = self.wiredtiger_open(dir, - 'create,error_prefix="{0}: ",{1}{2}'.format( - self.shortid(), encarg, extarg)) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + # Load the compression extension, skip the test if missing + extlist.skip_if_missing = True + extlist.extension('encryptors', self.sys_encrypt) - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' + def conn_config(self): + return 'encryption=(name={0}{1}),'.format( + self.sys_encrypt, self.sys_encrypt_args) def rot13(self, s): return codecs.encode(s, 'rot_13') diff --git a/test/suite/test_intpack.py b/test/suite/test_intpack.py index b0cece09494..ae391e68fca 100644 --- a/test/suite/test_intpack.py +++ b/test/suite/test_intpack.py @@ -126,8 +126,8 @@ class PackTester: class test_intpack(wttest.WiredTigerTestCase): name = 'test_intpack' - # We have to be a bit verbose here with naming, as there can be problems with - # case insensitive test names:w + # We have to be a bit verbose here with naming, scenario names are + # case insensitive and must be unique. scenarios = make_scenarios([ ('int8_t_b', dict(formatcode='b', low=-128, high=127, nbits=8)), diff --git a/test/suite/test_join01.py b/test/suite/test_join01.py index 2c4328dc7d3..bdd86a06d4f 100644 --- a/test/suite/test_join01.py +++ b/test/suite/test_join01.py @@ -69,7 +69,7 @@ class test_join01(wttest.WiredTigerTestCase): ] scenarios = make_scenarios(type_scen, bloom0_scen, bloom1_scen, projection_scen, nested_scen, stats_scen, - order_scen) + order_scen, prune=50, prunelong=1000) # We need statistics for these tests. conn_config = 'statistics=(all)' diff --git a/test/suite/test_join03.py b/test/suite/test_join03.py index edab7146a6b..dd8111f6ead 100644 --- a/test/suite/test_join03.py +++ b/test/suite/test_join03.py @@ -36,33 +36,9 @@ class test_join03(wttest.WiredTigerTestCase): table_name1 = 'test_join03' nentries = 100 - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name, libname) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' - - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor')]) - connarg = 'create,error_prefix="{0}: ",{1}'.format( - self.shortid(), extarg) - conn = self.wiredtiger_open(dir, connarg) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('extractors', 'csv') def gen_key(self, i): return [ i + 1 ] diff --git a/test/suite/test_join04.py b/test/suite/test_join04.py index a71418d9f05..e65b8b53333 100644 --- a/test/suite/test_join04.py +++ b/test/suite/test_join04.py @@ -36,33 +36,9 @@ class test_join04(wttest.WiredTigerTestCase): table_name1 = 'test_join04' nentries = 100 - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name, libname) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' - - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor')]) - connarg = 'create,error_prefix="{0}: ",{1}'.format( - self.shortid(), extarg) - conn = self.wiredtiger_open(dir, connarg) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('extractors', 'csv') # JIRA WT-2308: # Test extractors with equality joins diff --git a/test/suite/test_join07.py b/test/suite/test_join07.py index 2a32e678d72..8fae3539246 100644 --- a/test/suite/test_join07.py +++ b/test/suite/test_join07.py @@ -200,33 +200,9 @@ class test_join07(wttest.WiredTigerTestCase): scenarios = make_scenarios(extractscen) - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name, libname) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' - - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor')]) - connarg = 'create,error_prefix="{0}: ",{1}'.format( - self.shortid(), extarg) - conn = self.wiredtiger_open(dir, connarg) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('extractors', 'csv') def expect(self, token, expected): if token == None or token.kind not in expected: diff --git a/test/suite/test_jsondump02.py b/test/suite/test_jsondump02.py index 8482851fb94..60863c4aa97 100644 --- a/test/suite/test_jsondump02.py +++ b/test/suite/test_jsondump02.py @@ -234,6 +234,24 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): ('"ikey" : 4,\n"Skey" : "key4"', '"S1" : "val16",\n"i2" : 16,\n"S3" : "val64",\n"i4" : 64')) self.check_json(self.table_uri4, table4_json) + # This projection has 3 value fields reversed with a key at the end. + table4_json_projection = ( + ('"ikey" : 1,\n"Skey" : "key1"', + '"i4" : 1,\n"S3" : "val1",\n"i2" : 1,\n"ikey" : 1'), + ('"ikey" : 2,\n"Skey" : "key2"', + '"i4" : 8,\n"S3" : "val8",\n"i2" : 4,\n"ikey" : 2'), + ('"ikey" : 3,\n"Skey" : "key3"', + '"i4" : 27,\n"S3" : "val27",\n"i2" : 9,\n"ikey" : 3'), + ('"ikey" : 4,\n"Skey" : "key4"', + '"i4" : 64,\n"S3" : "val64",\n"i2" : 16,\n"ikey" : 4')) + # bad projection URI + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.check_json(self.table_uri4 + '(i4,S3,i2,ikey', + table4_json_projection), + '/Unbalanced brackets/') + # This projection should work. + self.check_json(self.table_uri4 + '(i4,S3,i2,ikey)', + table4_json_projection) # The dump config currently is not supported for the index type. self.check_json(uri4index1, ( ('"Skey" : "key1"', diff --git a/test/suite/test_lsm01.py b/test/suite/test_lsm01.py index b44df4bae14..f705b09b0a4 100644 --- a/test/suite/test_lsm01.py +++ b/test/suite/test_lsm01.py @@ -57,7 +57,7 @@ class test_lsm01(wttest.WiredTigerTestCase): scenarios = wtscenario.make_scenarios( chunk_size_scenarios, merge_max_scenarios, bloom_scenarios, bloom_bit_scenarios, bloom_hash_scenarios, record_count_scenarios, - prune=500) + prune=100, prunelong=500) # Test drop of an object. def test_lsm(self): diff --git a/test/suite/test_overwrite.py b/test/suite/test_overwrite.py index 4739abaa578..c894de99bd0 100644 --- a/test/suite/test_overwrite.py +++ b/test/suite/test_overwrite.py @@ -27,32 +27,47 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from wtdataset import SimpleDataSet +from wtdataset import SimpleDataSet, SimpleIndexDataSet +from wtdataset import SimpleLSMDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios # test_overwrite.py # cursor overwrite configuration method class test_overwrite(wttest.WiredTigerTestCase): name = 'overwrite' - scenarios = make_scenarios([ - ('file-r', dict(type='file:', keyfmt='r', dataset=SimpleDataSet)), - ('file-S', dict(type='file:', keyfmt='S', dataset=SimpleDataSet)), - ('lsm-S', dict(type='lsm:', keyfmt='S', dataset=SimpleDataSet)), - ('table-r', dict(type='table:', keyfmt='r', dataset=SimpleDataSet)), - ('table-S', dict(type='table:', keyfmt='S', dataset=SimpleDataSet)), - ]) + keyfmt = [ + ('integer', dict(keyfmt='i')), + ('recno', dict(keyfmt='r')), + ('string', dict(keyfmt='S')), + ] + types = [ + ('file', dict(uri='file:', ds=SimpleDataSet)), + ('lsm', dict(uri='lsm:', ds=SimpleDataSet)), + ('table-complex', dict(uri='table:', ds=ComplexDataSet)), + ('table-complex-lsm', dict(uri='table:', ds=ComplexLSMDataSet)), + ('table-index', dict(uri='table:', ds=SimpleIndexDataSet)), + ('table-simple', dict(uri='table:', ds=SimpleDataSet)), + ('table-simple-lsm', dict(uri='table:', ds=SimpleLSMDataSet)), + ] + scenarios = make_scenarios(types, keyfmt) + def skip(self): + return self.keyfmt == 'r' and \ + (self.ds.is_lsm() or self.uri == 'lsm') # Confirm a cursor configured with/without overwrite correctly handles # non-existent records during insert, remove and update operations. def test_overwrite_insert(self): - uri = self.type + self.name - ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + if self.skip(): + return + + uri = self.uri + self.name + ds = self.ds(self, uri, 100, key_format=self.keyfmt) ds.populate() # Insert of an existing record with overwrite off fails. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(5)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1000)) self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.insert()) # One additional test for the insert method: duplicate the cursor with @@ -63,30 +78,33 @@ class test_overwrite(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(5)) dupc = self.session.open_cursor(None, cursor, "overwrite=true") - dupc.set_value('XXXXXXXXXX') + dupc.set_value(ds.value(1001)) self.assertEquals(dupc.insert(), 0) # Insert of an existing record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(ds.key(6)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1002)) self.assertEquals(cursor.insert(), 0) # Insert of a non-existent record with overwrite off succeeds. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(200)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1003)) self.assertEquals(cursor.insert(), 0) # Insert of a non-existent record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(ds.key(201)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1004)) self.assertEquals(cursor.insert(), 0) def test_overwrite_remove(self): - uri = self.type + self.name - ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + if self.skip(): + return + + uri = self.uri + self.name + ds = self.ds(self, uri, 100, key_format=self.keyfmt) ds.populate() # Remove of an existing record with overwrite off succeeds. @@ -110,32 +128,35 @@ class test_overwrite(wttest.WiredTigerTestCase): self.assertEquals(cursor.remove(), 0) def test_overwrite_update(self): - uri = self.type + self.name - ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + if self.skip(): + return + + uri = self.uri + self.name + ds = self.ds(self, uri, 100, key_format=self.keyfmt) ds.populate() # Update of an existing record with overwrite off succeeds. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(5)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1005)) self.assertEquals(cursor.update(), 0) # Update of an existing record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(ds.key(6)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1006)) self.assertEquals(cursor.update(), 0) # Update of a non-existent record with overwrite off fails. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(200)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1007)) self.assertEquals(cursor.update(), wiredtiger.WT_NOTFOUND) # Update of a non-existent record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(ds.key(201)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1008)) self.assertEquals(cursor.update(), 0) if __name__ == '__main__': diff --git a/test/suite/test_perf001.py b/test/suite/test_perf001.py index b22ed2baeb0..6331a3f64d6 100644 --- a/test/suite/test_perf001.py +++ b/test/suite/test_perf001.py @@ -40,7 +40,8 @@ class test_perf001(wttest.WiredTigerTestCase): scenarios = make_scenarios([ #('file-file', dict(tabletype='file',indextype='file')), - ('file-lsm', dict(tabletype='file',indextype='lsm')), + ('file-lsm', dict(tabletype='file',indextype='lsm', cfg='', + conn_config="statistics=(fast),statistics_log=(wait=1)")), #('lsm-file', dict(tabletype='lsm',indextype='file')), #('lsm-lsm', dict(tabletype='lsm',indextype='lsm')), ]) diff --git a/test/suite/test_readonly01.py b/test/suite/test_readonly01.py index e4b431ca1da..f41280a3283 100644 --- a/test/suite/test_readonly01.py +++ b/test/suite/test_readonly01.py @@ -75,8 +75,7 @@ class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): scenarios = make_scenarios(basecfg_list, dir_list, log_list, types) - def conn_config(self, dir): - self.home = dir + def conn_config(self): params = \ 'error_prefix="%s",' % self.shortid() + \ '%s' % self.logcfg + \ diff --git a/test/suite/test_reconfig01.py b/test/suite/test_reconfig01.py index e76becac76a..cbc8bca5740 100644 --- a/test/suite/test_reconfig01.py +++ b/test/suite/test_reconfig01.py @@ -64,6 +64,18 @@ class test_reconfig01(wttest.WiredTigerTestCase): # same ops_max of 512 and thread of 8. self.conn.reconfigure("async=(enabled=true)") + def test_reconfig_eviction(self): + # Increase the max number of running threads (default 8). + self.conn.reconfigure("eviction=(threads_max=10)") + # Increase the min number of running threads (default 1). + self.conn.reconfigure("eviction=(threads_min=5)") + # Decrease the max number of running threads. + self.conn.reconfigure("eviction=(threads_max=7)") + # Decrease the min number of running threads. + self.conn.reconfigure("eviction=(threads_min=2)") + # Set min and max the same. + self.conn.reconfigure("eviction=(threads_min=6,threads_max=6)") + def test_reconfig_lsm_manager(self): # We create and populate a tiny LSM so that we can start off with # the LSM threads running and change the numbers of threads. diff --git a/test/suite/test_reconfig02.py b/test/suite/test_reconfig02.py index 36a78a1805f..042d3bbe71f 100644 --- a/test/suite/test_reconfig02.py +++ b/test/suite/test_reconfig02.py @@ -62,7 +62,7 @@ class test_reconfig02(wttest.WiredTigerTestCase): self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.conn.reconfigure("log=(path=foo)"), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.conn.reconfigure("log=(recovery=true)"), msg) + lambda: self.conn.reconfigure("log=(recover=true)"), msg) # Logging starts on, but prealloc is off. Verify it is off. # Reconfigure it on and run again, making sure that log files @@ -109,6 +109,7 @@ class test_reconfig02(wttest.WiredTigerTestCase): # Now turn on archive, sleep a bit to allow the archive thread # to run and then confirm that all original logs are gone. self.conn.reconfigure("log=(archive=true)") + self.session.checkpoint("force") time.sleep(2) cur_logs = fnmatch.filter(os.listdir('.'), "*Log*") for o in orig_logs: diff --git a/test/suite/test_reconfig04.py b/test/suite/test_reconfig04.py index be5e6d3729e..51d9b91c1f4 100644 --- a/test/suite/test_reconfig04.py +++ b/test/suite/test_reconfig04.py @@ -26,9 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import fnmatch, os, time import wiredtiger, wttest -from wtdataset import SimpleDataSet # test_reconfig04.py # Test WT_SESSION::reconfigure diff --git a/test/suite/test_schema05.py b/test/suite/test_schema05.py index 28ad51b3c92..d536a629373 100644 --- a/test/suite/test_schema05.py +++ b/test/suite/test_schema05.py @@ -57,33 +57,9 @@ class test_schema05(wttest.WiredTigerTestCase): ('index-after', { 'create_index' : 2 }), ]) - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name, libname) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' - - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor')]) - connarg = 'create,error_prefix="{0}: ",{1}'.format( - self.shortid(), extarg) - conn = self.wiredtiger_open(dir, connarg) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('extractors', 'csv') def create_indices(self): # Create self.nindices index files, each with a column from the CSV diff --git a/test/suite/test_schema07.py b/test/suite/test_schema07.py index ac397c6e1a1..3e4b1d28a4d 100644 --- a/test/suite/test_schema07.py +++ b/test/suite/test_schema07.py @@ -33,8 +33,7 @@ import wiredtiger, wttest class test_schema07(wttest.WiredTigerTestCase): tablename = 'table:test_schema07' - def conn_config(self, dir): - return 'cache_size=10MB' + conn_config = 'cache_size=10MB' @wttest.longtest("Creating many tables shouldn't fill the cache") def test_many_tables(self): diff --git a/test/suite/test_shared_cache01.py b/test/suite/test_shared_cache01.py index 70560a625ee..c3bd946cc4b 100644 --- a/test/suite/test_shared_cache01.py +++ b/test/suite/test_shared_cache01.py @@ -73,7 +73,7 @@ class test_shared_cache01(wttest.WiredTigerTestCase): os.mkdir(name) next_conn = self.wiredtiger_open( name, - 'create,error_prefix="' + self.shortid() + ': "' + + 'create,error_prefix="%s",' % self.shortid() + pool_opts + extra_opts) self.conns.append(next_conn) self.sessions.append(next_conn.open_session(None)) diff --git a/test/suite/test_shared_cache02.py b/test/suite/test_shared_cache02.py index 7cde6c86695..67f9bf7c6b7 100644 --- a/test/suite/test_shared_cache02.py +++ b/test/suite/test_shared_cache02.py @@ -73,7 +73,7 @@ class test_shared_cache02(wttest.WiredTigerTestCase): os.mkdir(name) next_conn = self.wiredtiger_open( name, - 'create,error_prefix="' + self.shortid() + ': "' + + 'create,error_prefix="%s",' % self.shortid() + pool_opts + extra_opts) self.conns.append(next_conn) self.sessions.append(next_conn.open_session(None)) diff --git a/test/suite/test_stat02.py b/test/suite/test_stat02.py index cecda7f1ddc..45af283ed02 100644 --- a/test/suite/test_stat02.py +++ b/test/suite/test_stat02.py @@ -59,7 +59,7 @@ class test_stat_cursor_config(wttest.WiredTigerTestCase): scenarios = make_scenarios(uri, data_config, cursor_config) # Turn on statistics for this test. - def conn_config(self, dir): + def conn_config(self): return 'statistics=(%s)' % self.data_config # For each database/cursor configuration, confirm the right combinations diff --git a/test/suite/test_sweep01.py b/test/suite/test_sweep01.py index 71f8fcb180e..5559190caca 100644 --- a/test/suite/test_sweep01.py +++ b/test/suite/test_sweep01.py @@ -116,10 +116,15 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): # Give slow machines time to process files. stat_cursor = self.session.open_cursor('statistics:', None, None) this_nfile = stat_cursor[stat.conn.file_open][2] + removed = stat_cursor[stat.conn.dh_sweep_remove][2] stat_cursor.close() self.pr("==== loop " + str(sleep)) self.pr("this_nfile " + str(this_nfile)) - if this_nfile == final_nfile: + self.pr("removed " + str(removed)) + # On slow machines there can be a lag where files get closed but + # the sweep server cannot yet remove the handles. So wait for the + # removed statistic to indicate forward progress too. + if this_nfile == final_nfile and removed != remove1: break c.close() self.pr("Sweep loop took " + str(sleep)) diff --git a/test/suite/test_truncate01.py b/test/suite/test_truncate01.py index 2319eeddbef..98b741ba6a4 100644 --- a/test/suite/test_truncate01.py +++ b/test/suite/test_truncate01.py @@ -128,6 +128,7 @@ class test_truncate_cursor_order(wttest.WiredTigerTestCase): msg = '/the start cursor position is after the stop cursor position/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.truncate(None, c1, c2, None), msg) + c1.set_key(ds.key(10)) c2.set_key(ds.key(20)) self.session.truncate(None, c1, c2, None) @@ -183,11 +184,11 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): # those tests to file objects. types = [ ('file', dict(type='file:', valuefmt='S', - config='allocation_size=512,leaf_page_max=512')), + config='allocation_size=512,leaf_page_max=512', P=0.25)), ('file8t', dict(type='file:', valuefmt='8t', - config='allocation_size=512,leaf_page_max=512')), + config='allocation_size=512,leaf_page_max=512', P=0.25)), ('table', dict(type='table:', valuefmt='S', - config='allocation_size=512,leaf_page_max=512')), + config='allocation_size=512,leaf_page_max=512', P=0.5)), ] keyfmt = [ ('integer', dict(keyfmt='i')), @@ -203,7 +204,8 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): ('big', dict(nentries=1000,skip=37)), ] - scenarios = make_scenarios(types, keyfmt, size, reopen) + scenarios = make_scenarios(types, keyfmt, size, reopen, + prune=10, prunelong=1000) # Set a cursor key. def cursorKey(self, ds, uri, key): diff --git a/test/suite/test_truncate02.py b/test/suite/test_truncate02.py index 73fed362354..729825b26d4 100644 --- a/test/suite/test_truncate02.py +++ b/test/suite/test_truncate02.py @@ -85,7 +85,8 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase): ('txn2', dict(commit=False)), ] - scenarios = make_scenarios(types, keyfmt, overflow, reads, writes, txn) + scenarios = make_scenarios(types, keyfmt, overflow, reads, writes, txn, + prune=20, prunelong=1000) # Return the number of records visible to the cursor; test both forward # and backward iteration, they are different code paths in this case. diff --git a/test/suite/test_txn02.py b/test/suite/test_txn02.py index a0c2c12a47c..01626057b9e 100644 --- a/test/suite/test_txn02.py +++ b/test/suite/test_txn02.py @@ -93,11 +93,10 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): checklog_calls = 100 if wttest.islongtest() else 2 checklog_mod = (len(scenarios) / checklog_calls + 1) - def setUpConnectionOpen(self, dir): - self.home = dir + def conn_config(self): # Cycle through the different transaction_sync values in a # deterministic manner. - self.txn_sync = self.sync_list[ + txn_sync = self.sync_list[ self.scenario_number % len(self.sync_list)] # # We don't want to run zero fill with only the same settings, such @@ -107,17 +106,9 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): zerofill = 'false' if self.scenario_number % freq == 0: zerofill = 'true' - self.backup_dir = os.path.join(self.home, "WT_BACKUP") - conn_params = \ - 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ - 'log=(zero_fill=%s),' % zerofill + \ - 'create,error_prefix="%s: ",' % self.shortid() + \ - 'transaction_sync="%s",' % self.txn_sync - # print "Creating conn at '%s' with config '%s'" % (dir, conn_params) - conn = self.wiredtiger_open(dir, conn_params) - self.pr(`conn`) - self.session2 = conn.open_session() - return conn + return 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ + 'log=(zero_fill=%s),' % zerofill + \ + 'transaction_sync="%s",' % txn_sync # Check that a cursor (optionally started in a new transaction), sees the # expected values. @@ -176,8 +167,10 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): backup_conn = self.wiredtiger_open(self.backup_dir, backup_conn_params) try: - self.check(backup_conn.open_session(), None, committed) + session = backup_conn.open_session() finally: + session.checkpoint("force") + self.check(backup_conn.open_session(), None, committed) # Sleep long enough so that the archive thread is guaranteed # to run before we close the connection. time.sleep(1.0) @@ -204,6 +197,8 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): self.assertEqual(cur_logs, pr_logs) def test_ops(self): + self.backup_dir = os.path.join(self.home, "WT_BACKUP") + self.session2 = self.conn.open_session() # print "Creating %s with config '%s'" % (self.uri, self.create_params) self.session.create(self.uri, self.create_params) # Set up the table with entries for 1, 2, 10 and 11. @@ -226,6 +221,7 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): # Close and reopen the connection and cursor. if reopen == 'reopen': self.reopen_conn() + self.session2 = self.conn.open_session() c = self.session.open_cursor(self.uri, None, 'overwrite') self.session.begin_transaction( diff --git a/test/suite/test_txn04.py b/test/suite/test_txn04.py index ade39272f84..d8f6774ded1 100644 --- a/test/suite/test_txn04.py +++ b/test/suite/test_txn04.py @@ -63,24 +63,15 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): txn1s = [('t1c', dict(txn1='commit')), ('t1r', dict(txn1='rollback'))] scenarios = make_scenarios(types, op1s, txn1s) - # Overrides WiredTigerTestCase - def setUpConnectionOpen(self, dir): - self.home = dir + + def conn_config(self): # Cycle through the different transaction_sync values in a # deterministic manner. - self.txn_sync = self.sync_list[ + txn_sync = self.sync_list[ self.scenario_number % len(self.sync_list)] - self.backup_dir = os.path.join(self.home, "WT_BACKUP") # Set archive false on the home directory. - conn_params = \ - 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ - 'create,error_prefix="%s: ",' % self.shortid() + \ - 'transaction_sync="%s",' % self.txn_sync - # print "Creating conn at '%s' with config '%s'" % (dir, conn_params) - conn = self.wiredtiger_open(dir, conn_params) - self.pr(`conn`) - self.session2 = conn.open_session() - return conn + return 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ + 'transaction_sync="%s",' % txn_sync # Check that a cursor (optionally started in a new transaction), sees the # expected values. @@ -146,6 +137,7 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): # The runWt command closes our connection and sessions so # we need to reopen them here. self.hot_backup(None, committed) + self.session2 = self.conn.open_session() c = self.session.open_cursor(self.uri, None, 'overwrite') c.set_value(1) # Then do the given modification. @@ -193,6 +185,8 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): self.hot_backup(self.uri, committed) def test_ops(self): + self.backup_dir = os.path.join(self.home, "WT_BACKUP") + self.session2 = self.conn.open_session() with self.expectedStdoutPattern('recreating metadata'): self.ops() diff --git a/test/suite/test_txn05.py b/test/suite/test_txn05.py index 9e84fe7d3fe..7aaff221ba4 100644 --- a/test/suite/test_txn05.py +++ b/test/suite/test_txn05.py @@ -64,23 +64,15 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): txn1s = [('t1c', dict(txn1='commit')), ('t1r', dict(txn1='rollback'))] scenarios = make_scenarios(types, op1s, txn1s) - # Overrides WiredTigerTestCase - def setUpConnectionOpen(self, dir): - self.home = dir + + def conn_config(self): # Cycle through the different transaction_sync values in a # deterministic manner. - self.txn_sync = self.sync_list[ + txn_sync = self.sync_list[ self.scenario_number % len(self.sync_list)] - self.backup_dir = os.path.join(self.home, "WT_BACKUP") - conn_params = \ - 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ - 'create,error_prefix="%s: ",' % self.shortid() + \ - 'transaction_sync="%s",' % self.txn_sync - # print "Creating conn at '%s' with config '%s'" % (dir, conn_params) - conn = self.wiredtiger_open(dir, conn_params) - self.pr(`conn`) - self.session2 = conn.open_session() - return conn + # Set archive false on the home directory. + return 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ + 'transaction_sync="%s",' % txn_sync # Check that a cursor (optionally started in a new transaction), sees the # expected values. @@ -139,8 +131,12 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): backup_conn = self.wiredtiger_open(self.backup_dir, backup_conn_params) try: - self.check(backup_conn.open_session(), None, committed) + session = backup_conn.open_session() finally: + self.check(session, None, committed) + # Force a checkpoint because we don't record the recovery + # checkpoint as available for archiving. + session.checkpoint("force") # Sleep long enough so that the archive thread is guaranteed # to run before we close the connection. time.sleep(1.0) @@ -163,6 +159,8 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): self.runWt(['-h', self.backup_dir, 'printlog'], outfilename='printlog.out') def test_ops(self): + self.backup_dir = os.path.join(self.home, "WT_BACKUP") + self.session2 = self.conn.open_session() # print "Creating %s with config '%s'" % (self.uri, self.create_params) self.session.create(self.uri, self.create_params) # Set up the table with entries for 1-5. diff --git a/test/suite/test_txn06.py b/test/suite/test_txn06.py index 2bff97f6aac..c91dc6a623b 100644 --- a/test/suite/test_txn06.py +++ b/test/suite/test_txn06.py @@ -40,10 +40,10 @@ class test_txn06(wttest.WiredTigerTestCase, suite_subprocess): source_uri = 'table:' + tablename + "_src" nrows = 100000 - def setUpConnectionOpen(self, *args): + def conn_config(self): if not wiredtiger.verbose_build(): self.skipTest('requires a verbose build') - return super(test_txn06, self).setUpConnectionOpen(*args) + return '' def test_long_running(self): # Populate a table diff --git a/test/suite/test_txn07.py b/test/suite/test_txn07.py index f9577bad7f2..e26cf5aaaea 100644 --- a/test/suite/test_txn07.py +++ b/test/suite/test_txn07.py @@ -70,43 +70,20 @@ class test_txn07(wttest.WiredTigerTestCase, suite_subprocess): ('none', dict(compress='')), ] - scenarios = make_scenarios(types, op1s, txn1s, compress) - # Overrides WiredTigerTestCase - def setUpConnectionOpen(self, dir): - self.home = dir - # Cycle through the different transaction_sync values in a - # deterministic manner. - self.txn_sync = self.sync_list[ - self.scenario_number % len(self.sync_list)] - self.backup_dir = os.path.join(self.home, "WT_BACKUP") - conn_params = \ - 'log=(archive=false,enabled,file_max=%s,' % self.logmax + \ - 'compressor=%s)' % self.compress + \ - self.extensionArg(self.compress) + \ - ',create,error_prefix="%s: ",' % self.shortid() + \ - "statistics=(fast)," + \ - 'transaction_sync="%s",' % self.txn_sync - # print "Creating conn at '%s' with config '%s'" % (dir, conn_params) - try: - conn = self.wiredtiger_open(dir, conn_params) - except wiredtiger.WiredTigerError as e: - print "Failed conn at '%s' with config '%s'" % (dir, conn_params) - self.pr(`conn`) - self.session2 = conn.open_session() - return conn - - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, name): - if name == None or name == '': - return '' - - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext/compressors') - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('compression extension "' + extfile + '" not built') - return ',extensions=["' + extfile + '"]' + scenarios = make_scenarios(types, op1s, txn1s, compress, + prune=30, prunelong=1000) + + def conn_config(self): + return 'log=(archive=false,enabled,file_max=%s,' % self.logmax + \ + 'compressor=%s)' % self.compress + \ + ',create,error_prefix="%s",' % self.shortid() + \ + "statistics=(fast)," + \ + 'transaction_sync="%s",' % \ + self.sync_list[self.scenario_number % len(self.sync_list)] + + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('compressors', self.compress) # Check that a cursor (optionally started in a new transaction), sees the # expected values. @@ -139,7 +116,7 @@ class test_txn07(wttest.WiredTigerTestCase, suite_subprocess): self.backup(self.backup_dir) backup_conn_params = 'log=(enabled,file_max=%s,' % self.logmax + \ 'compressor=%s)' % self.compress + \ - self.extensionArg(self.compress) + self.extensionsConfig() backup_conn = self.wiredtiger_open(self.backup_dir, backup_conn_params) try: self.check(backup_conn.open_session(), None, committed) @@ -147,6 +124,9 @@ class test_txn07(wttest.WiredTigerTestCase, suite_subprocess): backup_conn.close() def test_ops(self): + self.backup_dir = os.path.join(self.home, "WT_BACKUP") + self.session2 = self.conn.open_session() + # print "Creating %s with config '%s'" % (self.uri, self.create_params) self.session.create(self.uri, self.create_params) # Set up the table with entries for 1-5. diff --git a/test/suite/test_txn08.py b/test/suite/test_txn08.py index f0cdf08df07..04faed9d45a 100644 --- a/test/suite/test_txn08.py +++ b/test/suite/test_txn08.py @@ -41,7 +41,7 @@ class test_txn08(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename # Turn on logging for this test. - def conn_config(self, dir): + def conn_config(self): return 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ 'transaction_sync="(method=dsync,enabled)"' diff --git a/test/suite/test_txn09.py b/test/suite/test_txn09.py index cfad8270ab1..768d714e248 100644 --- a/test/suite/test_txn09.py +++ b/test/suite/test_txn09.py @@ -80,19 +80,9 @@ class test_txn09(wttest.WiredTigerTestCase, suite_subprocess): op1s, txn1s, op2s, txn2s, op3s, txn3s, op4s, txn4s, prune=20, prunelong=5000) - # Overrides WiredTigerTestCase - def setUpConnectionOpen(self, dir): - self.home = dir - conn_params = \ - 'create,error_prefix="%s: ",' % self.shortid() + \ - 'log=(archive=false,enabled=%s),' % int(self.log_enabled) + \ - 'transaction_sync=(enabled=false),' - - # print "Opening conn at '%s' with config '%s'" % (dir, conn_params) - conn = self.wiredtiger_open(dir, conn_params) - self.pr(`conn`) - self.session2 = conn.open_session() - return conn + def conn_config(self): + return 'log=(archive=false,enabled=%s),' % int(self.log_enabled) + \ + 'transaction_sync=(enabled=false)' # Check that a cursor (optionally started in a new transaction), sees the # expected values. @@ -141,6 +131,7 @@ class test_txn09(wttest.WiredTigerTestCase, suite_subprocess): # Close and reopen the connection and cursor, toggling the log self.log_enabled = not self.log_enabled self.reopen_conn() + self.session2 = self.conn.open_session() c = self.session.open_cursor(self.uri, None, 'overwrite') self.session.begin_transaction( diff --git a/test/suite/test_txn11.py b/test/suite/test_txn11.py index 147bf3a76c0..3c02b1e86e3 100644 --- a/test/suite/test_txn11.py +++ b/test/suite/test_txn11.py @@ -44,7 +44,7 @@ class test_txn11(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename # Turn on logging for this test. - def conn_config(self, dir): + def conn_config(self): return 'log=(archive=%s,' % self.archive + \ 'enabled,file_max=%s,prealloc=false),' % self.logmax + \ 'transaction_sync=(enabled=false),' diff --git a/test/suite/test_txn13.py b/test/suite/test_txn13.py index ae0250c06e8..2bf49486b3a 100644 --- a/test/suite/test_txn13.py +++ b/test/suite/test_txn13.py @@ -50,7 +50,7 @@ class test_txn13(wttest.WiredTigerTestCase, suite_subprocess): ]) # Turn on logging for this test. - def conn_config(self, dir): + def conn_config(self): return 'log=(archive=false,enabled,file_max=%s)' % self.logmax + \ ',cache_size=8G' diff --git a/test/suite/test_txn15.py b/test/suite/test_txn15.py index c061c093b02..a2bfb626338 100644 --- a/test/suite/test_txn15.py +++ b/test/suite/test_txn15.py @@ -41,7 +41,7 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): create_params = 'key_format=i,value_format=i' entries = 100 # Turn on logging for this test. - def conn_config(self, dir): + def conn_config(self): return 'statistics=(fast),' + \ 'log=(archive=false,enabled,file_max=100K),' + \ 'use_environment=false,' + \ diff --git a/test/suite/test_util14.py b/test/suite/test_util14.py new file mode 100644 index 00000000000..e2a9f41f0d4 --- /dev/null +++ b/test/suite/test_util14.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +from suite_subprocess import suite_subprocess +import wiredtiger, wttest + +# test_util14.py +# Utilities: wt truncate +class test_util14(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util14.a' + nentries = 1000 + + def test_truncate_process(self): + """ + Test truncate in a 'wt' process + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.assertTrue(os.path.exists(self.tablename + ".wt")) + cursor = self.session.open_cursor('table:' + self.tablename, None, None) + for i in range(0, self.nentries): + cursor[str(i)] = str(i) + cursor.close() + + self.runWt(["truncate", "table:" + self.tablename]) + + """ + Test to confirm table exists and is empty + """ + outfile="outfile.txt" + errfile="errfile.txt" + self.assertTrue(os.path.exists(self.tablename + ".wt")) + self.runWt(["read", 'table:' + self.tablename, 'NoMatch'], + outfilename=outfile, errfilename=errfile, failure=True) + self.check_empty_file(outfile) + self.check_file_contains(errfile, 'NoMatch: not found\n') + + """ + Tests for error cases + 1. Missing URI + 2. Invalid URI + 3. Valid but incorrect URI + 4. Double URI + """ + self.runWt(["truncate"], + outfilename=outfile, errfilename=errfile, failure=True) + self.check_empty_file(outfile) + self.check_file_contains(errfile, 'usage:') + + self.runWt(["truncate", "foobar"], + outfilename=outfile, errfilename=errfile, failure=True) + self.check_empty_file(outfile) + self.check_file_contains(errfile, 'No such file or directory') + + self.runWt(["truncate", 'table:xx' + self.tablename], + outfilename=outfile, errfilename=errfile, failure=True) + self.check_empty_file(outfile) + self.check_file_contains(errfile, 'No such file or directory') + + self.runWt(["truncate", 'table:' + self.tablename, 'table:' + self.tablename], + outfilename=outfile, errfilename=errfile, failure=True) + self.check_empty_file(outfile) + self.check_file_contains(errfile, 'usage:') + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_util15.py b/test/suite/test_util15.py new file mode 100644 index 00000000000..33096e71bee --- /dev/null +++ b/test/suite/test_util15.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +from suite_subprocess import suite_subprocess +import wiredtiger, wttest + +# test_util15.py +# Utilities: wt alter +class test_util15(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util15.a' + + def test_alter_process(self): + """ + Test alter in a 'wt' process + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.assertTrue(os.path.exists(self.tablename + ".wt")) + + """ + Alter access pattern and confirm + """ + acc_pat_seq="access_pattern_hint=sequential" + self.runWt(["alter", "table:" + self.tablename, acc_pat_seq]) + cursor = self.session.open_cursor("metadata:create", None, None) + cursor.set_key("table:" + self.tablename) + self.assertEqual(cursor.search(),0) + string = cursor.get_value() + cursor.close() + self.assertTrue(acc_pat_seq in string) + + """ + Alter access pattern again and confirm + """ + acc_pat_rand="access_pattern_hint=random" + self.runWt(["alter", "table:" + self.tablename, acc_pat_rand]) + cursor = self.session.open_cursor("metadata:create", None, None) + cursor.set_key("table:" + self.tablename) + self.assertEqual(cursor.search(),0) + string = cursor.get_value() + cursor.close() + self.assertTrue(acc_pat_rand in string) + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_util16.py b/test/suite/test_util16.py new file mode 100644 index 00000000000..00e68c1017a --- /dev/null +++ b/test/suite/test_util16.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +from suite_subprocess import suite_subprocess +import wiredtiger, wttest + +# test_util16.py +# Utilities: wt rename +class test_util16(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util16.a' + tablename2 = 'test_util16.b' + nentries = 1000 + + def test_rename_process(self): + """ + Test alter in a 'wt' process + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.assertTrue(os.path.exists(self.tablename + ".wt")) + cursor = self.session.open_cursor('table:' + self.tablename, None, None) + for i in range(0, self.nentries): + cursor[str(i)] = str(i) + cursor.close() + + self.runWt(["rename", "table:" + self.tablename, "table:" + self.tablename2]) + self.assertTrue(os.path.exists(self.tablename2 + ".wt")) + cursor = self.session.open_cursor('table:' + self.tablename2, None, None) + count = 0 + while cursor.next() == 0: + count +=1 + cursor.close() + self.assertEquals(self.nentries, count) + + self.runWt(["rename", "table:" + self.tablename2, "table:" + self.tablename]) + self.assertTrue(os.path.exists(self.tablename + ".wt")) + cursor = self.session.open_cursor('table:' + self.tablename, None, None) + count = 0 + while cursor.next() == 0: + count +=1 + cursor.close() + self.assertEquals(self.nentries, count) + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_util17.py b/test/suite/test_util17.py new file mode 100644 index 00000000000..decc1fabf1d --- /dev/null +++ b/test/suite/test_util17.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +from suite_subprocess import suite_subprocess +import wiredtiger, wttest + +# test_util17.py +# Utilities: wt stat +class test_util17(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util17.a' + + def test_stat_process(self): + """ + Test stat in a 'wt' process + This test is just here to confirm that stat produces a correct looking + output, it isn't here to do statistics validation. + """ + params = 'key_format=S,value_format=S' + outfile = "wt-stat.out" + expected_string = "cursor: cursor create calls=" + self.session.create('table:' + self.tablename, params) + self.assertTrue(os.path.exists(self.tablename + ".wt")) + self.runWt(["stat"], outfilename=outfile) + self.check_file_contains(outfile, expected_string) + + expected_string = "cache_walk: Entries in the root page=1" + self.runWt(["stat", "table:" + self.tablename ], outfilename=outfile) + self.check_file_contains(outfile, expected_string) + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/wtdataset.py b/test/suite/wtdataset.py index 74e07e24e93..946b97d995f 100644 --- a/test/suite/wtdataset.py +++ b/test/suite/wtdataset.py @@ -41,6 +41,7 @@ class BaseDataSet(object): self.key_format = kwargs.get('key_format', 'S') self.value_format = kwargs.get('value_format', 'S') self.config = kwargs.get('config', '') + self.projection = kwargs.get('projection', '') def create(self): self.testcase.session.create(self.uri, 'key_format=' + self.key_format @@ -103,7 +104,8 @@ class BaseDataSet(object): def check(self): self.testcase.pr('check: ' + self.uri) - cursor = self.testcase.session.open_cursor(self.uri, None) + cursor = self.testcase.session.open_cursor( + self.uri + self.projection, None, None) self.check_cursor(cursor) cursor.close() @@ -289,6 +291,94 @@ class ComplexLSMDataSet(ComplexDataSet): def is_lsm(cls): return True +class ProjectionDataSet(SimpleDataSet): + """ + ProjectionDataSet creates a table with predefined data identical to + SimpleDataSet (single key and value), but when checking it, uses + a cursor with a projection. + """ + def __init__(self, testcase, uri, rows, **kwargs): + kwargs['config'] = kwargs.get('config', '') + ',columns=(k,v0)' + kwargs['projection'] = '(v0,v0,v0)' + super(ProjectionDataSet, self).__init__(testcase, uri, rows, **kwargs) + + # A value suitable for checking the value returned by a cursor. + def comparable_value(self, i): + v0 = self.value(i) + return [v0, v0, v0] + + def check_cursor(self, cursor): + i = 0 + for key, got0, got1, got2 in cursor: + i += 1 + self.testcase.assertEqual(key, self.key(i)) + if cursor.value_format == '8t' and got0 == 0: # deleted + continue + self.testcase.assertEqual([got0, got1, got2], + self.comparable_value(i)) + self.testcase.assertEqual(i, self.rows) + +class ProjectionIndexDataSet(BaseDataSet): + """ + ProjectionIndexDataSet creates a table with three values and + an index. Checks are made against a projection of the main table + and a projection of the index. + """ + def __init__(self, testcase, uri, rows, **kwargs): + self.origconfig = kwargs.get('config', '') + self.indexname = 'index:' + uri.split(":")[1] + ':index0' + kwargs['config'] = self.origconfig + ',columns=(k,v0,v1,v2)' + kwargs['value_format'] = kwargs.get('value_format', 'SiS') + kwargs['projection'] = '(v1,v2,v0)' + super(ProjectionIndexDataSet, self).__init__( + testcase, uri, rows, **kwargs) + + def value(self, i): + return ('v0:' + str(i), i*i, 'v2:' + str(i)) + + # Suitable for checking the value returned by a cursor using a projection. + def comparable_value(self, i): + return [i*i, 'v2:' + str(i), 'v0:' + str(i)] + + def create(self): + super(ProjectionIndexDataSet, self).create() + self.testcase.session.create(self.indexname, 'columns=(v2,v1),' + + self.origconfig) + + def check_cursor(self, cursor): + i = 0 + for key, got0, got1, got2 in cursor: + i += 1 + self.testcase.assertEqual(key, self.key(i)) + if cursor.value_format == '8t' and got0 == 0: # deleted + continue + self.testcase.assertEqual([got0, got1, got2], + self.comparable_value(i)) + self.testcase.assertEqual(i, self.rows) + + def check_index_cursor(self, cursor): + for i in xrange(1, self.rows + 1): + k = self.key(i) + v = self.value(i) + ik = (v[2], v[1]) # The index key is (v2,v2) + expect = [v[1],k,v[2],v[0]] + self.testcase.assertEqual(expect, cursor[ik]) + + def check(self): + BaseDataSet.check(self) + + # Check values in the index. + idxcursor = self.testcase.session.open_cursor( + self.indexname + '(v1,k,v2,v0)') + self.check_index_cursor(idxcursor) + idxcursor.close() + + def index_count(self): + return 1 + + def index_name(self, i): + return self.indexname + # create a key based on a cursor as a shortcut to creating a SimpleDataSet def simple_key(cursor, i): return BaseDataSet.key_by_format(i, cursor.key_format) diff --git a/test/suite/wttest.py b/test/suite/wttest.py index 4d6df0bc8bd..e91838544b9 100644 --- a/test/suite/wttest.py +++ b/test/suite/wttest.py @@ -37,9 +37,8 @@ except ImportError: import unittest from contextlib import contextmanager -import os, re, shutil, sys, time, traceback -import wtscenario -import wiredtiger +import glob, os, re, shutil, sys, time, traceback +import wiredtiger, wtscenario def shortenWithEllipsis(s, maxlen): if len(s) > maxlen: @@ -152,6 +151,14 @@ class TestSuiteConnection(object): else: return getattr(self._conn, attr) +# Just like a list of strings, but with a convenience function +class ExtensionList(list): + skipIfMissing = False + def extension(self, dirname, name, extarg=None): + if name != None and name != 'none': + ext = '' if extarg == None else '=' + extarg + self.append(dirname + '/' + name + ext) + class WiredTigerTestCase(unittest.TestCase): _globalSetup = False _printOnceSeen = {} @@ -160,9 +167,16 @@ class WiredTigerTestCase(unittest.TestCase): # Can be a string or a callable function or lambda expression. conn_config = '' + # conn_extensions can be overridden to add a list of extensions to load. + # Each entry is a string (directory and extension name) and optional config. + # Example: + # conn_extensions = ('extractors/csv_extractor', + # 'test/fail_fs={allow_writes=100}') + conn_extensions = () + @staticmethod def globalSetup(preserveFiles = False, useTimestamp = False, - gdbSub = False, verbose = 1, dirarg = None, + gdbSub = False, verbose = 1, builddir = None, dirarg = None, longtest = False): WiredTigerTestCase._preserveFiles = preserveFiles d = 'WT_TEST' if dirarg == None else dirarg @@ -172,6 +186,7 @@ class WiredTigerTestCase(unittest.TestCase): os.makedirs(d) wtscenario.set_long_run(longtest) WiredTigerTestCase._parentTestdir = d + WiredTigerTestCase._builddir = builddir WiredTigerTestCase._origcwd = os.getcwd() WiredTigerTestCase._resultfile = open(os.path.join(d, 'results.txt'), "w", 0) # unbuffered WiredTigerTestCase._gdbSubprocess = gdbSub @@ -224,16 +239,70 @@ class WiredTigerTestCase(unittest.TestCase): return "%s.%s.%s" % (self.__module__, self.className(), self._testMethodName) - # Can be overridden, but first consider setting self.conn_config . + # Return the wiredtiger_open extension argument for + # any needed shared library. + def extensionsConfig(self): + exts = self.conn_extensions + if hasattr(exts, '__call__'): + exts = ExtensionList() + self.conn_extensions(exts) + result = '' + extfiles = {} + skipIfMissing = False + if hasattr(exts, 'skip_if_missing'): + skipIfMissing = exts.skip_if_missing + for ext in exts: + extconf = '' + if '=' in ext: + splits = ext.split('=', 1) + ext = splits[0] + extconf = '=' + splits[1] + splits = ext.split('/') + if len(splits) != 2: + raise Exception(self.shortid() + + ": " + ext + + ": extension is not named <dir>/<name>") + libname = splits[1] + dirname = splits[0] + pat = os.path.join(WiredTigerTestCase._builddir, 'ext', + dirname, libname, '.libs', 'libwiredtiger_*.so') + filenames = glob.glob(pat) + if len(filenames) == 0: + if skipIfMissing: + self.skipTest('extension "' + ext + '" not built') + continue + else: + raise Exception(self.shortid() + + ": " + ext + + ": no extensions library found matching: " + pat) + elif len(filenames) > 1: + raise Exception(self.shortid() + + ": " + ext + + ": multiple extensions libraries found matching: " + pat) + complete = '"' + filenames[0] + '"' + extconf + if ext in extfiles: + if extfiles[ext] != complete: + raise Exception(self.shortid() + + ": non-matching extension arguments in " + + str(exts)) + else: + extfiles[ext] = complete + if len(extfiles) != 0: + result = ',extensions=[' + ','.join(extfiles.values()) + ']' + return result + + # Can be overridden, but first consider setting self.conn_config + # or self.conn_extensions def setUpConnectionOpen(self, home): self.home = home config = self.conn_config if hasattr(config, '__call__'): - config = config(home) + config = self.conn_config() + config += self.extensionsConfig() # In case the open starts additional threads, flush first to # avoid confusion. sys.stdout.flush() - conn_param = 'create,error_prefix="%s: ",%s' % (self.shortid(), config) + conn_param = 'create,error_prefix="%s",%s' % (self.shortid(), config) try: conn = self.wiredtiger_open(home, conn_param) except wiredtiger.WiredTigerError as e: @@ -287,6 +356,7 @@ class WiredTigerTestCase(unittest.TestCase): self.testsubdir = self.className() + '.' + str(self.__class__.wt_ntests) self.testdir = os.path.join(WiredTigerTestCase._parentTestdir, self.testsubdir) self.__class__.wt_ntests += 1 + self.starttime = time.time() if WiredTigerTestCase._verbose > 2: self.prhead('started in ' + self.testdir, True) # tearDown needs connections list, set it here in case the open fails. @@ -355,6 +425,9 @@ class WiredTigerTestCase(unittest.TestCase): else: self.pr('preserving directory ' + self.testdir) + elapsed = time.time() - self.starttime + if elapsed > 0.001 and WiredTigerTestCase._verbose >= 2: + print "%s: %.2f seconds" % (str(self), elapsed) if not passed and not skipped: print "ERROR in " + str(self) self.pr('FAIL') diff --git a/test/thread/file.c b/test/thread/file.c index 81ec6ad44f8..7a7d16c4cd6 100644 --- a/test/thread/file.c +++ b/test/thread/file.c @@ -33,20 +33,18 @@ file_create(const char *name) { WT_SESSION *session; int ret; - char *p, *end, config[128]; + char config[128]; if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) testutil_die(ret, "conn.session"); - p = config; - end = config + sizeof(config); - p += snprintf(p, (size_t)(end - p), + testutil_check(__wt_snprintf(config, sizeof(config), "key_format=%s," "internal_page_max=%d," - "leaf_page_max=%d,", - ftype == ROW ? "u" : "r", 16 * 1024, 128 * 1024); - if (ftype == FIX) - (void)snprintf(p, (size_t)(end - p), ",value_format=3t"); + "leaf_page_max=%d," + "%s", + ftype == ROW ? "u" : "r", 16 * 1024, 128 * 1024, + ftype == FIX ? ",value_format=3t" : "")); if ((ret = session->create(session, name, config)) != 0) if (ret != EEXIST) @@ -62,9 +60,10 @@ load(const char *name) WT_CURSOR *cursor; WT_ITEM *key, _key, *value, _value; WT_SESSION *session; - char keybuf[64], valuebuf[64]; - u_int keyno; + uint64_t keyno; + size_t len; int ret; + char keybuf[64], valuebuf[64]; file_create(name); @@ -79,18 +78,22 @@ load(const char *name) value = &_value; for (keyno = 1; keyno <= nkeys; ++keyno) { if (ftype == ROW) { + testutil_check(__wt_snprintf_len_set( + keybuf, sizeof(keybuf), + &len, "%017" PRIu64, keyno)); key->data = keybuf; - key->size = (uint32_t) - snprintf(keybuf, sizeof(keybuf), "%017u", keyno); + key->size = (uint32_t)len; cursor->set_key(cursor, key); } else - cursor->set_key(cursor, (uint32_t)keyno); - value->data = valuebuf; + cursor->set_key(cursor, keyno); if (ftype == FIX) cursor->set_value(cursor, 0x01); else { - value->size = (uint32_t) - snprintf(valuebuf, sizeof(valuebuf), "%37u", keyno); + testutil_check(__wt_snprintf_len_set( + valuebuf, sizeof(valuebuf), + &len, "%37" PRIu64, keyno)); + value->data = valuebuf; + value->size = (uint32_t)len; cursor->set_value(cursor, value); } if ((ret = cursor->insert(cursor)) != 0) diff --git a/test/thread/rw.c b/test/thread/rw.c index c6107a06c49..e8a2650ca51 100644 --- a/test/thread/rw.c +++ b/test/thread/rw.c @@ -66,7 +66,8 @@ rw_start(u_int readers, u_int writers) for (i = 0; i < writers; ++i) { if (i == 0 || multiple_files) { run_info[i].name = dmalloc(64); - snprintf(run_info[i].name, 64, FNAME, i); + testutil_check(__wt_snprintf( + run_info[i].name, 64, FNAME, i)); /* Vary by orders of magnitude */ if (vary_nops) @@ -88,8 +89,8 @@ rw_start(u_int readers, u_int writers) run_info[offset].name = dmalloc(64); /* Have readers read from tables with writes. */ name_index = i % writers; - snprintf( - run_info[offset].name, 64, FNAME, name_index); + testutil_check(__wt_snprintf( + run_info[offset].name, 64, FNAME, name_index)); /* Vary by orders of magnitude */ if (vary_nops) @@ -158,7 +159,8 @@ static inline void reader_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) { WT_ITEM *key, _key; - u_int keyno; + size_t len; + uint64_t keyno; int ret; char keybuf[64]; @@ -166,17 +168,18 @@ reader_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) keyno = __wt_random(&s->rnd) % nkeys + 1; if (ftype == ROW) { + testutil_check(__wt_snprintf_len_set( + keybuf, sizeof(keybuf), &len, "%017" PRIu64, keyno)); key->data = keybuf; - key->size = (uint32_t) - snprintf(keybuf, sizeof(keybuf), "%017u", keyno); + key->size = (uint32_t)len; cursor->set_key(cursor, key); } else - cursor->set_key(cursor, (uint32_t)keyno); + cursor->set_key(cursor, keyno); if ((ret = cursor->search(cursor)) != 0 && ret != WT_NOTFOUND) testutil_die(ret, "cursor.search"); if (log_print) testutil_check(session->log_printf(session, - "Reader Thread %p key %017u", pthread_self(), keyno)); + "Reader Thread %p key %017" PRIu64, pthread_self(), keyno)); } /* @@ -195,7 +198,7 @@ reader(void *arg) id = (int)(uintptr_t)arg; s = &run_info[id]; - __wt_thread_id(tid, sizeof(tid)); + testutil_check(__wt_thread_id(tid, sizeof(tid))); __wt_random_init(&s->rnd); printf(" read thread %2d starting: tid: %s, file: %s\n", @@ -242,7 +245,8 @@ static inline void writer_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) { WT_ITEM *key, _key, *value, _value; - u_int keyno; + uint64_t keyno; + size_t len; int ret; char keybuf[64], valuebuf[64]; @@ -251,12 +255,13 @@ writer_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) keyno = __wt_random(&s->rnd) % nkeys + 1; if (ftype == ROW) { + testutil_check(__wt_snprintf_len_set( + keybuf, sizeof(keybuf), &len, "%017" PRIu64, keyno)); key->data = keybuf; - key->size = (uint32_t) - snprintf(keybuf, sizeof(keybuf), "%017u", keyno); + key->size = (uint32_t)len; cursor->set_key(cursor, key); } else - cursor->set_key(cursor, (uint32_t)keyno); + cursor->set_key(cursor, keyno); if (keyno % 5 == 0) { ++s->remove; if ((ret = @@ -268,8 +273,10 @@ writer_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) if (ftype == FIX) cursor->set_value(cursor, 0x10); else { - value->size = (uint32_t)snprintf( - valuebuf, sizeof(valuebuf), "XXX %37u", keyno); + testutil_check(__wt_snprintf_len_set( + valuebuf, sizeof(valuebuf), + &len, "XXX %37" PRIu64, keyno)); + value->size = (uint32_t)len; cursor->set_value(cursor, value); } if ((ret = cursor->update(cursor)) != 0) @@ -277,7 +284,7 @@ writer_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) } if (log_print) testutil_check(session->log_printf(session, - "Writer Thread %p key %017u", pthread_self(), keyno)); + "Writer Thread %p key %017" PRIu64, pthread_self(), keyno)); } /* @@ -296,7 +303,7 @@ writer(void *arg) id = (int)(uintptr_t)arg; s = &run_info[id]; - __wt_thread_id(tid, sizeof(tid)); + testutil_check(__wt_thread_id(tid, sizeof(tid))); __wt_random_init(&s->rnd); printf("write thread %2d starting: tid: %s, file: %s\n", diff --git a/test/thread/stats.c b/test/thread/stats.c index 67a2c02719b..839d65e8a4d 100644 --- a/test/thread/stats.c +++ b/test/thread/stats.c @@ -65,7 +65,8 @@ stats(void) /* File statistics. */ if (!multiple_files) { - (void)snprintf(name, sizeof(name), "statistics:" FNAME, 0); + testutil_check(__wt_snprintf( + name, sizeof(name), "statistics:" FNAME, 0)); if ((ret = session->open_cursor( session, name, NULL, NULL, &cursor)) != 0) testutil_die(ret, "session.open_cursor"); diff --git a/test/thread/t.c b/test/thread/t.c index baadbf2adb9..d2ed4c74bb7 100644 --- a/test/thread/t.c +++ b/test/thread/t.c @@ -37,7 +37,6 @@ int multiple_files; /* File per thread */ int session_per_op; /* New session per operation */ static char home[512]; /* Program working dir */ -static char *progname; /* Program name */ static FILE *logfp; /* Log file */ static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); @@ -59,10 +58,7 @@ main(int argc, char *argv[]) int ch, cnt, runs; char *config_open, *working_dir; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); config_open = NULL; working_dir = NULL; @@ -189,19 +185,15 @@ wt_connect(char *config_open) }; int ret; char config[512]; - size_t print_count; testutil_clean_work_dir(home); testutil_make_work_dir(home); - print_count = (size_t)snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "create,statistics=(all),error_prefix=\"%s\",%s%s", progname, config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open); - - if (print_count >= sizeof(config)) - testutil_die(EINVAL, "Config string too long"); + config_open == NULL ? "" : config_open)); if ((ret = wiredtiger_open(home, &event_handler, config, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); diff --git a/test/utility/misc.c b/test/utility/misc.c index 1491c9a6938..934dac86a7b 100644 --- a/test/utility/misc.c +++ b/test/utility/misc.c @@ -28,6 +28,7 @@ #include "test_util.h" void (*custom_die)(void) = NULL; +const char *progname = "program name not set"; /* * die -- @@ -42,7 +43,9 @@ testutil_die(int e, const char *fmt, ...) if (custom_die != NULL) (*custom_die)(); + fprintf(stderr, "%s: FAILED", progname); if (fmt != NULL) { + fprintf(stderr, ": "); va_start(ap, fmt); vfprintf(stderr, fmt, ap); va_end(ap); @@ -55,6 +58,20 @@ testutil_die(int e, const char *fmt, ...) } /* + * testutil_set_progname -- + * Set the global program name for error handling. + */ +const char * +testutil_set_progname(char * const *argv) +{ + if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) + progname = argv[0]; + else + ++progname; + return (progname); +} + +/* * testutil_work_dir_from_path -- * Takes a buffer, its size and the intended work directory. * Creates the full intended work directory in buffer. @@ -78,7 +95,7 @@ testutil_work_dir_from_path(char *buffer, size_t len, const char *dir) * Remove the work directory. */ void -testutil_clean_work_dir(char *dir) +testutil_clean_work_dir(const char *dir) { size_t len; int ret; @@ -91,14 +108,14 @@ testutil_clean_work_dir(char *dir) if ((buf = malloc(len)) == NULL) testutil_die(ENOMEM, "Failed to allocate memory"); - snprintf(buf, len, "%s %s %s %s", DIR_EXISTS_COMMAND, dir, - RM_COMMAND, dir); + testutil_check(__wt_snprintf( + buf, len, "%s %s %s %s", DIR_EXISTS_COMMAND, dir, RM_COMMAND, dir)); #else len = strlen(dir) + strlen(RM_COMMAND) + 1; if ((buf = malloc(len)) == NULL) testutil_die(ENOMEM, "Failed to allocate memory"); - snprintf(buf, len, "%s%s", RM_COMMAND, dir); + testutil_check(__wt_snprintf(buf, len, "%s%s", RM_COMMAND, dir)); #endif if ((ret = system(buf)) != 0 && ret != ENOENT) @@ -125,7 +142,7 @@ testutil_make_work_dir(char *dir) testutil_die(ENOMEM, "Failed to allocate memory"); /* mkdir shares syntax between Windows and Linux */ - snprintf(buf, len, "%s%s", MKDIR_COMMAND, dir); + testutil_check(__wt_snprintf(buf, len, "%s%s", MKDIR_COMMAND, dir)); if ((ret = system(buf)) != 0) testutil_die(ret, "%s", buf); free(buf); @@ -149,20 +166,25 @@ testutil_cleanup(TEST_OPTS *opts) } /* - * testutil_disable_long_tests -- - * Return if TESTUTIL_DISABLE_LONG_TESTS is set. + * testutil_enable_long_tests -- + * Return if TESTUTIL_ENABLE_LONG_TESTS is set. */ bool -testutil_disable_long_tests(void) +testutil_enable_long_tests(void) { const char *res; + bool enable_long_tests; if (__wt_getenv(NULL, - "TESTUTIL_DISABLE_LONG_TESTS", &res) == WT_NOTFOUND) + "TESTUTIL_ENABLE_LONG_TESTS", &res) == WT_NOTFOUND) return (false); + /* Accept anything other than "TESTUTIL_ENABLE_LONG_TESTS=0". */ + enable_long_tests = res[0] != '0'; + free((void *)res); - return (true); + + return (enable_long_tests); } /* diff --git a/test/utility/parse_opts.c b/test/utility/parse_opts.c index 74a1c021d5d..c3eff3360de 100644 --- a/test/utility/parse_opts.c +++ b/test/utility/parse_opts.c @@ -43,10 +43,7 @@ testutil_parse_opts(int argc, char * const *argv, TEST_OPTS *opts) opts->running = true; opts->verbose = false; - if ((opts->progname = strrchr(argv[0], DIR_DELIM)) == NULL) - opts->progname = argv[0]; - else - ++opts->progname; + opts->progname = testutil_set_progname(argv); while ((ch = __wt_getopt(opts->progname, argc, argv, "A:h:n:o:pR:T:t:vW:")) != EOF) @@ -118,13 +115,15 @@ testutil_parse_opts(int argc, char * const *argv, TEST_OPTS *opts) if (opts->home == NULL) { len = strlen("WT_TEST.") + strlen(opts->progname) + 10; opts->home = dmalloc(len); - snprintf(opts->home, len, "WT_TEST.%s", opts->progname); + testutil_check(__wt_snprintf( + opts->home, len, "WT_TEST.%s", opts->progname)); } /* Setup the default URI string */ len = strlen("table:") + strlen(opts->progname) + 10; opts->uri = dmalloc(len); - snprintf(opts->uri, len, "table:%s", opts->progname); + testutil_check(__wt_snprintf( + opts->uri, len, "table:%s", opts->progname)); return (0); } diff --git a/test/utility/test_util.h b/test/utility/test_util.h index f6a9cd68e02..406ed2c4961 100644 --- a/test/utility/test_util.h +++ b/test/utility/test_util.h @@ -48,7 +48,7 @@ /* Generic option parsing structure shared by all test cases. */ typedef struct { char *home; - char *progname; + const char *progname; enum { TABLE_COL=1, /* Fixed-length column store */ TABLE_FIX=2, /* Variable-length column store */ TABLE_ROW=3 /* Row-store */ @@ -183,12 +183,15 @@ void *dmalloc(size_t); void *drealloc(void *, size_t); void *dstrdup(const void *); void *dstrndup(const char *, size_t); -void testutil_clean_work_dir(char *); +void testutil_clean_work_dir(const char *); void testutil_cleanup(TEST_OPTS *); -bool testutil_disable_long_tests(void); +bool testutil_enable_long_tests(void); void testutil_make_work_dir(char *); int testutil_parse_opts(int, char * const *, TEST_OPTS *); void testutil_work_dir_from_path(char *, size_t, const char *); void *thread_append(void *); void *thread_insert_append(void *); void *thread_prev(void *); + +extern const char *progname; +const char *testutil_set_progname(char * const *); diff --git a/test/utility/thread.c b/test/utility/thread.c index 38465b2f02b..122ad554442 100644 --- a/test/utility/thread.c +++ b/test/utility/thread.c @@ -57,8 +57,8 @@ thread_append(void *arg) if (opts->table_type == TABLE_FIX) cursor->set_value(cursor, buf[0]); else { - snprintf(buf, sizeof(buf), - "%" PRIu64 " VALUE ------", recno); + testutil_check(__wt_snprintf(buf, sizeof(buf), + "%" PRIu64 " VALUE ------", recno)); cursor->set_value(cursor, buf); } testutil_check(cursor->insert(cursor)); @@ -94,7 +94,8 @@ thread_insert_append(void *arg) session, opts->uri, NULL, NULL, &cursor)); for (i = 0; i < opts->nrecords; ++i) { - snprintf(kbuf, sizeof(kbuf), "%010d KEY------", (int)i); + testutil_check(__wt_snprintf( + kbuf, sizeof(kbuf), "%010d KEY------", (int)i)); cursor->set_key(cursor, kbuf); cursor->set_value(cursor, "========== VALUE ======="); testutil_check(cursor->insert(cursor)); diff --git a/test/windows/windows_shim.h b/test/windows/windows_shim.h index 648b991b1a2..8985904fb19 100644 --- a/test/windows/windows_shim.h +++ b/test/windows/windows_shim.h @@ -36,6 +36,8 @@ #include <io.h> #include <process.h> +#include "wt_internal.h" + #define inline __inline /* Define some POSIX types */ @@ -52,12 +54,7 @@ typedef int u_int; /* snprintf does not exist on <= VS 2013 */ #if _MSC_VER < 1900 -#define snprintf _wt_snprintf - -_Check_return_opt_ int __cdecl _wt_snprintf( - _Out_writes_(_MaxCount) char * _DstBuf, - _In_ size_t _MaxCount, - _In_z_ _Printf_format_string_ const char * _Format, ...); +#define snprintf __wt_snprintf #endif /* diff --git a/test/wtperf/test_conf_dump.py b/test/wtperf/test_conf_dump.py new file mode 100644 index 00000000000..ef7f276a1d0 --- /dev/null +++ b/test/wtperf/test_conf_dump.py @@ -0,0 +1,296 @@ +# Usage: python test_conf_dump.py <optional-wtperf-config> +# +# This script tests if the config file dumped in the test directory corresponds +# correctly to the wtperf config file used. Command line options to wtperf are +# also taken into account. +# +# Following expectations are checked for: +# 1. If provided through multiple sources, "conn_config" and "table_config" +# configuration options are appended to each other. All other options get +# replaced by a higher precedent source. +# 2. The precedence order for the options in an increasing order is as follows: +# default option, +# provided through config file, +# provided through option -o +# provided through option -C (for conn_config) or -T (for table_config) +# +# Test fails if any config option is missing or has a wrong value. Test also +# fails if the value for the option is not replaced/appended in the correct +# order of precedence as stated above. + +import os, re, subprocess, sys + +OP_FILE = "WT_TEST/CONFIG.wtperf" +TMP_CONF = "__tmp.wtperf" +WTPERF_BIN = "./wtperf" +WTPERF_DIR = "../../build_posix/bench/wtperf/" + +CONF_NOT_PROVIDED = -2 + +# Generate a wtperf conf file to use +def generate_conf_file(file_name): + f = open(file_name, 'w') + f.write( +'''conn_config="cache_size=16GB,eviction=(threads_max=4),log=(enabled=false),session_max=33" +table_config="leaf_page_max=32k,internal_page_max=16k,allocation_size=4k,split_pct=90,type=file" +close_conn=false +icount=1500 +create=true +compression="snappy" +checkpoint_interval=5 +checkpoint_threads=1 +populate_threads=1 +report_interval=5 +session_count_idle=50 +session_count_idle=60 +session_count_idle=70 +session_count_idle=80 +run_time=5 +sample_interval=5 +sample_rate=1 +table_count=2 +threads=((count=6,updates=1)) +value_sz=1000 +warmup=2 +''') + f.close() + +# Build a command from the given options and execute wtperf +def execute_wtperf(conf_file, option_C = "", option_T = "", option_o = ""): + # Generate the command to run, execute wtperf + cmd = WTPERF_BIN + " -O " + conf_file + if option_C: + cmd += " -C " + option_C + if option_T: + cmd += " -T " + option_T + if option_o: + # Any quotes in option_o need to be escaped before providing it as part + # of the command + option_o_cmd_str = option_o.replace('"', '\\"') + cmd += " -o " + option_o_cmd_str + + print "Running: ", cmd + subprocess.check_call(cmd, shell=True) + print "=========================\n" + +# Build a dictionary of config key and it's value from the given config file. +# Optionally take -C, -T and -o and overwrite/append values as per correct +# precedence +def build_dict_from_conf( + conf_file, option_C = "", option_T = "", option_o = ""): + # Open given conf file and make a dictionary of passed arguments and values + with open(conf_file) as f: + lines = f.read().splitlines() + + # Maintain precedence order of config file, -o, -C/-T + # Build a dict of config options, appending values for table_config and + # conn_config, if specified multiple times. Replace with the latest in + # case of all other configuration keys. + key_val_dict = {} + for line in lines: + if re.match('^\s*#', line) is None: + key_val_pair = line.split('=', 1) + if ((key_val_pair[0] == 'table_config' or + key_val_pair[0] == 'conn_config') and + key_val_pair[0] in key_val_dict): + tmp_val = key_val_dict[key_val_pair[0]][:-1] + tmp_val += "," + tmp_val += key_val_pair[1][1:] + key_val_dict[key_val_pair[0]] = tmp_val + else: + key_val_dict[key_val_pair[0]] = key_val_pair[1] + + # If provided, put option o in the dict + if option_o: + opt_o_key_val_list = option_o.split(',') + for op_o_key_val in opt_o_key_val_list: + key_val_pair = op_o_key_val.split('=', 1) + if ((key_val_pair[0] == 'table_config' or + key_val_pair[0] == 'conn_config') and + key_val_pair[0] in key_val_dict): + tmp_val = key_val_dict[key_val_pair[0]][:-1] + tmp_val += "," + tmp_val += key_val_pair[1][1:] + key_val_dict[key_val_pair[0]] = tmp_val + else: + key_val_dict[key_val_pair[0]] = key_val_pair[1] + + # If provided, put option C in the dict + if option_C: + tmp_val = key_val_dict["conn_config"][:-1] + tmp_val += "," + tmp_val += option_C[1:] + key_val_dict["conn_config"] = tmp_val + + # If provided, put option T in the dict + if option_T: + tmp_val = key_val_dict["table_config"][:-1] + tmp_val += "," + tmp_val += option_T[1:] + key_val_dict["table_config"] = tmp_val + + return key_val_dict + +# Extract configuration value for the given key from the given config file +def extract_config_from_file(conf_file, key): + ret_val = "" + with open(conf_file) as f: + lines = f.read().splitlines() + for line in lines: + if re.match('^\s*#', line) is None: + key_val_pair = line.split('=', 1) + if key_val_pair[0] == key: + ret_val = key_val_pair[1] + return ret_val + +# Extract configuration value for the given key from the given "-o" string +def extract_config_from_opt_o(option_o, key): + ret_val = "" + opt_o_key_val_list = option_o.split(',') + for op_o_key_val in opt_o_key_val_list: + key_val_pair = op_o_key_val.split('=', 1) + if key_val_pair[0] == key: + ret_val = key_val_pair[1] + return ret_val + +# Execute test: +# Run wtperf with given config and check if the dumped config file matches the +# given inputs +def run_test(conf_file, option_C = "", option_T = "", option_o = ""): + # Run wtperf + execute_wtperf(conf_file, option_C, option_T, option_o) + + key_val_dict_ip = build_dict_from_conf( + conf_file, option_C, option_T, option_o) + key_val_dict_op = build_dict_from_conf(OP_FILE) + + conn_config_from_file = extract_config_from_file(conf_file, "conn_config") + table_config_from_file = extract_config_from_file(conf_file, "table_config") + conn_config_from_opt_o = "" + table_config_from_opt_o = "" + if option_o: + conn_config_from_opt_o = extract_config_from_opt_o( + option_o, "conn_config") + table_config_from_opt_o = extract_config_from_opt_o( + option_o, "table_config") + + # Check if dumped output conf matches with input file and options + match = True + for key in key_val_dict_ip: + match_itr = True + + # Check if we see this config key in the dumped file + if not key in key_val_dict_op: + print "Key '", key, "' not found in dumped file ", OP_FILE + match = match_itr = False + continue + + # Check if values from all sources of conn_config are presented in the + # conn_config in dumped file. Also check of their relative ordering as + # per precedence rules defined. + if (key == 'conn_config' and + (conn_config_from_file or conn_config_from_opt_o or option_C)): + # Should find these config in order: file < option o < option C + file_loc = CONF_NOT_PROVIDED + option_o_loc = CONF_NOT_PROVIDED + option_C_loc = CONF_NOT_PROVIDED + op_conn_config = key_val_dict_op['conn_config'] + + if conn_config_from_file: + file_loc = op_conn_config.find(conn_config_from_file[1:-1]) + if conn_config_from_opt_o: + option_o_loc = op_conn_config.find(conn_config_from_opt_o[1:-1]) + if option_C: + option_C_loc = op_conn_config.find(option_C[1:-1]) + + # Check if value from any of the sources is missing + if ((conn_config_from_file and file_loc == -1) or + (conn_config_from_opt_o and option_o_loc == -1) or + (option_C and option_C_loc == -1)): + print "Part of conn_config missing in dumped file ", OP_FILE + match_itr = False + + # Check if the values got appended in the correct order + if match_itr: + if ((option_o_loc != CONF_NOT_PROVIDED and + option_o_loc < file_loc) or + (option_C_loc != CONF_NOT_PROVIDED and + (option_C_loc < file_loc or option_C_loc < option_o_loc))): + print "Detected incorrect config append order:" + match_itr = False + + # Check if values from all sources of table_config are presented in the + # table_config in dumped file. Also check of their relative ordering as + # per precedence rules defined. + if (key == 'table_config' and + (table_config_from_file or table_config_from_opt_o or option_T)): + # Should find these config in order: file < option o < option T + file_loc = CONF_NOT_PROVIDED + option_o_loc = CONF_NOT_PROVIDED + option_T_loc = CONF_NOT_PROVIDED + op_table_config = key_val_dict_op['table_config'] + + if table_config_from_file: + file_loc = op_table_config.find(table_config_from_file[1:-1]) + if table_config_from_opt_o: + option_o_loc = op_table_config.find( + table_config_from_opt_o[1:-1]) + if option_T: + option_T_loc = op_table_config.find(option_T[1:-1]) + + # Check if value from any of the sources is missing + if ((table_config_from_file and file_loc == -1) or + (table_config_from_opt_o and option_o_loc == -1) or + (option_T and option_T_loc == -1)): + print "Part of table_config missing in dumped file ", OP_FILE + match_itr = False + + # Check if the values got appended in the correct order + if match_itr: + if ((option_o_loc != CONF_NOT_PROVIDED and + option_o_loc < file_loc) or + (option_T_loc != CONF_NOT_PROVIDED and + (option_T_loc < file_loc or option_T_loc < option_o_loc))): + print "Detected incorrect config append order:" + match_itr = False + + if (key != 'table_config' and key != 'conn_config' and + key_val_dict_ip[key] != key_val_dict_op[key]): + print "Config mismatch between:" + match_itr = False + + if match_itr is False: + print "Input Config:", key, '=', key_val_dict_ip[key] + print "Dumped Config:", key, '=', key_val_dict_op[key] + print "\n" + + match = match and match_itr + + return match + +# ----------------- Execute Test -------------- +# If a wtperf conf file is provided use it, else generate a temp conf file +os.chdir(WTPERF_DIR) +if len(sys.argv) == 2: + conf_file = sys.argv[1] +else: + conf_file = TMP_CONF + generate_conf_file(conf_file) + +# Run a test with no options +if not run_test(conf_file): + exit(-1) + +# Run a test with -C, -T, -o provided +option_o = "verbose=2,conn_config=\"session_max=135\",table_config=\"type=lsm\",sample_interval=2,run_time=0,sample_rate=2,readonly=false" +option_C = "\"cache_size=10GB,session_max=115\"" +option_T = "\"allocation_size=8k,split_pct=92\"" +if not run_test(conf_file, option_C, option_T, option_o): + exit(-1) + +# Cleanup generated temp files +subprocess.check_call("rm -rf WT_TEST/", shell=True) +if len(sys.argv) == 1 and conf_file == TMP_CONF: + subprocess.check_call("rm " + TMP_CONF, shell=True) + +print "All tests succeeded" |