diff options
Diffstat (limited to 'src/third_party/wiredtiger/test')
63 files changed, 3493 insertions, 542 deletions
diff --git a/src/third_party/wiredtiger/test/bloom/test_bloom.c b/src/third_party/wiredtiger/test/bloom/test_bloom.c index 04fc8d1c371..f95bc7faaf9 100644 --- a/src/third_party/wiredtiger/test/bloom/test_bloom.c +++ b/src/third_party/wiredtiger/test/bloom/test_bloom.c @@ -55,6 +55,8 @@ void usage(void); extern char *__wt_optarg; extern int __wt_optind; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { @@ -129,11 +131,9 @@ setup(void) "create,error_prefix=\"%s\",cache_size=%" PRIu32 "MB,%s", g.progname, g.c_cache, g.config_open == NULL ? "" : g.config_open); - if ((ret = wiredtiger_open(NULL, NULL, config, &conn)) != 0) - testutil_die(ret, "wiredtiger_open"); + testutil_check(wiredtiger_open(NULL, NULL, config, &conn)); - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - testutil_die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); g.wt_conn = conn; g.wt_session = session; @@ -153,39 +153,35 @@ run(void) /* Use the internal session handle to access private APIs. */ sess = (WT_SESSION_IMPL *)g.wt_session; - if ((ret = __wt_bloom_create( - sess, uri, NULL, g.c_ops, g.c_factor, g.c_k, &bloomp)) != 0) - testutil_die(ret, "__wt_bloom_create"); + testutil_check(__wt_bloom_create( + sess, uri, NULL, g.c_ops, g.c_factor, g.c_k, &bloomp)); item.size = g.c_key_max; for (i = 0; i < g.c_ops; i++) { item.data = g.entries[i]; if ((ret = __wt_bloom_insert(bloomp, &item)) != 0) - testutil_die(ret, "__wt_bloom_insert: %d", i); + testutil_die(ret, "__wt_bloom_insert: %" PRIu32, i); } - if ((ret = __wt_bloom_finalize(bloomp)) != 0) - testutil_die(ret, "__wt_bloom_finalize"); + testutil_check(__wt_bloom_finalize(bloomp)); for (i = 0; i < g.c_ops; i++) { item.data = g.entries[i]; if ((ret = __wt_bloom_get(bloomp, &item)) != 0) { - fprintf(stderr, "get failed at record: %d\n", i); + fprintf(stderr, + "get failed at record: %" PRIu32 "\n", i); testutil_die(ret, "__wt_bloom_get"); } } - if ((ret = __wt_bloom_close(bloomp)) != 0) - testutil_die(ret, "__wt_bloom_close"); - - if ((ret = g.wt_session->checkpoint(g.wt_session, NULL)) != 0) - testutil_die(ret, "WT_SESSION.checkpoint"); - if ((ret = __wt_bloom_open( - sess, uri, g.c_factor, g.c_k, NULL, &bloomp)) != 0) - testutil_die(ret, "__wt_bloom_open"); + testutil_check(__wt_bloom_close(bloomp)); + + testutil_check(g.wt_session->checkpoint(g.wt_session, NULL)); + testutil_check(__wt_bloom_open( + sess, uri, g.c_factor, g.c_k, NULL, &bloomp)); + for (i = 0; i < g.c_ops; i++) { item.data = g.entries[i]; - if ((ret = __wt_bloom_get(bloomp, &item)) != 0) - testutil_die(ret, "__wt_bloom_get"); + testutil_check(__wt_bloom_get(bloomp, &item)); } /* @@ -194,33 +190,34 @@ run(void) */ item.size = g.c_key_max + 10; item.data = calloc(item.size, 1); + if (item.data == NULL) + testutil_die(ENOMEM, "value buffer malloc"); memset((void *)item.data, 'a', item.size); for (i = 0, fp = 0; i < g.c_ops; i++) { ((uint8_t *)item.data)[i % item.size] = 'a' + ((uint8_t)rand() % 26); if ((ret = __wt_bloom_get(bloomp, &item)) == 0) ++fp; + if (ret != 0 && ret != WT_NOTFOUND) + testutil_die(ret, "__wt_bloom_get"); } free((void *)item.data); - printf("Out of %d ops, got %d false positives, %.4f%%\n", + printf( + "Out of %" PRIu32 " ops, got %" PRIu32 " false positives, %.4f%%\n", g.c_ops, fp, 100.0 * fp/g.c_ops); - if ((ret = __wt_bloom_drop(bloomp, NULL)) != 0) - testutil_die(ret, "__wt_bloom_drop"); + testutil_check(__wt_bloom_drop(bloomp, NULL)); } void cleanup(void) { uint32_t i; - int ret; for (i = 0; i < g.c_ops; i++) free(g.entries[i]); free(g.entries); - if ((ret = g.wt_session->close(g.wt_session, NULL)) != 0) - testutil_die(ret, "WT_SESSION.close"); - if ((g.wt_conn->close(g.wt_conn, NULL)) != 0) - testutil_die(ret, "WT_CONNECTION.close"); + testutil_check(g.wt_session->close(g.wt_session, NULL)); + testutil_check(g.wt_conn->close(g.wt_conn, NULL)); } /* diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c index 1914ad0188a..c5524b3c63e 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c @@ -41,6 +41,8 @@ static int wt_shutdown(void); extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { @@ -134,7 +136,7 @@ main(int argc, char *argv[]) printf("%s: process %" PRIu64 "\n", g.progname, (uint64_t)getpid()); for (cnt = 1; (runs == 0 || cnt <= runs) && g.status == 0; ++cnt) { - printf(" %d: %u workers, %u tables\n", + printf(" %d: %d workers, %d tables\n", cnt, g.nworkers, g.ntables); (void)cleanup(); /* Clean up previous runs */ diff --git a/src/third_party/wiredtiger/test/cursor_order/Makefile.am b/src/third_party/wiredtiger/test/cursor_order/Makefile.am new file mode 100644 index 00000000000..c0c0ed639bf --- /dev/null +++ b/src/third_party/wiredtiger/test/cursor_order/Makefile.am @@ -0,0 +1,13 @@ +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/test/utility + +noinst_PROGRAMS = cursor_order +cursor_order_LDADD = $(top_builddir)/libwiredtiger.la + +cursor_order_SOURCES = cursor_order_file.c cursor_order_ops.c cursor_order.c +cursor_order_LDFLAGS = -static + +TESTS = $(noinst_PROGRAMS) + +clean-local: + rm -rf WiredTiger* wt.* *.core __stats diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order.c new file mode 100644 index 00000000000..d8cfc0c1421 --- /dev/null +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order.c @@ -0,0 +1,307 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "cursor_order.h" + +static char home[512]; /* Program working dir */ +static char *progname; /* Program name */ +static FILE *logfp; /* Log file */ + +static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); +static int handle_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); +static void onint(int); +static void shutdown(void); +static int usage(void); +static void wt_connect(SHARED_CONFIG *, char *); +static void wt_shutdown(SHARED_CONFIG *); + +extern int __wt_optind; +extern char *__wt_optarg; + +void (*custom_die)(void) = NULL; + +int +main(int argc, char *argv[]) +{ + SHARED_CONFIG _cfg, *cfg; + int ch, cnt, runs; + char *config_open, *working_dir; + + if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) + progname = argv[0]; + else + ++progname; + + cfg = &_cfg; + config_open = NULL; + working_dir = NULL; + runs = 1; + + /* + * Explicitly initialize the shared configuration object before + * parsing command line options. + */ + cfg->append_inserters = 1; + cfg->conn = NULL; + cfg->ftype = ROW; + cfg->max_nops = 1000000; + cfg->multiple_files = false; + cfg->nkeys = 1000; + cfg->reverse_scanners = 5; + cfg->reverse_scan_ops = 10; + cfg->thread_finish = false; + cfg->vary_nops = false; + + while ((ch = __wt_getopt( + progname, argc, argv, "C:Fk:h:l:n:R:r:t:vw:W:")) != EOF) + switch (ch) { + case 'C': /* wiredtiger_open config */ + config_open = __wt_optarg; + break; + case 'F': /* multiple files */ + cfg->multiple_files = true; + break; + case 'h': + working_dir = __wt_optarg; + break; + case 'k': /* rows */ + cfg->nkeys = (uint64_t)atol(__wt_optarg); + break; + case 'l': /* log */ + if ((logfp = fopen(__wt_optarg, "w")) == NULL) { + fprintf(stderr, + "%s: %s\n", __wt_optarg, strerror(errno)); + return (EXIT_FAILURE); + } + break; + case 'n': /* operations */ + cfg->max_nops = (uint64_t)atol(__wt_optarg); + break; + case 'R': + cfg->reverse_scanners = (uint64_t)atol(__wt_optarg); + break; + case 'r': /* runs */ + runs = atoi(__wt_optarg); + break; + case 't': + switch (__wt_optarg[0]) { + case 'f': + cfg->ftype = FIX; + break; + case 'r': + cfg->ftype = ROW; + break; + case 'v': + cfg->ftype = VAR; + break; + default: + return (usage()); + } + break; + case 'v': /* vary operation count */ + cfg->vary_nops = true; + break; + case 'w': + cfg->reverse_scan_ops = (uint64_t)atol(__wt_optarg); + break; + case 'W': + cfg->append_inserters = (uint64_t)atol(__wt_optarg); + break; + default: + return (usage()); + } + + argc -= __wt_optind; + argv += __wt_optind; + if (argc != 0) + return (usage()); + + testutil_work_dir_from_path(home, 512, working_dir); + + if (cfg->vary_nops && !cfg->multiple_files) { + fprintf(stderr, + "Variable op counts only supported with multiple tables\n"); + return (usage()); + } + + /* Clean up on signal. */ + (void)signal(SIGINT, onint); + + printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); + for (cnt = 1; runs == 0 || cnt <= runs; ++cnt) { + printf( + " %d: %" PRIu64 + " reverse scanners, %" PRIu64 " writers\n", + cnt, cfg->reverse_scanners, cfg->append_inserters); + + shutdown(); /* Clean up previous runs */ + + wt_connect(cfg, config_open); /* WiredTiger connection */ + + if (ops_start(cfg)) + return (EXIT_FAILURE); + + wt_shutdown(cfg); /* WiredTiger shut down */ + } + return (0); +} + +/* + * wt_connect -- + * Configure the WiredTiger connection. + */ +static void +wt_connect(SHARED_CONFIG *cfg, char *config_open) +{ + static WT_EVENT_HANDLER event_handler = { + handle_error, + handle_message, + NULL, + NULL /* Close handler. */ + }; + int ret; + char config[512]; + size_t print_count; + + testutil_clean_work_dir(home); + testutil_make_work_dir(home); + + print_count = (size_t)snprintf(config, sizeof(config), + "create,statistics=(all),error_prefix=\"%s\",%s%s", + progname, + config_open == NULL ? "" : ",", + config_open == NULL ? "" : config_open); + + if (print_count >= sizeof(config)) + testutil_die(EINVAL, "Config string too long"); + + if ((ret = wiredtiger_open( + home, &event_handler, config, &cfg->conn)) != 0) + testutil_die(ret, "wiredtiger_open"); +} + +/* + * wt_shutdown -- + * Flush the file to disk and shut down the WiredTiger connection. + */ +static void +wt_shutdown(SHARED_CONFIG *cfg) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + int ret; + + conn = cfg->conn; + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + testutil_die(ret, "conn.session"); + + if ((ret = session->checkpoint(session, NULL)) != 0) + testutil_die(ret, "session.checkpoint"); + + if ((ret = conn->close(conn, NULL)) != 0) + testutil_die(ret, "conn.close"); +} + +/* + * shutdown -- + * Clean up from previous runs. + */ +static void +shutdown(void) +{ + testutil_clean_work_dir(home); +} + +static int +handle_error(WT_EVENT_HANDLER *handler, + WT_SESSION *session, int error, const char *errmsg) +{ + (void)(handler); + (void)(session); + (void)(error); + + return (fprintf(stderr, "%s\n", errmsg) < 0 ? -1 : 0); +} + +static int +handle_message(WT_EVENT_HANDLER *handler, + WT_SESSION *session, const char *message) +{ + (void)(handler); + (void)(session); + + if (logfp != NULL) + return (fprintf(logfp, "%s\n", message) < 0 ? -1 : 0); + + return (printf("%s\n", message) < 0 ? -1 : 0); +} + +/* + * onint -- + * Interrupt signal handler. + */ +static void +onint(int signo) +{ + (void)(signo); + + shutdown(); + + fprintf(stderr, "\n"); + exit(EXIT_FAILURE); +} + +/* + * usage -- + * Display usage statement and exit failure. + */ +static int +usage(void) +{ + fprintf(stderr, + "usage: %s " + "[-FLv] [-C wiredtiger-config] [-k keys] [-l log]\n\t" + "[-n ops] [-R reverse_scanners] [-r runs] [-t f|r|v] " + "[-W append_inserters]\n", + progname); + fprintf(stderr, "%s", + "\t-C specify wiredtiger_open configuration arguments\n" + "\t-F create a file per thread\n" + "\t-k set number of keys to load\n" + "\t-L log print per operation\n" + "\t-l specify a log file\n" + "\t-n set number of operations each thread does\n" + "\t-R set number of reverse scanner threads\n" + "\t-r set number of runs (0 for continuous)\n" + "\t-t set a file type (fix | row | var)\n" + "\t-v do a different number of operations on different tables\n" + "\t-w set number of items to walk in a reverse scan\n" + "\t-W set number of threads doing append inserts\n"); + return (EXIT_FAILURE); +} diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order.h b/src/third_party/wiredtiger/test/cursor_order/cursor_order.h new file mode 100644 index 00000000000..dd49fce124b --- /dev/null +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order.h @@ -0,0 +1,54 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <signal.h> + +#include "test_util.i" + +#define FNAME "file:cursor_order.%03d" /* File name */ + +typedef enum { FIX, ROW, VAR } __ftype; /* File type */ + +typedef struct { + uint64_t append_inserters; /* Number of append threads */ + WT_CONNECTION *conn; /* WiredTiger connection */ + __ftype ftype; + uint64_t key_range; /* Current key range */ + uint64_t max_nops; /* Operations per thread */ + bool multiple_files; /* File per thread */ + uint64_t nkeys; /* Keys to load */ + uint64_t reverse_scanners; /* Number of scan threads */ + uint64_t reverse_scan_ops; /* Keys to visit per scan */ + bool thread_finish; /* Signal to finish run. */ + bool vary_nops; /* Operations per thread */ + +} SHARED_CONFIG; + +void load(SHARED_CONFIG *, const char *); +int ops_start(SHARED_CONFIG *); +void verify(SHARED_CONFIG *, const char *); diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c new file mode 100644 index 00000000000..5dc7194b5fb --- /dev/null +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c @@ -0,0 +1,132 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "cursor_order.h" + +static void +file_create(SHARED_CONFIG *cfg, const char *name) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + int ret; + char *p, *end, config[128]; + + conn = cfg->conn; + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + testutil_die(ret, "conn.session"); + + p = config; + end = config + sizeof(config); + p += snprintf(p, (size_t)(end - p), + "key_format=%s," + "internal_page_max=%d," + "split_deepen_min_child=200," + "leaf_page_max=%d,", + cfg->ftype == ROW ? "S" : "r", 16 * 1024, 128 * 1024); + if (cfg->ftype == FIX) + (void)snprintf(p, (size_t)(end - p), ",value_format=3t"); + + if ((ret = session->create(session, name, config)) != 0) + if (ret != EEXIST) + testutil_die(ret, "session.create"); + + if ((ret = session->close(session, NULL)) != 0) + testutil_die(ret, "session.close"); +} + +void +load(SHARED_CONFIG *cfg, const char *name) +{ + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_ITEM *value, _value; + WT_SESSION *session; + char keybuf[64], valuebuf[64]; + int64_t keyno; + int ret; + + conn = cfg->conn; + + file_create(cfg, name); + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + testutil_die(ret, "conn.session"); + + if ((ret = + session->open_cursor(session, name, NULL, "bulk", &cursor)) != 0) + testutil_die(ret, "cursor.open"); + + value = &_value; + for (keyno = 1; keyno <= (int64_t)cfg->nkeys; ++keyno) { + if (cfg->ftype == ROW) { + snprintf(keybuf, sizeof(keybuf), "%016u", (u_int)keyno); + cursor->set_key(cursor, keybuf); + } else + cursor->set_key(cursor, (uint32_t)keyno); + value->data = valuebuf; + if (cfg->ftype == FIX) + cursor->set_value(cursor, 0x01); + else { + value->size = (uint32_t)snprintf( + valuebuf, sizeof(valuebuf), "%37u", (u_int)keyno); + cursor->set_value(cursor, value); + } + if ((ret = cursor->insert(cursor)) != 0) + testutil_die(ret, "cursor.insert"); + } + + /* Setup the starting key range for the workload phase. */ + cfg->key_range = cfg->nkeys; + if ((ret = cursor->close(cursor)) != 0) + testutil_die(ret, "cursor.close"); + if ((ret = session->checkpoint(session, NULL)) != 0) + testutil_die(ret, "session.checkpoint"); + + if ((ret = session->close(session, NULL)) != 0) + testutil_die(ret, "session.close"); +} + +void +verify(SHARED_CONFIG *cfg, const char *name) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + int ret; + + conn = cfg->conn; + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + testutil_die(ret, "conn.session"); + + if ((ret = session->verify(session, name, NULL)) != 0) + testutil_die(ret, "session.create"); + + if ((ret = session->close(session, NULL)) != 0) + testutil_die(ret, "session.close"); +} diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c new file mode 100644 index 00000000000..d44505ab2f3 --- /dev/null +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c @@ -0,0 +1,370 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "cursor_order.h" + +static void *append_insert(void *); +static void print_stats(SHARED_CONFIG *); +static void *reverse_scan(void *); + +typedef struct { + char *name; /* object name */ + uint64_t nops; /* Thread op count */ + + WT_RAND_STATE rnd; /* RNG */ + + int append_insert; /* cursor.insert */ + int reverse_scans; /* cursor.prev sequences */ + SHARED_CONFIG *cfg; +} INFO; + +static INFO *run_info; + +int +ops_start(SHARED_CONFIG *cfg) +{ + struct timeval start, stop; + double seconds; + pthread_t *tids; + uint64_t i, name_index, offset, total_nops; + int ret; + void *thread_ret; + + tids = NULL; /* Keep GCC 4.1 happy. */ + total_nops = 0; + + /* Create per-thread structures. */ + if ((run_info = calloc( + (size_t)(cfg->reverse_scanners + cfg->append_inserters), + sizeof(*run_info))) == NULL) + testutil_die(errno, "calloc"); + + if ((tids = calloc( + (size_t)(cfg->reverse_scanners + cfg->append_inserters), + sizeof(*tids))) == NULL) + testutil_die(errno, "calloc"); + + /* Create the files and load the initial records. */ + for (i = 0; i < cfg->append_inserters; ++i) { + run_info[i].cfg = cfg; + if (i == 0 || cfg->multiple_files) { + if ((run_info[i].name = malloc(64)) == NULL) + testutil_die(errno, "malloc"); + snprintf(run_info[i].name, 64, FNAME, (int)i); + + /* Vary by orders of magnitude */ + if (cfg->vary_nops) + run_info[i].nops = + WT_MAX(1000, cfg->max_nops >> i); + load(cfg, run_info[i].name); + } else + run_info[i].name = run_info[0].name; + + /* Setup op count if not varying ops. */ + if (run_info[i].nops == 0) + run_info[i].nops = cfg->max_nops; + total_nops += run_info[i].nops; + } + + /* Setup the reverse scanner configurations */ + for (i = 0; i < cfg->reverse_scanners; ++i) { + offset = i + cfg->append_inserters; + run_info[offset].cfg = cfg; + if (cfg->multiple_files) { + if ((run_info[offset].name = malloc(64)) == NULL) + testutil_die(errno, "malloc"); + /* Have reverse scans read from tables with writes. */ + name_index = i % cfg->append_inserters; + snprintf( + run_info[offset].name, 64, FNAME, (int)name_index); + + /* Vary by orders of magnitude */ + if (cfg->vary_nops) + run_info[offset].nops = + WT_MAX(1000, cfg->max_nops >> name_index); + } else + run_info[offset].name = run_info[0].name; + + /* Setup op count if not varying ops. */ + if (run_info[offset].nops == 0) + run_info[offset].nops = cfg->max_nops; + total_nops += run_info[offset].nops; + } + + (void)gettimeofday(&start, NULL); + + /* Create threads. */ + for (i = 0; i < cfg->reverse_scanners; ++i) + if ((ret = pthread_create( + &tids[i], NULL, reverse_scan, (void *)(uintptr_t)i)) != 0) + testutil_die(ret, "pthread_create"); + for (; i < cfg->reverse_scanners + cfg->append_inserters; ++i) { + if ((ret = pthread_create( + &tids[i], NULL, append_insert, (void *)(uintptr_t)i)) != 0) + testutil_die(ret, "pthread_create"); + } + + /* Wait for the threads. */ + for (i = 0; i < cfg->reverse_scanners + cfg->append_inserters; ++i) + (void)pthread_join(tids[i], &thread_ret); + + (void)gettimeofday(&stop, NULL); + seconds = (stop.tv_sec - start.tv_sec) + + (stop.tv_usec - start.tv_usec) * 1e-6; + fprintf(stderr, "timer: %.2lf seconds (%d ops/second)\n", + seconds, (int)(((cfg->reverse_scanners + cfg->append_inserters) * + total_nops) / seconds)); + + /* Verify the files. */ + for (i = 0; i < cfg->reverse_scanners + cfg->append_inserters; ++i) { + verify(cfg, run_info[i].name); + if (!cfg->multiple_files) + break; + } + + /* Output run statistics. */ + print_stats(cfg); + + /* Free allocated memory. */ + for (i = 0; i < cfg->reverse_scanners + cfg->append_inserters; ++i) { + free(run_info[i].name); + if (!cfg->multiple_files) + break; + } + + free(run_info); + free(tids); + + return (0); +} + +/* + * reverse_scan_op -- + * Walk a cursor back from the end of the file. + */ +static inline void +reverse_scan_op( + SHARED_CONFIG *cfg, WT_SESSION *session, WT_CURSOR *cursor, INFO *s) +{ + uint64_t i, initial_key_range, prev_key, this_key; + int ret; + char *strkey; + + WT_UNUSED(session); + WT_UNUSED(s); + + /* Make GCC 4.1 happy */ + prev_key = this_key = 0; + + /* Reset the cursor */ + if ((ret = cursor->reset(cursor)) != 0) + testutil_die(ret, "cursor.reset"); + + /* Save the key range. */ + initial_key_range = cfg->key_range - cfg->append_inserters; + + for (i = 0; i < cfg->reverse_scan_ops; i++) { + if ((ret = cursor->prev(cursor)) != 0) { + if (ret == WT_NOTFOUND) + break; + testutil_die(ret, "cursor.prev"); + } + + if (cfg->ftype == ROW) { + if ((ret = cursor->get_key(cursor, &strkey)) != 0) + testutil_die(ret, "cursor.get_key"); + this_key = (uint64_t)atol(strkey); + } else + if ((ret = cursor->get_key( + cursor, (uint64_t *)&this_key)) != 0) + testutil_die(ret, "cursor.get_key"); + + if (i == 0 && this_key < initial_key_range) + testutil_die(ret, + "cursor scan start range wrong first prev %" PRIu64 + " initial range: %" PRIu64, + this_key, initial_key_range); + if (i != 0 && this_key >= prev_key) + testutil_die(ret, + "cursor scan out of order this: %" PRIu64 + " prev: %" PRIu64, + this_key, prev_key); + prev_key = this_key; + } +} + +/* + * reverse_scan -- + * Reader thread start function. + */ +static void * +reverse_scan(void *arg) +{ + INFO *s; + SHARED_CONFIG *cfg; + WT_CURSOR *cursor; + WT_SESSION *session; + uintmax_t id; + uint64_t i; + int ret; + char tid[128]; + + id = (uintmax_t)arg; + s = &run_info[id]; + cfg = s->cfg; + __wt_thread_id(tid, sizeof(tid)); + __wt_random_init(&s->rnd); + + printf(" reverse scan thread %2" PRIuMAX + " starting: tid: %s, file: %s\n", + id, tid, s->name); + + __wt_yield(); /* Get all the threads created. */ + + if ((ret = cfg->conn->open_session( + cfg->conn, NULL, "isolation=snapshot", &session)) != 0) + testutil_die(ret, "conn.open_session"); + if ((ret = session->open_cursor( + session, s->name, NULL, NULL, &cursor)) != 0) + testutil_die(ret, "session.open_cursor"); + for (i = 0; i < s->nops && !cfg->thread_finish; + ++i, ++s->reverse_scans, __wt_yield()) + reverse_scan_op(cfg, session, cursor, s); + if ((ret = session->close(session, NULL)) != 0) + testutil_die(ret, "session.close"); + + printf(" reverse scan thread %2" PRIuMAX + " stopping: tid: %s, file: %s\n", + id, tid, s->name); + + /* Notify all other threads to finish once the first thread is done */ + cfg->thread_finish = true; + + return (NULL); +} + +/* + * append_insert_op -- + * Write operation. + */ +static inline void +append_insert_op( + SHARED_CONFIG *cfg, WT_SESSION *session, WT_CURSOR *cursor, INFO *s) +{ + WT_ITEM *value, _value; + uint64_t keyno; + int ret; + char keybuf[64], valuebuf[64]; + + WT_UNUSED(session); + + value = &_value; + + keyno = __wt_atomic_add64(&cfg->key_range, 1); + if (cfg->ftype == ROW) { + snprintf(keybuf, sizeof(keybuf), "%016u", (u_int)keyno); + cursor->set_key(cursor, keybuf); + } else + cursor->set_key(cursor, (uint32_t)keyno); + + ++s->append_insert; + value->data = valuebuf; + if (cfg->ftype == FIX) + cursor->set_value(cursor, 0x10); + else { + value->size = (uint32_t)snprintf( + valuebuf, sizeof(valuebuf), "XXX %37u", (u_int)keyno); + cursor->set_value(cursor, value); + } + if ((ret = cursor->insert(cursor)) != 0) + testutil_die(ret, "cursor.insert"); +} + +/* + * append_insert -- + * Writer thread start function. + */ +static void * +append_insert(void *arg) +{ + INFO *s; + SHARED_CONFIG *cfg; + WT_CURSOR *cursor; + WT_SESSION *session; + uintmax_t id; + uint64_t i; + int ret; + char tid[128]; + + id = (uintmax_t)arg; + s = &run_info[id]; + cfg = s->cfg; + __wt_thread_id(tid, sizeof(tid)); + __wt_random_init(&s->rnd); + + printf("write thread %2" PRIuMAX " starting: tid: %s, file: %s\n", + id, tid, s->name); + + __wt_yield(); /* Get all the threads created. */ + + if ((ret = cfg->conn->open_session( + cfg->conn, NULL, "isolation=snapshot", &session)) != 0) + testutil_die(ret, "conn.open_session"); + if ((ret = session->open_cursor( + session, s->name, NULL, NULL, &cursor)) != 0) + testutil_die(ret, "session.open_cursor"); + for (i = 0; i < s->nops && !cfg->thread_finish; ++i, __wt_yield()) + append_insert_op(cfg, session, cursor, s); + if ((ret = session->close(session, NULL)) != 0) + testutil_die(ret, "session.close"); + + printf("write thread %2" PRIuMAX " stopping: tid: %s, file: %s\n", + id, tid, s->name); + + /* Notify all other threads to finish once the first thread is done */ + cfg->thread_finish = true; + + return (NULL); +} + +/* + * print_stats -- + * Display reverse scan/writer thread stats. + */ +static void +print_stats(SHARED_CONFIG *cfg) +{ + INFO *s; + uint64_t id, total_threads; + + total_threads = cfg->reverse_scanners + cfg->append_inserters; + s = run_info; + for (id = 0; id < total_threads; ++id, ++s) + printf("%3d: reverse scans %6d, append inserts %6d\n", + (int)id, (int)s->reverse_scans, (int)s->append_insert); +} diff --git a/src/third_party/wiredtiger/test/fops/file.c b/src/third_party/wiredtiger/test/fops/file.c index 4cd92e7b590..ea15f1ee80d 100644 --- a/src/third_party/wiredtiger/test/fops/file.c +++ b/src/third_party/wiredtiger/test/fops/file.c @@ -147,7 +147,7 @@ obj_create_unique(int force) /* Generate a unique object name. */ if ((ret = pthread_rwlock_wrlock(&single)) != 0) testutil_die(ret, "pthread_rwlock_wrlock single"); - (void)snprintf(new_uri, sizeof(new_uri), "%s.%d", uri, ++uid); + (void)snprintf(new_uri, sizeof(new_uri), "%s.%u", uri, ++uid); if ((ret = pthread_rwlock_unlock(&single)) != 0) testutil_die(ret, "pthread_rwlock_unlock single"); diff --git a/src/third_party/wiredtiger/test/fops/fops.c b/src/third_party/wiredtiger/test/fops/fops.c index fbc9d9c6048..3333ff16858 100644 --- a/src/third_party/wiredtiger/test/fops/fops.c +++ b/src/third_party/wiredtiger/test/fops/fops.c @@ -109,7 +109,7 @@ fop(void *arg) __wt_random_init(&rnd); for (i = 0; i < nops; ++i, __wt_yield()) - switch (__wt_random(&rnd) % 9) { + switch (__wt_random(&rnd) % 10) { case 0: ++s->bulk; obj_bulk(); diff --git a/src/third_party/wiredtiger/test/fops/t.c b/src/third_party/wiredtiger/test/fops/t.c index 0881c23d7d4..24994404c7c 100644 --- a/src/third_party/wiredtiger/test/fops/t.c +++ b/src/third_party/wiredtiger/test/fops/t.c @@ -50,6 +50,8 @@ static void wt_shutdown(void); extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c index 748494bf841..2b1463bd0e3 100644 --- a/src/third_party/wiredtiger/test/format/backup.c +++ b/src/third_party/wiredtiger/test/format/backup.c @@ -37,20 +37,18 @@ check_copy(void) { WT_CONNECTION *conn; WT_SESSION *session; - int ret; wts_open(g.home_backup, 0, &conn); - if ((ret = conn->open_session( - conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session: %s", g.home_backup); + testutil_checkfmt( + conn->open_session(conn, NULL, NULL, &session), + "%s", g.home_backup); - ret = session->verify(session, g.uri, NULL); - if (ret != 0) - die(ret, "session.verify: %s: %s", g.home_backup, g.uri); + testutil_checkfmt( + session->verify(session, g.uri, NULL), + "%s: %s", g.home_backup, g.uri); - if ((ret = conn->close(conn, NULL)) != 0) - die(ret, "connection.close: %s", g.home_backup); + testutil_checkfmt(conn->close(conn, NULL), "%s", g.home_backup); } /* @@ -62,14 +60,19 @@ copy_file(const char *name) { size_t len; char *cmd; - int ret; len = strlen(g.home) + strlen(g.home_backup) + strlen(name) * 2 + 20; cmd = dmalloc(len); (void)snprintf(cmd, len, "cp %s/%s %s/%s", g.home, name, g.home_backup, name); - if ((ret = system(cmd)) != 0) - die(ret, "backup copy: %s", cmd); + testutil_checkfmt(system(cmd), "backup copy: %s", cmd); + free(cmd); + + len = strlen(g.home) + strlen(g.home_backup2) + strlen(name) * 2 + 20; + cmd = dmalloc(len); + (void)snprintf(cmd, len, + "cp %s/%s %s/%s", g.home, name, g.home_backup2, name); + testutil_checkfmt(system(cmd), "backup copy: %s", cmd); free(cmd); } @@ -96,8 +99,7 @@ backup(void *arg) return (NULL); /* Open a session. */ - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); /* * Perform a backup at somewhere under 10 seconds (so we get at @@ -113,12 +115,12 @@ backup(void *arg) break; /* Lock out named checkpoints */ - if ((ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0) - die(ret, "pthread_rwlock_wrlock: backup lock"); + testutil_check(pthread_rwlock_wrlock(&g.backup_lock)); /* Re-create the backup directory. */ - if ((ret = system(g.home_backup_init)) != 0) - die(ret, "backup directory creation failed"); + testutil_checkfmt( + system(g.home_backup_init), + "%s", "backup directory creation failed"); /* * open_cursor can return EBUSY if a metadata operation is @@ -128,26 +130,21 @@ backup(void *arg) "backup:", NULL, NULL, &backup_cursor)) == EBUSY) sleep(1); if (ret != 0) - die(ret, "session.open_cursor: backup"); + testutil_die(ret, "session.open_cursor: backup"); while ((ret = backup_cursor->next(backup_cursor)) == 0) { - if ((ret = - backup_cursor->get_key(backup_cursor, &key)) != 0) - die(ret, "cursor.get_key"); + testutil_check( + backup_cursor->get_key(backup_cursor, &key)); copy_file(key); } - if ((ret = backup_cursor->close(backup_cursor)) != 0) - die(ret, "cursor.close"); - - if ((ret = pthread_rwlock_unlock(&g.backup_lock)) != 0) - die(ret, "pthread_rwlock_unlock: backup lock"); + testutil_check(backup_cursor->close(backup_cursor)); + testutil_check(pthread_rwlock_unlock(&g.backup_lock)); check_copy(); } - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(session->close(session, NULL)); return (NULL); } diff --git a/src/third_party/wiredtiger/test/format/bdb.c b/src/third_party/wiredtiger/test/format/bdb.c index d7b4bca62f2..823fc8ff888 100644 --- a/src/third_party/wiredtiger/test/format/bdb.c +++ b/src/third_party/wiredtiger/test/format/bdb.c @@ -128,7 +128,7 @@ bdb_np(int next, if ((ret = dbc->get(dbc, &key, &value, next ? DB_NEXT : DB_PREV)) != 0) { if (ret != DB_NOTFOUND) - die(ret, "dbc.get: %s: {%.*s}", + testutil_die(ret, "dbc.get: %s: {%.*s}", next ? "DB_NEXT" : "DB_PREV", (int)key.size, (char *)key.data); *notfoundp = 1; @@ -154,7 +154,7 @@ bdb_read(uint64_t keyno, void *valuep, size_t *valuesizep, int *notfoundp) *notfoundp = 0; if ((ret = dbc->get(dbc, &key, &value, DB_SET)) != 0) { if (ret != DB_NOTFOUND) - die(ret, "dbc.get: DB_SET: {%.*s}", + testutil_die(ret, "dbc.get: DB_SET: {%.*s}", (int)key.size, (char *)key.data); *notfoundp = 1; } else { @@ -178,7 +178,7 @@ bdb_update(const void *arg_key, size_t arg_key_size, *notfoundp = 0; if ((ret = dbc->put(dbc, &key, &value, DB_KEYFIRST)) != 0) { if (ret != DB_NOTFOUND) { - die(ret, "dbc.put: DB_KEYFIRST: {%.*s}{%.*s}", + testutil_die(ret, "dbc.put: DB_KEYFIRST: {%.*s}{%.*s}", (int)key.size, (char *)key.data, (int)value.size, (char *)value.data); } @@ -204,7 +204,7 @@ bdb_remove(uint64_t keyno, int *notfoundp) if ((ret = dbc->del(dbc, 0)) != 0) { if (ret != DB_NOTFOUND) - die(ret, "dbc.del: {%.*s}", + testutil_die(ret, "dbc.del: {%.*s}", (int)key.size, (char *)key.data); *notfoundp = 1; } diff --git a/src/third_party/wiredtiger/test/format/bulk.c b/src/third_party/wiredtiger/test/format/bulk.c index 28189e25b65..64b005d294f 100644 --- a/src/third_party/wiredtiger/test/format/bulk.c +++ b/src/third_party/wiredtiger/test/format/bulk.c @@ -37,13 +37,11 @@ wts_load(void) WT_SESSION *session; uint8_t *keybuf, *valbuf; bool is_bulk; - int ret; conn = g.wts_conn; keybuf = valbuf = NULL; - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, @@ -61,9 +59,8 @@ wts_load(void) if (g.c_reverse) is_bulk = false; - if ((ret = session->open_cursor(session, g.uri, NULL, - is_bulk ? "bulk,append" : NULL, &cursor)) != 0) - die(ret, "session.open_cursor"); + testutil_check(session->open_cursor(session, g.uri, NULL, + is_bulk ? "bulk,append" : NULL, &cursor)); /* Set up the key/value buffers. */ key_gen_setup(&keybuf); @@ -120,8 +117,7 @@ wts_load(void) break; } - if ((ret = cursor->insert(cursor)) != 0) - die(ret, "cursor.insert"); + testutil_check(cursor->insert(cursor)); #ifdef HAVE_BERKELEY_DB if (SINGLETHREADED) @@ -129,15 +125,13 @@ wts_load(void) #endif } - if ((ret = cursor->close(cursor)) != 0) - die(ret, "cursor.close"); + testutil_check(cursor->close(cursor)); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== bulk load stop ==============="); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(session->close(session, NULL)); free(keybuf); free(valbuf); diff --git a/src/third_party/wiredtiger/test/format/compact.c b/src/third_party/wiredtiger/test/format/compact.c index fdfa597e07e..a75ee4f2adf 100644 --- a/src/third_party/wiredtiger/test/format/compact.c +++ b/src/third_party/wiredtiger/test/format/compact.c @@ -48,8 +48,7 @@ compact(void *arg) /* Open a session. */ conn = g.wts_conn; - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); /* * Perform compaction at somewhere under 15 seconds (so we get at @@ -66,11 +65,10 @@ compact(void *arg) if ((ret = session->compact( session, g.uri, NULL)) != 0 && ret != WT_ROLLBACK) - die(ret, "session.compact"); + testutil_die(ret, "session.compact"); } - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(session->close(session, NULL)); return (NULL); } diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index d431546f254..042316d8344 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -138,9 +138,10 @@ config_setup(void) /* Required shared libraries. */ if (DATASOURCE("helium") && access(HELIUM_PATH, R_OK) != 0) - die(errno, "Levyx/helium shared library: %s", HELIUM_PATH); + testutil_die(errno, + "Levyx/helium shared library: %s", HELIUM_PATH); if (DATASOURCE("kvsbdb") && access(KVS_BDB_PATH, R_OK) != 0) - die(errno, "kvsbdb shared library: %s", KVS_BDB_PATH); + testutil_die(errno, "kvsbdb shared library: %s", KVS_BDB_PATH); /* Some data-sources don't support user-specified collations. */ if (DATASOURCE("helium") || DATASOURCE("kvsbdb")) @@ -199,14 +200,15 @@ config_setup(void) if (!config_is_perm("key_max") && g.c_key_max < g.c_key_min) g.c_key_max = g.c_key_min; if (g.c_key_min > g.c_key_max) - die(EINVAL, "key_min may not be larger than key_max"); + testutil_die(EINVAL, "key_min may not be larger than key_max"); if (!config_is_perm("value_min") && g.c_value_min > g.c_value_max) g.c_value_min = g.c_value_max; if (!config_is_perm("value_max") && g.c_value_max < g.c_value_min) g.c_value_max = g.c_value_min; if (g.c_value_min > g.c_value_max) - die(EINVAL, "value_min may not be larger than value_max"); + testutil_die(EINVAL, + "value_min may not be larger than value_max"); /* Reset the key count. */ g.key_cnt = 0; @@ -412,7 +414,7 @@ config_lrt(void) */ if (g.type == FIX) { if (g.c_long_running_txn && config_is_perm("long_running_txn")) - die(EINVAL, + testutil_die(EINVAL, "long_running_txn not supported with fixed-length " "column store"); g.c_long_running_txn = 0; @@ -453,7 +455,7 @@ config_print(int error_display) fp = stdout; else if ((fp = fopen(g.home_config, "w")) == NULL) - die(errno, "fopen: %s", g.home_config); + testutil_die(errno, "fopen: %s", g.home_config); fprintf(fp, "############################################\n"); fprintf(fp, "# RUN PARAMETERS\n"); @@ -487,7 +489,7 @@ config_file(const char *name) char *p, buf[256]; if ((fp = fopen(name, "r")) == NULL) - die(errno, "fopen: %s", name); + testutil_die(errno, "fopen: %s", name); while (fgets(buf, sizeof(buf), fp) != NULL) { for (p = buf; *p != '\0' && *p != '\n'; ++p) ; @@ -582,7 +584,7 @@ config_single(const char *s, int perm) *cp->vstr = strdup(ep); } if (*cp->vstr == NULL) - die(errno, "malloc"); + testutil_die(errno, "malloc"); return; } @@ -625,7 +627,7 @@ config_map_file_type(const char *s, u_int *vp) strcmp(s, "row-store") == 0) *vp = ROW; else - die(EINVAL, "illegal file type configuration: %s", s); + testutil_die(EINVAL, "illegal file type configuration: %s", s); } /* @@ -642,7 +644,7 @@ config_map_checksum(const char *s, u_int *vp) else if (strcmp(s, "uncompressed") == 0) *vp = CHECKSUM_UNCOMPRESSED; else - die(EINVAL, "illegal checksum configuration: %s", s); + testutil_die(EINVAL, "illegal checksum configuration: %s", s); } /* @@ -667,7 +669,8 @@ config_map_compression(const char *s, u_int *vp) else if (strcmp(s, "zlib-noraw") == 0) *vp = COMPRESS_ZLIB_NO_RAW; else - die(EINVAL, "illegal compression configuration: %s", s); + testutil_die(EINVAL, + "illegal compression configuration: %s", s); } /* @@ -682,7 +685,7 @@ config_map_encryption(const char *s, u_int *vp) else if (strcmp(s, "rotn-7") == 0) *vp = ENCRYPT_ROTN_7; else - die(EINVAL, "illegal encryption configuration: %s", s); + testutil_die(EINVAL, "illegal encryption configuration: %s", s); } /* @@ -701,7 +704,7 @@ config_map_isolation(const char *s, u_int *vp) else if (strcmp(s, "snapshot") == 0) *vp = ISOLATION_SNAPSHOT; else - die(EINVAL, "illegal isolation configuration: %s", s); + testutil_die(EINVAL, "illegal isolation configuration: %s", s); } /* diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index d8b11b005d4..a17614bc044 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -246,6 +246,10 @@ static CONFIG c[] = { "minimum gain before prefix compression is used", 0x0, 0, 8, 256, &g.c_prefix_compression_min, NULL }, + { "quiet", + "quiet run (same as -q)", + C_IGNORE|C_BOOL, 0, 0, 0, &g.c_quiet, NULL }, + { "repeat_data_pct", "percent duplicate values in row- or var-length column-stores", 0x0, 0, 90, 90, &g.c_repeat_data_pct, NULL }, diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 41c9de3dd30..a129c5395fd 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -109,6 +109,7 @@ typedef struct { char *home; /* Home directory */ char *home_backup; /* Hot-backup directory */ + char *home_backup2; /* Saved Hot-backup directory */ char *home_backup_init; /* Initialize backup command */ char *home_bdb; /* BDB directory */ char *home_config; /* Run CONFIG file path */ @@ -142,7 +143,6 @@ typedef struct { FILE *logfp; /* Log file */ int replay; /* Replaying a run. */ - int track; /* Track progress */ int workers_finished; /* Operations completed */ pthread_rwlock_t backup_lock; /* Hot backup running */ @@ -210,6 +210,7 @@ typedef struct { uint32_t c_merge_max; uint32_t c_mmap; uint32_t c_ops; + uint32_t c_quiet; uint32_t c_prefix_compression; uint32_t c_prefix_compression_min; uint32_t c_repeat_data_pct; @@ -334,12 +335,6 @@ void wts_salvage(void); void wts_stats(void); void wts_verify(const char *); -void die(int, const char *, ...) -#if defined(__GNUC__) -__attribute__((__noreturn__)) -#endif -; - /* * mmrand -- * Return a random value between a min/max pair. diff --git a/src/third_party/wiredtiger/test/format/lrt.c b/src/third_party/wiredtiger/test/format/lrt.c index b7392829d30..451d2f4fa3c 100644 --- a/src/third_party/wiredtiger/test/format/lrt.c +++ b/src/third_party/wiredtiger/test/format/lrt.c @@ -60,11 +60,9 @@ lrt(void *arg) /* Open a session and cursor. */ conn = g.wts_conn; - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); - if ((ret = session->open_cursor( - session, g.uri, NULL, NULL, &cursor)) != 0) - die(ret, "session.open_cursor"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->open_cursor( + session, g.uri, NULL, NULL, &cursor)); for (pinned = 0;;) { if (pinned) { @@ -73,7 +71,8 @@ lrt(void *arg) &key, saved_keyno, 1)) == WT_ROLLBACK) ; if (ret != 0) - die(ret, "read_row %" PRIu64, saved_keyno); + testutil_die(ret, + "read_row %" PRIu64, saved_keyno); /* Compare the previous value with the current one. */ if (g.type == FIX) { @@ -83,21 +82,19 @@ lrt(void *arg) } else ret = cursor->get_value(cursor, &value); if (ret != 0) - die(ret, + testutil_die(ret, "cursor.get_value: %" PRIu64, saved_keyno); if (buf_size != value.size || memcmp(buf, value.data, value.size) != 0) - die(0, "mismatched start/stop values"); + testutil_die(0, "mismatched start/stop values"); /* End the transaction. */ - if ((ret = - session->commit_transaction(session, NULL)) != 0) - die(ret, "session.commit_transaction"); + testutil_check( + session->commit_transaction(session, NULL)); /* Reset the cursor, releasing our pin. */ - if ((ret = cursor->reset(cursor)) != 0) - die(ret, "cursor.reset"); + testutil_check(cursor->reset(cursor)); pinned = 0; } else { /* @@ -106,9 +103,8 @@ lrt(void *arg) * positioned. As soon as the cursor loses its position * a new snapshot will be allocated. */ - if ((ret = session->begin_transaction( - session, "isolation=snapshot")) != 0) - die(ret, "session.begin_transaction"); + testutil_check(session->begin_transaction( + session, "isolation=snapshot")); /* Read a record at the end of the table. */ do { @@ -120,7 +116,8 @@ lrt(void *arg) ; } while (ret == WT_NOTFOUND); if (ret != 0) - die(ret, "read_row %" PRIu64, saved_keyno); + testutil_die(ret, + "read_row %" PRIu64, saved_keyno); /* Copy the cursor's value. */ if (g.type == FIX) { @@ -130,11 +127,11 @@ lrt(void *arg) } else ret = cursor->get_value(cursor, &value); if (ret != 0) - die(ret, + testutil_die(ret, "cursor.get_value: %" PRIu64, saved_keyno); if (buf_len < value.size && (buf = realloc(buf, buf_len = value.size)) == NULL) - die(errno, "malloc"); + testutil_die(errno, "malloc"); memcpy(buf, value.data, buf_size = value.size); /* @@ -149,7 +146,7 @@ lrt(void *arg) ; } while (ret == WT_NOTFOUND); if (ret != 0) - die(ret, "read_row %" PRIu64, keyno); + testutil_die(ret, "read_row %" PRIu64, keyno); pinned = 1; } @@ -166,8 +163,7 @@ lrt(void *arg) break; } - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(session->close(session, NULL)); free(keybuf); free(buf); diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 36d56df1505..5d66f4d5391 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -56,7 +56,7 @@ wts_ops(int lastrun) pthread_t backup_tid, compact_tid, lrt_tid; int64_t fourths, thread_ops; uint32_t i; - int ret, running; + int running; conn = g.wts_conn; @@ -97,36 +97,32 @@ wts_ops(int lastrun) /* Open a session. */ if (g.logging != 0) { - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); (void)g.wt_api->msg_printf(g.wt_api, session, "=============== thread ops start ==============="); } /* Create thread structure; start the worker threads. */ if ((tinfo = calloc((size_t)g.c_threads, sizeof(*tinfo))) == NULL) - die(errno, "calloc"); + testutil_die(errno, "calloc"); for (i = 0; i < g.c_threads; ++i) { tinfo[i].id = (int)i + 1; tinfo[i].state = TINFO_RUNNING; - if ((ret = - pthread_create(&tinfo[i].tid, NULL, ops, &tinfo[i])) != 0) - die(ret, "pthread_create"); + testutil_check( + pthread_create(&tinfo[i].tid, NULL, ops, &tinfo[i])); } /* * If a multi-threaded run, start optional backup, compaction and * long-running reader threads. */ - if (g.c_backups && - (ret = pthread_create(&backup_tid, NULL, backup, NULL)) != 0) - die(ret, "pthread_create: backup"); - if (g.c_compact && - (ret = pthread_create(&compact_tid, NULL, compact, NULL)) != 0) - die(ret, "pthread_create: compaction"); - if (!SINGLETHREADED && g.c_long_running_txn && - (ret = pthread_create(&lrt_tid, NULL, lrt, NULL)) != 0) - die(ret, "pthread_create: long-running reader"); + if (g.c_backups) + testutil_check(pthread_create(&backup_tid, NULL, backup, NULL)); + if (g.c_compact) + testutil_check( + pthread_create(&compact_tid, NULL, compact, NULL)); + if (!SINGLETHREADED && g.c_long_running_txn) + testutil_check(pthread_create(&lrt_tid, NULL, lrt, NULL)); /* Spin on the threads, calculating the totals. */ for (;;) { @@ -192,8 +188,7 @@ wts_ops(int lastrun) if (g.logging != 0) { (void)g.wt_api->msg_printf(g.wt_api, session, "=============== thread ops stop ==============="); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(session->close(session, NULL)); } } @@ -234,7 +229,7 @@ ops(void *arg) uint32_t op; uint8_t *keybuf, *valbuf; u_int np; - int ckpt_available, dir, insert, intxn, notfound, readonly, ret; + int ckpt_available, dir, insert, intxn, notfound, readonly; char *ckpt_config, ckpt_name[64]; tinfo = arg; @@ -269,9 +264,8 @@ ops(void *arg) */ if (intxn && (tinfo->ops == ckpt_op || tinfo->ops == session_op)) { - if ((ret = session->commit_transaction( - session, NULL)) != 0) - die(ret, "session.commit_transaction"); + testutil_check( + session->commit_transaction(session, NULL)); ++tinfo->commit; intxn = 0; } @@ -279,13 +273,11 @@ ops(void *arg) /* Open up a new session and cursors. */ if (tinfo->ops == session_op || session == NULL || cursor == NULL) { - if (session != NULL && - (ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + if (session != NULL) + testutil_check(session->close(session, NULL)); - if ((ret = conn->open_session(conn, NULL, - ops_session_config(&tinfo->rnd), &session)) != 0) - die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, + ops_session_config(&tinfo->rnd), &session)); /* * 10% of the time, perform some read-only operations @@ -300,9 +292,8 @@ ops(void *arg) */ if (!SINGLETHREADED && !DATASOURCE("lsm") && ckpt_available && mmrand(&tinfo->rnd, 1, 10) == 1) { - if ((ret = session->open_cursor(session, - g.uri, NULL, ckpt_name, &cursor)) != 0) - die(ret, "session.open_cursor"); + testutil_check(session->open_cursor(session, + g.uri, NULL, ckpt_name, &cursor)); /* Pick the next session/cursor close/open. */ session_op += 250; @@ -323,13 +314,12 @@ ops(void *arg) * want to have to specify the record number, * which requires an append configuration. */ - if ((ret = session->open_cursor(session, g.uri, - NULL, "overwrite", &cursor)) != 0) - die(ret, "session.open_cursor"); - if ((g.type == FIX || g.type == VAR) && - (ret = session->open_cursor(session, g.uri, - NULL, "append", &cursor_insert)) != 0) - die(ret, "session.open_cursor"); + testutil_check(session->open_cursor(session, + g.uri, NULL, "overwrite", &cursor)); + if (g.type == FIX || g.type == VAR) + testutil_check(session->open_cursor( + session, g.uri, + NULL, "append", &cursor_insert)); /* Pick the next session/cursor close/open. */ session_op += mmrand(&tinfo->rnd, 100, 5000); @@ -358,21 +348,17 @@ ops(void *arg) } /* Named checkpoints lock out backups */ - if (ckpt_config != NULL && - (ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0) - die(ret, - "pthread_rwlock_wrlock: backup lock"); - - if ((ret = - session->checkpoint(session, ckpt_config)) != 0) - die(ret, "session.checkpoint%s%s", - ckpt_config == NULL ? "" : ": ", - ckpt_config == NULL ? "" : ckpt_config); - - if (ckpt_config != NULL && - (ret = pthread_rwlock_unlock(&g.backup_lock)) != 0) - die(ret, - "pthread_rwlock_wrlock: backup lock"); + if (ckpt_config != NULL) + testutil_check( + pthread_rwlock_wrlock(&g.backup_lock)); + + testutil_checkfmt( + session->checkpoint(session, ckpt_config), + "%s", ckpt_config == NULL ? "" : ckpt_config); + + if (ckpt_config != NULL) + testutil_check( + pthread_rwlock_unlock(&g.backup_lock)); /* Rephrase the checkpoint name for cursor open. */ if (ckpt_config == NULL) @@ -393,8 +379,7 @@ ops(void *arg) * have to do the reset outside of a transaction. */ if (tinfo->ops > reset_op && !intxn) { - if ((ret = session->reset(session)) != 0) - die(ret, "session.reset"); + testutil_check(session->reset(session)); /* Pick the next reset operation. */ reset_op += mmrand(&tinfo->rnd, 20000, 50000); @@ -406,9 +391,8 @@ ops(void *arg) */ if (!SINGLETHREADED && !intxn && mmrand(&tinfo->rnd, 1, 10) >= 8) { - if ((ret = - session->begin_transaction(session, NULL)) != 0) - die(ret, "session.begin_transaction"); + testutil_check( + session->begin_transaction(session, NULL)); intxn = 1; } @@ -466,9 +450,8 @@ ops(void *arg) if (col_insert(tinfo, cursor_insert, &key, &value, &keyno)) goto deadlock; - if ((ret = - cursor_insert->reset(cursor_insert)) != 0) - die(ret, "cursor.reset"); + testutil_check( + cursor_insert->reset(cursor_insert)); insert = 1; break; @@ -518,8 +501,7 @@ skip_insert: if (col_update(tinfo, goto deadlock; /* Reset the cursor: there is no reason to keep pages pinned. */ - if ((ret = cursor->reset(cursor)) != 0) - die(ret, "cursor.reset"); + testutil_check(cursor->reset(cursor)); /* * If we're in the transaction, commit 40% of the time and @@ -528,9 +510,8 @@ skip_insert: if (col_update(tinfo, if (intxn) switch (mmrand(&tinfo->rnd, 1, 10)) { case 1: case 2: case 3: case 4: /* 40% */ - if ((ret = session->commit_transaction( - session, NULL)) != 0) - die(ret, "session.commit_transaction"); + testutil_check(session->commit_transaction( + session, NULL)); ++tinfo->commit; intxn = 0; break; @@ -538,10 +519,8 @@ skip_insert: if (col_update(tinfo, if (0) { deadlock: ++tinfo->deadlock; } - if ((ret = session->rollback_transaction( - session, NULL)) != 0) - die(ret, - "session.rollback_transaction"); + testutil_check(session->rollback_transaction( + session, NULL)); ++tinfo->rollback; intxn = 0; break; @@ -550,8 +529,8 @@ deadlock: ++tinfo->deadlock; } } - if (session != NULL && (ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + if (session != NULL) + testutil_check(session->close(session, NULL)); free(keybuf); free(valbuf); @@ -573,7 +552,6 @@ wts_read_scan(void) WT_SESSION *session; uint64_t cnt, last_cnt; uint8_t *keybuf; - int ret; conn = g.wts_conn; @@ -581,12 +559,10 @@ wts_read_scan(void) key_gen_setup(&keybuf); /* Open a session and cursor pair. */ - if ((ret = conn->open_session( - conn, NULL, ops_session_config(NULL), &session)) != 0) - die(ret, "connection.open_session"); - if ((ret = session->open_cursor( - session, g.uri, NULL, NULL, &cursor)) != 0) - die(ret, "session.open_cursor"); + testutil_check(conn->open_session( + conn, NULL, ops_session_config(NULL), &session)); + testutil_check(session->open_cursor( + session, g.uri, NULL, NULL, &cursor)); /* Check a random subset of the records using the key. */ for (last_cnt = cnt = 0; cnt < g.key_cnt;) { @@ -599,12 +575,11 @@ wts_read_scan(void) } key.data = keybuf; - if ((ret = read_row(cursor, &key, cnt, 0)) != 0) - die(ret, "read_scan"); + testutil_checkfmt( + read_row(cursor, &key, cnt, 0), "%s", "read_scan"); } - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(session->close(session, NULL)); free(keybuf); } @@ -666,7 +641,7 @@ read_row(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int notfound_err) return (WT_NOTFOUND); break; default: - die(ret, "read_row: read row %" PRIu64, keyno); + testutil_die(ret, "read_row: read row %" PRIu64, keyno); } #ifdef HAVE_BERKELEY_DB @@ -703,7 +678,7 @@ read_row(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int notfound_err) "read_row: value mismatch %" PRIu64 ":\n", keyno); print_item("bdb", &bdb_value); print_item(" wt", &value); - die(0, NULL); + testutil_die(0, NULL); } } #endif @@ -748,7 +723,7 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp) break; } if (ret != 0 && ret != WT_NOTFOUND) - die(ret, "%s", which); + testutil_die(ret, "%s", which); *notfoundp = (ret == WT_NOTFOUND); #ifdef HAVE_BERKELEY_DB @@ -777,7 +752,7 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp) fprintf(stderr, "nextprev: %s key mismatch:\n", which); print_item("bdb-key", &bdb_key); print_item(" wt-key", &key); - die(0, NULL); + testutil_die(0, NULL); } } else { if (keyno != (uint64_t)atoll(bdb_key.data)) { @@ -787,7 +762,7 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp) "nextprev: %s key mismatch: %.*s != %" PRIu64 "\n", which, (int)bdb_key.size, (char *)bdb_key.data, keyno); - die(0, NULL); + testutil_die(0, NULL); } } if (value.size != bdb_value.size || @@ -795,7 +770,7 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp) fprintf(stderr, "nextprev: %s value mismatch:\n", which); print_item("bdb-value", &bdb_value); print_item(" wt-value", &value); - die(0, NULL); + testutil_die(0, NULL); } if (g.logging == LOG_OPS) @@ -851,7 +826,8 @@ row_update(TINFO *tinfo, if (ret == WT_ROLLBACK) return (WT_ROLLBACK); if (ret != 0 && ret != WT_NOTFOUND) - die(ret, "row_update: update row %" PRIu64 " by key", keyno); + testutil_die(ret, + "row_update: update row %" PRIu64 " by key", keyno); #ifdef HAVE_BERKELEY_DB if (!SINGLETHREADED) @@ -905,7 +881,7 @@ col_update(TINFO *tinfo, if (ret == WT_ROLLBACK) return (WT_ROLLBACK); if (ret != 0 && ret != WT_NOTFOUND) - die(ret, "col_update: %" PRIu64, keyno); + testutil_die(ret, "col_update: %" PRIu64, keyno); #ifdef HAVE_BERKELEY_DB if (!SINGLETHREADED) @@ -937,7 +913,7 @@ table_append_init(void) free(g.append); if ((g.append = calloc(g.append_max, sizeof(uint64_t))) == NULL) - die(errno, "calloc"); + testutil_die(errno, "calloc"); } /* @@ -948,7 +924,7 @@ static void table_append(uint64_t keyno) { uint64_t *p, *ep; - int done, ret; + int done; ep = g.append + g.append_max; @@ -979,8 +955,7 @@ table_append(uint64_t keyno) * and we find a slot. */ for (done = 0;;) { - if ((ret = pthread_rwlock_wrlock(&g.append_lock)) != 0) - die(ret, "pthread_rwlock_wrlock: append_lock"); + testutil_check(pthread_rwlock_wrlock(&g.append_lock)); /* * If this is the thread we've been waiting for, and its record @@ -1017,8 +992,7 @@ table_append(uint64_t keyno) break; } - if ((ret = pthread_rwlock_unlock(&g.append_lock)) != 0) - die(ret, "pthread_rwlock_unlock: append_lock"); + testutil_check(pthread_rwlock_unlock(&g.append_lock)); if (done) break; @@ -1055,7 +1029,8 @@ row_insert(TINFO *tinfo, if (ret == WT_ROLLBACK) return (WT_ROLLBACK); if (ret != 0 && ret != WT_NOTFOUND) - die(ret, "row_insert: insert row %" PRIu64 " by key", keyno); + testutil_die(ret, + "row_insert: insert row %" PRIu64 " by key", keyno); #ifdef HAVE_BERKELEY_DB if (!SINGLETHREADED) @@ -1094,10 +1069,9 @@ col_insert(TINFO *tinfo, if ((ret = cursor->insert(cursor)) != 0) { if (ret == WT_ROLLBACK) return (WT_ROLLBACK); - die(ret, "cursor.insert"); + testutil_die(ret, "cursor.insert"); } - if ((ret = cursor->get_key(cursor, &keyno)) != 0) - die(ret, "cursor.get_key"); + testutil_check(cursor->get_key(cursor, &keyno)); *keynop = (uint32_t)keyno; table_append(keyno); /* Extend the object. */ @@ -1157,7 +1131,8 @@ row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp) if (ret == WT_ROLLBACK) return (WT_ROLLBACK); if (ret != 0 && ret != WT_NOTFOUND) - die(ret, "row_remove: remove %" PRIu64 " by key", keyno); + testutil_die(ret, + "row_remove: remove %" PRIu64 " by key", keyno); *notfoundp = (ret == WT_NOTFOUND); #ifdef HAVE_BERKELEY_DB @@ -1200,7 +1175,8 @@ col_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp) if (ret == WT_ROLLBACK) return (WT_ROLLBACK); if (ret != 0 && ret != WT_NOTFOUND) - die(ret, "col_remove: remove %" PRIu64 " by key", keyno); + testutil_die(ret, + "col_remove: remove %" PRIu64 " by key", keyno); *notfoundp = (ret == WT_NOTFOUND); #ifdef HAVE_BERKELEY_DB @@ -1245,7 +1221,7 @@ notfound_chk(const char *f, int wt_ret, int bdb_notfound, uint64_t keyno) fprintf(stderr, " row %" PRIu64 ":", keyno); fprintf(stderr, " not found in Berkeley DB, found in WiredTiger\n"); - die(0, NULL); + testutil_die(0, NULL); } if (wt_ret == WT_NOTFOUND) { fprintf(stderr, "%s: %s:", g.progname, f); @@ -1253,7 +1229,7 @@ notfound_chk(const char *f, int wt_ret, int bdb_notfound, uint64_t keyno) fprintf(stderr, " row %" PRIu64 ":", keyno); fprintf(stderr, " found in Berkeley DB, not found in WiredTiger\n"); - die(0, NULL); + testutil_die(0, NULL); } return (0); } diff --git a/src/third_party/wiredtiger/test/format/rebalance.c b/src/third_party/wiredtiger/test/format/rebalance.c index 8e8fa1a371f..d35dcec1d53 100644 --- a/src/third_party/wiredtiger/test/format/rebalance.c +++ b/src/third_party/wiredtiger/test/format/rebalance.c @@ -33,7 +33,6 @@ wts_rebalance(void) { WT_CONNECTION *conn; WT_SESSION *session; - int ret; char cmd[1024]; if (g.c_rebalance == 0) @@ -45,26 +44,23 @@ wts_rebalance(void) (void)snprintf(cmd, sizeof(cmd), "../../wt -h %s dump -f %s/rebalance.orig %s", g.home, g.home, g.uri); - if ((ret = system(cmd)) != 0) - die(ret, "command failed: %s", cmd); + testutil_checkfmt(system(cmd), "command failed: %s", cmd); /* Rebalance, then verify the object. */ wts_reopen(); conn = g.wts_conn; - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== rebalance start ==============="); - if ((ret = session->rebalance(session, g.uri, NULL)) != 0) - die(ret, "session.rebalance: %s: %s", g.uri); + testutil_checkfmt( + session->rebalance(session, g.uri, NULL), "%s", g.uri); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== rebalance stop ==============="); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(session->close(session, NULL)); wts_verify("post-rebalance verify"); wts_close(); @@ -72,13 +68,11 @@ wts_rebalance(void) (void)snprintf(cmd, sizeof(cmd), "../../wt -h %s dump -f %s/rebalance.new %s", g.home, g.home, g.uri); - if ((ret = system(cmd)) != 0) - die(ret, "command failed: %s", cmd); + testutil_checkfmt(system(cmd), "command failed: %s", cmd); /* Compare the old/new versions of the object. */ (void)snprintf(cmd, sizeof(cmd), "cmp %s/rebalance.orig %s/rebalance.new > /dev/null", g.home, g.home); - if ((ret = system(cmd)) != 0) - die(ret, "command failed: %s", cmd); + testutil_checkfmt(system(cmd), "command failed: %s", cmd); } diff --git a/src/third_party/wiredtiger/test/format/salvage.c b/src/third_party/wiredtiger/test/format/salvage.c index d0358e998b4..526e1563390 100644 --- a/src/third_party/wiredtiger/test/format/salvage.c +++ b/src/third_party/wiredtiger/test/format/salvage.c @@ -42,12 +42,10 @@ salvage(void) conn = g.wts_conn; track("salvage", 0ULL, NULL); - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); if ((ret = session->salvage(session, g.uri, "force=true")) != 0) - die(ret, "session.salvage: %s", g.uri); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_die(ret, "session.salvage: %s", g.uri); + testutil_check(session->close(session, NULL)); } /* @@ -101,37 +99,37 @@ corrupt(void) return (0); found: if (fstat(fd, &sb) == -1) - die(errno, "salvage-corrupt: fstat"); + testutil_die(errno, "salvage-corrupt: fstat"); offset = mmrand(NULL, 0, (u_int)sb.st_size); len = (size_t)(20 + (sb.st_size / 100) * 2); (void)snprintf(buf, sizeof(buf), "%s/slvg.corrupt", g.home); if ((fp = fopen(buf, "w")) == NULL) - die(errno, "salvage-corrupt: open: %s", buf); + testutil_die(errno, "salvage-corrupt: open: %s", buf); (void)fprintf(fp, "salvage-corrupt: offset %" PRIuMAX ", length " SIZET_FMT "\n", (uintmax_t)offset, len); fclose_and_clear(&fp); if (lseek(fd, offset, SEEK_SET) == -1) - die(errno, "salvage-corrupt: lseek"); + testutil_die(errno, "salvage-corrupt: lseek"); memset(buf, 'z', sizeof(buf)); for (; len > 0; len -= nw) { nw = (size_t)(len > sizeof(buf) ? sizeof(buf) : len); if (write(fd, buf, nw) == -1) - die(errno, "salvage-corrupt: write"); + testutil_die(errno, "salvage-corrupt: write"); } if (close(fd) == -1) - die(errno, "salvage-corrupt: close"); + testutil_die(errno, "salvage-corrupt: close"); /* * Save a copy of the corrupted file so we can replay the salvage step * as necessary. */ if ((ret = system(copycmd)) != 0) - die(ret, "salvage corrupt copy step failed"); + testutil_die(ret, "salvage corrupt copy step failed"); return (1); } @@ -157,7 +155,7 @@ wts_salvage(void) * step as necessary. */ if ((ret = system(g.home_salvage_copy)) != 0) - die(ret, "salvage copy step failed"); + testutil_die(ret, "salvage copy step failed"); /* Salvage, then verify. */ wts_open(g.home, 1, &g.wts_conn); diff --git a/src/third_party/wiredtiger/test/format/t.c b/src/third_party/wiredtiger/test/format/t.c index ccbc0442e4a..28c22e23cb8 100644 --- a/src/third_party/wiredtiger/test/format/t.c +++ b/src/third_party/wiredtiger/test/format/t.c @@ -30,17 +30,20 @@ GLOBAL g; +static void format_die(void); static void startup(void); static void usage(void); extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = format_die; /* Local death handler. */ + int main(int argc, char *argv[]) { time_t start; - int ch, i, onerun, reps, ret; + int ch, i, onerun, reps; const char *config, *home; config = NULL; @@ -64,7 +67,7 @@ main(int argc, char *argv[]) #endif /* Track progress unless we're re-directing output to a file. */ - g.track = isatty(1) ? 1 : 0; + g.c_quiet = isatty(1) ? 0 : 1; /* Set values from the command line. */ home = NULL; @@ -99,7 +102,7 @@ main(int argc, char *argv[]) g.logging = LOG_OPS; break; case 'q': /* Quiet */ - g.track = 0; + g.c_quiet = 1; break; case 'r': /* Replay a run */ g.replay = 1; @@ -125,9 +128,9 @@ main(int argc, char *argv[]) /* If it's a replay, use the home directory's CONFIG file. */ if (g.replay) { if (config != NULL) - die(EINVAL, "-c incompatible with -r"); + testutil_die(EINVAL, "-c incompatible with -r"); if (access(g.home_config, R_OK) != 0) - die(ENOENT, "%s", g.home_config); + testutil_die(ENOENT, "%s", g.home_config); config = g.home_config; } @@ -176,12 +179,9 @@ main(int argc, char *argv[]) * Initialize locks to single-thread named checkpoints and backups, last * last-record updates, and failures. */ - if ((ret = pthread_rwlock_init(&g.append_lock, NULL)) != 0) - die(ret, "pthread_rwlock_init: append lock"); - if ((ret = pthread_rwlock_init(&g.backup_lock, NULL)) != 0) - die(ret, "pthread_rwlock_init: backup lock"); - if ((ret = pthread_rwlock_init(&g.death_lock, NULL)) != 0) - die(ret, "pthread_rwlock_init: death lock"); + testutil_check(pthread_rwlock_init(&g.append_lock, NULL)); + testutil_check(pthread_rwlock_init(&g.backup_lock, NULL)); + testutil_check(pthread_rwlock_init(&g.death_lock, NULL)); printf("%s: process %" PRIdMAX "\n", g.progname, (intmax_t)getpid()); while (++g.run_cnt <= g.c_runs || g.c_runs == 0 ) { @@ -259,7 +259,7 @@ main(int argc, char *argv[]) wts_salvage(); /* Overwrite the progress line with a completion line. */ - if (g.track) + if (!g.c_quiet) printf("\r%78s\r", " "); printf("%4d: %s, %s (%.0f seconds)\n", g.run_cnt, g.c_data_source, @@ -273,10 +273,8 @@ main(int argc, char *argv[]) config_print(0); - if ((ret = pthread_rwlock_destroy(&g.append_lock)) != 0) - die(ret, "pthread_rwlock_destroy: append lock"); - if ((ret = pthread_rwlock_destroy(&g.backup_lock)) != 0) - die(ret, "pthread_rwlock_destroy: backup lock"); + testutil_check(pthread_rwlock_destroy(&g.append_lock)); + testutil_check(pthread_rwlock_destroy(&g.backup_lock)); config_clear(); @@ -298,41 +296,33 @@ startup(void) /* Create or initialize the home and data-source directories. */ if ((ret = system(g.home_init)) != 0) - die(ret, "home directory initialization failed"); + testutil_die(ret, "home directory initialization failed"); /* Open/truncate the logging file. */ if (g.logging != 0 && (g.logfp = fopen(g.home_log, "w")) == NULL) - die(errno, "fopen: %s", g.home_log); + testutil_die(errno, "fopen: %s", g.home_log); /* Open/truncate the random number logging file. */ if ((g.randfp = fopen(g.home_rand, g.replay ? "r" : "w")) == NULL) - die(errno, "%s", g.home_rand); + testutil_die(errno, "%s", g.home_rand); } /* * die -- - * Report an error and quit, dumping the configuration. + * Report an error, dumping the configuration. */ -void -die(int e, const char *fmt, ...) +static void +format_die(void) { - va_list ap; - - /* Single-thread error handling. */ + /* + * Single-thread error handling, our caller exits after calling + * us - don't release the lock. + */ (void)pthread_rwlock_wrlock(&g.death_lock); /* Try and turn off tracking so it doesn't obscure the error message. */ - if (g.track) { - g.track = 0; - fprintf(stderr, "\n"); - } - if (fmt != NULL) { /* Death message. */ - fprintf(stderr, "%s: ", g.progname); - va_start(ap, fmt); - vfprintf(stderr, fmt, ap); - va_end(ap); - if (e != 0) - fprintf(stderr, ": %s", wiredtiger_strerror(e)); + if (!g.c_quiet) { + g.c_quiet = 1; fprintf(stderr, "\n"); } @@ -343,8 +333,6 @@ die(int e, const char *fmt, ...) /* Display the configuration that failed. */ if (g.run_cnt) config_print(1); - - exit(EXIT_FAILURE); } /* diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index 2b6b9d67fc3..2e4c869366c 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -42,7 +42,7 @@ dmalloc(size_t len) void *p; if ((p = malloc(len)) == NULL) - die(errno, "malloc"); + testutil_die(errno, "malloc"); return (p); } @@ -56,7 +56,7 @@ dstrdup(const char *str) char *p; if ((p = strdup(str)) == NULL) - die(errno, "strdup"); + testutil_die(errno, "strdup"); return (p); } @@ -236,7 +236,7 @@ track(const char *tag, uint64_t cnt, TINFO *tinfo) int len; char msg[128]; - if (!g.track || tag == NULL) + if (g.c_quiet || tag == NULL) return; if (tinfo == NULL && cnt == 0) @@ -268,9 +268,9 @@ track(const char *tag, uint64_t cnt, TINFO *tinfo) lastlen = len; if (printf("%s\r", msg) < 0) - die(EIO, "printf"); + testutil_die(EIO, "printf"); if (fflush(stdout) == EOF) - die(errno, "fflush"); + testutil_die(errno, "fflush"); } /* @@ -310,6 +310,10 @@ path_setup(const char *home) g.home_backup = dmalloc(len); snprintf(g.home_backup, len, "%s/%s", g.home, "BACKUP"); + len = strlen(g.home) + strlen("BACKUP2") + 2; + g.home_backup2 = dmalloc(len); + snprintf(g.home_backup2, len, "%s/%s", g.home, "BACKUP2"); + /* BDB directory. */ len = strlen(g.home) + strlen("bdb") + 2; g.home_bdb = dmalloc(len); @@ -340,13 +344,15 @@ path_setup(const char *home) /* Backup directory initialize command, remove and re-create it. */ #undef CMD #ifdef _WIN32 -#define CMD "del /s /q >:nul && mkdir %s" +#define CMD "del /s /q >:nul && mkdir %s %s" #else -#define CMD "rm -rf %s && mkdir %s" +#define CMD "rm -rf %s %s && mkdir %s %s" #endif - len = strlen(g.home_backup) * 2 + strlen(CMD) + 1; + len = strlen(g.home_backup) * 2 + + strlen(g.home_backup2) * 2 + strlen(CMD) + 1; g.home_backup_init = dmalloc(len); - snprintf(g.home_backup_init, len, CMD, g.home_backup, g.home_backup); + snprintf(g.home_backup_init, len, CMD, g.home_backup, g.home_backup2, + g.home_backup, g.home_backup2); /* * Salvage command, save the interesting files so we can replay the @@ -407,7 +413,7 @@ rng(WT_RAND_STATE *rnd) "\n" "end of random number log reached\n"); exit(EXIT_SUCCESS); } - die(errno, "random number log"); + testutil_die(errno, "random number log"); } return ((uint32_t)strtoul(buf, NULL, 10)); @@ -435,6 +441,6 @@ fclose_and_clear(FILE **fpp) return; *fpp = NULL; if (fclose(fp) != 0) - die(errno, "fclose"); + testutil_die(errno, "fclose"); return; } diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index 9d4d3fe5cb8..81e484296e2 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -53,7 +53,8 @@ compressor(uint32_t compress_flag) default: break; } - die(EINVAL, "illegal compression flag: 0x%x", compress_flag); + testutil_die(EINVAL, + "illegal compression flag: %#" PRIx32, compress_flag); } /* @@ -71,7 +72,8 @@ encryptor(uint32_t encrypt_flag) default: break; } - die(EINVAL, "illegal encryption flag: 0x%x", encrypt_flag); + testutil_die(EINVAL, + "illegal encryption flag: %#" PRIx32, encrypt_flag); } static int @@ -222,7 +224,8 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) p += snprintf(p, REMAIN(p, end), ",%s", g.config_open); if (REMAIN(p, end) == 0) - die(ENOMEM, "wiredtiger_open configuration buffer too small"); + testutil_die(ENOMEM, + "wiredtiger_open configuration buffer too small"); /* * Direct I/O may not work with backups, doing copies through the buffer @@ -233,8 +236,8 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) if (strstr(config, "direct_io") != NULL) g.c_backups = 0; - if ((ret = wiredtiger_open(home, &event_handler, config, &conn)) != 0) - die(ret, "wiredtiger_open: %s", home); + testutil_checkfmt( + wiredtiger_open(home, &event_handler, config, &conn), "%s", home); if (set_api) g.wt_api = conn->get_extension_api(conn); @@ -247,7 +250,7 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) */ if (DATASOURCE("helium")) { if (g.helium_mount == NULL) - die(EINVAL, "no Helium mount point specified"); + testutil_die(EINVAL, "no Helium mount point specified"); (void)snprintf(helium_config, sizeof(helium_config), "entry=wiredtiger_extension_init,config=[" "helium_verbose=0," @@ -256,7 +259,7 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) g.helium_mount); if ((ret = conn->load_extension( conn, HELIUM_PATH, helium_config)) != 0) - die(ret, + testutil_die(ret, "WT_CONNECTION.load_extension: %s:%s", HELIUM_PATH, helium_config); } @@ -270,11 +273,8 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) void wts_reopen(void) { - int ret; - - if ((ret = wiredtiger_open(g.home, - &event_handler, g.wiredtiger_open_config, &g.wts_conn)) != 0) - die(ret, "wiredtiger_open: %s", g.home); + testutil_checkfmt(wiredtiger_open(g.home, &event_handler, + g.wiredtiger_open_config, &g.wts_conn), "%s", g.home); } /* @@ -287,7 +287,6 @@ wts_create(void) WT_CONNECTION *conn; WT_SESSION *session; uint32_t maxintlpage, maxintlkey, maxleafpage, maxleafkey, maxleafvalue; - int ret; char config[4096], *end, *p; conn = g.wts_conn; @@ -316,7 +315,7 @@ wts_create(void) p += snprintf(p, REMAIN(p, end), "key_format=%s," "allocation_size=512,%s" - "internal_page_max=%d,leaf_page_max=%d", + "internal_page_max=%" PRIu32 ",leaf_page_max=%" PRIu32, (g.type == ROW) ? "u" : "r", g.c_firstfit ? "block_allocation=first," : "", maxintlpage, maxleafpage); @@ -328,15 +327,15 @@ wts_create(void) maxintlkey = mmrand(NULL, maxintlpage / 50, maxintlpage / 40); if (maxintlkey > 20) p += snprintf(p, REMAIN(p, end), - ",internal_key_max=%d", maxintlkey); + ",internal_key_max=%" PRIu32, maxintlkey); maxleafkey = mmrand(NULL, maxleafpage / 50, maxleafpage / 40); if (maxleafkey > 20) p += snprintf(p, REMAIN(p, end), - ",leaf_key_max=%d", maxleafkey); + ",leaf_key_max=%" PRIu32, maxleafkey); maxleafvalue = mmrand(NULL, maxleafpage * 10, maxleafpage / 40); if (maxleafvalue > 40 && maxleafvalue < 100 * 1024) p += snprintf(p, REMAIN(p, end), - ",leaf_value_max=%d", maxleafvalue); + ",leaf_value_max=%" PRIu32, maxleafvalue); switch (g.type) { case FIX: @@ -364,7 +363,7 @@ wts_create(void) ",huffman_value=english"); if (g.c_dictionary) p += snprintf(p, REMAIN(p, end), - ",dictionary=%d", mmrand(NULL, 123, 517)); + ",dictionary=%" PRIu32, mmrand(NULL, 123, 517)); break; } @@ -431,32 +430,28 @@ wts_create(void) } if (REMAIN(p, end) == 0) - die(ENOMEM, "WT_SESSION.create configuration buffer too small"); + testutil_die(ENOMEM, + "WT_SESSION.create configuration buffer too small"); /* * Create the underlying store. */ - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); - if ((ret = session->create(session, g.uri, config)) != 0) - die(ret, "session.create: %s", g.uri); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_checkfmt(session->create(session, g.uri, config), "%s", g.uri); + testutil_check(session->close(session, NULL)); } void wts_close(void) { WT_CONNECTION *conn; - int ret; const char *config; conn = g.wts_conn; config = g.c_leak_memory ? "leak_memory" : NULL; - if ((ret = conn->close(conn, config)) != 0) - die(ret, "connection.close"); + testutil_check(conn->close(conn, config)); g.wts_conn = NULL; g.wt_api = NULL; } @@ -466,7 +461,6 @@ wts_dump(const char *tag, int dump_bdb) { #ifdef HAVE_BERKELEY_DB size_t len; - int ret; char *cmd; /* @@ -491,8 +485,7 @@ wts_dump(const char *tag, int dump_bdb) g.uri == NULL ? "" : "-n", g.uri == NULL ? "" : g.uri); - if ((ret = system(cmd)) != 0) - die(ret, "%s: dump comparison failed", tag); + testutil_checkfmt(system(cmd), "%s: dump comparison failed", tag); free(cmd); #else (void)tag; /* [-Wunused-variable] */ @@ -513,8 +506,7 @@ wts_verify(const char *tag) conn = g.wts_conn; track("verify", 0ULL, NULL); - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== verify start ==============="); @@ -522,13 +514,12 @@ wts_verify(const char *tag) /* Session operations for LSM can return EBUSY. */ ret = session->verify(session, g.uri, "strict"); if (ret != 0 && !(ret == EBUSY && DATASOURCE("lsm"))) - die(ret, "session.verify: %s: %s", g.uri, tag); + testutil_die(ret, "session.verify: %s: %s", g.uri, tag); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== verify stop ==============="); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(session->close(session, NULL)); } /* @@ -558,49 +549,43 @@ wts_stats(void) conn = g.wts_conn; track("stat", 0ULL, NULL); - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); if ((fp = fopen(g.home_stats, "w")) == NULL) - die(errno, "fopen: %s", g.home_stats); + testutil_die(errno, "fopen: %s", g.home_stats); /* Connection statistics. */ fprintf(fp, "====== Connection statistics:\n"); - if ((ret = session->open_cursor(session, - "statistics:", NULL, NULL, &cursor)) != 0) - die(ret, "session.open_cursor"); + testutil_check(session->open_cursor( + session, "statistics:", NULL, NULL, &cursor)); while ((ret = cursor->next(cursor)) == 0 && (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) if (fprintf(fp, "%s=%s\n", desc, pval) < 0) - die(errno, "fprintf"); + testutil_die(errno, "fprintf"); if (ret != WT_NOTFOUND) - die(ret, "cursor.next"); - if ((ret = cursor->close(cursor)) != 0) - die(ret, "cursor.close"); + testutil_die(ret, "cursor.next"); + testutil_check(cursor->close(cursor)); /* Data source statistics. */ fprintf(fp, "\n\n====== Data source statistics:\n"); stat_name = dmalloc(strlen("statistics:") + strlen(g.uri) + 1); sprintf(stat_name, "statistics:%s", g.uri); - if ((ret = session->open_cursor( - session, stat_name, NULL, NULL, &cursor)) != 0) - die(ret, "session.open_cursor"); + testutil_check(session->open_cursor( + session, stat_name, NULL, NULL, &cursor)); free(stat_name); while ((ret = cursor->next(cursor)) == 0 && (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) if (fprintf(fp, "%s=%s\n", desc, pval) < 0) - die(errno, "fprintf"); + testutil_die(errno, "fprintf"); if (ret != WT_NOTFOUND) - die(ret, "cursor.next"); - if ((ret = cursor->close(cursor)) != 0) - die(ret, "cursor.close"); + testutil_die(ret, "cursor.next"); + testutil_check(cursor->close(cursor)); fclose_and_clear(&fp); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + testutil_check(session->close(session, NULL)); } diff --git a/src/third_party/wiredtiger/test/huge/huge.c b/src/third_party/wiredtiger/test/huge/huge.c index d09f6f375fb..ad19035ff99 100644 --- a/src/third_party/wiredtiger/test/huge/huge.c +++ b/src/third_party/wiredtiger/test/huge/huge.c @@ -167,6 +167,8 @@ run(CONFIG *cp, int bigkey, size_t bytes) extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/src/third_party/wiredtiger/test/manydbs/Makefile.am b/src/third_party/wiredtiger/test/manydbs/Makefile.am new file mode 100644 index 00000000000..53559b25243 --- /dev/null +++ b/src/third_party/wiredtiger/test/manydbs/Makefile.am @@ -0,0 +1,13 @@ +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/test/utility + +noinst_PROGRAMS = t +t_SOURCES = manydbs.c +t_LDADD = $(top_builddir)/libwiredtiger.la +t_LDFLAGS = -static + +# Run this during a "make check" smoke test. +TESTS = smoke.sh + +clean-local: + rm -rf WiredTiger* *.core __* diff --git a/src/third_party/wiredtiger/test/manydbs/manydbs.c b/src/third_party/wiredtiger/test/manydbs/manydbs.c new file mode 100644 index 00000000000..1d3412a7b06 --- /dev/null +++ b/src/third_party/wiredtiger/test/manydbs/manydbs.c @@ -0,0 +1,264 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <sys/wait.h> +#include <errno.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#ifndef _WIN32 +#include <unistd.h> +#endif + +#include <wiredtiger.h> + +#include "test_util.i" + +#define HOME_SIZE 512 +#define HOME_BASE "WT_HOME" +static char home[HOME_SIZE]; /* Base home directory */ +static char hometmp[HOME_SIZE]; /* Each conn home directory */ +static const char *progname; /* Program name */ +static const char * const uri = "table:main"; + +#define WTOPEN_CFG_COMMON \ + "create,log=(file_max=10M,archive=false,enabled)," \ + "statistics=(fast),statistics_log=(wait=5)," +#define WT_CONFIG0 \ + WTOPEN_CFG_COMMON \ + "transaction_sync=(enabled=false)" +#define WT_CONFIG1 \ + WTOPEN_CFG_COMMON \ + "transaction_sync=(enabled,method=none)" +#define WT_CONFIG2 \ + WTOPEN_CFG_COMMON \ + "transaction_sync=(enabled,method=fsync)" + +#define MAX_DBS 10 +#define MAX_IDLE_TIME 30 +#define IDLE_INCR 5 + +#define MAX_KV 100 +#define MAX_VAL 128 + +static void +usage(void) +{ + fprintf(stderr, + "usage: %s [-I] [-D maxdbs] [-h dir]\n", progname); + exit(EXIT_FAILURE); +} + +extern int __wt_optind; +extern char *__wt_optarg; + +void (*custom_die)(void) = NULL; + +WT_CONNECTION **connections = NULL; +WT_CURSOR **cursors = NULL; +WT_RAND_STATE rnd; +WT_SESSION **sessions = NULL; + +static int +get_stat(WT_SESSION *stat_session, int stat_field, uint64_t *valuep) +{ + WT_CURSOR *statc; + const char *desc, *pvalue; + int ret; + + testutil_check(stat_session->open_cursor(stat_session, + "statistics:", NULL, NULL, &statc)); + statc->set_key(statc, stat_field); + if ((ret = statc->search(statc)) != 0) + return (ret); + + ret = statc->get_value(statc, &desc, &pvalue, valuep); + testutil_check(statc->close(statc)); + return (ret); +} + +static int +run_ops(int dbs) +{ + WT_ITEM data; + int db_set, i, key; + uint32_t db; + uint8_t buf[MAX_VAL]; + + memset(buf, 0, sizeof(buf)); + for (i = 0; i < MAX_VAL; ++i) + buf[i] = (uint8_t)__wt_random(&rnd); + data.data = buf; + /* + * Write a small amount of data into a random subset of the databases. + */ + db_set = dbs / 4; + for (i = 0; i < db_set; ++i) { + db = __wt_random(&rnd) % (uint32_t)dbs; + printf("Write to database %" PRIu32 "\n", db); + for (key = 0; key < MAX_KV; ++key) { + data.size = __wt_random(&rnd) % MAX_VAL; + cursors[db]->set_key(cursors[db], key); + cursors[db]->set_value(cursors[db], &data); + testutil_check(cursors[db]->insert(cursors[db])); + } + } + return (0); +} + +int +main(int argc, char *argv[]) +{ + uint64_t cond_reset, cond_wait; + uint64_t *cond_reset_orig; + int cfg, ch, dbs, i; + bool idle; + const char *working_dir, *wt_cfg; + char cmd[128]; + + if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) + progname = argv[0]; + else + ++progname; + dbs = MAX_DBS; + working_dir = HOME_BASE; + idle = false; + while ((ch = __wt_getopt(progname, argc, argv, "D:h:I")) != EOF) + switch (ch) { + case 'D': + dbs = atoi(__wt_optarg); + break; + case 'h': + working_dir = __wt_optarg; + break; + case 'I': + idle = true; + break; + default: + usage(); + } + argc -= __wt_optind; + argv += __wt_optind; + if (argc != 0) + usage(); + + /* + * Allocate arrays for connection handles, sessions, statistics + * cursors and, if needed, data cursors. + */ + if ((connections = calloc( + (size_t)dbs, sizeof(WT_CONNECTION *))) == NULL) + testutil_die(ENOMEM, "connection array malloc"); + if ((sessions = calloc( + (size_t)dbs, sizeof(WT_SESSION *))) == NULL) + testutil_die(ENOMEM, "session array malloc"); + if ((cond_reset_orig = calloc((size_t)dbs, sizeof(uint64_t))) == NULL) + testutil_die(ENOMEM, "orig stat malloc"); + if (!idle && ((cursors = calloc( + (size_t)dbs, sizeof(WT_CURSOR *))) == NULL)) + testutil_die(ENOMEM, "cursor array malloc"); + memset(cmd, 0, sizeof(cmd)); + /* + * Set up all the directory names. + */ + testutil_work_dir_from_path(home, HOME_SIZE, working_dir); + testutil_make_work_dir(home); + __wt_random_init(&rnd); + for (i = 0; i < dbs; ++i) { + snprintf(hometmp, HOME_SIZE, "%s/%s.%d", home, HOME_BASE, i); + testutil_make_work_dir(hometmp); + /* + * Open each database. Rotate different configurations + * among them. Open a session and statistics cursor. + * If writing data, create the table and open a data cursor. + */ + cfg = i % 3; + if (cfg == 0) + wt_cfg = WT_CONFIG0; + else if (cfg == 1) + wt_cfg = WT_CONFIG1; + else + wt_cfg = WT_CONFIG2; + testutil_check(wiredtiger_open( + hometmp, NULL, wt_cfg, &connections[i])); + testutil_check(connections[i]->open_session(connections[i], + NULL, NULL, &sessions[i])); + if (!idle) { + testutil_check(sessions[i]->create(sessions[i], + uri, "key_format=Q,value_format=u")); + testutil_check(sessions[i]->open_cursor(sessions[i], + uri, NULL, NULL, &cursors[i])); + } + } + + sleep(10); + + /* + * Record original reset setting. There could have been some + * activity during the creation period. + */ + for (i = 0; i < dbs; ++i) + testutil_check(get_stat(sessions[i], + WT_STAT_CONN_COND_AUTO_WAIT_RESET, &cond_reset_orig[i])); + for (i = 0; i < MAX_IDLE_TIME; i += IDLE_INCR) { + if (!idle) + testutil_check(run_ops(dbs)); + printf("Sleep %d (%d of %d)\n", IDLE_INCR, i, MAX_IDLE_TIME); + sleep(IDLE_INCR); + } + for (i = 0; i < dbs; ++i) { + testutil_check(get_stat(sessions[i], + WT_STAT_CONN_COND_AUTO_WAIT_RESET, &cond_reset)); + testutil_check(get_stat(sessions[i], + WT_STAT_CONN_COND_AUTO_WAIT, &cond_wait)); + /* + * On an idle workload there should be no resets of condition + * variables during the idle period. Even with a light + * workload, resets should not be very common. We look for 5%. + */ + if (idle && cond_reset != cond_reset_orig[i]) + testutil_die(ERANGE, + "condition reset on idle connection %d of %" PRIu64, + i, cond_reset); + if (!idle && cond_reset > cond_wait / 20) + testutil_die(ERANGE, "connection %d condition reset %" + PRIu64 " exceeds 5%% of %" PRIu64, + i, cond_reset, cond_wait); + testutil_check(connections[i]->close(connections[i], NULL)); + } + + /* Cleanup allocated memory. */ + free(connections); + free(sessions); + free(cond_reset_orig); + if (!idle) + free(cursors); + + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/test/manydbs/smoke.sh b/src/third_party/wiredtiger/test/manydbs/smoke.sh new file mode 100755 index 00000000000..c0e2976f154 --- /dev/null +++ b/src/third_party/wiredtiger/test/manydbs/smoke.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +set -e + +# Smoke-test format as part of running "make check". +# Run with: +# 1. The defaults +# 2. Set idle flag to turn off operations. +# 3. More dbs. +# +echo "manydbs: default with operations turned on" +$TEST_WRAPPER ./t +echo "manydbs: totally idle databases" +$TEST_WRAPPER ./t -I +echo "manydbs: 40 databases with operations" +$TEST_WRAPPER ./t -D 40 +echo "manydbs: 40 idle databases" +$TEST_WRAPPER ./t -I -D 40 diff --git a/src/third_party/wiredtiger/test/readonly/Makefile.am b/src/third_party/wiredtiger/test/readonly/Makefile.am new file mode 100644 index 00000000000..3abcd2386a1 --- /dev/null +++ b/src/third_party/wiredtiger/test/readonly/Makefile.am @@ -0,0 +1,13 @@ +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/test/utility + +noinst_PROGRAMS = t +t_SOURCES = readonly.c +t_LDADD = $(top_builddir)/libwiredtiger.la +t_LDFLAGS = -static + +# Run this during a "make check" smoke test. +TESTS = smoke.sh + +clean-local: + rm -rf WT_RD* WiredTiger* *.core __* diff --git a/src/third_party/wiredtiger/test/readonly/readonly.c b/src/third_party/wiredtiger/test/readonly/readonly.c new file mode 100644 index 00000000000..41400da2605 --- /dev/null +++ b/src/third_party/wiredtiger/test/readonly/readonly.c @@ -0,0 +1,409 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <sys/wait.h> +#include <errno.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#ifndef _WIN32 +#include <unistd.h> +#endif + +#include <wiredtiger.h> + +#include "test_util.i" + +#define HOME_SIZE 512 +static char home[HOME_SIZE]; /* Program working dir lock file */ +#define HOME_WR_SUFFIX ".WRNOLOCK" /* Writable dir copy no lock file */ +static char home_wr[HOME_SIZE + sizeof(HOME_WR_SUFFIX)]; +#define HOME_RD_SUFFIX ".RD" /* Read-only dir */ +static char home_rd[HOME_SIZE + sizeof(HOME_RD_SUFFIX)]; +#define HOME_RD2_SUFFIX ".RDNOLOCK" /* Read-only dir no lock file */ +static char home_rd2[HOME_SIZE + sizeof(HOME_RD2_SUFFIX)]; + +static const char *progname; /* Program name */ +static const char *saved_argv0; /* Program command */ +static const char * const uri = "table:main"; + +#define ENV_CONFIG \ + "create,log=(file_max=10M,archive=false,enabled)," \ + "transaction_sync=(enabled,method=none)" +#define ENV_CONFIG_RD "readonly=true" +#define ENV_CONFIG_WR "readonly=false" +#define MAX_VAL 4096 +#define MAX_KV 10000 + +#define EXPECT_ERR 1 +#define EXPECT_SUCCESS 0 + +#define OP_READ 0 +#define OP_WRITE 1 + +static void +usage(void) +{ + fprintf(stderr, "usage: %s [-h dir]\n", progname); + exit(EXIT_FAILURE); +} + +static int +run_child(const char *homedir, int op, int expect) +{ + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_SESSION *session; + int i, ret; + const char *cfg; + + /* + * We expect the read-only database will allow the second read-only + * handle to succeed because no one can create or set the lock file. + */ + if (op == OP_READ) + cfg = ENV_CONFIG_RD; + else + cfg = ENV_CONFIG_WR; + if ((ret = wiredtiger_open(homedir, NULL, cfg, &conn)) == 0) { + if (expect == EXPECT_ERR) + testutil_die( + ret, "wiredtiger_open expected error, succeeded"); + } else { + if (expect == EXPECT_SUCCESS) + testutil_die( + ret, "wiredtiger_open expected success, error"); + /* + * If we expect an error and got one, we're done. + */ + return (0); + } + + /* + * Make sure we can read the data. + */ + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + testutil_die(ret, "WT_CONNECTION:open_session"); + + if ((ret = + session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0) + testutil_die(ret, "WT_SESSION.open_cursor: %s", uri); + + i = 0; + while ((ret = cursor->next(cursor)) == 0) + ++i; + if (i != MAX_KV) + testutil_die(EPERM, "cursor walk"); + if ((ret = conn->close(conn, NULL)) != 0) + testutil_die(ret, "conn_close"); + return (0); +} + +/* + * Child process opens both databases readonly. + */ +static void +open_dbs(int op, const char *dir, + const char *dir_wr, const char *dir_rd, const char *dir_rd2) +{ + int expect, ret; + + /* + * The parent has an open connection to all directories. + * We expect opening the writeable homes to return an error. + * It is a failure if the child successfully opens that. + */ + expect = EXPECT_ERR; + if ((ret = run_child(dir, op, expect)) != 0) + testutil_die(ret, "wiredtiger_open readonly allowed"); + if ((ret = run_child(dir_wr, op, expect)) != 0) + testutil_die(ret, "wiredtiger_open readonly allowed"); + + /* + * The parent must have a read-only connection open to the + * read-only databases. If the child is opening read-only + * too, we expect success. Otherwise an error if the child + * attempts to open read/write (permission error). + */ + if (op == OP_READ) + expect = EXPECT_SUCCESS; + if ((ret = run_child(dir_rd, op, expect)) != 0) + testutil_die(ret, "run child 1"); + if ((ret = run_child(dir_rd2, op, expect)) != 0) + testutil_die(ret, "run child 2"); + exit(EXIT_SUCCESS); +} + +extern int __wt_optind; +extern char *__wt_optarg; + +void (*custom_die)(void) = NULL; + +int +main(int argc, char *argv[]) +{ + WT_CONNECTION *conn, *conn2, *conn3, *conn4; + WT_CURSOR *cursor; + WT_ITEM data; + WT_SESSION *session; + uint64_t i; + int ch, status, op, ret; + bool child; + const char *working_dir; + char cmd[512]; + uint8_t buf[MAX_VAL]; + + if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) + progname = argv[0]; + else + ++progname; + /* + * Needed unaltered for system command later. + */ + saved_argv0 = argv[0]; + + working_dir = "WT_RD"; + child = false; + op = OP_READ; + while ((ch = __wt_getopt(progname, argc, argv, "Rh:W")) != EOF) + switch (ch) { + case 'R': + child = true; + op = OP_READ; + break; + case 'W': + child = true; + op = OP_WRITE; + break; + case 'h': + working_dir = __wt_optarg; + break; + default: + usage(); + } + argc -= __wt_optind; + argv += __wt_optind; + if (argc != 0) + usage(); + + /* + * Set up all the directory names. + */ + testutil_work_dir_from_path(home, sizeof(home), working_dir); + (void)snprintf(home_wr, sizeof(home_wr), "%s%s", home, HOME_WR_SUFFIX); + (void)snprintf(home_rd, sizeof(home_rd), "%s%s", home, HOME_RD_SUFFIX); + (void)snprintf( + home_rd2, sizeof(home_rd2), "%s%s", home, HOME_RD2_SUFFIX); + if (!child) { + testutil_make_work_dir(home); + testutil_make_work_dir(home_wr); + testutil_make_work_dir(home_rd); + testutil_make_work_dir(home_rd2); + } else + /* + * We are a child process, we just want to call + * the open_dbs with the directories we have. + * The child function will exit. + */ + open_dbs(op, home, home_wr, home_rd, home_rd2); + + /* + * Parent creates a database and table. Then cleanly shuts down. + * Then copy database to read-only directory and chmod. + * Also copy database to read-only directory and remove the lock + * file. One read-only database will have a lock file in the + * file system and the other will not. + * Parent opens all databases with read-only configuration flag. + * Parent forks off child who tries to also open all databases + * with the read-only flag. It should error on the writeable + * directory, but allow it on the read-only directories. + * The child then confirms it can read all the data. + */ + /* + * Run in the home directory and create the table. + */ + if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG, &conn)) != 0) + testutil_die(ret, "wiredtiger_open"); + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + testutil_die(ret, "WT_CONNECTION:open_session"); + if ((ret = session->create(session, + uri, "key_format=Q,value_format=u")) != 0) + testutil_die(ret, "WT_SESSION.create: %s", uri); + if ((ret = + session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0) + testutil_die(ret, "WT_SESSION.open_cursor: %s", uri); + + /* + * Write data into the table and then cleanly shut down connection. + */ + memset(buf, 0, sizeof(buf)); + data.data = buf; + data.size = MAX_VAL; + for (i = 0; i < MAX_KV; ++i) { + cursor->set_key(cursor, i); + cursor->set_value(cursor, &data); + if ((ret = cursor->insert(cursor)) != 0) + testutil_die(ret, "WT_CURSOR.insert"); + } + if ((ret = conn->close(conn, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + + /* + * Copy the database. Remove any lock file from one copy + * and chmod the copies to be read-only permissions. + */ + (void)snprintf(cmd, sizeof(cmd), + "cp -rp %s/* %s; rm -f %s/WiredTiger.lock", + home, home_wr, home_wr); + (void)system(cmd); + + (void)snprintf(cmd, sizeof(cmd), + "cp -rp %s/* %s; chmod 0555 %s; chmod -R 0444 %s/*", + home, home_rd, home_rd, home_rd); + (void)system(cmd); + + (void)snprintf(cmd, sizeof(cmd), + "cp -rp %s/* %s; rm -f %s/WiredTiger.lock; " + "chmod 0555 %s; chmod -R 0444 %s/*", + home, home_rd2, home_rd2, home_rd2, home_rd2); + (void)system(cmd); + + /* + * Run four scenarios. Sometimes expect errors, sometimes success. + * The writable database directories should always fail to allow the + * child to open due to the lock file. The read-only ones will only + * succeed when the child attempts read-only. + * + * 1. Parent has read-only handle to all databases. Child opens + * read-only also. + * 2. Parent has read-only handle to all databases. Child opens + * read-write. + * 3. Parent has read-write handle to writable databases and + * read-only to read-only databases. Child opens read-only. + * 4. Parent has read-write handle to writable databases and + * read-only to read-only databases. Child opens read-write. + */ + /* + * Open a connection handle to all databases. + */ + fprintf(stderr, " *** Expect several error messages from WT ***\n"); + /* + * Scenario 1. + */ + if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG_RD, &conn)) != 0) + testutil_die(ret, "wiredtiger_open original home"); + if ((ret = wiredtiger_open(home_wr, NULL, ENV_CONFIG_RD, &conn2)) != 0) + testutil_die(ret, "wiredtiger_open write nolock"); + if ((ret = wiredtiger_open(home_rd, NULL, ENV_CONFIG_RD, &conn3)) != 0) + testutil_die(ret, "wiredtiger_open readonly"); + if ((ret = wiredtiger_open(home_rd2, NULL, ENV_CONFIG_RD, &conn4)) != 0) + testutil_die(ret, "wiredtiger_open readonly nolock"); + + /* + * Create a child to also open a connection handle to the databases. + * We cannot use fork here because using fork the child inherits the + * same memory image. Therefore the WT process structure is set in + * the child even though it should not be. So use 'system' to spawn + * an entirely new process. + */ + (void)snprintf( + cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir); + if ((status = system(cmd)) < 0) + testutil_die(status, "system"); + /* + * The child will exit with success if its test passes. + */ + if (WEXITSTATUS(status) != 0) + testutil_die(WEXITSTATUS(status), "system"); + + /* + * Scenario 2. Run child with writable config. + */ + (void)snprintf( + cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir); + if ((status = system(cmd)) < 0) + testutil_die(status, "system"); + + if (WEXITSTATUS(status) != 0) + testutil_die(WEXITSTATUS(status), "system"); + + /* + * Reopen the two writable directories and rerun the child. + */ + if ((ret = conn->close(conn, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + if ((ret = conn2->close(conn2, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG_RD, &conn)) != 0) + testutil_die(ret, "wiredtiger_open original home"); + if ((ret = wiredtiger_open(home_wr, NULL, ENV_CONFIG_RD, &conn2)) != 0) + testutil_die(ret, "wiredtiger_open write nolock"); + /* + * Scenario 3. Child read-only. + */ + (void)snprintf( + cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir); + if ((status = system(cmd)) < 0) + testutil_die(status, "system"); + if (WEXITSTATUS(status) != 0) + testutil_die(WEXITSTATUS(status), "system"); + + /* + * Scenario 4. Run child with writable config. + */ + (void)snprintf( + cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir); + if ((status = system(cmd)) < 0) + testutil_die(status, "system"); + if (WEXITSTATUS(status) != 0) + testutil_die(WEXITSTATUS(status), "system"); + + /* + * Clean-up. + */ + if ((ret = conn->close(conn, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + if ((ret = conn2->close(conn2, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + if ((ret = conn3->close(conn3, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + if ((ret = conn4->close(conn4, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + /* + * We need to chmod the read-only databases back so that they can + * be removed by scripts. + */ + (void)snprintf(cmd, sizeof(cmd), "chmod 0777 %s %s", home_rd, home_rd2); + (void)system(cmd); + (void)snprintf(cmd, sizeof(cmd), "chmod -R 0666 %s/* %s/*", + home_rd, home_rd2); + (void)system(cmd); + printf(" *** Readonly test successful ***\n"); + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/test/readonly/smoke.sh b/src/third_party/wiredtiger/test/readonly/smoke.sh new file mode 100755 index 00000000000..740deb5743a --- /dev/null +++ b/src/third_party/wiredtiger/test/readonly/smoke.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +trap 'chmod -R u+w WT_*; exit 0' 0 1 2 3 13 15 + +set -e + +# Smoke-test format as part of running "make check". +$TEST_WRAPPER ./t diff --git a/src/third_party/wiredtiger/test/recovery/random-abort.c b/src/third_party/wiredtiger/test/recovery/random-abort.c index ddcafbc80fd..f9c3ed28814 100644 --- a/src/third_party/wiredtiger/test/recovery/random-abort.c +++ b/src/third_party/wiredtiger/test/recovery/random-abort.c @@ -42,7 +42,7 @@ static char home[512]; /* Program working dir */ static const char *progname; /* Program name */ -static const char *uri = "table:main"; +static const char * const uri = "table:main"; #define RECORDS_FILE "records" @@ -88,7 +88,8 @@ fill_db(void) /* * Run in the home directory so that the records file is in there too. */ - chdir(home); + if (chdir(home) != 0) + testutil_die(errno, "chdir: %s", home); if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) @@ -109,7 +110,7 @@ fill_db(void) /* * Set to no buffering. */ - setvbuf(fp, NULL, _IONBF, 0); + (void)setvbuf(fp, NULL, _IONBF, 0); /* * Write data into the table until we are killed by the parent. @@ -135,6 +136,8 @@ fill_db(void) extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { @@ -201,13 +204,15 @@ main(int argc, char *argv[]) printf("Kill child\n"); if (kill(pid, SIGKILL) != 0) testutil_die(errno, "kill"); - waitpid(pid, &status, 0); + if (waitpid(pid, &status, 0) == -1) + testutil_die(errno, "waitpid"); /* * !!! If we wanted to take a copy of the directory before recovery, * this is the place to do it. */ - chdir(home); + if (chdir(home) != 0) + testutil_die(errno, "chdir: %s", home); printf("Open database, run recovery and verify content\n"); if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); @@ -239,13 +244,15 @@ main(int argc, char *argv[]) ++absent; } } - fclose(fp); + if (fclose(fp) != 0) + testutil_die(errno, "fclose"); if ((ret = conn->close(conn, NULL)) != 0) testutil_die(ret, "WT_CONNECTION:close"); if (absent) { - printf("%u record(s) absent from %u\n", absent, count); + printf("%" PRIu32 " record(s) absent from %" PRIu32 "\n", + absent, count); return (EXIT_FAILURE); } - printf("%u records verified\n", count); + printf("%" PRIu32 " records verified\n", count); return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/recovery/truncated-log.c b/src/third_party/wiredtiger/test/recovery/truncated-log.c index 4add7a61f66..67fdb932c27 100644 --- a/src/third_party/wiredtiger/test/recovery/truncated-log.c +++ b/src/third_party/wiredtiger/test/recovery/truncated-log.c @@ -45,7 +45,7 @@ static char home[512]; /* Program working dir */ static const char *progname; /* Program name */ -static const char *uri = "table:main"; +static const char * const uri = "table:main"; #define RECORDS_FILE "records" @@ -54,7 +54,6 @@ static const char *uri = "table:main"; "transaction_sync=(enabled,method=none)" #define ENV_CONFIG_REC "log=(recover=on)" #define LOG_FILE_1 "WiredTigerLog.0000000001" -#define MAX_VAL 4096 #define K_SIZE 16 #define V_SIZE 256 @@ -86,7 +85,8 @@ fill_db(void) /* * Run in the home directory so that the records file is in there too. */ - chdir(home); + if (chdir(home) != 0) + testutil_die(errno, "chdir: %s", home); if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) @@ -107,7 +107,7 @@ fill_db(void) /* * Set to no buffering. */ - setvbuf(fp, NULL, _IONBF, 0); + (void)setvbuf(fp, NULL, _IONBF, 0); save_lsn.l.file = 0; /* @@ -156,18 +156,23 @@ fill_db(void) "%" PRIu32 " %" PRIu32 "\n", save_lsn.l.offset, i - 1) == -1) testutil_die(errno, "fprintf"); - fclose(fp); - abort(); + break; } } first = false; } } + if (fclose(fp) != 0) + testutil_die(errno, "fclose"); + abort(); + /* NOTREACHED */ } extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { @@ -218,26 +223,32 @@ main(int argc, char *argv[]) /* parent */ /* Wait for child to kill itself. */ - waitpid(pid, &status, 0); + if (waitpid(pid, &status, 0) == -1) + testutil_die(errno, "waitpid"); /* * !!! If we wanted to take a copy of the directory before recovery, * this is the place to do it. */ - chdir(home); + if (chdir(home) != 0) + testutil_die(errno, "chdir: %s", home); + printf("Open database, run recovery and verify content\n"); if ((fp = fopen(RECORDS_FILE, "r")) == NULL) testutil_die(errno, "fopen"); ret = fscanf(fp, "%" SCNu64 " %" SCNu32 "\n", &offset, &max_key); - fclose(fp); if (ret != 2) testutil_die(errno, "fscanf"); + if (fclose(fp) != 0) + testutil_die(errno, "fclose"); /* * The offset is the beginning of the last record. Truncate to * the middle of that last record (i.e. ahead of that offset). */ + if (offset > UINT64_MAX - V_SIZE) + testutil_die(ERANGE, "offset"); new_offset = offset + V_SIZE; - printf("Parent: Truncate to %u\n", (uint32_t)new_offset); + printf("Parent: Truncate to %" PRIu64 "\n", new_offset); if ((ret = truncate(LOG_FILE_1, (wt_off_t)new_offset)) != 0) testutil_die(errno, "truncate"); @@ -260,9 +271,10 @@ main(int argc, char *argv[]) if ((ret = conn->close(conn, NULL)) != 0) testutil_die(ret, "WT_CONNECTION:close"); if (count > max_key) { - printf("expected %u records found %u\n", max_key, count); + printf("expected %" PRIu32 " records found %" PRIu32 "\n", + max_key, count); return (EXIT_FAILURE); } - printf("%u records verified\n", count); + printf("%" PRIu32 " records verified\n", count); return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/salvage/salvage.c b/src/third_party/wiredtiger/test/salvage/salvage.c index c2ad6224b11..a1517d70787 100644 --- a/src/third_party/wiredtiger/test/salvage/salvage.c +++ b/src/third_party/wiredtiger/test/salvage/salvage.c @@ -64,6 +64,8 @@ static int verbose; /* -v flag */ extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/src/third_party/wiredtiger/test/suite/helper.py b/src/third_party/wiredtiger/test/suite/helper.py index 3c460e23d08..f85d708880f 100644 --- a/src/third_party/wiredtiger/test/suite/helper.py +++ b/src/third_party/wiredtiger/test/suite/helper.py @@ -107,7 +107,10 @@ def copy_wiredtiger_home(olddir, newdir, aligned=True): for fname in os.listdir(olddir): fullname = os.path.join(olddir, fname) # Skip lock file, on Windows it is locked. - if os.path.isfile(fullname) and "WiredTiger.lock" not in fullname: + # Skip temporary log files. + if os.path.isfile(fullname) and "WiredTiger.lock" not in fullname and \ + "WiredTigerTmplog" not in fullname and \ + "WiredTigerPreplog" not in fullname: # Use a dd command that does not align on a block boundary. if aligned: shutil.copy(fullname, newdir) @@ -196,31 +199,36 @@ def complex_populate_index_count(): # config: prefix of the session.create configuration string # rows: entries to insert def complex_populate(self, uri, config, rows): - complex_populate_type(self, uri, config, rows, '') + complex_populate_type(self, uri, config, '', rows, '') +def complex_populate_cgconfig(self, uri, config, rows): + complex_populate_type(self, uri, config, config, rows, '') def complex_populate_lsm(self, uri, config, rows): - complex_populate_type(self, uri, config, rows, 'type=lsm') -def complex_populate_type(self, uri, config, rows, type): + complex_populate_type(self, uri, config, '', rows, 'type=lsm') +def complex_populate_cgconfig_lsm(self, uri, config, rows): + complex_populate_type(self, uri, config, config, rows, 'type=lsm') +def complex_populate_type(self, uri, config, cgconfig, rows, type): self.session.create(uri, config + ',value_format=SiSS,' + 'columns=(record,column2,column3,column4,column5),' + 'colgroups=(cgroup1,cgroup2,cgroup3,cgroup4,cgroup5,cgroup6)') cgname = 'colgroup:' + uri.split(":")[1] - self.session.create(cgname + ':cgroup1', 'columns=(column2)' + ',' + type) - self.session.create(cgname + ':cgroup2', 'columns=(column3)' + ',' + type) - self.session.create(cgname + ':cgroup3', 'columns=(column4)' + ',' + type) + cgcfg = ',' + cgconfig + ',' + type + self.session.create(cgname + ':cgroup1', 'columns=(column2)' + ',' + cgcfg) + self.session.create(cgname + ':cgroup2', 'columns=(column3)' + ',' + cgcfg) + self.session.create(cgname + ':cgroup3', 'columns=(column4)' + ',' + cgcfg) self.session.create( - cgname + ':cgroup4', 'columns=(column2,column3)' + ',' + type) + cgname + ':cgroup4', 'columns=(column2,column3)' + ',' + cgcfg) self.session.create( - cgname + ':cgroup5', 'columns=(column3,column4)' + ',' + type) + cgname + ':cgroup5', 'columns=(column3,column4)' + ',' + cgcfg) self.session.create( - cgname + ':cgroup6', 'columns=(column2,column4,column5)' + ',' + type) + cgname + ':cgroup6', 'columns=(column2,column4,column5)' + ',' + cgcfg) indxname = 'index:' + uri.split(":")[1] - self.session.create(indxname + ':indx1', 'columns=(column2)' + ',' + type) - self.session.create(indxname + ':indx2', 'columns=(column3)' + ',' + type) - self.session.create(indxname + ':indx3', 'columns=(column4)' + ',' + type) + self.session.create(indxname + ':indx1', 'columns=(column2)' + ',' + cgcfg) + self.session.create(indxname + ':indx2', 'columns=(column3)' + ',' + cgcfg) + self.session.create(indxname + ':indx3', 'columns=(column4)' + ',' + cgcfg) self.session.create( - indxname + ':indx4', 'columns=(column2,column4)' + ',' + type) + indxname + ':indx4', 'columns=(column2,column4)' + ',' + cgcfg) cursor = self.session.open_cursor(uri, None) for i in range(1, rows + 1): cursor[key_populate(cursor, i)] = \ @@ -228,9 +236,9 @@ def complex_populate_type(self, uri, config, rows, type): cursor.close() # add some indices after populating self.session.create( - indxname + ':indx5', 'columns=(column3,column5)' + ',' + type) + indxname + ':indx5', 'columns=(column3,column5)' + ',' + cgcfg) self.session.create( - indxname + ':indx6', 'columns=(column3,column5,column4)' + ',' + type) + indxname + ':indx6', 'columns=(column3,column5,column4)' + ',' + cgcfg) def complex_populate_colgroup_name(self, uri, i): return 'colgroup:' + uri.split(":")[1] + ':cgroup' + str(i + 1) diff --git a/src/third_party/wiredtiger/test/suite/test_backup05.py b/src/third_party/wiredtiger/test/suite/test_backup05.py index 8b176d0f7d7..991a9f71b19 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup05.py +++ b/src/third_party/wiredtiger/test/suite/test_backup05.py @@ -44,14 +44,6 @@ class test_backup05(wttest.WiredTigerTestCase, suite_subprocess): create_params = 'key_format=i,value_format=i' freq = 5 - def copy_windows(self, olddir, newdir): - os.mkdir(newdir) - for fname in os.listdir(olddir): - fullname = os.path.join(olddir, fname) - # Skip lock file on Windows since it is locked - if os.path.isfile(fullname) and "WiredTiger.lock" not in fullname: - shutil.copy(fullname, newdir) - def check_manual_backup(self, i, olddir, newdir): ''' Simulate a manual backup from olddir and restart in newdir. ''' self.session.checkpoint() @@ -71,7 +63,7 @@ class test_backup05(wttest.WiredTigerTestCase, suite_subprocess): session.verify(self.uri) conn.close() - def test_backup(self): + def backup(self): '''Check manual fsyncLock backup strategy''' # Here's the strategy: @@ -95,5 +87,9 @@ class test_backup05(wttest.WiredTigerTestCase, suite_subprocess): else: self.session.verify(self.uri) + def test_backup(self): + with self.expectedStdoutPattern('Recreating metadata'): + self.backup() + if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bug008.py b/src/third_party/wiredtiger/test/suite/test_bug008.py index 8f0526d9cef..0243887e258 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug008.py +++ b/src/third_party/wiredtiger/test/suite/test_bug008.py @@ -33,65 +33,208 @@ import wiredtiger, wttest from helper import simple_populate, key_populate, value_populate from wtscenario import check_scenarios -# Tests for invisible updates. +# Test search/search-near operations, including invisible values and keys +# past the end of the table. class test_bug008(wttest.WiredTigerTestCase): + uri = 'file:test_bug008' # This is a btree layer test. scenarios = check_scenarios([ - ('fix', dict(fmt='key_format=r,value_format=8t', empty=1)), - ('row', dict(fmt='key_format=S', empty=0)), - ('var', dict(fmt='key_format=r', empty=0)) + ('fix', dict(fmt='key_format=r,value_format=8t', empty=1, colvar=0)), + ('row', dict(fmt='key_format=S', empty=0, colvar=0)), + ('var', dict(fmt='key_format=r', empty=0, colvar=1)) ]) + # Verify cursor search and search-near operations in an empty table. + def test_search_empty(self): + # Create the object and open a cursor. + self.session.create(self.uri, self.fmt) + cursor = self.session.open_cursor(self.uri, None) + + # Search for a record past the end of the table, which should fail. + cursor.set_key(key_populate(cursor, 100)) + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + + # Search-near for a record past the end of the table, which should fail. + cursor.set_key(key_populate(cursor, 100)) + self.assertEqual(cursor.search_near(), wiredtiger.WT_NOTFOUND) + + # Verify cursor search and search-near operations at and past the end of + # a file, with a set of on-page visible records. + def test_search_eot(self): + # Populate the tree and reopen the connection, forcing it to disk + # and moving the records to an on-page format. + simple_populate(self, self.uri, self.fmt, 100) + self.reopen_conn() + + # Open a cursor. + cursor = self.session.open_cursor(self.uri, None) + + # Search for a record at the end of the table, which should succeed. + cursor.set_key(key_populate(cursor, 100)) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) + self.assertEqual(cursor.get_value(), value_populate(cursor, 100)) + + # Search-near for a record at the end of the table, which should + # succeed, returning the last record. + cursor.set_key(key_populate(cursor, 100)) + self.assertEqual(cursor.search_near(), 0) + self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) + self.assertEqual(cursor.get_value(), value_populate(cursor, 100)) + + # Search for a record past the end of the table, which should fail. + cursor.set_key(key_populate(cursor, 200)) + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + + # Search-near for a record past the end of the table, which should + # succeed, returning the last record. + cursor.set_key(key_populate(cursor, 200)) + self.assertEqual(cursor.search_near(), -1) + self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) + self.assertEqual(cursor.get_value(), value_populate(cursor, 100)) + + # Verify cursor search-near operations before and after a set of + # column-store duplicates. + def test_search_duplicate(self): + if self.colvar == 0: + return + + # Populate the tree. + simple_populate(self, self.uri, self.fmt, 105) + + # Set up deleted records before and after a set of duplicate records, + # and make sure search/search-near returns the correct record. + cursor = self.session.open_cursor(self.uri, None) + for i in range(20, 100): + cursor[key_populate(cursor, i)] = '=== IDENTICAL VALUE ===' + for i in range(15, 25): + cursor.set_key(key_populate(cursor, i)) + self.assertEqual(cursor.remove(), 0) + for i in range(95, 106): + cursor.set_key(key_populate(cursor, i)) + self.assertEqual(cursor.remove(), 0) + cursor.close() + + # Reopen the connection, forcing it to disk and moving the records to + # an on-page format. + self.reopen_conn() + + # Open a cursor. + cursor = self.session.open_cursor(self.uri, None) + + # Search-near for a record in the deleted set before the duplicate set, + # which should succeed, returning the first record in the duplicate set. + cursor.set_key(key_populate(cursor, 18)) + self.assertEqual(cursor.search_near(), 1) + self.assertEqual(cursor.get_key(), key_populate(cursor, 25)) + + # Search-near for a record in the deleted set after the duplicate set, + # which should succeed, returning the last record in the duplicate set. + cursor.set_key(key_populate(cursor, 98)) + self.assertEqual(cursor.search_near(), -1) + self.assertEqual(cursor.get_key(), key_populate(cursor, 94)) + # Verify cursor search and search-near operations on a file with a set of # on-page visible records, and a set of insert-list invisible records. def test_search_invisible_one(self): - uri = 'file:test_bug008' # This is a btree layer test. + # Populate the tree. + simple_populate(self, self.uri, self.fmt, 100) - # Populate the tree and reopen the connection, forcing it to disk - # and moving the records to an on-page format. - simple_populate(self, uri, self.fmt, 100) + # Delete a range of records. + for i in range(5, 10): + cursor = self.session.open_cursor(self.uri, None) + cursor.set_key(key_populate(cursor, i)) + self.assertEqual(cursor.remove(), 0) + + # Reopen the connection, forcing it to disk and moving the records to + # an on-page format. self.reopen_conn() - # Begin a transaction, and add some additional records. + # Add updates to the existing records (in both the deleted an undeleted + # range), as well as some new records after the end. Put the updates in + # a separate transaction so they're invisible to another cursor. self.session.begin_transaction() - cursor = self.session.open_cursor(uri, None) + cursor = self.session.open_cursor(self.uri, None) + for i in range(5, 10): + cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000) + for i in range(30, 40): + cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000) for i in range(100, 140): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000) # Open a separate session and cursor. s = self.conn.open_session() - cursor = s.open_cursor(uri, None) + cursor = s.open_cursor(self.uri, None) - # Search for an invisible record. - cursor.set_key(key_populate(cursor, 130)) - if self.empty: - # Invisible updates to fixed-length column-store objects are - # invisible to the reader, but the fact that they exist past - # the end of the initial records causes the instantiation of - # empty records: confirm successful return of an empty row. - cursor.search() - self.assertEqual(cursor.get_key(), 130) - self.assertEqual(cursor.get_value(), 0) - else: - # Otherwise, we should not find any matching records. - self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + # Search for an existing record in the deleted range, should not find + # it. + for i in range(5, 10): + cursor.set_key(key_populate(cursor, i)) + if self.empty: + # Fixed-length column-store rows always exist. + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_key(), i) + self.assertEqual(cursor.get_value(), 0) + else: + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) - # Search-near for an invisible record, which should succeed, returning - # the last visible record. - cursor.set_key(key_populate(cursor, 130)) - cursor.search_near() - if self.empty: - # Invisible updates to fixed-length column-store objects are - # invisible to the reader, but the fact that they exist past - # the end of the initial records causes the instantiation of - # empty records: confirm successful return of an empty row. - cursor.search() - self.assertEqual(cursor.get_key(), 130) - self.assertEqual(cursor.get_value(), 0) - else: - # Otherwise, we should find the closest record for which we can see - # the value. - self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) - self.assertEqual(cursor.get_value(), value_populate(cursor, 100)) + # Search for an existing record in the updated range, should see the + # original value. + for i in range(30, 40): + cursor.set_key(key_populate(cursor, i)) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_key(), key_populate(cursor, i)) + + # Search for a added record, should not find it. + for i in range(120, 130): + cursor.set_key(key_populate(cursor, i)) + if self.empty: + # Invisible updates to fixed-length column-store objects are + # invisible to the reader, but the fact that they exist past + # the end of the initial records causes the instantiation of + # empty records: confirm successful return of an empty row. + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_key(), i) + self.assertEqual(cursor.get_value(), 0) + else: + # Otherwise, we should not find any matching records. + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + + # Search-near for an existing record in the deleted range, should find + # the next largest record. (This depends on the implementation behavior + # which currently includes a bias to prefix search.) + for i in range(5, 10): + cursor.set_key(key_populate(cursor, i)) + if self.empty: + # Fixed-length column-store rows always exist. + self.assertEqual(cursor.search_near(), 0) + self.assertEqual(cursor.get_key(), i) + self.assertEqual(cursor.get_value(), 0) + else: + self.assertEqual(cursor.search_near(), 1) + self.assertEqual(cursor.get_key(), key_populate(cursor, 10)) + + # Search-near for an existing record in the updated range, should see + # the original value. + for i in range(30, 40): + cursor.set_key(key_populate(cursor, i)) + self.assertEqual(cursor.search_near(), 0) + self.assertEqual(cursor.get_key(), key_populate(cursor, i)) + + # Search-near for an added record, should find the previous largest + # record. + for i in range(120, 130): + cursor.set_key(key_populate(cursor, i)) + if self.empty: + # Invisible updates to fixed-length column-store objects are + # invisible to the reader, but the fact that they exist past + # the end of the initial records causes the instantiation of + # empty records: confirm successful return of an empty row. + self.assertEqual(cursor.search_near(), 0) + self.assertEqual(cursor.get_key(), i) + self.assertEqual(cursor.get_value(), 0) + else: + self.assertEqual(cursor.search_near(), -1) + self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) # Verify cursor search and search-near operations on a file with a set of # on-page visible records, a set of insert-list visible records, and a set @@ -101,28 +244,26 @@ class test_bug008(wttest.WiredTigerTestCase): # fallback happens, whether the correct position is in the page slots or # the insert list.) def test_search_invisible_two(self): - uri = 'file:test_bug008' # This is a btree layer test. - # Populate the tree and reopen the connection, forcing it to disk # and moving the records to an on-page format. - simple_populate(self, uri, self.fmt, 100) + simple_populate(self, self.uri, self.fmt, 100) self.reopen_conn() # Add some additional visible records. - cursor = self.session.open_cursor(uri, None) + cursor = self.session.open_cursor(self.uri, None) for i in range(100, 120): cursor[key_populate(cursor, i)] = value_populate(cursor, i) cursor.close() # Begin a transaction, and add some additional records. self.session.begin_transaction() - cursor = self.session.open_cursor(uri, None) + cursor = self.session.open_cursor(self.uri, None) for i in range(120, 140): cursor[key_populate(cursor, i)] = value_populate(cursor, i) # Open a separate session and cursor. s = self.conn.open_session() - cursor = s.open_cursor(uri, None) + cursor = s.open_cursor(self.uri, None) # Search for an invisible record. cursor.set_key(key_populate(cursor, 130)) diff --git a/src/third_party/wiredtiger/test/suite/test_bulk02.py b/src/third_party/wiredtiger/test/suite/test_bulk02.py index eeca6a56967..fe8118209f2 100644 --- a/src/third_party/wiredtiger/test/suite/test_bulk02.py +++ b/src/third_party/wiredtiger/test/suite/test_bulk02.py @@ -49,8 +49,7 @@ class test_bulkload_checkpoint(wttest.WiredTigerTestCase, suite_subprocess): scenarios = number_scenarios(multiply_scenarios('.', types, ckpt_type)) - # Bulk-load handles return EBUSY to the checkpoint code, causing the - # checkpoint call to find a handle anyway, and create fake checkpoint. + # Bulk-load handles are skipped by checkpoints. # Named and unnamed checkpoint versions. def test_bulkload_checkpoint(self): # Open a bulk cursor and insert a few records. @@ -72,11 +71,8 @@ class test_bulkload_checkpoint(wttest.WiredTigerTestCase, suite_subprocess): # In the case of named checkpoints, verify they're still there, # reflecting an empty file. if self.ckpt_type == 'named': - cursor = self.session.open_cursor( - self.uri, None, 'checkpoint=myckpt') - self.assertEquals(cursor.next(), wiredtiger.WT_NOTFOUND) - cursor.close() - + self.assertRaises(wiredtiger.WiredTigerError, + lambda: self.session.open_cursor(self.uri, None, 'checkpoint=myckpt')) # test_bulkload_backup # Test bulk-load with hot-backup. diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint01.py b/src/third_party/wiredtiger/test/suite/test_checkpoint01.py index 7d4503b84b7..6e1ad7814ed 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint01.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint01.py @@ -185,7 +185,7 @@ class test_checkpoint_cursor(wttest.WiredTigerTestCase): # Check dropping all checkpoints fails. msg = '/checkpoints cannot be dropped/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.checkpoint("name=checkpoint-2"), msg) + lambda: self.session.checkpoint("force,name=checkpoint-2"), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.checkpoint("drop=(checkpoint-2)"), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, @@ -265,9 +265,13 @@ class test_checkpoint_cursor_update(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt") cursor.set_key(key_populate(cursor, 10)) cursor.set_value("XXX") - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.insert()) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.remove()) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.update()) + msg = "/Unsupported cursor/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: cursor.insert(), msg) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: cursor.remove(), msg) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: cursor.update(), msg) cursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_collator.py b/src/third_party/wiredtiger/test/suite/test_collator.py new file mode 100644 index 00000000000..34b5c20247f --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_collator.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +import wiredtiger, wttest, run +from wtscenario import check_scenarios, number_scenarios + +# test_collator.py +# Test indices using a custom extractor and collator. +class test_collator(wttest.WiredTigerTestCase): + """ + Test indices with a custom extractor to create an index, + with our own collator. + Our set of rows looks like a multiplication table: + row '0': '0,0,0,0' + row '1': '0,1,2,3' + row '2': '0,2,4,6' + with the twist that entries are mod 100. So, looking further: + row '40': '0,40,80,20' + + Each column is placed into its own index. Our collator reverses + the values. + """ + nentries = 100 + nindices = 4 + + # Return the wiredtiger_open extension argument for a shared library. + def extensionArg(self, exts): + extfiles = [] + for ext in exts: + (dirname, name, libname) = ext + if name != None and name != 'none': + testdir = os.path.dirname(__file__) + extdir = os.path.join(run.wt_builddir, 'ext', dirname) + extfile = os.path.join( + extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') + if not os.path.exists(extfile): + self.skipTest('extension "' + extfile + '" not built') + if not extfile in extfiles: + extfiles.append(extfile) + if len(extfiles) == 0: + return '' + else: + return ',extensions=["' + '","'.join(extfiles) + '"]' + + # Override WiredTigerTestCase, we have extensions. + def setUpConnectionOpen(self, dir): + extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor'), + ('collators', 'revint', 'revint_collator')]) + connarg = 'create,error_prefix="{0}: ",{1}'.format( + self.shortid(), extarg) + conn = self.wiredtiger_open(dir, connarg) + self.pr(`conn`) + return conn + + def create_indices(self): + # Create self.nindices index files, each with a column from the CSV + for i in range(0, self.nindices): + si = str(i) + self.session.create('index:collator:x' + si, + 'key_format=i,columns=(key),' + + 'collator=revint,' + + 'extractor=csv,app_metadata={"format" : "i",' + + '"field" : "' + si + '"}') + + def drop_indices(self): + for i in range(0, self.nindices): + self.session.drop("index:collator:x" + str(i)) + + def csv(self, s, i): + return s.split(',')[i] + + def expected_main_value(self, i): + return ','.join([str((i*j)%100) for j in range(0, self.nindices)]) + + # We split the population into two phases + # (in anticipation of future tests that create + # indices between the two population steps). + def populate(self): + cursor = self.session.open_cursor('table:collator', None, None) + for i in range(0, self.nentries): + cursor[i] = self.expected_main_value(i) + cursor.close() + + def check_entries(self): + cursor = self.session.open_cursor('table:collator', None, None) + icursor = [] + for i in range(0, self.nindices): + icursor.append(self.session.open_cursor('index:collator:x' + str(i), + None, None)) + i = 0 + for primkey, value in cursor: + # Check main table + expect = self.expected_main_value(i) + self.assertEqual(i, primkey) + self.assertEqual(value, expect) + for idx in range(0, self.nindices): + c = icursor[idx] + indexkey = (i*idx)%100 + c.set_key(indexkey) + self.assertEqual(c.search(), 0) + value = c.get_value() + key = c.get_key() + while value != expect and key == indexkey and \ + self.csv(value, idx) == self.csv(expect, idx): + self.assertEqual(0, c.next()) + value = c.get_value() + key = c.get_key() + self.assertEqual(value, expect) + i += 1 + self.assertEqual(self.nentries, i) + for i in range(0, self.nindices): + c = icursor[i] + c.reset() + expected = set(range(0, self.nentries)) + for key, val in c: + primkey = int(val.split(',')[1]) + expected.remove(primkey) + self.assertEquals(0, len(expected)) + c.close() + + def test_index(self): + self.session.create("table:collator", "key_format=i,value_format=S," + "columns=(primarykey,value)") + self.create_indices() + self.populate() + self.check_entries() + + # Drop and recreate all indices, everything should be there. + self.drop_indices() + self.create_indices() + self.check_entries() + + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_compact02.py b/src/third_party/wiredtiger/test/suite/test_compact02.py index 14781b0f050..7ad05cd2536 100644 --- a/src/third_party/wiredtiger/test/suite/test_compact02.py +++ b/src/third_party/wiredtiger/test/suite/test_compact02.py @@ -50,11 +50,12 @@ class test_compact02(wttest.WiredTigerTestCase): # being stored: compaction doesn't work on tables with many overflow items # because we don't rewrite them. Experimentally, 8KB is as small as the test # can go. Additionally, we can't set the maximum page size too large because - # there won't be enough pages to rewrite. Experimentally, 32KB (the default) - # is as large as the test can go. + # there won't be enough pages to rewrite. Experimentally, 128KB works. fileConfig = [ ('default', dict(fileConfig='')), ('8KB', dict(fileConfig='leaf_page_max=8kb')), + ('64KB', dict(fileConfig='leaf_page_max=64KB')), + ('128KB', dict(fileConfig='leaf_page_max=128KB')), ] scenarios = \ number_scenarios(multiply_scenarios('.', types, cacheSize, fileConfig)) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor06.py b/src/third_party/wiredtiger/test/suite/test_cursor06.py index ff7c1144344..5545c862dd7 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor06.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor06.py @@ -89,10 +89,11 @@ class test_cursor06(wttest.WiredTigerTestCase): self.session.drop(uri, "force") self.populate(uri) cursor = self.session.open_cursor(uri, None, open_config) + msg = '/Unsupported cursor/' if open_config == "readonly=1": self.set_kv(cursor) - self.assertRaises(wiredtiger.WiredTigerError, - lambda: cursor.update()) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: cursor.update(), msg) else: self.set_kv(cursor) cursor.update() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_random.py b/src/third_party/wiredtiger/test/suite/test_cursor_random.py index 2cef62b218a..16ce5cae685 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_random.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_random.py @@ -51,15 +51,21 @@ class test_cursor_random(wttest.WiredTigerTestCase): uri = self.type self.session.create(uri, 'key_format=S,value_format=S') cursor = self.session.open_cursor(uri, None, self.config) - self.assertRaises( - wiredtiger.WiredTigerError, lambda: cursor.compare(cursor)) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.insert()) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.prev()) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.remove()) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.search()) - self.assertRaises( - wiredtiger.WiredTigerError, lambda: cursor.search_near()) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.update()) + msg = "/Unsupported cursor/" + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.compare(cursor), msg) + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.insert(), msg) + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.prev(), msg) + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.remove(), msg) + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.search(), msg) + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.search_near(), msg) + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.update(), msg) self.assertTrue(cursor.next(), wiredtiger.WT_NOTFOUND) self.assertEquals(cursor.reconfigure(), 0) @@ -137,7 +143,7 @@ class test_cursor_random_column(wttest.WiredTigerTestCase): def test_cursor_random_column(self): self.session.create(self.uri, 'key_format=r,value_format=S') - msg = '/Operation not supported/' + msg = '/next_random .* not supported/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.open_cursor(self.uri, None, "next_random=true"), msg) diff --git a/src/third_party/wiredtiger/test/suite/test_drop.py b/src/third_party/wiredtiger/test/suite/test_drop.py index 5663b85d661..52ea7251ab5 100644 --- a/src/third_party/wiredtiger/test/suite/test_drop.py +++ b/src/third_party/wiredtiger/test/suite/test_drop.py @@ -41,12 +41,11 @@ class test_drop(wttest.WiredTigerTestCase): scenarios = check_scenarios([ ('file', dict(uri='file:')), ('table', dict(uri='table:')), - #Not yet: drop failing with an open cursor needs handle locking - #('table-lsm', dict(uri='table:', extra_config=',type=lsm')), + ('table-lsm', dict(uri='table:', extra_config=',type=lsm')), ]) # Populate an object, remove it and confirm it no longer exists. - def drop(self, populate, with_cursor, close_session, drop_index): + def drop(self, populate, with_cursor, reopen, drop_index): uri = self.uri + self.name populate(self, uri, 'key_format=S' + self.extra_config, 10) @@ -57,7 +56,7 @@ class test_drop(wttest.WiredTigerTestCase): lambda: self.session.drop(uri, None)) cursor.close() - if close_session: + if reopen: self.reopen_conn() if drop_index: @@ -73,17 +72,17 @@ class test_drop(wttest.WiredTigerTestCase): # Try all combinations except dropping the index, the simple # case has no indices. for with_cursor in [False, True]: - for close_session in [False, True]: - self.drop(simple_populate, with_cursor, close_session, False) + for reopen in [False, True]: + self.drop(simple_populate, with_cursor, reopen, False) # A complex, multi-file table object. # Try all test combinations. if self.uri == "table:": for with_cursor in [False, True]: - for close_session in [False, True]: + for reopen in [False, True]: for drop_index in [False, True]: self.drop(complex_populate, with_cursor, - close_session, drop_index) + reopen, drop_index) # Test drop of a non-existent object: force succeeds, without force fails. def test_drop_dne(self): diff --git a/src/third_party/wiredtiger/test/suite/test_drop02.py b/src/third_party/wiredtiger/test/suite/test_drop02.py new file mode 100644 index 00000000000..677ba3866b2 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_drop02.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from helper import simple_populate + +# test_drop02.py +# Test dropping an LSM tree on first open. There was a bug where this +# would cause an assertion failure: WT-2501 +class test_drop02(wttest.WiredTigerTestCase): + name = 'test_drop02' + + # Populate an object, remove it and confirm it no longer exists. + def test_drop(self): + uri = 'lsm:' + self.name + simple_populate(self, uri, 'key_format=S', 100000) + self.reopen_conn() + + self.session.drop(uri, None) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_dump.py b/src/third_party/wiredtiger/test/suite/test_dump.py index c850d1b5d3f..fc1422155e2 100644 --- a/src/third_party/wiredtiger/test/suite/test_dump.py +++ b/src/third_party/wiredtiger/test/suite/test_dump.py @@ -29,8 +29,8 @@ import os import wiredtiger, wttest from helper import \ - complex_populate, complex_populate_check_cursor,\ - simple_populate, simple_populate_check_cursor + complex_populate, complex_populate_check, \ + simple_populate, simple_populate_check from suite_subprocess import suite_subprocess from wtscenario import multiply_scenarios, number_scenarios @@ -54,15 +54,24 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): ('string', dict(keyfmt='S')) ] types = [ - ('file', dict(type='file:', + ('file', dict(uri='file:', config='', lsm=False, populate=simple_populate, - populate_check=simple_populate_check_cursor)), - ('table-simple', dict(type='table:', + populate_check=simple_populate_check)), + ('lsm', dict(uri='lsm:', config='', lsm=True, populate=simple_populate, - populate_check=simple_populate_check_cursor)), - ('table-complex', dict(type='table:', + populate_check=simple_populate_check)), + ('table-simple', dict(uri='table:', config='', lsm=False, + populate=simple_populate, + populate_check=simple_populate_check)), + ('table-simple-lsm', dict(uri='table:', config='type=lsm', lsm=True, + populate=simple_populate, + populate_check=simple_populate_check)), + ('table-complex', dict(uri='table:', config='', lsm=False, + populate=complex_populate, + populate_check=complex_populate_check)), + ('table-complex-lsm', dict(uri='table:', config='type=lsm', lsm=True, populate=complex_populate, - populate_check=complex_populate_check_cursor)) + populate_check=complex_populate_check)) ] scenarios = number_scenarios( multiply_scenarios('.', types, keyfmt, dumpfmt)) @@ -94,9 +103,14 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): # Dump, re-load and do a content comparison. def test_dump(self): + # LSM and column-store isn't a valid combination. + if self.lsm and self.keyfmt == 'r': + return + # Create the object. - uri = self.type + self.name - self.populate(self, uri, 'key_format=' + self.keyfmt, self.nentries) + uri = self.uri + self.name + self.populate(self, uri, + self.config + ',key_format=' + self.keyfmt, self.nentries) # Dump the object. os.mkdir(self.dir) @@ -108,11 +122,17 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): # Re-load the object. self.runWt(['-h', self.dir, 'load', '-f', 'dump.out']) - # Check the contents + # Check the database contents + self.runWt(['list'], outfilename='list.out') + self.runWt(['-h', self.dir, 'list'], outfilename='list.out.new') + s1 = set(open('list.out').read().split()) + s2 = set(open('list.out.new').read().split()) + self.assertEqual(not s1.symmetric_difference(s2), True) + + # Check the object's contents conn = self.wiredtiger_open(self.dir) session = conn.open_session() - cursor = session.open_cursor(uri, None, None) - self.populate_check(self, cursor, self.nentries) + self.populate_check(self, uri, self.nentries) conn.close() # Re-load the object again. @@ -121,8 +141,7 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): # Check the contents, they shouldn't have changed. conn = self.wiredtiger_open(self.dir) session = conn.open_session() - cursor = session.open_cursor(uri, None, None) - self.populate_check(self, cursor, self.nentries) + self.populate_check(self, uri, self.nentries) conn.close() # Re-load the object again, but confirm -n (no overwrite) fails. @@ -130,7 +149,7 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): 'load', '-n', '-f', 'dump.out'], errfilename='errfile.out') self.check_non_empty_file('errfile.out') - # If there is are indices, dump one of them and check the output. + # If there are indices, dump one of them and check the output. if self.populate == complex_populate: indexuri = 'index:' + self.name + ':indx1' hexopt = ['-x'] if self.hex == 1 else [] diff --git a/src/third_party/wiredtiger/test/suite/test_join01.py b/src/third_party/wiredtiger/test/suite/test_join01.py index f03c7c6f06c..4aa2bc6e269 100644 --- a/src/third_party/wiredtiger/test/suite/test_join01.py +++ b/src/third_party/wiredtiger/test/suite/test_join01.py @@ -33,7 +33,6 @@ from wtscenario import check_scenarios, multiply_scenarios, number_scenarios # Join operations # Basic tests for join class test_join01(wttest.WiredTigerTestCase): - table_name1 = 'test_join01' nentries = 100 scenarios = [ @@ -75,8 +74,18 @@ class test_join01(wttest.WiredTigerTestCase): # the join cursor and iterating again. def stats(self, jc, which): statcur = self.session.open_cursor('statistics:join', jc, None) - self.check_stats(statcur, 0, 'join: index:join01:index1: ' + - 'bloom filter false positives') + # pick a stat we always expect to see + statdesc = 'bloom filter false positives' + expectstats = [ + 'join: index:join01:index1: ' + statdesc, + 'join: index:join01:index2: ' + statdesc ] + if self.ref == 'index': + expectstats.append('join: index:join01:index0: ' + statdesc) + else: + expectstats.append('join: table:join01: ' + statdesc) + self.check_stats(statcur, expectstats) + statcur.reset() + self.check_stats(statcur, expectstats) statcur.close() def statstr_to_int(self, str): @@ -87,16 +96,14 @@ class test_join01(wttest.WiredTigerTestCase): parts = str.rpartition('(') return int(parts[2].rstrip(')')) - # string should appear with a minimum value of least "min". - def check_stats(self, statcursor, min, lookfor): + # All of the expect strings should appear + def check_stats(self, statcursor, expectstats): stringclass = ''.__class__ intclass = (0).__class__ # Reset the cursor, we're called multiple times. statcursor.reset() - found = False - foundval = 0 self.printVerbose(3, 'statistics:') for id, desc, valstr, val in statcursor: self.assertEqual(type(desc), stringclass) @@ -105,12 +112,11 @@ class test_join01(wttest.WiredTigerTestCase): self.assertEqual(val, self.statstr_to_int(valstr)) self.printVerbose(3, ' stat: \'' + desc + '\', \'' + valstr + '\', ' + str(val)) - if desc == lookfor: - found = True - foundval = val + if desc in expectstats: + expectstats.remove(desc) - self.assertTrue(found, 'in stats, did not see: ' + lookfor) - self.assertTrue(foundval >= min) + self.assertTrue(len(expectstats) == 0, + 'missing expected values in stats: ' + str(expectstats)) # Common function for testing the most basic functionality # of joins @@ -142,7 +148,8 @@ class test_join01(wttest.WiredTigerTestCase): # and examine primary keys 2,5,8,...,95,98,1,4,7,...,94,97. jc = self.session.open_cursor('join:table:join01' + proj_suffix, None, None) - c2 = self.session.open_cursor('index:join01:index2', None, None) + # Adding a projection to a reference cursor should be allowed. + c2 = self.session.open_cursor('index:join01:index2(v1)', None, None) c2.set_key(99) # skips all entries w/ primary key divisible by three self.assertEquals(0, c2.search()) self.session.join(jc, c2, 'compare=gt') @@ -160,12 +167,12 @@ class test_join01(wttest.WiredTigerTestCase): # Then select all numbers whose reverse string representation # is in '20' < x < '40'. - c1a = self.session.open_cursor('index:join01:index1', None, None) + c1a = self.session.open_cursor('index:join01:index1(v1)', None, None) c1a.set_key('21') self.assertEquals(0, c1a.search()) self.session.join(jc, c1a, 'compare=gt' + joincfg1) - c1b = self.session.open_cursor('index:join01:index1', None, None) + c1b = self.session.open_cursor('index:join01:index1(v1)', None, None) c1b.set_key('41') self.assertEquals(0, c1b.search()) self.session.join(jc, c1b, 'compare=lt' + joincfg1) @@ -342,11 +349,12 @@ class test_join01(wttest.WiredTigerTestCase): '/index cursor is being used in a join/') # Only a small number of operations allowed on a join cursor - self.assertRaises(wiredtiger.WiredTigerError, - lambda: jc.search()) + msg = "/Unsupported cursor/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: jc.search(), msg) - self.assertRaises(wiredtiger.WiredTigerError, - lambda: jc.prev()) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: jc.prev(), msg) self.assertEquals(jc.next(), 0) self.assertEquals(jc.next(), wiredtiger.WT_NOTFOUND) @@ -390,6 +398,7 @@ class test_join01(wttest.WiredTigerTestCase): def test_cursor_close2(self): self.cursor_close_common(False) + # test statistics using the framework set up for this test def test_stats(self): bloomcfg1000 = ',strategy=bloom,count=1000' bloomcfg10 = ',strategy=bloom,count=10' @@ -399,6 +408,40 @@ class test_join01(wttest.WiredTigerTestCase): # statistics should pick up some false positives. self.join_common(bloomcfg10, bloomcfg10, False, True) + # test statistics with a simple one index join cursor + def test_simple_stats(self): + self.session.create("table:join01b", + "key_format=i,value_format=i,columns=(k,v)") + self.session.create("index:join01b:index", "columns=(v)") + + cursor = self.session.open_cursor("table:join01b", None, None) + cursor[1] = 11 + cursor[2] = 12 + cursor[3] = 13 + cursor.close() + + cursor = self.session.open_cursor("index:join01b:index", None, None) + cursor.set_key(11) + cursor.search() + + jcursor = self.session.open_cursor("join:table:join01b", None, None) + self.session.join(jcursor, cursor, "compare=gt") + + while jcursor.next() == 0: + [k] = jcursor.get_keys() + [v] = jcursor.get_values() + + statcur = self.session.open_cursor("statistics:join", jcursor, None) + found = False + while statcur.next() == 0: + [desc, pvalue, value] = statcur.get_values() + #self.tty(str(desc) + "=" + str(pvalue)) + found = True + self.assertEquals(found, True) + + jcursor.close() + cursor.close() + if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_join02.py b/src/third_party/wiredtiger/test/suite/test_join02.py index d122de8a0eb..a691c499cf6 100644 --- a/src/third_party/wiredtiger/test/suite/test_join02.py +++ b/src/third_party/wiredtiger/test/suite/test_join02.py @@ -179,15 +179,16 @@ class test_join02(wttest.WiredTigerTestCase): c.close() # Use the primary table in one of the joins. + # Use various projections, which should not matter for ref cursors c0a = self.session.open_cursor('table:join02', None, None) - c0b = self.session.open_cursor('table:join02', None, None) - c1a = self.session.open_cursor('index:join02:index1', None, None) + c0b = self.session.open_cursor('table:join02(v4)', None, None) + c1a = self.session.open_cursor('index:join02:index1(v0)', None, None) c1b = self.session.open_cursor('index:join02:index1', None, None) c2a = self.session.open_cursor('index:join02:index2', None, None) c2b = self.session.open_cursor('index:join02:index2', None, None) - c3a = self.session.open_cursor('index:join02:index3', None, None) - c3b = self.session.open_cursor('index:join02:index3', None, None) - c4a = self.session.open_cursor('index:join02:index4', None, None) + c3a = self.session.open_cursor('index:join02:index3(v4)', None, None) + c3b = self.session.open_cursor('index:join02:index3(v0)', None, None) + c4a = self.session.open_cursor('index:join02:index4(v1)', None, None) # Attach extra properties to each cursor. For cursors that # may appear on the 'left' side of a range CA < x < CB, diff --git a/src/third_party/wiredtiger/test/suite/test_join05.py b/src/third_party/wiredtiger/test/suite/test_join05.py new file mode 100644 index 00000000000..ef2be4c6460 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_join05.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import check_scenarios, multiply_scenarios, number_scenarios + +# test_join05.py +# Tests based on JIRA reports +class test_join05(wttest.WiredTigerTestCase): + + # test join having the first index just be lt/le + def test_wt_2384(self): + self.session.create("table:test_2384", + "key_format=i,value_format=i,columns=(k,v)") + self.session.create("index:test_2384:index", "columns=(v)") + cursor = self.session.open_cursor("table:test_2384", None, None) + cursor[1] = 11 + cursor[2] = 12 + cursor[3] = 13 + cursor.close() + + cursor = self.session.open_cursor("index:test_2384:index", None, None) + cursor.set_key(13) + self.assertEquals(cursor.search(), 0) + + jcursor = self.session.open_cursor("join:table:test_2384", None, None) + self.session.join(jcursor, cursor, "compare=lt") + + nr_found = 0 + while jcursor.next() == 0: + [k] = jcursor.get_keys() + [v] = jcursor.get_values() + #self.tty("jcursor: k=" + str(k) + ", v=" + str(v)) + nr_found += 1 + + self.assertEquals(nr_found, 2) + jcursor.close() + cursor.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_join06.py b/src/third_party/wiredtiger/test/suite/test_join06.py new file mode 100644 index 00000000000..9af6f93792f --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_join06.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +import wiredtiger, wttest, run +from wtscenario import check_scenarios, multiply_scenarios, number_scenarios + +# test_join06.py +# Join operations +# Joins with a read-uncommitted +class test_join06(wttest.WiredTigerTestCase): + nentries = 1000 + + isoscen = [ + ('isolation_read_uncommitted', dict(uncommitted=True)), + ('isolation_default', dict(uncommitted=False)) + ] + + bloomscen = [ + ('bloom', dict(bloom=True)), + ('nobloom', dict(bloom=False)) + ] + + scenarios = number_scenarios(multiply_scenarios('.', isoscen, bloomscen)) + + def gen_values(self, i): + s = str(i) # 345 => "345" + f = s[0:1] + s[0:1] + s[0:1] # 345 => "333" + return [s, f] + + def gen_values2(self, i): + s = str(i) # 345 => "345" + l = s[-1:] + s[-1:] + s[-1:] # 345 => "555" + return [s, l] + + def populate(self, s, gen_values): + c = s.open_cursor('table:join06', None, None) + for i in range(0, self.nentries): + c.set_key(i) + c.set_value(*gen_values(i)) + c.insert() + c.close() + + # Common function for testing the most basic functionality + # of joins + def test_join(self): + self.session.create('table:join06', + 'columns=(k,v0,v1),key_format=i,value_format=SS') + self.session.create('index:join06:index0','columns=(v0)') + self.session.create('index:join06:index1','columns=(v1)') + + self.populate(self.session, self.gen_values) + + # TODO: needed? + #self.reopen_conn() + + if self.uncommitted: + self.session.begin_transaction('isolation=read-uncommitted') + + jc = self.session.open_cursor('join:table:join06', None, None) + c0 = self.session.open_cursor('index:join06:index0', None, None) + c0.set_key('520') + self.assertEquals(0, c0.search()) + self.session.join(jc, c0, 'compare=ge') + + joinconfig = 'compare=eq' + if self.bloom: + joinconfig += ',strategy=bloom,count=1000' + c1 = self.session.open_cursor('index:join06:index1', None, None) + c1.set_key('555') + self.assertEquals(0, c1.search()) + self.session.join(jc, c1, joinconfig) + + if self.uncommitted and self.bloom: + # Make sure that read-uncommitted with Bloom is not allowed. + # This is detected on the first next() operation. + msg = '/cannot be used with read-uncommitted/' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: jc.next(), msg) + return + + # Changes made in another session may or may not be visible to us, + # depending on the isolation level. + if self.uncommitted: + # isolation level is read-uncommitted, so we will see + # additions deletions made in our other session. + mbr = set(range(525,1000,10)) | set(range(55,100,10)) | set([520]) + else: + # default isolation level, so we should see a consistent + # set at the time we begin iteration. + mbr = set(range(520,600)) | set(range(53,60)) + + altered = False + + while jc.next() == 0: + [k] = jc.get_keys() + [v0,v1] = jc.get_values() + #self.tty('GOT: ' + str(k) + ': ' + str(jc.get_values())) + if altered and self.uncommitted: + self.assertEquals(self.gen_values2(k), [v0, v1]) + else: + self.assertEquals(self.gen_values(k), [v0, v1]) + if not k in mbr: + self.tty('**** ERROR: result ' + str(k) + ' is not in: ' + + str(mbr)) + self.assertTrue(k in mbr) + mbr.remove(k) + + # In another session, we remove entries for keys ending in 6, + # and add entries for keys ending in 5. Depending on the + # isolation level for the transaction, these changes may or + # may not be visible for the original session. + if not altered: + s = self.conn.open_session(None) + s.begin_transaction(None) + self.populate(s, self.gen_values2) + s.commit_transaction() + s.close() + altered = True + + if len(mbr) != 0: + self.tty('**** ERROR: did not see these: ' + str(mbr)) + self.assertEquals(0, len(mbr)) + + jc.close() + c1.close() + c0.close() + if self.uncommitted: + self.session.commit_transaction() + self.session.drop('table:join06') + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_lsm03.py b/src/third_party/wiredtiger/test/suite/test_lsm03.py new file mode 100644 index 00000000000..448d864c646 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_lsm03.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wtscenario, wttest +from helper import simple_populate + +# test_lsm03.py +# Check to make sure that LSM schema operations don't get EBUSY when +# there are no user operations active. +class test_lsm03(wttest.WiredTigerTestCase): + name = 'test_lsm03' + + # Use small pages so we generate some internal layout + # Setup LSM so multiple chunks are present + config = 'key_format=S,allocation_size=512,internal_page_max=512' + \ + ',leaf_page_max=1k,lsm=(chunk_size=512k,merge_min=10)' + + # Populate an object then drop it. + def test_lsm_drop_active(self): + uri = 'lsm:' + self.name + simple_populate(self, uri, self.config, 10000) + + # Force to disk + self.reopen_conn() + + # An open cursors should cause failure. + cursor = self.session.open_cursor(uri, None, None) + self.assertRaises(wiredtiger.WiredTigerError, + lambda: self.session.drop(uri, None)) + cursor.close() + + # Add enough records that a merge should be running + simple_populate(self, uri, self.config, 50000) + # The drop should succeed even when LSM work units are active + self.session.drop(uri) diff --git a/src/third_party/wiredtiger/test/suite/test_readonly01.py b/src/third_party/wiredtiger/test/suite/test_readonly01.py new file mode 100644 index 00000000000..59e9743ab7e --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_readonly01.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +# +# Public Domain 2016-2016 MongoDB, Inc. +# Public Domain 2008-2016 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_readonly01.py +# Readonly: Test readonly mode. +# + +import fnmatch, os, shutil, time +from suite_subprocess import suite_subprocess +from wtscenario import multiply_scenarios, number_scenarios, prune_scenarios +import wttest + +class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_readonly01' + create = True + entries = 10000 + + # + # We want a list of directory writable or readonly. + # + basecfg_list = [ + ('basecfg', dict(basecfg='config_base=true,')), + ('no_basecfg', dict(basecfg='config_base=false,')), + ] + dir_list = [ + ('write', dict(dirchmod=False)), + ('readonly', dict(dirchmod=True)), + ] + log_list = [ + ('logging', dict(logcfg='log=(archive=false,enabled,file_max=100K),')), + ('no_logging', dict(logcfg='log=(enabled=false),')), + ] + + types = [ + ('lsm', dict(tabletype='lsm', uri='lsm', + create_params = 'key_format=i,value_format=i')), + ('file-row', dict(tabletype='row', uri='file', + create_params = 'key_format=i,value_format=i')), + ('file-var', dict(tabletype='var', uri='file', + create_params = 'key_format=r,value_format=i')), + ('file-fix', dict(tabletype='fix', uri='file', + create_params = 'key_format=r,value_format=8t')), + ('table-row', dict(tabletype='row', uri='table', + create_params = 'key_format=i,value_format=i')), + ('table-var', dict(tabletype='var', uri='table', + create_params = 'key_format=r,value_format=i')), + ('table-fix', dict(tabletype='fix', uri='table', + create_params = 'key_format=r,value_format=8t')), + ] + + scenarios = multiply_scenarios('.', + basecfg_list, dir_list, log_list, types) + + def conn_config(self, dir): + self.home = dir + params = \ + 'error_prefix="%s",' % self.shortid() + \ + '%s' % self.logcfg + \ + '%s' % self.basecfg + if self.create: + conn_params = 'create,' + params + else: + conn_params = 'readonly=true,' + params + return conn_params + + def close_reopen(self): + ''' Close the connection and reopen readonly''' + # + # close the original connection. If needed, chmod the + # database directory to readonly mode. Then reopen the + # connection with readonly. + # + self.close_conn() + # + # The chmod command is not fully portable to windows. + # + if self.dirchmod and os.name == 'posix': + for f in os.listdir(self.home): + if os.path.isfile(f): + os.chmod(f, 0444) + os.chmod(self.home, 0555) + self.conn = self.setUpConnectionOpen(self.home) + self.session = self.setUpSessionOpen(self.conn) + + def readonly(self): + # Here's the strategy: + # - Create a table. + # - Insert data into table. + # - Close connection. + # - Possibly chmod to readonly + # - Open connection readonly + # - Confirm we can read the data. + # + tablearg = self.uri + ':' + self.tablename + self.session.create(tablearg, self.create_params) + c = self.session.open_cursor(tablearg, None, None) + for i in range(self.entries): + c[i+1] = i % 255 + # Close the connection. Reopen readonly + self.create = False + self.close_reopen() + c = self.session.open_cursor(tablearg, None, None) + i = 0 + for key, value in c: + self.assertEqual(i+1, key) + self.assertEqual(i % 255, value) + i += 1 + self.assertEqual(i, self.entries) + self.pr('Read %d entries' % i) + c.close() + self.create = True + + def test_readonly(self): + if self.dirchmod and os.name == 'posix': + with self.expectedStderrPattern('Permission'): + self.readonly() + else: + self.readonly() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_readonly02.py b/src/third_party/wiredtiger/test/suite/test_readonly02.py new file mode 100644 index 00000000000..0df5465642d --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_readonly02.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# +# Public Domain 2016-2016 MongoDB, Inc. +# Public Domain 2008-2016 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_readonly02.py +# Readonly: Test readonly mode with illegal config combinations +# and error checking during updates. +# + +from helper import copy_wiredtiger_home +from suite_subprocess import suite_subprocess +import os, wiredtiger, wttest + +class test_readonly02(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'table:test_readonly02' + create = True + create_params = 'key_format=i,value_format=i' + entries = 10 + + conn_params = \ + 'create,statistics=(fast),log=(enabled,file_max=100K,zero_fill=true),' + conn_params_rd = \ + 'create,readonly=true,statistics=(fast),log=(enabled,zero_fill=false),' + conn_params_rdcfg = \ + 'create,readonly=true,statistics=(fast),log=(enabled),' + + # + # Run to make sure incompatible configuration options return an error. + # The situations that cause failures (instead of silent overrides) are: + # 1. setting readonly on a new database directory + # 2. an unclean shutdown and reopening readonly + # 3. logging with zero-fill enabled and readonly + # + badcfg1 = 'log=(enabled,zero_fill=true)' + + def setUpConnectionOpen(self, dir): + self.home = dir + rdonlydir = dir + '.rdonly' + # + # First time through check readonly on a non-existent database. + # + if self.create: + # 1. setting readonly on a new database directory + # Setting readonly prevents creation so we should see an + # error because the lock file does not exist. + msg = '/No such file/' + if os.name != 'posix': + msg = '/cannot find the file/' + os.mkdir(rdonlydir) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.wiredtiger_open( + rdonlydir, self.conn_params_rd), msg) + + self.create = False + conn = self.wiredtiger_open(dir, self.conn_params) + return conn + + def check_unclean(self): + backup = "WT_COPYDIR" + copy_wiredtiger_home(self.home, backup, True) + msg = '/needs recovery/' + # 2. an unclean shutdown and reopening readonly + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.wiredtiger_open(backup, self.conn_params_rd), msg) + + def close_checkerror(self, cfg): + ''' Close the connection and reopen readonly''' + # + # Close the original connection. Reopen readonly and also with + # the given configuration string. + # + self.close_conn() + conn_params = self.conn_params_rd + cfg + msg = '/Invalid argument/' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.wiredtiger_open(self.home, conn_params), msg) + + def test_readonly(self): + tablearg = self.tablename + self.session.create(tablearg, self.create_params) + c = self.session.open_cursor(tablearg, None, None) + for i in range(self.entries): + c[i+1] = i % 255 + # Check for an error on an unclean recovery/restart. + self.check_unclean() + + # Close the connection. Reopen readonly with other bad settings. + # 3. logging with zero-fill enabled and readonly + self.close_checkerror(self.badcfg1) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_readonly03.py b/src/third_party/wiredtiger/test/suite/test_readonly03.py new file mode 100644 index 00000000000..d9930e8f553 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_readonly03.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +# +# Public Domain 2016-2016 MongoDB, Inc. +# Public Domain 2008-2016 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_readonly03.py +# Readonly: Test connection readonly mode with modifying methods. Confirm +# all return ENOTSUP. +# + +from helper import simple_populate +from suite_subprocess import suite_subprocess +import os, sys, wiredtiger, wttest + +class test_readonly03(wttest.WiredTigerTestCase, suite_subprocess): + uri = 'table:test_readonly03' + uri2 = 'table:test_readonly03_2' + create = True + + conn_params = 'create,log=(enabled),' + conn_params_rd = 'readonly=true' + + session_ops = [ 'create', 'compact', 'drop', 'log_flush', 'log_printf', + 'rebalance', 'rename', 'salvage', 'truncate', 'upgrade', ] + cursor_ops = [ 'insert', 'remove', 'update', ] + + def setUpConnectionOpen(self, dir): + self.home = dir + if self.create: + conn_cfg = self.conn_params + else: + conn_cfg = self.conn_params_rd + conn = self.wiredtiger_open(dir, conn_cfg) + self.create = False + return conn + + + def test_readonly(self): + create_params = 'key_format=i,value_format=i' + entries = 10 + # Create a database and a table. + simple_populate(self, self.uri, create_params, entries) + + # + # Now close and reopen. Note that the connection function + # above will reopen it readonly. + self.reopen_conn() + msg = '/Unsupported/' + c = self.session.open_cursor(self.uri, None, None) + for op in self.cursor_ops: + c.set_key(1) + c.set_value(1) + if op == 'insert': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: c.insert(), msg) + elif op == 'remove': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: c.remove(), msg) + elif op == 'update': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: c.update(), msg) + else: + self.fail('Unknown cursor operation: ' + op) + c.close() + for op in self.session_ops: + if op == 'create': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.create(self.uri2, create_params), + msg) + elif op == 'compact': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.compact(self.uri, None), msg) + elif op == 'drop': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.drop(self.uri, None), msg) + elif op == 'log_flush': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.log_flush(None), msg) + elif op == 'log_printf': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.log_printf("test"), msg) + elif op == 'rebalance': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.rebalance(self.uri, None), msg) + elif op == 'rename': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.rename(self.uri, self.uri2, None), msg) + elif op == 'salvage': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.salvage(self.uri, None), msg) + elif op == 'truncate': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.truncate(self.uri, None, None, None), + msg) + elif op == 'upgrade': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.upgrade(self.uri, None), msg) + else: + self.fail('Unknown session method: ' + op) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_rebalance.py b/src/third_party/wiredtiger/test/suite/test_rebalance.py index 80cce6ed514..f2167e864c9 100644 --- a/src/third_party/wiredtiger/test/suite/test_rebalance.py +++ b/src/third_party/wiredtiger/test/suite/test_rebalance.py @@ -59,7 +59,7 @@ class test_rebalance(wttest.WiredTigerTestCase): if with_cursor: cursor = self.session.open_cursor(uri, None, None) self.assertRaises(wiredtiger.WiredTigerError, - lambda: self.session.drop(uri, None)) + lambda: self.session.rebalance(uri, None)) cursor.close() self.session.rebalance(uri, None) diff --git a/src/third_party/wiredtiger/test/suite/test_schema07.py b/src/third_party/wiredtiger/test/suite/test_schema07.py new file mode 100644 index 00000000000..ac397c6e1a1 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_schema07.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest + +# test_schema07.py +# Test that long-running tests don't fill the cache with metadata +class test_schema07(wttest.WiredTigerTestCase): + tablename = 'table:test_schema07' + + def conn_config(self, dir): + return 'cache_size=10MB' + + @wttest.longtest("Creating many tables shouldn't fill the cache") + def test_many_tables(self): + s = self.session + # We have a 10MB cache, metadata is (well) over 512B per table, + # if we can create 20K tables, something must be cleaning up. + for i in xrange(20000): + uri = '%s-%06d' % (self.tablename, i) + s.create(uri) + c = s.open_cursor(uri) + # This will block if the metadata fills the cache + c["key"] = "value" + c.close() + self.session.drop(uri) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_sweep01.py b/src/third_party/wiredtiger/test/suite/test_sweep01.py index f996dbfa06d..bccd2bce012 100644 --- a/src/third_party/wiredtiger/test/suite/test_sweep01.py +++ b/src/third_party/wiredtiger/test/suite/test_sweep01.py @@ -40,7 +40,7 @@ import wttest class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): tablebase = 'test_sweep01' uri = 'table:' + tablebase - numfiles = 50 + numfiles = 30 numkv = 1000 conn_config = 'file_manager=(close_handle_minimum=0,' + \ 'close_idle_time=6,close_scan_interval=2),' + \ @@ -87,7 +87,7 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): # # We've configured checkpoints to run every 5 seconds, sweep server to # run every 2 seconds and idle time to be 6 seconds. It should take - # about 8 seconds for a handle to be closed. Sleep for 12 seconds to be + # about 8 seconds for a handle to be closed. Sleep for double to be # safe. # uri = '%s.test' % self.uri @@ -105,13 +105,24 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri, None) k = 0 sleep = 0 - while sleep < 12: + max = 60 + final_nfile = 4 + while sleep < max: self.session.checkpoint() k = k+1 c[k] = 1 sleep += 2 time.sleep(2) + # Give slow machines time to process files. + stat_cursor = self.session.open_cursor('statistics:', None, None) + this_nfile = stat_cursor[stat.conn.file_open][2] + stat_cursor.close() + self.pr("==== loop " + str(sleep)) + self.pr("this_nfile " + str(this_nfile)) + if this_nfile == final_nfile: + break c.close() + self.pr("Sweep loop took " + str(sleep)) stat_cursor = self.session.open_cursor('statistics:', None, None) close2 = stat_cursor[stat.conn.dh_sweep_close][2] @@ -177,7 +188,7 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): self.assertEqual(nfile2 < nfile1, True) # The only files that should be left are the metadata, the lookaside # file, the lock file, and the active file. - if (nfile2 != 4): + if (nfile2 != final_nfile): print "close1: " + str(close1) + " close2: " + str(close2) print "remove1: " + str(remove1) + " remove2: " + str(remove2) print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2) @@ -186,7 +197,7 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): print "tod1: " + str(tod1) + " tod2: " + str(tod2) print "ref1: " + str(ref1) + " ref2: " + str(ref2) print "XX2: nfile1: " + str(nfile1) + " nfile2: " + str(nfile2) - self.assertEqual(nfile2 == 4, True) + self.assertEqual(nfile2 == final_nfile, True) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_txn04.py b/src/third_party/wiredtiger/test/suite/test_txn04.py index de49c5fe235..bbd6ce8c4e2 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn04.py +++ b/src/third_party/wiredtiger/test/suite/test_txn04.py @@ -121,17 +121,14 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): cmd += self.backup_dir self.runWt(cmd.split()) - self.exception='false' backup_conn_params = 'log=(enabled,file_max=%s)' % self.logmax backup_conn = self.wiredtiger_open(self.backup_dir, backup_conn_params) try: self.check(backup_conn.open_session(), None, committed) - except: - self.exception='true' finally: backup_conn.close() - def test_ops(self): + def ops(self): self.session.create(self.uri, self.create_params) c = self.session.open_cursor(self.uri, None, 'overwrite') # Set up the table with entries for 1-5. @@ -149,7 +146,6 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): # The runWt command closes our connection and sessions so # we need to reopen them here. self.hot_backup(None, committed) - self.assertEqual(True, self.exception == 'false') c = self.session.open_cursor(self.uri, None, 'overwrite') c.set_value(1) # Then do the given modification. @@ -192,14 +188,13 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): # Check the state after each commit/rollback. self.check_all(current, committed) - # Backup the target we modified. We expect that running - # recovery now will generate an exception if we committed. + # Backup the target we modified and verify the data. # print 'Call hot_backup with ' + self.uri self.hot_backup(self.uri, committed) - if txn == 'commit': - self.assertEqual(True, self.exception == 'true') - else: - self.assertEqual(True, self.exception == 'false') + + def test_ops(self): + with self.expectedStdoutPattern('Recreating metadata'): + self.ops() if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_util13.py b/src/third_party/wiredtiger/test/suite/test_util13.py new file mode 100644 index 00000000000..222f42cd7f1 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_util13.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os, re, string +from suite_subprocess import suite_subprocess +import itertools, wiredtiger, wttest + +from helper import complex_populate_cgconfig, complex_populate_cgconfig_lsm +from helper import simple_populate +from helper import complex_populate_check, simple_populate_check +from wtscenario import multiply_scenarios, number_scenarios + +# test_util13.py +# Utilities: wt dump, as well as the dump cursor +# Test that dump and load retain table configuration information. +# +class test_util13(wttest.WiredTigerTestCase, suite_subprocess): + """ + Test wt dump. We check for specific output and preservation of + non-default table create parameters. + """ + + pfx = 'test_util13' + nentries = 100 + dir = "dump_dir" + # + # Select table configuration settings that are not the default. + # + types = [ + ('file-simple', dict(uri='file:' + pfx, pop=simple_populate, + populate_check=simple_populate_check, + table_config='prefix_compression_min=3', cfg='')), + ('lsm-simple', dict(uri='lsm:' + pfx, pop=simple_populate, + populate_check=simple_populate_check, + table_config='lsm=(bloom_bit_count=29)', + cfg='bloom_bit_count=29')), + ('table-simple', dict(uri='table:' + pfx, pop=simple_populate, + populate_check=simple_populate_check, + table_config='split_pct=50', cfg='')), + ('table-complex', + dict(uri='table:' + pfx, pop=complex_populate_cgconfig, + populate_check=complex_populate_check, + table_config='allocation_size=512B', cfg='')), + ('table-complex-lsm', + dict(uri='table:' + pfx, pop=complex_populate_cgconfig_lsm, + populate_check=complex_populate_check, + table_config='lsm=(merge_max=5)', + cfg='merge_max=5')), + ] + + scenarios = number_scenarios(multiply_scenarios('.', types)) + + def compare_config(self, expected_cfg, actual_cfg): + # Replace '(' characters so configuration groups don't break parsing. + # If we ever want to look for config groups this will need to change. + #print "compare_config Actual config " + #print actual_cfg + #print "compare_config Expected config " + #print expected_cfg + cfg_orig = actual_cfg + if self.pop != simple_populate: + # + # If we have a complex config, strip out the colgroups and + # columns from the config. Doing so allows us to keep the + # split commands below usable because those two items don't + # have assignments in them. + # + nocolgrp = re.sub("colgroups=\((.+?)\),", '', actual_cfg) + cfg_orig = re.sub("columns=\((.+?)\),", '', nocolgrp) + + #print "Using original config " + #print cfg_orig + da = dict(kv.split('=') for kv in + cfg_orig.strip().replace('(',',').split(',')) + dx = dict(kv.split('=') for kv in + expected_cfg.strip().replace('(',',').split(',')) + + # Check that all items in our expected config subset are in + # the actual configuration and they match. + match = all(item in da.items() for item in dx.items()) + if match == False: + print "MISMATCH:" + print "Original dict: " + print da + print "Expected config: " + print dx + return match + + def compare_files(self, expect_subset, dump_out): + inheader = isconfig = False + for l1, l2 in zip(open(expect_subset, "rb"), open(dump_out, "rb")): + if isconfig: + if not self.compare_config(l1, l2): + return False + if inheader: + # This works because the expected subset has a format + # of URI and config lines alternating. + isconfig = not isconfig + if l1.strip() == 'Header': + inheader = True + if l1.strip() == 'Data': + break + return True + + def load_recheck(self, expect_subset, dump_out): + newdump = "newdump.out" + os.mkdir(self.dir) + self.runWt(['-h', self.dir, 'load', '-f', dump_out]) + # Check the contents + conn = self.wiredtiger_open(self.dir) + session = conn.open_session() + cursor = session.open_cursor(self.uri, None, None) + self.populate_check + conn.close() + dumpargs = ["-h"] + dumpargs.append(self.dir) + dumpargs.append("dump") + dumpargs.append(self.uri) + self.runWt(dumpargs, outfilename=newdump) + + self.assertTrue(self.compare_files(expect_subset, newdump)) + return True + + def test_dump_config(self): + # The number of btree_entries reported is influenced by the + # number of column groups and indices. Each insert will have + # a multiplied effect. + self.pop(self, self.uri, + 'key_format=S,value_format=S,' + self.table_config, self.nentries) + + ver = wiredtiger.wiredtiger_version() + verstring = str(ver[1]) + '.' + str(ver[2]) + '.' + str(ver[3]) + expectfile="expect.out" + with open(expectfile, "w") as expectout: + # Note: this output is sensitive to the precise output format + # generated by wt dump. If this is likely to change, we should + # make this test more accommodating. + expectout.write( + 'WiredTiger Dump (WiredTiger Version ' + verstring + ')\n') + expectout.write('Format=print\n') + expectout.write('Header\n') + expectout.write(self.uri + '\n') + # Check the config on the colgroup itself for complex tables. + if self.pop != simple_populate: + expectout.write('key_format=S\n') + expectout.write('colgroup:' + self.pfx + ':cgroup1\n') + if self.cfg == '': + expectout.write(self.table_config + '\n') + else: + expectout.write(self.cfg + '\n') + expectout.write('Data\n') + + self.pr('calling dump') + outfile="dump.out" + dumpargs = ["dump"] + dumpargs.append(self.uri) + self.runWt(dumpargs, outfilename=outfile) + + self.assertTrue(self.compare_files(expectfile, outfile)) + self.assertTrue(self.load_recheck(expectfile, outfile)) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index b5a58d1566f..a1945b4325d 100644 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -335,6 +335,14 @@ class WiredTigerTestCase(unittest.TestCase): # always get back to original directory os.chdir(self.origcwd) + # Make sure no read-only files or directories were left behind + os.chmod(self.testdir, 0777) + for root, dirs, files in os.walk(self.testdir): + for d in dirs: + os.chmod(os.path.join(root, d), 0777) + for f in files: + os.chmod(os.path.join(root, f), 0666) + # Clean up unless there's a failure if (passed or skipped) and not WiredTigerTestCase._preserveFiles: shutil.rmtree(self.testdir, ignore_errors=True) diff --git a/src/third_party/wiredtiger/test/thread/t.c b/src/third_party/wiredtiger/test/thread/t.c index e72b54bf62a..22334076ee1 100644 --- a/src/third_party/wiredtiger/test/thread/t.c +++ b/src/third_party/wiredtiger/test/thread/t.c @@ -51,6 +51,8 @@ static void wt_shutdown(void); extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/src/third_party/wiredtiger/test/utility/test_util.i b/src/third_party/wiredtiger/test/utility/test_util.i index 3b88d375381..c5cebadcb5c 100644 --- a/src/third_party/wiredtiger/test/utility/test_util.i +++ b/src/third_party/wiredtiger/test/utility/test_util.i @@ -42,25 +42,60 @@ #define DEFAULT_DIR "WT_TEST" #define MKDIR_COMMAND "mkdir " +/* Allow tests to add their own death handling. */ +extern void (*custom_die)(void); + +static void testutil_die(int, const char *, ...) +#if defined(__GNUC__) +__attribute__((__noreturn__)) +#endif +; + /* * die -- * Report an error and quit. */ -static inline void +static void testutil_die(int e, const char *fmt, ...) { va_list ap; + /* Allow test programs to cleanup on fatal error. */ + if (custom_die != NULL) + (*custom_die)(); + va_start(ap, fmt); vfprintf(stderr, fmt, ap); va_end(ap); if (e != 0) fprintf(stderr, ": %s", wiredtiger_strerror(e)); fprintf(stderr, "\n"); + exit(EXIT_FAILURE); } /* + * testutil_check -- + * Complain and quit if a function call fails. + */ +#define testutil_check(call) do { \ + int __r; \ + if ((__r = (call)) != 0) \ + testutil_die(__r, "%s/%d: %s", __func__, __LINE__, #call);\ +} while (0) + +/* + * testutil_checkfmt -- + * Complain and quit if a function call fails, with additional arguments. + */ +#define testutil_checkfmt(call, fmt, ...) do { \ + int __r; \ + if ((__r = (call)) != 0) \ + testutil_die(__r, "%s/%d: %s: " fmt, \ + __func__, __LINE__, #call, __VA_ARGS__); \ +} while (0) + +/* * testutil_work_dir_from_path -- * Takes a buffer, its size and the intended work directory. * Creates the full intended work directory in buffer. diff --git a/src/third_party/wiredtiger/test/windows/windows_shim.h b/src/third_party/wiredtiger/test/windows/windows_shim.h index c35c27cb7b0..f32edce88e7 100644 --- a/src/third_party/wiredtiger/test/windows/windows_shim.h +++ b/src/third_party/wiredtiger/test/windows/windows_shim.h @@ -44,6 +44,11 @@ typedef int u_int; #define R_OK 04 #define X_OK R_OK +/* MSVC Doesn't provide __func__, it has __FUNCTION__ */ +#ifdef _MSC_VER +#define __func__ __FUNCTION__ +#endif + /* snprintf does not exist on <= VS 2013 */ #if _MSC_VER < 1900 #define snprintf _wt_snprintf |