diff options
author | Luke Chen <luke.chen@mongodb.com> | 2019-08-21 05:23:37 +0000 |
---|---|---|
committer | evergreen <evergreen@mongodb.com> | 2019-08-21 05:23:37 +0000 |
commit | ac41c65f6355f83aac70136324c98561ac79daa1 (patch) | |
tree | a7c3f7ef090b59c6a06838a02c96bd1d49e1c729 /src/third_party/wiredtiger/test | |
parent | f54709196711c63a429b71f47c584661286d675f (diff) | |
download | mongo-ac41c65f6355f83aac70136324c98561ac79daa1.tar.gz |
Import wiredtiger: 7dfd9391862bc9a6d84868c4dc51689c45a3aacf from branch mongodb-4.4
ref: c809757d8b..7dfd939186
for: 4.3.1
WT-4658 Apply Clang Format
WT-4810 Adding WT_ERR_ASSERT and WT_RET_ASSERT macros
WT-5046 Prepared transactions aren't properly cleared from global table with WT_CONN_LOG_DEBUG_MODE enabled
Diffstat (limited to 'src/third_party/wiredtiger/test')
86 files changed, 17344 insertions, 18938 deletions
diff --git a/src/third_party/wiredtiger/test/bloom/test_bloom.c b/src/third_party/wiredtiger/test/bloom/test_bloom.c index 29806d5e488..093a231c063 100644 --- a/src/third_party/wiredtiger/test/bloom/test_bloom.c +++ b/src/third_party/wiredtiger/test/bloom/test_bloom.c @@ -29,28 +29,27 @@ #include "test_util.h" static struct { - WT_CONNECTION *wt_conn; /* WT_CONNECTION handle */ - WT_SESSION *wt_session; /* WT_SESSION handle */ + WT_CONNECTION *wt_conn; /* WT_CONNECTION handle */ + WT_SESSION *wt_session; /* WT_SESSION handle */ - char *config_open; /* Command-line configuration */ + char *config_open; /* Command-line configuration */ - uint32_t c_cache; /* Config values */ - uint32_t c_key_max; - uint32_t c_ops; - uint32_t c_k; /* Number of hash iterations */ - uint32_t c_factor; /* Number of bits per item */ + uint32_t c_cache; /* Config values */ + uint32_t c_key_max; + uint32_t c_ops; + uint32_t c_k; /* Number of hash iterations */ + uint32_t c_factor; /* Number of bits per item */ - WT_RAND_STATE rand; + WT_RAND_STATE rand; - uint8_t **entries; + uint8_t **entries; } g; void cleanup(void); void populate_entries(void); void run(void); void setup(void); -void usage(void) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); extern char *__wt_optarg; extern int __wt_optind; @@ -58,191 +57,186 @@ extern int __wt_optind; int main(int argc, char *argv[]) { - int ch; - - (void)testutil_set_progname(argv); - - /* Set default configuration values. */ - g.c_cache = 10; - g.c_ops = 100000; - g.c_key_max = 100; - g.c_k = 8; - g.c_factor = 16; - - /* Set values from the command line. */ - while ((ch = __wt_getopt(progname, argc, argv, "c:f:k:o:")) != EOF) - switch (ch) { - case 'c': /* Cache size */ - g.c_cache = (u_int)atoi(__wt_optarg); - break; - case 'f': /* Factor */ - g.c_factor = (u_int)atoi(__wt_optarg); - break; - case 'k': /* Number of hash functions */ - g.c_k = (u_int)atoi(__wt_optarg); - break; - case 'o': /* Number of ops */ - g.c_ops = (u_int)atoi(__wt_optarg); - break; - default: - usage(); - } - - argc -= __wt_optind; - if (argc != 0) - usage(); - - setup(); - run(); - cleanup(); - - return (EXIT_SUCCESS); + int ch; + + (void)testutil_set_progname(argv); + + /* Set default configuration values. */ + g.c_cache = 10; + g.c_ops = 100000; + g.c_key_max = 100; + g.c_k = 8; + g.c_factor = 16; + + /* Set values from the command line. */ + while ((ch = __wt_getopt(progname, argc, argv, "c:f:k:o:")) != EOF) + switch (ch) { + case 'c': /* Cache size */ + g.c_cache = (u_int)atoi(__wt_optarg); + break; + case 'f': /* Factor */ + g.c_factor = (u_int)atoi(__wt_optarg); + break; + case 'k': /* Number of hash functions */ + g.c_k = (u_int)atoi(__wt_optarg); + break; + case 'o': /* Number of ops */ + g.c_ops = (u_int)atoi(__wt_optarg); + break; + default: + usage(); + } + + argc -= __wt_optind; + if (argc != 0) + usage(); + + setup(); + run(); + cleanup(); + + return (EXIT_SUCCESS); } void setup(void) { - WT_CONNECTION *conn; - WT_SESSION *session; - char config[512]; + WT_CONNECTION *conn; + WT_SESSION *session; + char config[512]; - testutil_check(system("rm -f WiredTiger* *.bf")); + testutil_check(system("rm -f WiredTiger* *.bf")); - /* - * This test doesn't test public Wired Tiger functionality, it still - * needs connection and session handles. - */ + /* + * This test doesn't test public Wired Tiger functionality, it still needs connection and + * session handles. + */ - /* - * Open configuration -- put command line configuration options at the - * end so they can override "standard" configuration. - */ - testutil_check(__wt_snprintf(config, sizeof(config), - "create,error_prefix=\"%s\",cache_size=%" PRIu32 "MB,%s", - progname, g.c_cache, g.config_open == NULL ? "" : g.config_open)); + /* + * Open configuration -- put command line configuration options at the end so they can override + * "standard" configuration. + */ + testutil_check(__wt_snprintf(config, sizeof(config), + "create,error_prefix=\"%s\",cache_size=%" PRIu32 "MB,%s", progname, g.c_cache, + g.config_open == NULL ? "" : g.config_open)); - testutil_check(wiredtiger_open(NULL, NULL, config, &conn)); + testutil_check(wiredtiger_open(NULL, NULL, config, &conn)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); - g.wt_conn = conn; - g.wt_session = session; - populate_entries(); + g.wt_conn = conn; + g.wt_session = session; + populate_entries(); } void run(void) { - WT_BLOOM *bloomp; - WT_ITEM item; - WT_SESSION_IMPL *sess; - uint32_t fp, i; - int ret; - const char *uri = "file:my_bloom.bf"; - - /* Use the internal session handle to access private APIs. */ - sess = (WT_SESSION_IMPL *)g.wt_session; - - testutil_check(__wt_bloom_create( - sess, uri, NULL, g.c_ops, g.c_factor, g.c_k, &bloomp)); - - item.size = g.c_key_max; - for (i = 0; i < g.c_ops; i++) { - item.data = g.entries[i]; - __wt_bloom_insert(bloomp, &item); - } - - testutil_check(__wt_bloom_finalize(bloomp)); - - for (i = 0; i < g.c_ops; i++) { - item.data = g.entries[i]; - if ((ret = __wt_bloom_get(bloomp, &item)) != 0) { - fprintf(stderr, - "get failed at record: %" PRIu32 "\n", i); - testutil_die(ret, "__wt_bloom_get"); - } - } - testutil_check(__wt_bloom_close(bloomp)); - - testutil_check(g.wt_session->checkpoint(g.wt_session, NULL)); - testutil_check(__wt_bloom_open( - sess, uri, g.c_factor, g.c_k, NULL, &bloomp)); - - for (i = 0; i < g.c_ops; i++) { - item.data = g.entries[i]; - testutil_check(__wt_bloom_get(bloomp, &item)); - } - - /* - * Try out some values we didn't insert - choose a different size to - * ensure the value doesn't overlap with existing values. - */ - item.size = g.c_key_max + 10; - item.data = dcalloc(item.size, 1); - memset((void *)item.data, 'a', item.size); - for (i = 0, fp = 0; i < g.c_ops; i++) { - ((uint8_t *)item.data)[i % item.size] = - 'a' + (__wt_random(&g.rand) % 26); - if ((ret = __wt_bloom_get(bloomp, &item)) == 0) - ++fp; - if (ret != 0 && ret != WT_NOTFOUND) - testutil_die(ret, "__wt_bloom_get"); - } - free((void *)item.data); - printf( - "Out of %" PRIu32 " ops, got %" PRIu32 " false positives, %.4f%%\n", - g.c_ops, fp, 100.0 * fp/g.c_ops); - testutil_check(__wt_bloom_drop(bloomp, NULL)); + WT_BLOOM *bloomp; + WT_ITEM item; + WT_SESSION_IMPL *sess; + uint32_t fp, i; + int ret; + const char *uri = "file:my_bloom.bf"; + + /* Use the internal session handle to access private APIs. */ + sess = (WT_SESSION_IMPL *)g.wt_session; + + testutil_check(__wt_bloom_create(sess, uri, NULL, g.c_ops, g.c_factor, g.c_k, &bloomp)); + + item.size = g.c_key_max; + for (i = 0; i < g.c_ops; i++) { + item.data = g.entries[i]; + __wt_bloom_insert(bloomp, &item); + } + + testutil_check(__wt_bloom_finalize(bloomp)); + + for (i = 0; i < g.c_ops; i++) { + item.data = g.entries[i]; + if ((ret = __wt_bloom_get(bloomp, &item)) != 0) { + fprintf(stderr, "get failed at record: %" PRIu32 "\n", i); + testutil_die(ret, "__wt_bloom_get"); + } + } + testutil_check(__wt_bloom_close(bloomp)); + + testutil_check(g.wt_session->checkpoint(g.wt_session, NULL)); + testutil_check(__wt_bloom_open(sess, uri, g.c_factor, g.c_k, NULL, &bloomp)); + + for (i = 0; i < g.c_ops; i++) { + item.data = g.entries[i]; + testutil_check(__wt_bloom_get(bloomp, &item)); + } + + /* + * Try out some values we didn't insert - choose a different size to ensure the value doesn't + * overlap with existing values. + */ + item.size = g.c_key_max + 10; + item.data = dcalloc(item.size, 1); + memset((void *)item.data, 'a', item.size); + for (i = 0, fp = 0; i < g.c_ops; i++) { + ((uint8_t *)item.data)[i % item.size] = 'a' + (__wt_random(&g.rand) % 26); + if ((ret = __wt_bloom_get(bloomp, &item)) == 0) + ++fp; + if (ret != 0 && ret != WT_NOTFOUND) + testutil_die(ret, "__wt_bloom_get"); + } + free((void *)item.data); + printf("Out of %" PRIu32 " ops, got %" PRIu32 " false positives, %.4f%%\n", g.c_ops, fp, + 100.0 * fp / g.c_ops); + testutil_check(__wt_bloom_drop(bloomp, NULL)); } void cleanup(void) { - uint32_t i; + uint32_t i; - for (i = 0; i < g.c_ops; i++) - free(g.entries[i]); - free(g.entries); - testutil_check(g.wt_session->close(g.wt_session, NULL)); - testutil_check(g.wt_conn->close(g.wt_conn, NULL)); + for (i = 0; i < g.c_ops; i++) + free(g.entries[i]); + free(g.entries); + testutil_check(g.wt_session->close(g.wt_session, NULL)); + testutil_check(g.wt_conn->close(g.wt_conn, NULL)); } /* - * Create and keep all the strings used to populate the bloom filter, so that - * we can do validation with the same set of entries. + * Create and keep all the strings used to populate the bloom filter, so that we can do validation + * with the same set of entries. */ void populate_entries(void) { - uint32_t i, j; - uint8_t **entries; + uint32_t i, j; + uint8_t **entries; - __wt_random_init_seed(NULL, &g.rand); + __wt_random_init_seed(NULL, &g.rand); - entries = dcalloc(g.c_ops, sizeof(uint8_t *)); + entries = dcalloc(g.c_ops, sizeof(uint8_t *)); - for (i = 0; i < g.c_ops; i++) { - entries[i] = dcalloc(g.c_key_max, sizeof(uint8_t)); - for (j = 0; j < g.c_key_max; j++) - entries[i][j] = 'a' + (__wt_random(&g.rand) % 26); - } + for (i = 0; i < g.c_ops; i++) { + entries[i] = dcalloc(g.c_key_max, sizeof(uint8_t)); + for (j = 0; j < g.c_key_max; j++) + entries[i][j] = 'a' + (__wt_random(&g.rand) % 26); + } - g.entries = entries; + g.entries = entries; } /* * usage -- - * Display usage statement and exit failure. + * Display usage statement and exit failure. */ void usage(void) { - fprintf(stderr, "usage: %s [-cfko]\n", progname); - fprintf(stderr, "%s", - "\t-c cache size\n" - "\t-f number of bits per item\n" - "\t-k size of entry strings\n" - "\t-o number of operations to perform\n"); - - exit(EXIT_FAILURE); + fprintf(stderr, "usage: %s [-cfko]\n", progname); + fprintf(stderr, "%s", + "\t-c cache size\n" + "\t-f number of bits per item\n" + "\t-k size of entry strings\n" + "\t-o number of operations to perform\n"); + + exit(EXIT_FAILURE); } diff --git a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c index 311c21eff5e..2c311e46af5 100644 --- a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c +++ b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c @@ -30,8 +30,7 @@ static WT_THREAD_RET checkpointer(void *); static WT_THREAD_RET clock_thread(void *); -static int compare_cursors( - WT_CURSOR *, const char *, WT_CURSOR *, const char *); +static int compare_cursors(WT_CURSOR *, const char *, WT_CURSOR *, const char *); static int diagnose_key_error(WT_CURSOR *, int, WT_CURSOR *, int); static int real_checkpointer(void); static int verify_consistency(WT_SESSION *, bool); @@ -43,13 +42,11 @@ static int verify_consistency(WT_SESSION *, bool); void start_checkpoints(void) { - testutil_check(__wt_thread_create(NULL, - &g.checkpoint_thread, checkpointer, NULL)); - if (g.use_timestamps) { - testutil_check(__wt_rwlock_init(NULL, &g.clock_lock)); - testutil_check(__wt_thread_create(NULL, - &g.clock_thread, clock_thread, NULL)); - } + testutil_check(__wt_thread_create(NULL, &g.checkpoint_thread, checkpointer, NULL)); + if (g.use_timestamps) { + testutil_check(__wt_rwlock_init(NULL, &g.clock_lock)); + testutil_check(__wt_thread_create(NULL, &g.clock_thread, clock_thread, NULL)); + } } /* @@ -59,262 +56,245 @@ start_checkpoints(void) void end_checkpoints(void) { - testutil_check(__wt_thread_join(NULL, &g.checkpoint_thread)); - if (g.use_timestamps) { - testutil_check(__wt_thread_join(NULL, &g.clock_thread)); - __wt_rwlock_destroy(NULL, &g.clock_lock); - } + testutil_check(__wt_thread_join(NULL, &g.checkpoint_thread)); + if (g.use_timestamps) { + testutil_check(__wt_thread_join(NULL, &g.clock_thread)); + __wt_rwlock_destroy(NULL, &g.clock_lock); + } } /* * clock_thread -- - * Clock thread: ticks up timestamps. + * Clock thread: ticks up timestamps. */ static WT_THREAD_RET clock_thread(void *arg) { - WT_RAND_STATE rnd; - WT_SESSION *wt_session; - WT_SESSION_IMPL *session; - uint64_t delay; - char buf[128]; - - WT_UNUSED(arg); - - __wt_random_init(&rnd); - testutil_check(g.conn->open_session(g.conn, NULL, NULL, &wt_session)); - session = (WT_SESSION_IMPL *)wt_session; - - g.ts = 0; - while (g.running) { - __wt_writelock(session, &g.clock_lock); - ++g.ts; - testutil_check(__wt_snprintf( - buf, sizeof(buf), - "oldest_timestamp=%x,stable_timestamp=%x", g.ts, g.ts)); - testutil_check(g.conn->set_timestamp(g.conn, buf)); - if (g.ts % 997 == 0) { - /* - * Random value between 6 and 10 seconds. - */ - delay = __wt_random(&rnd) % 5; - __wt_sleep(delay + 6, 0); - } - __wt_writeunlock(session, &g.clock_lock); - /* - * Random value between 5000 and 10000. - */ - delay = __wt_random(&rnd) % 5001; - __wt_sleep(0, delay + 5000); - } - - testutil_check(wt_session->close(wt_session, NULL)); - - return (WT_THREAD_RET_VALUE); + WT_RAND_STATE rnd; + WT_SESSION *wt_session; + WT_SESSION_IMPL *session; + uint64_t delay; + char buf[128]; + + WT_UNUSED(arg); + + __wt_random_init(&rnd); + testutil_check(g.conn->open_session(g.conn, NULL, NULL, &wt_session)); + session = (WT_SESSION_IMPL *)wt_session; + + g.ts = 0; + while (g.running) { + __wt_writelock(session, &g.clock_lock); + ++g.ts; + testutil_check( + __wt_snprintf(buf, sizeof(buf), "oldest_timestamp=%x,stable_timestamp=%x", g.ts, g.ts)); + testutil_check(g.conn->set_timestamp(g.conn, buf)); + if (g.ts % 997 == 0) { + /* + * Random value between 6 and 10 seconds. + */ + delay = __wt_random(&rnd) % 5; + __wt_sleep(delay + 6, 0); + } + __wt_writeunlock(session, &g.clock_lock); + /* + * Random value between 5000 and 10000. + */ + delay = __wt_random(&rnd) % 5001; + __wt_sleep(0, delay + 5000); + } + + testutil_check(wt_session->close(wt_session, NULL)); + + return (WT_THREAD_RET_VALUE); } /* * checkpointer -- - * Checkpoint thread start function. + * Checkpoint thread start function. */ static WT_THREAD_RET checkpointer(void *arg) { - char tid[128]; + char tid[128]; - WT_UNUSED(arg); + WT_UNUSED(arg); - testutil_check(__wt_thread_str(tid, sizeof(tid))); - printf("checkpointer thread starting: tid: %s\n", tid); + testutil_check(__wt_thread_str(tid, sizeof(tid))); + printf("checkpointer thread starting: tid: %s\n", tid); - (void)real_checkpointer(); - return (WT_THREAD_RET_VALUE); + (void)real_checkpointer(); + return (WT_THREAD_RET_VALUE); } /* * real_checkpointer -- - * Do the work of creating checkpoints and then verifying them. Also - * responsible for finishing in a timely fashion. + * Do the work of creating checkpoints and then verifying them. Also responsible for finishing + * in a timely fashion. */ static int real_checkpointer(void) { - WT_RAND_STATE rnd; - WT_SESSION *session; - uint64_t delay; - int ret; - char buf[128], *checkpoint_config; - - if (g.running == 0) - return (log_print_err( - "Checkpoint thread started stopped\n", EINVAL, 1)); - - __wt_random_init(&rnd); - while (g.ntables > g.ntables_created) - __wt_yield(); - - if ((ret = g.conn->open_session(g.conn, NULL, NULL, &session)) != 0) - return (log_print_err("conn.open_session", ret, 1)); - - if (WT_PREFIX_MATCH(g.checkpoint_name, "WiredTigerCheckpoint")) - checkpoint_config = NULL; - else { - testutil_check(__wt_snprintf( - buf, sizeof(buf), "name=%s", g.checkpoint_name)); - checkpoint_config = buf; - } - - while (g.running) { - /* Check for consistency of online data */ - if ((ret = verify_consistency(session, false)) != 0) - return (log_print_err( - "verify_consistency (online)", ret, 1)); - - /* Execute a checkpoint */ - if ((ret = session->checkpoint( - session, checkpoint_config)) != 0) - return (log_print_err("session.checkpoint", ret, 1)); - printf("Finished a checkpoint\n"); - fflush(stdout); - - if (!g.running) - goto done; - - /* Verify the content of the checkpoint. */ - if ((ret = verify_consistency(session, true)) != 0) - return (log_print_err( - "verify_consistency (offline)", ret, 1)); - - /* - * Random value between 4 and 8 seconds. - */ - if (g.sweep_stress) { - delay = __wt_random(&rnd) % 5; - __wt_sleep(delay + 4, 0); - } - } - -done: if ((ret = session->close(session, NULL)) != 0) - return (log_print_err("session.close", ret, 1)); - - return (0); + WT_RAND_STATE rnd; + WT_SESSION *session; + uint64_t delay; + int ret; + char buf[128], *checkpoint_config; + + if (g.running == 0) + return (log_print_err("Checkpoint thread started stopped\n", EINVAL, 1)); + + __wt_random_init(&rnd); + while (g.ntables > g.ntables_created) + __wt_yield(); + + if ((ret = g.conn->open_session(g.conn, NULL, NULL, &session)) != 0) + return (log_print_err("conn.open_session", ret, 1)); + + if (WT_PREFIX_MATCH(g.checkpoint_name, "WiredTigerCheckpoint")) + checkpoint_config = NULL; + else { + testutil_check(__wt_snprintf(buf, sizeof(buf), "name=%s", g.checkpoint_name)); + checkpoint_config = buf; + } + + while (g.running) { + /* Check for consistency of online data */ + if ((ret = verify_consistency(session, false)) != 0) + return (log_print_err("verify_consistency (online)", ret, 1)); + + /* Execute a checkpoint */ + if ((ret = session->checkpoint(session, checkpoint_config)) != 0) + return (log_print_err("session.checkpoint", ret, 1)); + printf("Finished a checkpoint\n"); + fflush(stdout); + + if (!g.running) + goto done; + + /* Verify the content of the checkpoint. */ + if ((ret = verify_consistency(session, true)) != 0) + return (log_print_err("verify_consistency (offline)", ret, 1)); + + /* + * Random value between 4 and 8 seconds. + */ + if (g.sweep_stress) { + delay = __wt_random(&rnd) % 5; + __wt_sleep(delay + 4, 0); + } + } + +done: + if ((ret = session->close(session, NULL)) != 0) + return (log_print_err("session.close", ret, 1)); + + return (0); } /* * verify_consistency -- - * Open a cursor on each table at the last checkpoint and walk through - * the tables in parallel. The key/values should match across all tables. + * Open a cursor on each table at the last checkpoint and walk through the tables in parallel. + * The key/values should match across all tables. */ static int verify_consistency(WT_SESSION *session, bool use_checkpoint) { - WT_CURSOR **cursors; - uint64_t key_count; - int i, ret, t_ret; - const char *ckpt, *type0, *typei; - char ckpt_buf[128], next_uri[128]; - - ret = t_ret = 0; - key_count = 0; - cursors = calloc((size_t)g.ntables, sizeof(*cursors)); - if (cursors == NULL) - return (log_print_err("verify_consistency", ENOMEM, 1)); - - if (use_checkpoint) { - testutil_check(__wt_snprintf(ckpt_buf, sizeof(ckpt_buf), - "checkpoint=%s", g.checkpoint_name)); - ckpt = ckpt_buf; - } else { - ckpt = NULL; - testutil_check(session->begin_transaction( - session, "isolation=snapshot")); - } - - for (i = 0; i < g.ntables; i++) { - /* - * TODO: LSM doesn't currently support reading from - * checkpoints. - */ - if (use_checkpoint && g.cookies[i].type == LSM) - continue; - testutil_check(__wt_snprintf( - next_uri, sizeof(next_uri), "table:__wt%04d", i)); - if ((ret = session->open_cursor( - session, next_uri, NULL, ckpt, &cursors[i])) != 0) { - (void)log_print_err( - "verify_consistency:session.open_cursor", ret, 1); - goto err; - } - } - - /* There's no way to verify LSM-only runs. */ - if (cursors[0] == NULL) { - printf("LSM-only, skipping checkpoint verification\n"); - goto err; - } - - while (ret == 0) { - ret = cursors[0]->next(cursors[0]); - if (ret == 0) - ++key_count; - else if (ret != WT_NOTFOUND) { - (void)log_print_err("cursor->next", ret, 1); - goto err; - } - /* - * Check to see that all remaining cursors have the - * same key/value pair. - */ - for (i = 1; i < g.ntables; i++) { - /* - * TODO: LSM doesn't currently support reading from - * checkpoints. - */ - if (g.cookies[i].type == LSM) - continue; - t_ret = cursors[i]->next(cursors[i]); - if (t_ret != 0 && t_ret != WT_NOTFOUND) { - (void)log_print_err("cursor->next", t_ret, 1); - goto err; - } - - if (ret == WT_NOTFOUND && t_ret == WT_NOTFOUND) - continue; - else if (ret == WT_NOTFOUND || t_ret == WT_NOTFOUND) { - (void)log_print_err( - "verify_consistency tables with different" - " amount of data", EFAULT, 1); - goto err; - } - - type0 = type_to_string(g.cookies[0].type); - typei = type_to_string(g.cookies[i].type); - if ((ret = compare_cursors( - cursors[0], type0, cursors[i], typei)) != 0) { - (void)diagnose_key_error( - cursors[0], 0, cursors[i], i); - (void)log_print_err( - "verify_consistency - mismatching data", - EFAULT, 1); - goto err; - } - } - } - printf("Finished verifying a %s with %d tables and %" PRIu64 - " keys\n", use_checkpoint ? "checkpoint" : "snapshot", - g.ntables, key_count); - fflush(stdout); - -err: for (i = 0; i < g.ntables; i++) { - if (cursors[i] != NULL && - (ret = cursors[i]->close(cursors[i])) != 0) - (void)log_print_err( - "verify_consistency:cursor close", ret, 1); - } - if (!use_checkpoint) - testutil_check(session->commit_transaction(session, NULL)); - free(cursors); - return (ret); + WT_CURSOR **cursors; + uint64_t key_count; + int i, ret, t_ret; + char ckpt_buf[128], next_uri[128]; + const char *ckpt, *type0, *typei; + + ret = t_ret = 0; + key_count = 0; + cursors = calloc((size_t)g.ntables, sizeof(*cursors)); + if (cursors == NULL) + return (log_print_err("verify_consistency", ENOMEM, 1)); + + if (use_checkpoint) { + testutil_check( + __wt_snprintf(ckpt_buf, sizeof(ckpt_buf), "checkpoint=%s", g.checkpoint_name)); + ckpt = ckpt_buf; + } else { + ckpt = NULL; + testutil_check(session->begin_transaction(session, "isolation=snapshot")); + } + + for (i = 0; i < g.ntables; i++) { + /* + * TODO: LSM doesn't currently support reading from checkpoints. + */ + if (use_checkpoint && g.cookies[i].type == LSM) + continue; + testutil_check(__wt_snprintf(next_uri, sizeof(next_uri), "table:__wt%04d", i)); + if ((ret = session->open_cursor(session, next_uri, NULL, ckpt, &cursors[i])) != 0) { + (void)log_print_err("verify_consistency:session.open_cursor", ret, 1); + goto err; + } + } + + /* There's no way to verify LSM-only runs. */ + if (cursors[0] == NULL) { + printf("LSM-only, skipping checkpoint verification\n"); + goto err; + } + + while (ret == 0) { + ret = cursors[0]->next(cursors[0]); + if (ret == 0) + ++key_count; + else if (ret != WT_NOTFOUND) { + (void)log_print_err("cursor->next", ret, 1); + goto err; + } + /* + * Check to see that all remaining cursors have the same key/value pair. + */ + for (i = 1; i < g.ntables; i++) { + /* + * TODO: LSM doesn't currently support reading from checkpoints. + */ + if (g.cookies[i].type == LSM) + continue; + t_ret = cursors[i]->next(cursors[i]); + if (t_ret != 0 && t_ret != WT_NOTFOUND) { + (void)log_print_err("cursor->next", t_ret, 1); + goto err; + } + + if (ret == WT_NOTFOUND && t_ret == WT_NOTFOUND) + continue; + else if (ret == WT_NOTFOUND || t_ret == WT_NOTFOUND) { + (void)log_print_err( + "verify_consistency tables with different" + " amount of data", + EFAULT, 1); + goto err; + } + + type0 = type_to_string(g.cookies[0].type); + typei = type_to_string(g.cookies[i].type); + if ((ret = compare_cursors(cursors[0], type0, cursors[i], typei)) != 0) { + (void)diagnose_key_error(cursors[0], 0, cursors[i], i); + (void)log_print_err("verify_consistency - mismatching data", EFAULT, 1); + goto err; + } + } + } + printf("Finished verifying a %s with %d tables and %" PRIu64 " keys\n", + use_checkpoint ? "checkpoint" : "snapshot", g.ntables, key_count); + fflush(stdout); + +err: + for (i = 0; i < g.ntables; i++) { + if (cursors[i] != NULL && (ret = cursors[i]->close(cursors[i])) != 0) + (void)log_print_err("verify_consistency:cursor close", ret, 1); + } + if (!use_checkpoint) + testutil_check(session->commit_transaction(session, NULL)); + free(cursors); + return (ret); } /* @@ -322,156 +302,140 @@ err: for (i = 0; i < g.ntables; i++) { * Compare the key/value pairs from two cursors. */ static int -compare_cursors( - WT_CURSOR *cursor1, const char *type1, - WT_CURSOR *cursor2, const char *type2) +compare_cursors(WT_CURSOR *cursor1, const char *type1, WT_CURSOR *cursor2, const char *type2) { - uint64_t key1, key2; - int ret; - char buf[128], *val1, *val2; + uint64_t key1, key2; + int ret; + char buf[128], *val1, *val2; - ret = 0; - memset(buf, 0, 128); + ret = 0; + memset(buf, 0, 128); - if (cursor1->get_key(cursor1, &key1) != 0 || - cursor2->get_key(cursor2, &key2) != 0) - return (log_print_err("Error getting keys", EINVAL, 1)); + if (cursor1->get_key(cursor1, &key1) != 0 || cursor2->get_key(cursor2, &key2) != 0) + return (log_print_err("Error getting keys", EINVAL, 1)); - if (cursor1->get_value(cursor1, &val1) != 0 || - cursor2->get_value(cursor2, &val2) != 0) - return (log_print_err("Error getting values", EINVAL, 1)); + if (cursor1->get_value(cursor1, &val1) != 0 || cursor2->get_value(cursor2, &val2) != 0) + return (log_print_err("Error getting values", EINVAL, 1)); - if (g.logfp != NULL) - fprintf(g.logfp, "k1: %" PRIu64 " k2: %" PRIu64 - " val1: %s val2: %s \n", key1, key2, val1, val2); + if (g.logfp != NULL) + fprintf( + g.logfp, "k1: %" PRIu64 " k2: %" PRIu64 " val1: %s val2: %s \n", key1, key2, val1, val2); - if (key1 != key2) - ret = ERR_KEY_MISMATCH; - else if (strlen(val1) != strlen(val2) || strcmp(val1, val2) != 0) - ret = ERR_DATA_MISMATCH; - else - return (0); + if (key1 != key2) + ret = ERR_KEY_MISMATCH; + else if (strlen(val1) != strlen(val2) || strcmp(val1, val2) != 0) + ret = ERR_DATA_MISMATCH; + else + return (0); - printf("Key/value mismatch: %" PRIu64 "/%s from a %s table is not %" - PRIu64 "/%s from a %s table\n", - key1, val1, type1, key2, val2, type2); + printf("Key/value mismatch: %" PRIu64 "/%s from a %s table is not %" PRIu64 + "/%s from a %s table\n", + key1, val1, type1, key2, val2, type2); - return (ret); + return (ret); } /* * diagnose_key_error -- - * Dig a bit deeper on failure. Continue after some failures here to - * extract as much information as we can. + * Dig a bit deeper on failure. Continue after some failures here to extract as much information + * as we can. */ static int -diagnose_key_error( - WT_CURSOR *cursor1, int index1, - WT_CURSOR *cursor2, int index2) +diagnose_key_error(WT_CURSOR *cursor1, int index1, WT_CURSOR *cursor2, int index2) { - WT_CURSOR *c; - WT_SESSION *session; - uint64_t key1, key1_orig, key2, key2_orig; - int ret; - char ckpt[128], next_uri[128]; - - /* Hack to avoid passing session as parameter. */ - session = cursor1->session; - key1_orig = key2_orig = 0; - - testutil_check(__wt_snprintf( - ckpt, sizeof(ckpt), "checkpoint=%s", g.checkpoint_name)); - - /* Save the failed keys. */ - if (cursor1->get_key(cursor1, &key1_orig) != 0 || - cursor2->get_key(cursor2, &key2_orig) != 0) { - (void)log_print_err("Error retrieving key.", EINVAL, 0); - goto live_check; - } - - if (key1_orig == key2_orig) - goto live_check; - - /* See if previous values are still valid. */ - if (cursor1->prev(cursor1) != 0 || cursor2->prev(cursor2) != 0) - return (1); - if (cursor1->get_key(cursor1, &key1) != 0 || - cursor2->get_key(cursor2, &key2) != 0) - (void)log_print_err("Error decoding key", EINVAL, 1); - else if (key1 != key2) - (void)log_print_err("Now previous keys don't match", EINVAL, 0); - - if (cursor1->next(cursor1) != 0 || cursor2->next(cursor2) != 0) - return (1); - if (cursor1->get_key(cursor1, &key1) != 0 || - cursor2->get_key(cursor2, &key2) != 0) - (void)log_print_err("Error decoding key", EINVAL, 1); - else if (key1 == key2) - (void)log_print_err("After prev/next keys match", EINVAL, 0); - - if (cursor1->next(cursor1) != 0 || cursor2->next(cursor2) != 0) - return (1); - if (cursor1->get_key(cursor1, &key1) != 0 || - cursor2->get_key(cursor2, &key2) != 0) - (void)log_print_err("Error decoding key", EINVAL, 1); - else if (key1 == key2) - (void)log_print_err( - "After prev/next/next keys match", EINVAL, 0); - - /* - * Now try opening new cursors on the checkpoints and see if we - * get the same missing key via searching. - */ - testutil_check(__wt_snprintf( - next_uri, sizeof(next_uri), "table:__wt%04d", index1)); - if (session->open_cursor(session, next_uri, NULL, ckpt, &c) != 0) - return (1); - c->set_key(c, key1_orig); - if ((ret = c->search(c)) != 0) - (void)log_print_err("1st cursor didn't find 1st key", ret, 0); - c->set_key(c, key2_orig); - if ((ret = c->search(c)) != 0) - (void)log_print_err("1st cursor didn't find 2nd key", ret, 0); - if (c->close(c) != 0) - return (1); - - testutil_check(__wt_snprintf( - next_uri, sizeof(next_uri), "table:__wt%04d", index2)); - if (session->open_cursor(session, next_uri, NULL, ckpt, &c) != 0) - return (1); - c->set_key(c, key1_orig); - if ((ret = c->search(c)) != 0) - (void)log_print_err("2nd cursor didn't find 1st key", ret, 0); - c->set_key(c, key2_orig); - if ((ret = c->search(c)) != 0) - (void)log_print_err("2nd cursor didn't find 2nd key", ret, 0); - if (c->close(c) != 0) - return (1); + WT_CURSOR *c; + WT_SESSION *session; + uint64_t key1, key1_orig, key2, key2_orig; + int ret; + char ckpt[128], next_uri[128]; + + /* Hack to avoid passing session as parameter. */ + session = cursor1->session; + key1_orig = key2_orig = 0; + + testutil_check(__wt_snprintf(ckpt, sizeof(ckpt), "checkpoint=%s", g.checkpoint_name)); + + /* Save the failed keys. */ + if (cursor1->get_key(cursor1, &key1_orig) != 0 || cursor2->get_key(cursor2, &key2_orig) != 0) { + (void)log_print_err("Error retrieving key.", EINVAL, 0); + goto live_check; + } + + if (key1_orig == key2_orig) + goto live_check; + + /* See if previous values are still valid. */ + if (cursor1->prev(cursor1) != 0 || cursor2->prev(cursor2) != 0) + return (1); + if (cursor1->get_key(cursor1, &key1) != 0 || cursor2->get_key(cursor2, &key2) != 0) + (void)log_print_err("Error decoding key", EINVAL, 1); + else if (key1 != key2) + (void)log_print_err("Now previous keys don't match", EINVAL, 0); + + if (cursor1->next(cursor1) != 0 || cursor2->next(cursor2) != 0) + return (1); + if (cursor1->get_key(cursor1, &key1) != 0 || cursor2->get_key(cursor2, &key2) != 0) + (void)log_print_err("Error decoding key", EINVAL, 1); + else if (key1 == key2) + (void)log_print_err("After prev/next keys match", EINVAL, 0); + + if (cursor1->next(cursor1) != 0 || cursor2->next(cursor2) != 0) + return (1); + if (cursor1->get_key(cursor1, &key1) != 0 || cursor2->get_key(cursor2, &key2) != 0) + (void)log_print_err("Error decoding key", EINVAL, 1); + else if (key1 == key2) + (void)log_print_err("After prev/next/next keys match", EINVAL, 0); + + /* + * Now try opening new cursors on the checkpoints and see if we get the same missing key via + * searching. + */ + testutil_check(__wt_snprintf(next_uri, sizeof(next_uri), "table:__wt%04d", index1)); + if (session->open_cursor(session, next_uri, NULL, ckpt, &c) != 0) + return (1); + c->set_key(c, key1_orig); + if ((ret = c->search(c)) != 0) + (void)log_print_err("1st cursor didn't find 1st key", ret, 0); + c->set_key(c, key2_orig); + if ((ret = c->search(c)) != 0) + (void)log_print_err("1st cursor didn't find 2nd key", ret, 0); + if (c->close(c) != 0) + return (1); + + testutil_check(__wt_snprintf(next_uri, sizeof(next_uri), "table:__wt%04d", index2)); + if (session->open_cursor(session, next_uri, NULL, ckpt, &c) != 0) + return (1); + c->set_key(c, key1_orig); + if ((ret = c->search(c)) != 0) + (void)log_print_err("2nd cursor didn't find 1st key", ret, 0); + c->set_key(c, key2_orig); + if ((ret = c->search(c)) != 0) + (void)log_print_err("2nd cursor didn't find 2nd key", ret, 0); + if (c->close(c) != 0) + return (1); live_check: - /* - * Now try opening cursors on the live checkpoint to see if we get the - * same missing key via searching. - */ - testutil_check(__wt_snprintf( - next_uri, sizeof(next_uri), "table:__wt%04d", index1)); - if (session->open_cursor(session, next_uri, NULL, NULL, &c) != 0) - return (1); - c->set_key(c, key1_orig); - if ((ret = c->search(c)) != 0) - (void)log_print_err("1st cursor didn't find 1st key", ret, 0); - if (c->close(c) != 0) - return (1); - - testutil_check(__wt_snprintf( - next_uri, sizeof(next_uri), "table:__wt%04d", index2)); - if (session->open_cursor(session, next_uri, NULL, NULL, &c) != 0) - return (1); - c->set_key(c, key2_orig); - if ((ret = c->search(c)) != 0) - (void)log_print_err("2nd cursor didn't find 2nd key", ret, 0); - if (c->close(c) != 0) - return (1); - - return (0); + /* + * Now try opening cursors on the live checkpoint to see if we get the same missing key via + * searching. + */ + testutil_check(__wt_snprintf(next_uri, sizeof(next_uri), "table:__wt%04d", index1)); + if (session->open_cursor(session, next_uri, NULL, NULL, &c) != 0) + return (1); + c->set_key(c, key1_orig); + if ((ret = c->search(c)) != 0) + (void)log_print_err("1st cursor didn't find 1st key", ret, 0); + if (c->close(c) != 0) + return (1); + + testutil_check(__wt_snprintf(next_uri, sizeof(next_uri), "table:__wt%04d", index2)); + if (session->open_cursor(session, next_uri, NULL, NULL, &c) != 0) + return (1); + c->set_key(c, key2_orig); + if ((ret = c->search(c)) != 0) + (void)log_print_err("2nd cursor didn't find 2nd key", ret, 0); + if (c->close(c) != 0) + return (1); + + return (0); } diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c index c30af666c5c..08b1e236b7c 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c @@ -30,14 +30,13 @@ GLOBAL g; -static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); -static int handle_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); -static void onint(int) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); +static int handle_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); +static void onint(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void cleanup(bool); -static int usage(void); -static int wt_connect(const char *); -static int wt_shutdown(void); +static int usage(void); +static int wt_connect(const char *); +static int wt_shutdown(void); extern int __wt_optind; extern char *__wt_optarg; @@ -45,329 +44,312 @@ extern char *__wt_optarg; int main(int argc, char *argv[]) { - table_type ttype; - int ch, cnt, ret, runs; - char *working_dir; - const char *config_open; - - (void)testutil_set_progname(argv); - - config_open = NULL; - ret = 0; - working_dir = NULL; - ttype = MIX; - g.checkpoint_name = "WiredTigerCheckpoint"; - g.debug_mode = false; - g.home = dmalloc(512); - g.nkeys = 10000; - g.nops = 100000; - g.ntables = 3; - g.nworkers = 1; - g.sweep_stress = g.use_timestamps = false; - runs = 1; - - while ((ch = __wt_getopt( - progname, argc, argv, "C:c:Dh:k:l:n:r:sT:t:W:x")) != EOF) - switch (ch) { - case 'c': - g.checkpoint_name = __wt_optarg; - break; - case 'C': /* wiredtiger_open config */ - config_open = __wt_optarg; - break; - case 'D': - g.debug_mode = true; - break; - case 'h': /* wiredtiger_open config */ - working_dir = __wt_optarg; - break; - case 'k': /* rows */ - g.nkeys = (u_int)atoi(__wt_optarg); - break; - case 'l': /* log */ - if ((g.logfp = fopen(__wt_optarg, "w")) == NULL) { - fprintf(stderr, - "%s: %s\n", __wt_optarg, strerror(errno)); - return (EXIT_FAILURE); - } - break; - case 'n': /* operations */ - g.nops = (u_int)atoi(__wt_optarg); - break; - case 'r': /* runs */ - runs = atoi(__wt_optarg); - break; - case 's': - g.sweep_stress = true; - break; - case 't': - switch (__wt_optarg[0]) { - case 'c': - ttype = COL; - break; - case 'l': - ttype = LSM; - break; - case 'm': - ttype = MIX; - break; - case 'r': - ttype = ROW; - break; - default: - return (usage()); - } - break; - case 'T': - g.ntables = atoi(__wt_optarg); - break; - case 'W': - g.nworkers = atoi(__wt_optarg); - break; - case 'x': - g.use_timestamps = true; - break; - default: - return (usage()); - } - - argc -= __wt_optind; - if (argc != 0) - return (usage()); - - /* Clean up on signal. */ - (void)signal(SIGINT, onint); - - testutil_work_dir_from_path(g.home, 512, working_dir); - - printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); - for (cnt = 1; (runs == 0 || cnt <= runs) && g.status == 0; ++cnt) { - cleanup(cnt == 1); /* Clean up previous runs */ - - printf(" %d: %d workers, %d tables\n", - cnt, g.nworkers, g.ntables); - - /* Setup a fresh set of cookies in the global array. */ - if ((g.cookies = calloc( - (size_t)(g.ntables), sizeof(COOKIE))) == NULL) { - (void)log_print_err("No memory", ENOMEM, 1); - break; - } - - g.running = 1; - - if ((ret = wt_connect(config_open)) != 0) { - (void)log_print_err("Connection failed", ret, 1); - break; - } - - start_checkpoints(); - if ((ret = start_workers(ttype)) != 0) { - (void)log_print_err("Start workers failed", ret, 1); - break; - } - - g.running = 0; - end_checkpoints(); - - free(g.cookies); - g.cookies = NULL; - if ((ret = wt_shutdown()) != 0) { - (void)log_print_err("Start workers failed", ret, 1); - break; - } - } - if (g.logfp != NULL) - (void)fclose(g.logfp); - - /* Ensure that cleanup is done on error. */ - (void)wt_shutdown(); - free(g.cookies); - return (g.status); + table_type ttype; + int ch, cnt, ret, runs; + char *working_dir; + const char *config_open; + + (void)testutil_set_progname(argv); + + config_open = NULL; + ret = 0; + working_dir = NULL; + ttype = MIX; + g.checkpoint_name = "WiredTigerCheckpoint"; + g.debug_mode = false; + g.home = dmalloc(512); + g.nkeys = 10000; + g.nops = 100000; + g.ntables = 3; + g.nworkers = 1; + g.sweep_stress = g.use_timestamps = false; + runs = 1; + + while ((ch = __wt_getopt(progname, argc, argv, "C:c:Dh:k:l:n:r:sT:t:W:x")) != EOF) + switch (ch) { + case 'c': + g.checkpoint_name = __wt_optarg; + break; + case 'C': /* wiredtiger_open config */ + config_open = __wt_optarg; + break; + case 'D': + g.debug_mode = true; + break; + case 'h': /* wiredtiger_open config */ + working_dir = __wt_optarg; + break; + case 'k': /* rows */ + g.nkeys = (u_int)atoi(__wt_optarg); + break; + case 'l': /* log */ + if ((g.logfp = fopen(__wt_optarg, "w")) == NULL) { + fprintf(stderr, "%s: %s\n", __wt_optarg, strerror(errno)); + return (EXIT_FAILURE); + } + break; + case 'n': /* operations */ + g.nops = (u_int)atoi(__wt_optarg); + break; + case 'r': /* runs */ + runs = atoi(__wt_optarg); + break; + case 's': + g.sweep_stress = true; + break; + case 't': + switch (__wt_optarg[0]) { + case 'c': + ttype = COL; + break; + case 'l': + ttype = LSM; + break; + case 'm': + ttype = MIX; + break; + case 'r': + ttype = ROW; + break; + default: + return (usage()); + } + break; + case 'T': + g.ntables = atoi(__wt_optarg); + break; + case 'W': + g.nworkers = atoi(__wt_optarg); + break; + case 'x': + g.use_timestamps = true; + break; + default: + return (usage()); + } + + argc -= __wt_optind; + if (argc != 0) + return (usage()); + + /* Clean up on signal. */ + (void)signal(SIGINT, onint); + + testutil_work_dir_from_path(g.home, 512, working_dir); + + printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); + for (cnt = 1; (runs == 0 || cnt <= runs) && g.status == 0; ++cnt) { + cleanup(cnt == 1); /* Clean up previous runs */ + + printf(" %d: %d workers, %d tables\n", cnt, g.nworkers, g.ntables); + + /* Setup a fresh set of cookies in the global array. */ + if ((g.cookies = calloc((size_t)(g.ntables), sizeof(COOKIE))) == NULL) { + (void)log_print_err("No memory", ENOMEM, 1); + break; + } + + g.running = 1; + + if ((ret = wt_connect(config_open)) != 0) { + (void)log_print_err("Connection failed", ret, 1); + break; + } + + start_checkpoints(); + if ((ret = start_workers(ttype)) != 0) { + (void)log_print_err("Start workers failed", ret, 1); + break; + } + + g.running = 0; + end_checkpoints(); + + free(g.cookies); + g.cookies = NULL; + if ((ret = wt_shutdown()) != 0) { + (void)log_print_err("Start workers failed", ret, 1); + break; + } + } + if (g.logfp != NULL) + (void)fclose(g.logfp); + + /* Ensure that cleanup is done on error. */ + (void)wt_shutdown(); + free(g.cookies); + return (g.status); } -#define DEBUG_MODE_CFG \ -",debug_mode=(eviction=true,table_logging=true)" +#define DEBUG_MODE_CFG ",debug_mode=(eviction=true,table_logging=true)" /* * wt_connect -- - * Configure the WiredTiger connection. + * Configure the WiredTiger connection. */ static int wt_connect(const char *config_open) { - static WT_EVENT_HANDLER event_handler = { - handle_error, - handle_message, - NULL, - NULL /* Close handler. */ - }; - int ret; - char config[512]; - - /* - * If we want to stress sweep, we have a lot of additional - * configuration settings to set. - */ - if (g.sweep_stress) - testutil_check(__wt_snprintf(config, sizeof(config), - "create,cache_cursors=false,statistics=(fast)," \ - "statistics_log=(json,wait=1),error_prefix=\"%s\"," \ - "file_manager=(close_handle_minimum=1,close_idle_time=1,"\ - "close_scan_interval=1),log=(enabled),cache_size=1GB,"\ - "timing_stress_for_test=(aggressive_sweep)%s%s%s", - progname, - g.debug_mode ? DEBUG_MODE_CFG : "", - config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open)); - else - testutil_check(__wt_snprintf(config, sizeof(config), - "create,cache_cursors=false,statistics=(fast)," \ - "statistics_log=(json,wait=1),error_prefix=\"%s\"" \ - "%s%s%s", - progname, - g.debug_mode ? DEBUG_MODE_CFG : "", - config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open)); - - if ((ret = wiredtiger_open( - g.home, &event_handler, config, &g.conn)) != 0) - return (log_print_err("wiredtiger_open", ret, 1)); - return (0); + static WT_EVENT_HANDLER event_handler = { + handle_error, handle_message, NULL, NULL /* Close handler. */ + }; + int ret; + char config[512]; + + /* + * If we want to stress sweep, we have a lot of additional configuration settings to set. + */ + if (g.sweep_stress) + testutil_check(__wt_snprintf(config, sizeof(config), + "create,cache_cursors=false,statistics=(fast)," + "statistics_log=(json,wait=1),error_prefix=\"%s\"," + "file_manager=(close_handle_minimum=1,close_idle_time=1," + "close_scan_interval=1),log=(enabled),cache_size=1GB," + "timing_stress_for_test=(aggressive_sweep)%s%s%s", + progname, g.debug_mode ? DEBUG_MODE_CFG : "", config_open == NULL ? "" : ",", + config_open == NULL ? "" : config_open)); + else + testutil_check(__wt_snprintf(config, sizeof(config), + "create,cache_cursors=false,statistics=(fast)," + "statistics_log=(json,wait=1),error_prefix=\"%s\"" + "%s%s%s", + progname, g.debug_mode ? DEBUG_MODE_CFG : "", config_open == NULL ? "" : ",", + config_open == NULL ? "" : config_open)); + + if ((ret = wiredtiger_open(g.home, &event_handler, config, &g.conn)) != 0) + return (log_print_err("wiredtiger_open", ret, 1)); + return (0); } /* * wt_shutdown -- - * Shut down the WiredTiger connection. + * Shut down the WiredTiger connection. */ static int wt_shutdown(void) { - int ret; + int ret; - if (g.conn == NULL) - return (0); + if (g.conn == NULL) + return (0); - printf("Closing connection\n"); - ret = g.conn->close(g.conn, NULL); - g.conn = NULL; - if (ret != 0) - return (log_print_err("conn.close", ret, 1)); - return (0); + printf("Closing connection\n"); + ret = g.conn->close(g.conn, NULL); + g.conn = NULL; + if (ret != 0) + return (log_print_err("conn.close", ret, 1)); + return (0); } /* * cleanup -- - * Clean up from previous runs. + * Clean up from previous runs. */ static void cleanup(bool remove_dir) { - g.running = 0; - g.ntables_created = 0; - g.ts = 0; + g.running = 0; + g.ntables_created = 0; + g.ts = 0; - if (remove_dir) - testutil_make_work_dir(g.home); + if (remove_dir) + testutil_make_work_dir(g.home); } static int -handle_error(WT_EVENT_HANDLER *handler, - WT_SESSION *session, int error, const char *errmsg) +handle_error(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const char *errmsg) { - WT_UNUSED(handler); - WT_UNUSED(session); - WT_UNUSED(error); + WT_UNUSED(handler); + WT_UNUSED(session); + WT_UNUSED(error); - return (fprintf(stderr, "%s\n", errmsg) < 0 ? -1 : 0); + return (fprintf(stderr, "%s\n", errmsg) < 0 ? -1 : 0); } static int -handle_message(WT_EVENT_HANDLER *handler, - WT_SESSION *session, const char *message) +handle_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message) { - WT_UNUSED(handler); - WT_UNUSED(session); + WT_UNUSED(handler); + WT_UNUSED(session); - if (g.logfp != NULL) - return (fprintf(g.logfp, "%s\n", message) < 0 ? -1 : 0); + if (g.logfp != NULL) + return (fprintf(g.logfp, "%s\n", message) < 0 ? -1 : 0); - return (printf("%s\n", message) < 0 ? -1 : 0); + return (printf("%s\n", message) < 0 ? -1 : 0); } /* * onint -- - * Interrupt signal handler. + * Interrupt signal handler. */ static void onint(int signo) { - WT_UNUSED(signo); + WT_UNUSED(signo); - cleanup(false); + cleanup(false); - fprintf(stderr, "\n"); - exit(EXIT_FAILURE); + fprintf(stderr, "\n"); + exit(EXIT_FAILURE); } /* * log_print_err -- - * Report an error and return the error. + * Report an error and return the error. */ int log_print_err(const char *m, int e, int fatal) { - if (fatal) { - g.running = 0; - g.status = e; - } - fprintf(stderr, "%s: %s: %s\n", progname, m, wiredtiger_strerror(e)); - if (g.logfp != NULL) - fprintf(g.logfp, "%s: %s: %s\n", - progname, m, wiredtiger_strerror(e)); - return (e); + if (fatal) { + g.running = 0; + g.status = e; + } + fprintf(stderr, "%s: %s: %s\n", progname, m, wiredtiger_strerror(e)); + if (g.logfp != NULL) + fprintf(g.logfp, "%s: %s: %s\n", progname, m, wiredtiger_strerror(e)); + return (e); } /* * path_setup -- - * Build the standard paths and shell commands we use. + * Build the standard paths and shell commands we use. */ const char * type_to_string(table_type type) { - if (type == COL) - return ("COL"); - if (type == LSM) - return ("LSM"); - if (type == ROW) - return ("ROW"); - if (type == MIX) - return ("MIX"); - return ("INVALID"); + if (type == COL) + return ("COL"); + if (type == LSM) + return ("LSM"); + if (type == ROW) + return ("ROW"); + if (type == MIX) + return ("MIX"); + return ("INVALID"); } /* * usage -- - * Display usage statement and exit failure. + * Display usage statement and exit failure. */ static int usage(void) { - fprintf(stderr, - "usage: %s " - "[-C wiredtiger-config] [-c checkpoint] [-h home] [-k keys]\n\t" - "[-l log] [-n ops] [-r runs] [-T table-config] [-t f|r|v]\n\t" - "[-W workers]\n", - progname); - fprintf(stderr, "%s", - "\t-C specify wiredtiger_open configuration arguments\n" - "\t-c checkpoint name to used named checkpoints\n" - "\t-h set a database home directory\n" - "\t-k set number of keys to load\n" - "\t-l specify a log file\n" - "\t-n set number of operations each thread does\n" - "\t-r set number of runs (0 for continuous)\n" - "\t-T specify a table configuration\n" - "\t-t set a file type ( col | mix | row | lsm )\n" - "\t-W set number of worker threads\n"); - return (EXIT_FAILURE); + fprintf(stderr, + "usage: %s " + "[-C wiredtiger-config] [-c checkpoint] [-h home] [-k keys]\n\t" + "[-l log] [-n ops] [-r runs] [-T table-config] [-t f|r|v]\n\t" + "[-W workers]\n", + progname); + fprintf(stderr, "%s", + "\t-C specify wiredtiger_open configuration arguments\n" + "\t-c checkpoint name to used named checkpoints\n" + "\t-h set a database home directory\n" + "\t-k set number of keys to load\n" + "\t-l specify a log file\n" + "\t-n set number of operations each thread does\n" + "\t-r set number of runs (0 for continuous)\n" + "\t-T specify a table configuration\n" + "\t-t set a file type ( col | mix | row | lsm )\n" + "\t-W set number of worker threads\n"); + return (EXIT_FAILURE); } diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h index 087c2d4be19..0bf5d4f669e 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h @@ -30,52 +30,52 @@ #include <signal.h> -#define URI_BASE "table:__wt" /* File name */ +#define URI_BASE "table:__wt" /* File name */ -#define ERR_KEY_MISMATCH 0x200001 -#define ERR_DATA_MISMATCH 0x200002 +#define ERR_KEY_MISMATCH 0x200001 +#define ERR_DATA_MISMATCH 0x200002 /* - * There are three different table types in the test, and a 'special' type - * of mixed (i.e a mixture of the other three types. + * There are three different table types in the test, and a 'special' type of mixed (i.e a mixture + * of the other three types. */ -#define MAX_TABLE_TYPE 3 -typedef enum { MIX = 0, COL, LSM, ROW } table_type; /* File type */ +#define MAX_TABLE_TYPE 3 +typedef enum { MIX = 0, COL, LSM, ROW } table_type; /* File type */ /* * Per-table cookie structure. */ typedef struct { - int id; - table_type type; /* Type for table. */ - char uri[128]; + int id; + table_type type; /* Type for table. */ + char uri[128]; } COOKIE; typedef struct { - char *home; /* Home directory */ - const char *checkpoint_name; /* Checkpoint name */ - WT_CONNECTION *conn; /* WiredTiger connection */ - bool debug_mode; /* Lookaside stress test */ - u_int nkeys; /* Keys to load */ - u_int nops; /* Operations per thread */ - FILE *logfp; /* Message log file. */ - int nworkers; /* Number workers configured */ - int ntables; /* Number tables configured */ - int ntables_created; /* Number tables opened */ - volatile int running; /* Whether to stop */ - int status; /* Exit status */ - bool sweep_stress; /* Sweep stress test */ - u_int ts; /* Current timestamp */ - bool use_timestamps; /* Use txn timestamps */ - COOKIE *cookies; /* Per-thread info */ - WT_RWLOCK clock_lock; /* Clock synchronization */ - wt_thread_t checkpoint_thread; /* Checkpoint thread */ - wt_thread_t clock_thread; /* Clock thread */ + char *home; /* Home directory */ + const char *checkpoint_name; /* Checkpoint name */ + WT_CONNECTION *conn; /* WiredTiger connection */ + bool debug_mode; /* Lookaside stress test */ + u_int nkeys; /* Keys to load */ + u_int nops; /* Operations per thread */ + FILE *logfp; /* Message log file. */ + int nworkers; /* Number workers configured */ + int ntables; /* Number tables configured */ + int ntables_created; /* Number tables opened */ + volatile int running; /* Whether to stop */ + int status; /* Exit status */ + bool sweep_stress; /* Sweep stress test */ + u_int ts; /* Current timestamp */ + bool use_timestamps; /* Use txn timestamps */ + COOKIE *cookies; /* Per-thread info */ + WT_RWLOCK clock_lock; /* Clock synchronization */ + wt_thread_t checkpoint_thread; /* Checkpoint thread */ + wt_thread_t clock_thread; /* Clock thread */ } GLOBAL; extern GLOBAL g; -void end_checkpoints(void); -int log_print_err(const char *, int, int); -void start_checkpoints(void); -int start_workers(table_type); +void end_checkpoints(void); +int log_print_err(const char *, int, int); +void start_checkpoints(void); +int start_workers(table_type); const char *type_to_string(table_type); diff --git a/src/third_party/wiredtiger/test/checkpoint/workers.c b/src/third_party/wiredtiger/test/checkpoint/workers.c index e9966cec145..1b5a78bbdef 100644 --- a/src/third_party/wiredtiger/test/checkpoint/workers.c +++ b/src/third_party/wiredtiger/test/checkpoint/workers.c @@ -38,292 +38,266 @@ static WT_THREAD_RET worker(void *); static int create_table(WT_SESSION *session, COOKIE *cookie) { - int ret; - char config[256]; - - /* - * If we're using timestamps, turn off logging for the table. - */ - if (g.use_timestamps) - testutil_check(__wt_snprintf(config, sizeof(config), - "key_format=%s,value_format=S,allocation_size=512," \ - "leaf_page_max=1KB,internal_page_max=1KB," \ - "memory_page_max=64KB,log=(enabled=false),%s", - cookie->type == COL ? "r" : "q", - cookie->type == LSM ? ",type=lsm" : "")); - else - testutil_check(__wt_snprintf(config, sizeof(config), - "key_format=%s,value_format=S,%s", - cookie->type == COL ? "r" : "q", - cookie->type == LSM ? ",type=lsm" : "")); - - if ((ret = session->create(session, cookie->uri, config)) != 0) - if (ret != EEXIST) - return (log_print_err("session.create", ret, 1)); - ++g.ntables_created; - return (0); + int ret; + char config[256]; + + /* + * If we're using timestamps, turn off logging for the table. + */ + if (g.use_timestamps) + testutil_check(__wt_snprintf(config, sizeof(config), + "key_format=%s,value_format=S,allocation_size=512," + "leaf_page_max=1KB,internal_page_max=1KB," + "memory_page_max=64KB,log=(enabled=false),%s", + cookie->type == COL ? "r" : "q", cookie->type == LSM ? ",type=lsm" : "")); + else + testutil_check(__wt_snprintf(config, sizeof(config), "key_format=%s,value_format=S,%s", + cookie->type == COL ? "r" : "q", cookie->type == LSM ? ",type=lsm" : "")); + + if ((ret = session->create(session, cookie->uri, config)) != 0) + if (ret != EEXIST) + return (log_print_err("session.create", ret, 1)); + ++g.ntables_created; + return (0); } /* * start_workers -- - * Setup the configuration for the tables being populated, then start - * the worker thread(s) and wait for them to finish. + * Setup the configuration for the tables being populated, then start the worker thread(s) and + * wait for them to finish. */ int start_workers(table_type type) { - struct timeval start, stop; - WT_SESSION *session; - wt_thread_t *tids; - double seconds; - int i, ret; - - ret = 0; - - /* Create statistics and thread structures. */ - if ((tids = calloc((size_t)(g.nworkers), sizeof(*tids))) == NULL) - return (log_print_err("calloc", errno, 1)); - - if ((ret = g.conn->open_session(g.conn, NULL, NULL, &session)) != 0) { - (void)log_print_err("conn.open_session", ret, 1); - goto err; - } - /* Setup the cookies */ - for (i = 0; i < g.ntables; ++i) { - g.cookies[i].id = i; - if (type == MIX) - g.cookies[i].type = - (table_type)((i % MAX_TABLE_TYPE) + 1); - else - g.cookies[i].type = type; - testutil_check(__wt_snprintf( - g.cookies[i].uri, sizeof(g.cookies[i].uri), - "%s%04d", URI_BASE, g.cookies[i].id)); - - /* Should probably be atomic to avoid races. */ - if ((ret = create_table(session, &g.cookies[i])) != 0) - goto err; - } - - testutil_check(session->close(session, NULL)); - - (void)gettimeofday(&start, NULL); - - /* Create threads. */ - for (i = 0; i < g.nworkers; ++i) - testutil_check(__wt_thread_create( - NULL, &tids[i], worker, &g.cookies[i])); - - /* Wait for the threads. */ - for (i = 0; i < g.nworkers; ++i) - testutil_check(__wt_thread_join(NULL, &tids[i])); - - (void)gettimeofday(&stop, NULL); - seconds = (stop.tv_sec - start.tv_sec) + - (stop.tv_usec - start.tv_usec) * 1e-6; - printf("Ran workers for: %f seconds\n", seconds); - -err: free(tids); - - return (ret); + struct timeval start, stop; + WT_SESSION *session; + wt_thread_t *tids; + double seconds; + int i, ret; + + ret = 0; + + /* Create statistics and thread structures. */ + if ((tids = calloc((size_t)(g.nworkers), sizeof(*tids))) == NULL) + return (log_print_err("calloc", errno, 1)); + + if ((ret = g.conn->open_session(g.conn, NULL, NULL, &session)) != 0) { + (void)log_print_err("conn.open_session", ret, 1); + goto err; + } + /* Setup the cookies */ + for (i = 0; i < g.ntables; ++i) { + g.cookies[i].id = i; + if (type == MIX) + g.cookies[i].type = (table_type)((i % MAX_TABLE_TYPE) + 1); + else + g.cookies[i].type = type; + testutil_check(__wt_snprintf( + g.cookies[i].uri, sizeof(g.cookies[i].uri), "%s%04d", URI_BASE, g.cookies[i].id)); + + /* Should probably be atomic to avoid races. */ + if ((ret = create_table(session, &g.cookies[i])) != 0) + goto err; + } + + testutil_check(session->close(session, NULL)); + + (void)gettimeofday(&start, NULL); + + /* Create threads. */ + for (i = 0; i < g.nworkers; ++i) + testutil_check(__wt_thread_create(NULL, &tids[i], worker, &g.cookies[i])); + + /* Wait for the threads. */ + for (i = 0; i < g.nworkers; ++i) + testutil_check(__wt_thread_join(NULL, &tids[i])); + + (void)gettimeofday(&stop, NULL); + seconds = (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec) * 1e-6; + printf("Ran workers for: %f seconds\n", seconds); + +err: + free(tids); + + return (ret); } /* * worker_op -- - * Write operation. + * Write operation. */ static inline int worker_op(WT_CURSOR *cursor, uint64_t keyno, u_int new_val) { - int cmp, ret; - char valuebuf[64]; - - cursor->set_key(cursor, keyno); - /* Roughly half inserts, then balanced inserts / range removes. */ - if (new_val > g.nops / 2 && new_val % 39 == 0) { - if ((ret = cursor->search_near(cursor, &cmp)) != 0) { - if (ret == WT_NOTFOUND) - return (0); - if (ret == WT_ROLLBACK) - return (WT_ROLLBACK); - return (log_print_err("cursor.search_near", ret, 1)); - } - if (cmp < 0) { - if ((ret = cursor->next(cursor)) != 0) { - if (ret == WT_NOTFOUND) - return (0); - if (ret == WT_ROLLBACK) - return (WT_ROLLBACK); - return (log_print_err("cursor.next", ret, 1)); - } - } - for (int i = 10; i > 0; i--) { - if ((ret = cursor->remove(cursor)) != 0) { - if (ret == WT_ROLLBACK) - return (WT_ROLLBACK); - return (log_print_err("cursor.remove", ret, 1)); - } - if ((ret = cursor->next(cursor)) != 0) { - if (ret == WT_NOTFOUND) - return (0); - if (ret == WT_ROLLBACK) - return (WT_ROLLBACK); - return (log_print_err("cursor.next", ret, 1)); - } - } - if (g.sweep_stress) - testutil_check(cursor->reset(cursor)); - } else if (new_val % 39 < 10) { - if ((ret = cursor->search(cursor)) != 0 && ret != WT_NOTFOUND) { - if (ret == WT_ROLLBACK) - return (WT_ROLLBACK); - return (log_print_err("cursor.search", ret, 1)); - } - if (g.sweep_stress) - testutil_check(cursor->reset(cursor)); - } else { - testutil_check(__wt_snprintf( - valuebuf, sizeof(valuebuf), "%052u", new_val)); - cursor->set_value(cursor, valuebuf); - if ((ret = cursor->insert(cursor)) != 0) { - if (ret == WT_ROLLBACK) - return (WT_ROLLBACK); - return (log_print_err("cursor.insert", ret, 1)); - } - } - - return (0); + int cmp, ret; + char valuebuf[64]; + + cursor->set_key(cursor, keyno); + /* Roughly half inserts, then balanced inserts / range removes. */ + if (new_val > g.nops / 2 && new_val % 39 == 0) { + if ((ret = cursor->search_near(cursor, &cmp)) != 0) { + if (ret == WT_NOTFOUND) + return (0); + if (ret == WT_ROLLBACK) + return (WT_ROLLBACK); + return (log_print_err("cursor.search_near", ret, 1)); + } + if (cmp < 0) { + if ((ret = cursor->next(cursor)) != 0) { + if (ret == WT_NOTFOUND) + return (0); + if (ret == WT_ROLLBACK) + return (WT_ROLLBACK); + return (log_print_err("cursor.next", ret, 1)); + } + } + for (int i = 10; i > 0; i--) { + if ((ret = cursor->remove(cursor)) != 0) { + if (ret == WT_ROLLBACK) + return (WT_ROLLBACK); + return (log_print_err("cursor.remove", ret, 1)); + } + if ((ret = cursor->next(cursor)) != 0) { + if (ret == WT_NOTFOUND) + return (0); + if (ret == WT_ROLLBACK) + return (WT_ROLLBACK); + return (log_print_err("cursor.next", ret, 1)); + } + } + if (g.sweep_stress) + testutil_check(cursor->reset(cursor)); + } else if (new_val % 39 < 10) { + if ((ret = cursor->search(cursor)) != 0 && ret != WT_NOTFOUND) { + if (ret == WT_ROLLBACK) + return (WT_ROLLBACK); + return (log_print_err("cursor.search", ret, 1)); + } + if (g.sweep_stress) + testutil_check(cursor->reset(cursor)); + } else { + testutil_check(__wt_snprintf(valuebuf, sizeof(valuebuf), "%052u", new_val)); + cursor->set_value(cursor, valuebuf); + if ((ret = cursor->insert(cursor)) != 0) { + if (ret == WT_ROLLBACK) + return (WT_ROLLBACK); + return (log_print_err("cursor.insert", ret, 1)); + } + } + + return (0); } /* * worker -- - * Worker thread start function. + * Worker thread start function. */ static WT_THREAD_RET worker(void *arg) { - char tid[128]; + char tid[128]; - WT_UNUSED(arg); + WT_UNUSED(arg); - testutil_check(__wt_thread_str(tid, sizeof(tid))); - printf("worker thread starting: tid: %s\n", tid); + testutil_check(__wt_thread_str(tid, sizeof(tid))); + printf("worker thread starting: tid: %s\n", tid); - (void)real_worker(); - return (WT_THREAD_RET_VALUE); + (void)real_worker(); + return (WT_THREAD_RET_VALUE); } /* * real_worker -- - * A single worker thread that transactionally updates all tables with - * consistent values. + * A single worker thread that transactionally updates all tables with consistent values. */ static int real_worker(void) { - WT_CURSOR **cursors; - WT_RAND_STATE rnd; - WT_SESSION *session; - u_int i, keyno; - int j, ret, t_ret; - const char *begin_cfg; - char buf[128]; - bool has_cursors; - - ret = t_ret = 0; - - if ((cursors = calloc( - (size_t)(g.ntables), sizeof(WT_CURSOR *))) == NULL) - return (log_print_err("malloc", ENOMEM, 1)); - - if ((ret = g.conn->open_session( - g.conn, NULL, "isolation=snapshot", &session)) != 0) { - (void)log_print_err("conn.open_session", ret, 1); - goto err; - } - - __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); - - for (j = 0; j < g.ntables; j++) - if ((ret = session->open_cursor(session, - g.cookies[j].uri, NULL, NULL, &cursors[j])) != 0) { - (void)log_print_err("session.open_cursor", ret, 1); - goto err; - } - has_cursors = true; - - if (g.use_timestamps) - begin_cfg = "read_timestamp=1,roundup_timestamps=(read=true)"; - else - begin_cfg = NULL; - - for (i = 0; i < g.nops && g.running; ++i, __wt_yield()) { - if ((ret = - session->begin_transaction(session, begin_cfg)) != 0) { - (void)log_print_err( - "real_worker:begin_transaction", ret, 1); - goto err; - } - keyno = __wt_random(&rnd) % g.nkeys + 1; - if (g.use_timestamps && i % 23 == 0) { - if (__wt_try_readlock( - (WT_SESSION_IMPL *)session, &g.clock_lock) != 0) { - testutil_check( - session->commit_transaction(session, NULL)); - for (j = 0; j < g.ntables; j++) - testutil_check( - cursors[j]->close(cursors[j])); - has_cursors = false; - __wt_readlock( - (WT_SESSION_IMPL *)session, &g.clock_lock); - testutil_check(session->begin_transaction( - session, begin_cfg)); - } - testutil_check(__wt_snprintf( - buf, sizeof(buf), "commit_timestamp=%x", g.ts + 1)); - testutil_check( - session->timestamp_transaction(session, buf)); - __wt_readunlock( - (WT_SESSION_IMPL *)session, &g.clock_lock); - - for (j = 0; !has_cursors && j < g.ntables; j++) - if ((ret = session->open_cursor( - session, g.cookies[j].uri, - NULL, NULL, &cursors[j])) != 0) { - (void)log_print_err( - "session.open_cursor", ret, 1); - goto err; - } - has_cursors = true; - } - for (j = 0; ret == 0 && j < g.ntables; j++) { - ret = worker_op(cursors[j], keyno, i); - } - if (ret != 0 && ret != WT_ROLLBACK) { - (void)log_print_err("worker op failed", ret, 1); - goto err; - } else if (ret == 0 && __wt_random(&rnd) % 7 != 0) { - if ((ret = session->commit_transaction( - session, NULL)) != 0) { - (void)log_print_err( - "real_worker:commit_transaction", ret, 1); - goto err; - } - } else { - if ((ret = session->rollback_transaction( - session, NULL)) != 0) { - (void)log_print_err( - "real_worker:rollback_transaction", ret, 1); - goto err; - } - } - } - -err: if ((t_ret = session->close(session, NULL)) != 0 && ret == 0) { - ret = t_ret; - (void)log_print_err("session.close", ret, 1); - } - free(cursors); - - return (ret); + WT_CURSOR **cursors; + WT_RAND_STATE rnd; + WT_SESSION *session; + u_int i, keyno; + int j, ret, t_ret; + char buf[128]; + const char *begin_cfg; + bool has_cursors; + + ret = t_ret = 0; + + if ((cursors = calloc((size_t)(g.ntables), sizeof(WT_CURSOR *))) == NULL) + return (log_print_err("malloc", ENOMEM, 1)); + + if ((ret = g.conn->open_session(g.conn, NULL, "isolation=snapshot", &session)) != 0) { + (void)log_print_err("conn.open_session", ret, 1); + goto err; + } + + __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); + + for (j = 0; j < g.ntables; j++) + if ((ret = session->open_cursor(session, g.cookies[j].uri, NULL, NULL, &cursors[j])) != 0) { + (void)log_print_err("session.open_cursor", ret, 1); + goto err; + } + has_cursors = true; + + if (g.use_timestamps) + begin_cfg = "read_timestamp=1,roundup_timestamps=(read=true)"; + else + begin_cfg = NULL; + + for (i = 0; i < g.nops && g.running; ++i, __wt_yield()) { + if ((ret = session->begin_transaction(session, begin_cfg)) != 0) { + (void)log_print_err("real_worker:begin_transaction", ret, 1); + goto err; + } + keyno = __wt_random(&rnd) % g.nkeys + 1; + if (g.use_timestamps && i % 23 == 0) { + if (__wt_try_readlock((WT_SESSION_IMPL *)session, &g.clock_lock) != 0) { + testutil_check(session->commit_transaction(session, NULL)); + for (j = 0; j < g.ntables; j++) + testutil_check(cursors[j]->close(cursors[j])); + has_cursors = false; + __wt_readlock((WT_SESSION_IMPL *)session, &g.clock_lock); + testutil_check(session->begin_transaction(session, begin_cfg)); + } + testutil_check(__wt_snprintf(buf, sizeof(buf), "commit_timestamp=%x", g.ts + 1)); + testutil_check(session->timestamp_transaction(session, buf)); + __wt_readunlock((WT_SESSION_IMPL *)session, &g.clock_lock); + + for (j = 0; !has_cursors && j < g.ntables; j++) + if ((ret = session->open_cursor( + session, g.cookies[j].uri, NULL, NULL, &cursors[j])) != 0) { + (void)log_print_err("session.open_cursor", ret, 1); + goto err; + } + has_cursors = true; + } + for (j = 0; ret == 0 && j < g.ntables; j++) { + ret = worker_op(cursors[j], keyno, i); + } + if (ret != 0 && ret != WT_ROLLBACK) { + (void)log_print_err("worker op failed", ret, 1); + goto err; + } else if (ret == 0 && __wt_random(&rnd) % 7 != 0) { + if ((ret = session->commit_transaction(session, NULL)) != 0) { + (void)log_print_err("real_worker:commit_transaction", ret, 1); + goto err; + } + } else { + if ((ret = session->rollback_transaction(session, NULL)) != 0) { + (void)log_print_err("real_worker:rollback_transaction", ret, 1); + goto err; + } + } + } + +err: + if ((t_ret = session->close(session, NULL)) != 0 && ret == 0) { + ret = t_ret; + (void)log_print_err("session.close", ret, 1); + } + free(cursors); + + return (ret); } diff --git a/src/third_party/wiredtiger/test/csuite/random_abort/main.c b/src/third_party/wiredtiger/test/csuite/random_abort/main.c index 98402f0d233..8bc365d75c1 100644 --- a/src/third_party/wiredtiger/test/csuite/random_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/random_abort/main.c @@ -31,182 +31,171 @@ #include <sys/wait.h> #include <signal.h> -static char home[1024]; /* Program working dir */ +static char home[1024]; /* Program working dir */ /* * These two names for the URI and file system must be maintained in tandem. */ -static const char * const uri = "table:main"; +static const char *const uri = "table:main"; static bool compat; static bool inmem; -#define MAX_TH 12 -#define MIN_TH 5 -#define MAX_TIME 40 -#define MIN_TIME 10 -#define RECORDS_FILE "records-%" PRIu32 +#define MAX_TH 12 +#define MIN_TH 5 +#define MAX_TIME 40 +#define MIN_TIME 10 +#define RECORDS_FILE "records-%" PRIu32 -#define ENV_CONFIG_COMPAT ",compatibility=(release=\"2.9\")" -#define ENV_CONFIG_DEF \ - "create,log=(file_max=10M,enabled)" -#define ENV_CONFIG_TXNSYNC \ - "create,log=(file_max=10M,enabled)," \ +#define ENV_CONFIG_COMPAT ",compatibility=(release=\"2.9\")" +#define ENV_CONFIG_DEF "create,log=(file_max=10M,enabled)" +#define ENV_CONFIG_TXNSYNC \ + "create,log=(file_max=10M,enabled)," \ "transaction_sync=(enabled,method=none)" -#define ENV_CONFIG_REC "log=(recover=on)" -#define MAX_VAL 4096 +#define ENV_CONFIG_REC "log=(recover=on)" +#define MAX_VAL 4096 -static void handler(int) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); -static void usage(void) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void handler(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void usage(void) { - fprintf(stderr, "usage: %s [-h dir] [-T threads]\n", progname); - exit(EXIT_FAILURE); + fprintf(stderr, "usage: %s [-h dir] [-T threads]\n", progname); + exit(EXIT_FAILURE); } typedef struct { - WT_CONNECTION *conn; - uint64_t start; - uint32_t id; + WT_CONNECTION *conn; + uint64_t start; + uint32_t id; } WT_THREAD_DATA; static WT_THREAD_RET thread_run(void *arg) { - FILE *fp; - WT_CURSOR *cursor; - WT_ITEM data; - WT_RAND_STATE rnd; - WT_SESSION *session; - WT_THREAD_DATA *td; - size_t lsize; - uint64_t i; - char buf[MAX_VAL], kname[64], lgbuf[8]; - char large[128*1024]; + FILE *fp; + WT_CURSOR *cursor; + WT_ITEM data; + WT_RAND_STATE rnd; + WT_SESSION *session; + WT_THREAD_DATA *td; + size_t lsize; + uint64_t i; + char buf[MAX_VAL], kname[64], lgbuf[8]; + char large[128 * 1024]; - __wt_random_init(&rnd); - memset(buf, 0, sizeof(buf)); - memset(kname, 0, sizeof(kname)); - lsize = sizeof(large); - memset(large, 0, lsize); + __wt_random_init(&rnd); + memset(buf, 0, sizeof(buf)); + memset(kname, 0, sizeof(kname)); + lsize = sizeof(large); + memset(large, 0, lsize); - td = (WT_THREAD_DATA *)arg; - /* - * The value is the name of the record file with our id appended. - */ - testutil_check(__wt_snprintf(buf, sizeof(buf), RECORDS_FILE, td->id)); - /* - * Set up a large value putting our id in it. Write it in there a - * bunch of times, but the rest of the buffer can just be zero. - */ - testutil_check(__wt_snprintf( - lgbuf, sizeof(lgbuf), "th-%" PRIu32, td->id)); - for (i = 0; i < 128; i += strlen(lgbuf)) - testutil_check(__wt_snprintf( - &large[i], lsize - i, "%s", lgbuf)); - /* - * Keep a separate file with the records we wrote for checking. - */ - (void)unlink(buf); - if ((fp = fopen(buf, "w")) == NULL) - testutil_die(errno, "fopen"); - /* - * Set to line buffering. But that is advisory only. We've seen - * cases where the result files end up with partial lines. - */ - __wt_stream_set_line_buffer(fp); - testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); - testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); - data.data = buf; - data.size = sizeof(buf); - /* - * Write our portion of the key space until we're killed. - */ - printf("Thread %" PRIu32 " starts at %" PRIu64 "\n", - td->id, td->start); - for (i = td->start; ; ++i) { - testutil_check(__wt_snprintf( - kname, sizeof(kname), "%" PRIu64, i)); - cursor->set_key(cursor, kname); - /* - * Every 30th record write a very large record that exceeds the - * log buffer size. This forces us to use the unbuffered path. - */ - if (i % 30 == 0) { - data.size = 128 * 1024; - data.data = large; - } else { - data.size = __wt_random(&rnd) % MAX_VAL; - data.data = buf; - } - cursor->set_value(cursor, &data); - testutil_check(cursor->insert(cursor)); - /* - * Save the key separately for checking later. - */ - if (fprintf(fp, "%" PRIu64 "\n", i) == -1) - testutil_die(errno, "fprintf"); - } - /* NOTREACHED */ + td = (WT_THREAD_DATA *)arg; + /* + * The value is the name of the record file with our id appended. + */ + testutil_check(__wt_snprintf(buf, sizeof(buf), RECORDS_FILE, td->id)); + /* + * Set up a large value putting our id in it. Write it in there a bunch of times, but the rest + * of the buffer can just be zero. + */ + testutil_check(__wt_snprintf(lgbuf, sizeof(lgbuf), "th-%" PRIu32, td->id)); + for (i = 0; i < 128; i += strlen(lgbuf)) + testutil_check(__wt_snprintf(&large[i], lsize - i, "%s", lgbuf)); + /* + * Keep a separate file with the records we wrote for checking. + */ + (void)unlink(buf); + if ((fp = fopen(buf, "w")) == NULL) + testutil_die(errno, "fopen"); + /* + * Set to line buffering. But that is advisory only. We've seen cases where the result files end + * up with partial lines. + */ + __wt_stream_set_line_buffer(fp); + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); + data.data = buf; + data.size = sizeof(buf); + /* + * Write our portion of the key space until we're killed. + */ + printf("Thread %" PRIu32 " starts at %" PRIu64 "\n", td->id, td->start); + for (i = td->start;; ++i) { + testutil_check(__wt_snprintf(kname, sizeof(kname), "%" PRIu64, i)); + cursor->set_key(cursor, kname); + /* + * Every 30th record write a very large record that exceeds the log buffer size. This forces + * us to use the unbuffered path. + */ + if (i % 30 == 0) { + data.size = 128 * 1024; + data.data = large; + } else { + data.size = __wt_random(&rnd) % MAX_VAL; + data.data = buf; + } + cursor->set_value(cursor, &data); + testutil_check(cursor->insert(cursor)); + /* + * Save the key separately for checking later. + */ + if (fprintf(fp, "%" PRIu64 "\n", i) == -1) + testutil_die(errno, "fprintf"); + } + /* NOTREACHED */ } /* - * Child process creates the database and table, and then creates worker - * threads to add data until it is killed by the parent. + * Child process creates the database and table, and then creates worker threads to add data until + * it is killed by the parent. */ -static void fill_db(uint32_t) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void fill_db(uint32_t) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void fill_db(uint32_t nth) { - WT_CONNECTION *conn; - WT_SESSION *session; - WT_THREAD_DATA *td; - wt_thread_t *thr; - uint32_t i; - char envconf[512]; + WT_CONNECTION *conn; + WT_SESSION *session; + WT_THREAD_DATA *td; + wt_thread_t *thr; + uint32_t i; + char envconf[512]; - thr = dcalloc(nth, sizeof(*thr)); - td = dcalloc(nth, sizeof(WT_THREAD_DATA)); - if (chdir(home) != 0) - testutil_die(errno, "Child chdir: %s", home); - if (inmem) - strcpy(envconf, ENV_CONFIG_DEF); - else - strcpy(envconf, ENV_CONFIG_TXNSYNC); - if (compat) - strcat(envconf, ENV_CONFIG_COMPAT); + thr = dcalloc(nth, sizeof(*thr)); + td = dcalloc(nth, sizeof(WT_THREAD_DATA)); + if (chdir(home) != 0) + testutil_die(errno, "Child chdir: %s", home); + if (inmem) + strcpy(envconf, ENV_CONFIG_DEF); + else + strcpy(envconf, ENV_CONFIG_TXNSYNC); + if (compat) + strcat(envconf, ENV_CONFIG_COMPAT); - testutil_check(wiredtiger_open(NULL, NULL, envconf, &conn)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->create( - session, uri, "key_format=S,value_format=u")); - testutil_check(session->close(session, NULL)); + testutil_check(wiredtiger_open(NULL, NULL, envconf, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->create(session, uri, "key_format=S,value_format=u")); + testutil_check(session->close(session, NULL)); - printf("Create %" PRIu32 " writer threads\n", nth); - for (i = 0; i < nth; ++i) { - td[i].conn = conn; - td[i].start = WT_BILLION * (uint64_t)i; - td[i].id = i; - testutil_check(__wt_thread_create( - NULL, &thr[i], thread_run, &td[i])); - } - printf("Spawned %" PRIu32 " writer threads\n", nth); - fflush(stdout); - /* - * The threads never exit, so the child will just wait here until - * it is killed. - */ - for (i = 0; i < nth; ++i) - testutil_check(__wt_thread_join(NULL, &thr[i])); - /* - * NOTREACHED - */ - free(thr); - free(td); - exit(EXIT_SUCCESS); + printf("Create %" PRIu32 " writer threads\n", nth); + for (i = 0; i < nth; ++i) { + td[i].conn = conn; + td[i].start = WT_BILLION * (uint64_t)i; + td[i].id = i; + testutil_check(__wt_thread_create(NULL, &thr[i], thread_run, &td[i])); + } + printf("Spawned %" PRIu32 " writer threads\n", nth); + fflush(stdout); + /* + * The threads never exit, so the child will just wait here until it is killed. + */ + for (i = 0; i < nth; ++i) + testutil_check(__wt_thread_join(NULL, &thr[i])); + /* + * NOTREACHED + */ + free(thr); + free(td); + exit(EXIT_SUCCESS); } extern int __wt_optind; @@ -215,246 +204,226 @@ extern char *__wt_optarg; static void handler(int sig) { - pid_t pid; + pid_t pid; - WT_UNUSED(sig); - pid = wait(NULL); - /* - * The core file will indicate why the child exited. Choose EINVAL here. - */ - testutil_die(EINVAL, - "Child process %" PRIu64 " abnormally exited", (uint64_t)pid); + WT_UNUSED(sig); + pid = wait(NULL); + /* + * The core file will indicate why the child exited. Choose EINVAL here. + */ + testutil_die(EINVAL, "Child process %" PRIu64 " abnormally exited", (uint64_t)pid); } int main(int argc, char *argv[]) { - struct sigaction sa; - struct stat sb; - FILE *fp; - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_RAND_STATE rnd; - WT_SESSION *session; - pid_t pid; - uint64_t absent, count, key, last_key, middle; - uint32_t i, nth, timeout; - int ch, status, ret; - char buf[1024], fname[64], kname[64]; - const char *working_dir; - bool fatal, rand_th, rand_time, verify_only; + struct sigaction sa; + struct stat sb; + FILE *fp; + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_RAND_STATE rnd; + WT_SESSION *session; + pid_t pid; + uint64_t absent, count, key, last_key, middle; + uint32_t i, nth, timeout; + int ch, status, ret; + char buf[1024], fname[64], kname[64]; + const char *working_dir; + bool fatal, rand_th, rand_time, verify_only; - (void)testutil_set_progname(argv); + (void)testutil_set_progname(argv); - compat = inmem = false; - nth = MIN_TH; - rand_th = rand_time = true; - timeout = MIN_TIME; - verify_only = false; - working_dir = "WT_TEST.random-abort"; + compat = inmem = false; + nth = MIN_TH; + rand_th = rand_time = true; + timeout = MIN_TIME; + verify_only = false; + working_dir = "WT_TEST.random-abort"; - while ((ch = __wt_getopt(progname, argc, argv, "Ch:mT:t:v")) != EOF) - switch (ch) { - case 'C': - compat = true; - break; - case 'h': - working_dir = __wt_optarg; - break; - case 'm': - inmem = true; - break; - case 'T': - rand_th = false; - nth = (uint32_t)atoi(__wt_optarg); - break; - case 't': - rand_time = false; - timeout = (uint32_t)atoi(__wt_optarg); - break; - case 'v': - verify_only = true; - break; - default: - usage(); - } - argc -= __wt_optind; - if (argc != 0) - usage(); + while ((ch = __wt_getopt(progname, argc, argv, "Ch:mT:t:v")) != EOF) + switch (ch) { + case 'C': + compat = true; + break; + case 'h': + working_dir = __wt_optarg; + break; + case 'm': + inmem = true; + break; + case 'T': + rand_th = false; + nth = (uint32_t)atoi(__wt_optarg); + break; + case 't': + rand_time = false; + timeout = (uint32_t)atoi(__wt_optarg); + break; + case 'v': + verify_only = true; + break; + default: + usage(); + } + argc -= __wt_optind; + if (argc != 0) + usage(); - testutil_work_dir_from_path(home, sizeof(home), working_dir); - /* - * If the user wants to verify they need to tell us how many threads - * there were so we can find the old record files. - */ - if (verify_only && rand_th) { - fprintf(stderr, - "Verify option requires specifying number of threads\n"); - exit (EXIT_FAILURE); - } - if (!verify_only) { - testutil_make_work_dir(home); + testutil_work_dir_from_path(home, sizeof(home), working_dir); + /* + * If the user wants to verify they need to tell us how many threads there were so we can find + * the old record files. + */ + if (verify_only && rand_th) { + fprintf(stderr, "Verify option requires specifying number of threads\n"); + exit(EXIT_FAILURE); + } + if (!verify_only) { + testutil_make_work_dir(home); - __wt_random_init_seed(NULL, &rnd); - if (rand_time) { - timeout = __wt_random(&rnd) % MAX_TIME; - if (timeout < MIN_TIME) - timeout = MIN_TIME; - } - if (rand_th) { - nth = __wt_random(&rnd) % MAX_TH; - if (nth < MIN_TH) - nth = MIN_TH; - } - printf("Parent: Compatibility %s in-mem log %s\n", - compat ? "true" : "false", inmem ? "true" : "false"); - printf("Parent: Create %" PRIu32 - " threads; sleep %" PRIu32 " seconds\n", nth, timeout); - printf("CONFIG: %s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n", - progname, - compat ? " -C" : "", - inmem ? " -m" : "", - working_dir, nth, timeout); - /* - * Fork a child to insert as many items. We will then randomly - * kill the child, run recovery and make sure all items we wrote - * exist after recovery runs. - */ - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = handler; - testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); - if ((pid = fork()) < 0) - testutil_die(errno, "fork"); + __wt_random_init_seed(NULL, &rnd); + if (rand_time) { + timeout = __wt_random(&rnd) % MAX_TIME; + if (timeout < MIN_TIME) + timeout = MIN_TIME; + } + if (rand_th) { + nth = __wt_random(&rnd) % MAX_TH; + if (nth < MIN_TH) + nth = MIN_TH; + } + printf("Parent: Compatibility %s in-mem log %s\n", compat ? "true" : "false", + inmem ? "true" : "false"); + printf("Parent: Create %" PRIu32 " threads; sleep %" PRIu32 " seconds\n", nth, timeout); + printf("CONFIG: %s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n", progname, + compat ? " -C" : "", inmem ? " -m" : "", working_dir, nth, timeout); + /* + * Fork a child to insert as many items. We will then randomly kill the child, run recovery + * and make sure all items we wrote exist after recovery runs. + */ + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = handler; + testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); + if ((pid = fork()) < 0) + testutil_die(errno, "fork"); - if (pid == 0) { /* child */ - fill_db(nth); - return (EXIT_SUCCESS); - } + if (pid == 0) { /* child */ + fill_db(nth); + return (EXIT_SUCCESS); + } - /* parent */ - /* - * Sleep for the configured amount of time before killing - * the child. Start the timeout from the time we notice that - * the child workers have created their record files. That - * allows the test to run correctly on really slow machines. - */ - i = 0; - while (i < nth) { - /* - * Wait for each record file to exist. - */ - testutil_check(__wt_snprintf( - fname, sizeof(fname), RECORDS_FILE, i)); - testutil_check(__wt_snprintf( - buf, sizeof(buf),"%s/%s", home, fname)); - while (stat(buf, &sb) != 0) - testutil_sleep_wait(1, pid); - ++i; - } - sleep(timeout); - sa.sa_handler = SIG_DFL; - testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); + /* parent */ + /* + * Sleep for the configured amount of time before killing the child. Start the timeout from + * the time we notice that the child workers have created their record files. That allows + * the test to run correctly on really slow machines. + */ + i = 0; + while (i < nth) { + /* + * Wait for each record file to exist. + */ + testutil_check(__wt_snprintf(fname, sizeof(fname), RECORDS_FILE, i)); + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s/%s", home, fname)); + while (stat(buf, &sb) != 0) + testutil_sleep_wait(1, pid); + ++i; + } + sleep(timeout); + sa.sa_handler = SIG_DFL; + testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); - /* - * !!! It should be plenty long enough to make sure more than - * one log file exists. If wanted, that check would be added - * here. - */ - printf("Kill child\n"); - if (kill(pid, SIGKILL) != 0) - testutil_die(errno, "kill"); - if (waitpid(pid, &status, 0) == -1) - testutil_die(errno, "waitpid"); - } - /* - * !!! If we wanted to take a copy of the directory before recovery, - * this is the place to do it. - */ - if (chdir(home) != 0) - testutil_die(errno, "parent chdir: %s", home); + /* + * !!! It should be plenty long enough to make sure more than + * one log file exists. If wanted, that check would be added + * here. + */ + printf("Kill child\n"); + if (kill(pid, SIGKILL) != 0) + testutil_die(errno, "kill"); + if (waitpid(pid, &status, 0) == -1) + testutil_die(errno, "waitpid"); + } + /* + * !!! If we wanted to take a copy of the directory before recovery, + * this is the place to do it. + */ + if (chdir(home) != 0) + testutil_die(errno, "parent chdir: %s", home); - testutil_check(__wt_snprintf(buf, sizeof(buf), - "rm -rf ../%s.SAVE; mkdir ../%s.SAVE; " - "cp -p WiredTigerLog.* ../%s.SAVE;", - home, home, home)); - if ((status = system(buf)) < 0) - testutil_die(status, "system: %s", buf); + testutil_check(__wt_snprintf(buf, sizeof(buf), + "rm -rf ../%s.SAVE; mkdir ../%s.SAVE; " + "cp -p WiredTigerLog.* ../%s.SAVE;", + home, home, home)); + if ((status = system(buf)) < 0) + testutil_die(status, "system: %s", buf); - printf("Open database, run recovery and verify content\n"); - testutil_check(wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); + printf("Open database, run recovery and verify content\n"); + testutil_check(wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); - absent = count = 0; - fatal = false; - for (i = 0; i < nth; ++i) { - middle = 0; - testutil_check(__wt_snprintf( - fname, sizeof(fname), RECORDS_FILE, i)); - if ((fp = fopen(fname, "r")) == NULL) - testutil_die(errno, "fopen: %s", fname); + absent = count = 0; + fatal = false; + for (i = 0; i < nth; ++i) { + middle = 0; + testutil_check(__wt_snprintf(fname, sizeof(fname), RECORDS_FILE, i)); + if ((fp = fopen(fname, "r")) == NULL) + testutil_die(errno, "fopen: %s", fname); - /* - * For every key in the saved file, verify that the key exists - * in the table after recovery. If we're doing in-memory - * log buffering we never expect a record missing in the middle, - * but records may be missing at the end. If we did - * write-no-sync, we expect every key to have been recovered. - */ - for (last_key = UINT64_MAX;; ++count, last_key = key) { - ret = fscanf(fp, "%" SCNu64 "\n", &key); - /* - * Consider anything other than clear success in - * getting the key to be EOF. We've seen file system - * issues where the file ends with zeroes on a 4K - * boundary and does not return EOF but a ret of zero. - */ - if (ret != 1) - break; - /* - * If we're unlucky, the last line may be a partially - * written key at the end that can result in a false - * negative error for a missing record. Detect it. - */ - if (last_key != UINT64_MAX && key != last_key + 1) { - printf("%s: Ignore partial record %" PRIu64 - " last valid key %" PRIu64 "\n", - fname, key, last_key); - break; - } - testutil_check(__wt_snprintf( - kname, sizeof(kname), "%" PRIu64, key)); - cursor->set_key(cursor, kname); - if ((ret = cursor->search(cursor)) != 0) { - if (ret != WT_NOTFOUND) - testutil_die(ret, "search"); - if (!inmem) - printf("%s: no record with key %" - PRIu64 "\n", fname, key); - absent++; - middle = key; - } else if (middle != 0) { - /* - * We should never find an existing key after - * we have detected one missing. - */ - printf("%s: after absent record at %" PRIu64 - " key %" PRIu64 " exists\n", - fname, middle, key); - fatal = true; - } - } - if (fclose(fp) != 0) - testutil_die(errno, "fclose"); - } - testutil_check(conn->close(conn, NULL)); - if (fatal) - return (EXIT_FAILURE); - if (!inmem && absent) { - printf("%" PRIu64 " record(s) absent from %" PRIu64 "\n", - absent, count); - return (EXIT_FAILURE); - } - printf("%" PRIu64 " records verified\n", count); - return (EXIT_SUCCESS); + /* + * For every key in the saved file, verify that the key exists in the table after recovery. + * If we're doing in-memory log buffering we never expect a record missing in the middle, + * but records may be missing at the end. If we did write-no-sync, we expect every key to + * have been recovered. + */ + for (last_key = UINT64_MAX;; ++count, last_key = key) { + ret = fscanf(fp, "%" SCNu64 "\n", &key); + /* + * Consider anything other than clear success in getting the key to be EOF. We've seen + * file system issues where the file ends with zeroes on a 4K boundary and does not + * return EOF but a ret of zero. + */ + if (ret != 1) + break; + /* + * If we're unlucky, the last line may be a partially written key at the end that can + * result in a false negative error for a missing record. Detect it. + */ + if (last_key != UINT64_MAX && key != last_key + 1) { + printf("%s: Ignore partial record %" PRIu64 " last valid key %" PRIu64 "\n", fname, + key, last_key); + break; + } + testutil_check(__wt_snprintf(kname, sizeof(kname), "%" PRIu64, key)); + cursor->set_key(cursor, kname); + if ((ret = cursor->search(cursor)) != 0) { + if (ret != WT_NOTFOUND) + testutil_die(ret, "search"); + if (!inmem) + printf("%s: no record with key %" PRIu64 "\n", fname, key); + absent++; + middle = key; + } else if (middle != 0) { + /* + * We should never find an existing key after we have detected one missing. + */ + printf("%s: after absent record at %" PRIu64 " key %" PRIu64 " exists\n", fname, + middle, key); + fatal = true; + } + } + if (fclose(fp) != 0) + testutil_die(errno, "fclose"); + } + testutil_check(conn->close(conn, NULL)); + if (fatal) + return (EXIT_FAILURE); + if (!inmem && absent) { + printf("%" PRIu64 " record(s) absent from %" PRIu64 "\n", absent, count); + return (EXIT_FAILURE); + } + printf("%" PRIu64 " records verified\n", count); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/random_directio/main.c b/src/third_party/wiredtiger/test/csuite/random_directio/main.c index 894d704a7cf..73c7a8c6316 100644 --- a/src/third_party/wiredtiger/test/csuite/random_directio/main.c +++ b/src/third_party/wiredtiger/test/csuite/random_directio/main.c @@ -73,38 +73,36 @@ #include <signal.h> #include <sys/wait.h> -static char home[1024]; /* Program working dir */ +static char home[1024]; /* Program working dir */ -static const char * const uri_main = "table:main"; -static const char * const uri_rev = "table:rev"; +static const char *const uri_main = "table:main"; +static const char *const uri_rev = "table:rev"; /* - * The number of threads cannot be more than 16, we are using a hex digit - * to encode this in the key. + * The number of threads cannot be more than 16, we are using a hex digit to encode this in the key. */ -#define MAX_TH 16 -#define MIN_TH 5 +#define MAX_TH 16 +#define MIN_TH 5 -#define MAX_TIME 40 -#define MIN_TIME 10 +#define MAX_TIME 40 +#define MIN_TIME 10 -#define LARGE_WRITE_SIZE (128*1024) -#define MIN_DATA_SIZE 30 -#define DEFAULT_DATA_SIZE 50 +#define LARGE_WRITE_SIZE (128 * 1024) +#define MIN_DATA_SIZE 30 +#define DEFAULT_DATA_SIZE 50 -#define DEFAULT_CYCLES 5 -#define DEFAULT_INTERVAL 3 +#define DEFAULT_CYCLES 5 +#define DEFAULT_INTERVAL 3 -#define KEY_SEP "_" /* Must be one char string */ +#define KEY_SEP "_" /* Must be one char string */ -#define ENV_CONFIG \ - "create,log=(file_max=10M,enabled)," \ +#define ENV_CONFIG \ + "create,log=(file_max=10M,enabled)," \ "transaction_sync=(enabled,method=%s)" -#define ENV_CONFIG_REC "log=(recover=on)" +#define ENV_CONFIG_REC "log=(recover=on)" /* 64 spaces */ -#define SPACES \ - " " +#define SPACES " " /* * Set the "schema operation frequency" higher to be less stressful for schema @@ -155,32 +153,30 @@ static const char * const uri_rev = "table:rev"; * that has schema operations happens again at id 200, assuming frequency * set to 100. So it is a good test of schema operations 'in flight'. */ -#define SCHEMA_OP_FREQUENCY 100 +#define SCHEMA_OP_FREQUENCY 100 -#define TEST_STREQ(expect, got, message) \ - do { \ - if (!WT_STREQ(expect, got)) { \ - printf("FAIL: %s: expect %s, got %s", message, \ - expect, got); \ - testutil_assert(WT_STREQ(expect, got)); \ - } \ - } while (0) +#define TEST_STREQ(expect, got, message) \ + do { \ + if (!WT_STREQ(expect, got)) { \ + printf("FAIL: %s: expect %s, got %s", message, expect, got); \ + testutil_assert(WT_STREQ(expect, got)); \ + } \ + } while (0) /* * Values for flags used in various places. */ -#define SCHEMA_CREATE 0x0001 -#define SCHEMA_CREATE_CHECK 0x0002 -#define SCHEMA_DATA_CHECK 0x0004 -#define SCHEMA_DROP 0x0008 -#define SCHEMA_DROP_CHECK 0x0010 -#define SCHEMA_INTEGRATED 0x0020 -#define SCHEMA_RENAME 0x0040 -#define SCHEMA_VERBOSE 0x0080 -#define SCHEMA_ALL \ - (SCHEMA_CREATE | SCHEMA_CREATE_CHECK | \ - SCHEMA_DATA_CHECK | SCHEMA_DROP | \ - SCHEMA_DROP_CHECK | SCHEMA_INTEGRATED | SCHEMA_RENAME) +#define SCHEMA_CREATE 0x0001 +#define SCHEMA_CREATE_CHECK 0x0002 +#define SCHEMA_DATA_CHECK 0x0004 +#define SCHEMA_DROP 0x0008 +#define SCHEMA_DROP_CHECK 0x0010 +#define SCHEMA_INTEGRATED 0x0020 +#define SCHEMA_RENAME 0x0040 +#define SCHEMA_VERBOSE 0x0080 +#define SCHEMA_ALL \ + (SCHEMA_CREATE | SCHEMA_CREATE_CHECK | SCHEMA_DATA_CHECK | SCHEMA_DROP | SCHEMA_DROP_CHECK | \ + SCHEMA_INTEGRATED | SCHEMA_RENAME) extern int __wt_optind; extern char *__wt_optarg; @@ -188,569 +184,511 @@ extern char *__wt_optarg; static void handler(int); typedef struct { - WT_CONNECTION *conn; - char *data; - uint32_t datasize; - uint32_t id; + WT_CONNECTION *conn; + char *data; + uint32_t datasize; + uint32_t id; - uint32_t flags; /* Uses SCHEMA_* values above */ + uint32_t flags; /* Uses SCHEMA_* values above */ } WT_THREAD_DATA; /* * usage -- - * Print usage and exit. + * Print usage and exit. */ -static void usage(void) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void usage(void) { - fprintf(stderr, "usage: %s [options]\n", progname); - fprintf(stderr, "options:\n"); - fprintf(stderr, " %-20s%s\n", "-d data_size", - "approximate size of keys and values [1000]"); - fprintf(stderr, " %-20s%s\n", "-h home", - "WiredTiger home directory [WT_TEST.directio]"); - fprintf(stderr, " %-20s%s\n", "-i interval", - "interval timeout between copy/recover cycles [3]"); - fprintf(stderr, " %-20s%s\n", "-m method", - "sync method: fsync, dsync, none [none]"); - fprintf(stderr, " %-20s%s\n", "-n num_cycles", - "number of copy/recover cycles [5]"); - fprintf(stderr, " %-20s%s\n", "-p", "populate only [false]"); - fprintf(stderr, " %-20s%s\n", "-S arg1,arg2,...", - "comma separated schema operations, from the following:"); - fprintf(stderr, " %-5s%-15s%s\n", "", "none", - "no schema operations [default]"); - fprintf(stderr, " %-5s%-15s%s\n", "", "all", - "all of the below operations, except verbose"); - fprintf(stderr, " %-5s%-15s%s\n", "", "create", - "create tables"); - fprintf(stderr, " %-5s%-15s%s\n", "", "create_check", - "newly created tables are checked (requires create)"); - fprintf(stderr, " %-5s%-15s%s\n", "", "data_check", - "check contents of files for various ops (requires create)"); - fprintf(stderr, " %-5s%-15s%s\n", "", "integrated", - "schema operations are integrated into main table transactions"); - fprintf(stderr, " %-5s%-15s%s\n", "", "rename", - "rename tables (requires create)"); - fprintf(stderr, " %-5s%-15s%s\n", "", "drop", - "drop tables (requires create)"); - fprintf(stderr, " %-5s%-15s%s\n", "", "drop_check", - "after recovery, dropped tables are checked (requires drop)"); - fprintf(stderr, " %-5s%-15s%s\n", "", "", - "that they no longer exist (requires drop)"); - fprintf(stderr, " %-5s%-15s%s\n", "", "verbose", - "verbose print during schema operation checks,"); - fprintf(stderr, " %-5s%-15s%s\n", "", "", - "done after recovery, so does not effect test timing"); - fprintf(stderr, " %-20s%s\n", "-T num_threads", - "number of threads in writer [random]"); - fprintf(stderr, " %-20s%s\n", "-t timeout", - "initial timeout before first copy [random]"); - fprintf(stderr, " %-20s%s\n", "-v", "verify only [false]"); - exit(EXIT_FAILURE); + fprintf(stderr, "usage: %s [options]\n", progname); + fprintf(stderr, "options:\n"); + fprintf(stderr, " %-20s%s\n", "-d data_size", "approximate size of keys and values [1000]"); + fprintf(stderr, " %-20s%s\n", "-h home", "WiredTiger home directory [WT_TEST.directio]"); + fprintf( + stderr, " %-20s%s\n", "-i interval", "interval timeout between copy/recover cycles [3]"); + fprintf(stderr, " %-20s%s\n", "-m method", "sync method: fsync, dsync, none [none]"); + fprintf(stderr, " %-20s%s\n", "-n num_cycles", "number of copy/recover cycles [5]"); + fprintf(stderr, " %-20s%s\n", "-p", "populate only [false]"); + fprintf(stderr, " %-20s%s\n", "-S arg1,arg2,...", + "comma separated schema operations, from the following:"); + fprintf(stderr, " %-5s%-15s%s\n", "", "none", "no schema operations [default]"); + fprintf(stderr, " %-5s%-15s%s\n", "", "all", "all of the below operations, except verbose"); + fprintf(stderr, " %-5s%-15s%s\n", "", "create", "create tables"); + fprintf(stderr, " %-5s%-15s%s\n", "", "create_check", + "newly created tables are checked (requires create)"); + fprintf(stderr, " %-5s%-15s%s\n", "", "data_check", + "check contents of files for various ops (requires create)"); + fprintf(stderr, " %-5s%-15s%s\n", "", "integrated", + "schema operations are integrated into main table transactions"); + fprintf(stderr, " %-5s%-15s%s\n", "", "rename", "rename tables (requires create)"); + fprintf(stderr, " %-5s%-15s%s\n", "", "drop", "drop tables (requires create)"); + fprintf(stderr, " %-5s%-15s%s\n", "", "drop_check", + "after recovery, dropped tables are checked (requires drop)"); + fprintf(stderr, " %-5s%-15s%s\n", "", "", "that they no longer exist (requires drop)"); + fprintf( + stderr, " %-5s%-15s%s\n", "", "verbose", "verbose print during schema operation checks,"); + fprintf( + stderr, " %-5s%-15s%s\n", "", "", "done after recovery, so does not effect test timing"); + fprintf(stderr, " %-20s%s\n", "-T num_threads", "number of threads in writer [random]"); + fprintf(stderr, " %-20s%s\n", "-t timeout", "initial timeout before first copy [random]"); + fprintf(stderr, " %-20s%s\n", "-v", "verify only [false]"); + exit(EXIT_FAILURE); } /* * has_schema_operation -- - * Return true if a schema operation should be performed for this id. - * See the comment above describing schema operation frequency. + * Return true if a schema operation should be performed for this id. See the comment above + * describing schema operation frequency. */ static bool has_schema_operation(uint64_t id, uint32_t offset) { - return (id >= offset && - (id - offset) % SCHEMA_OP_FREQUENCY < 10); + return (id >= offset && (id - offset) % SCHEMA_OP_FREQUENCY < 10); } /* * large_buf -- - * Fill or check a large buffer. + * Fill or check a large buffer. */ static void large_buf(char *large, size_t lsize, uint32_t id, bool fill) { - size_t len; - uint64_t i; - char lgbuf[1024 + 20]; - - /* - * Set up a large value putting our id in it every 1024 bytes or so. - */ - testutil_check(__wt_snprintf( - lgbuf, sizeof(lgbuf), "th-%" PRIu32 - "%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s", id, - SPACES, SPACES, SPACES, SPACES, - SPACES, SPACES, SPACES, SPACES, - SPACES, SPACES, SPACES, SPACES, - SPACES, SPACES, SPACES, SPACES)); - - len = strlen(lgbuf); - for (i = 0; i < lsize - len; i += len) - if (fill) - testutil_check(__wt_snprintf( - &large[i], lsize - i, "%s", lgbuf)); - else - testutil_check(strncmp(&large[i], lgbuf, len)); + size_t len; + uint64_t i; + char lgbuf[1024 + 20]; + + /* + * Set up a large value putting our id in it every 1024 bytes or so. + */ + testutil_check(__wt_snprintf(lgbuf, sizeof(lgbuf), + "th-%" PRIu32 "%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s", id, SPACES, SPACES, SPACES, SPACES, SPACES, + SPACES, SPACES, SPACES, SPACES, SPACES, SPACES, SPACES, SPACES, SPACES, SPACES, SPACES)); + + len = strlen(lgbuf); + for (i = 0; i < lsize - len; i += len) + if (fill) + testutil_check(__wt_snprintf(&large[i], lsize - i, "%s", lgbuf)); + else + testutil_check(strncmp(&large[i], lgbuf, len)); } /* * reverse -- - * Reverse a string in place. + * Reverse a string in place. */ static void reverse(char *s) { - size_t i, j, len; - char tmp; - - len = strlen(s); - for (i = 0, j = len - 1; i < len / 2; i++, j--) { - tmp = s[i]; - s[i] = s[j]; - s[j] = tmp; - } + size_t i, j, len; + char tmp; + + len = strlen(s); + for (i = 0, j = len - 1; i < len / 2; i++, j--) { + tmp = s[i]; + s[i] = s[j]; + s[j] = tmp; + } } /* * gen_kv -- - * Generate a key/value. + * Generate a key/value. */ static void -gen_kv(char *buf, size_t buf_size, uint64_t id, uint32_t threadid, - const char *large, bool forward) +gen_kv(char *buf, size_t buf_size, uint64_t id, uint32_t threadid, const char *large, bool forward) { - size_t keyid_size, large_size; - char keyid[64]; - - testutil_check(__wt_snprintf(keyid, sizeof(keyid), - "%10.10" PRIu64, id)); - keyid_size = strlen(keyid); - if (!forward) - reverse(keyid); - testutil_assert(keyid_size + 4 <= buf_size); - large_size = (buf_size - 4) - keyid_size; - testutil_check(__wt_snprintf(buf, buf_size, - "%s" KEY_SEP "%1.1x" KEY_SEP "%.*s", - keyid, threadid, (int)large_size, large)); + size_t keyid_size, large_size; + char keyid[64]; + + testutil_check(__wt_snprintf(keyid, sizeof(keyid), "%10.10" PRIu64, id)); + keyid_size = strlen(keyid); + if (!forward) + reverse(keyid); + testutil_assert(keyid_size + 4 <= buf_size); + large_size = (buf_size - 4) - keyid_size; + testutil_check(__wt_snprintf( + buf, buf_size, "%s" KEY_SEP "%1.1x" KEY_SEP "%.*s", keyid, threadid, (int)large_size, large)); } /* * gen_table_name -- - * Generate a table name used for the schema test. + * Generate a table name used for the schema test. */ static void gen_table_name(char *buf, size_t buf_size, uint64_t id, uint32_t threadid) { - testutil_check(__wt_snprintf(buf, buf_size, - "table:A%" PRIu64 "-%" PRIu32, id, threadid)); + testutil_check(__wt_snprintf(buf, buf_size, "table:A%" PRIu64 "-%" PRIu32, id, threadid)); } /* * gen_table2_name -- - * Generate a second table name used for the schema test. + * Generate a second table name used for the schema test. */ static void -gen_table2_name(char *buf, size_t buf_size, uint64_t id, uint32_t threadid, - uint32_t flags) +gen_table2_name(char *buf, size_t buf_size, uint64_t id, uint32_t threadid, uint32_t flags) { - if (!LF_ISSET(SCHEMA_RENAME)) - /* table is not renamed, so use original table name */ - gen_table_name(buf, buf_size, id, threadid); - else - testutil_check(__wt_snprintf(buf, buf_size, - "table:B%" PRIu64 "-%" PRIu32, id, threadid)); + if (!LF_ISSET(SCHEMA_RENAME)) + /* table is not renamed, so use original table name */ + gen_table_name(buf, buf_size, id, threadid); + else + testutil_check(__wt_snprintf(buf, buf_size, "table:B%" PRIu64 "-%" PRIu32, id, threadid)); } static int -schema_operation(WT_SESSION *session, uint32_t threadid, uint64_t id, - uint32_t op, uint32_t flags) +schema_operation(WT_SESSION *session, uint32_t threadid, uint64_t id, uint32_t op, uint32_t flags) { - WT_CURSOR *cursor; - WT_DECL_RET; - const char *retry_opname; - char uri1[50], uri2[50]; - - if (!has_schema_operation(id, op)) - return (0); - - id -= op; - retry_opname = NULL; - - switch (op) { - case 0: - /* Create a table. */ - gen_table_name(uri1, sizeof(uri1), id, threadid); - /* - fprintf(stderr, "CREATE: %s\n", uri1); - */ - testutil_check(session->create(session, uri1, - "key_format=S,value_format=S")); - break; - case 1: - /* Insert a value into the table. */ - gen_table_name(uri1, sizeof(uri1), id, threadid); - /* - fprintf(stderr, "INSERT: %s\n", uri1); - */ - testutil_check(session->open_cursor( - session, uri1, NULL, NULL, &cursor)); - cursor->set_key(cursor, uri1); - cursor->set_value(cursor, uri1); - testutil_check(cursor->insert(cursor)); - testutil_check(cursor->close(cursor)); - break; - case 2: - /* Rename the table. */ - if (LF_ISSET(SCHEMA_RENAME)) { - gen_table_name(uri1, sizeof(uri1), id, threadid); - gen_table2_name(uri2, sizeof(uri2), id, threadid, - flags); - retry_opname = "rename"; - /* - fprintf(stderr, "RENAME: %s->%s\n", uri1, uri2); - */ - ret = session->rename(session, uri1, uri2, NULL); - } - break; - case 3: - /* Update the single value in the table. */ - gen_table_name(uri1, sizeof(uri1), id, threadid); - gen_table2_name(uri2, sizeof(uri2), id, threadid, flags); - testutil_check(session->open_cursor(session, - uri2, NULL, NULL, &cursor)); - cursor->set_key(cursor, uri1); - cursor->set_value(cursor, uri2); - /* - fprintf(stderr, "UPDATE: %s\n", uri2); - */ - testutil_check(cursor->update(cursor)); - testutil_check(cursor->close(cursor)); - break; - case 4: - /* Drop the table. */ - if (LF_ISSET(SCHEMA_DROP)) { - gen_table2_name(uri1, sizeof(uri1), id, threadid, - flags); - retry_opname = "drop"; - /* - fprintf(stderr, "DROP: %s\n", uri1); - */ - ret = session->drop(session, uri1, NULL); - } - } - /* - * XXX - * We notice occasional EBUSY errors from - * rename or drop, even though neither URI should be - * used by any other thread. Report it, and retry. - */ - if (retry_opname != NULL && ret == EBUSY) - printf("%s(\"%s\", ....) failed, retrying transaction\n", - retry_opname, uri1); - else if (ret != 0) { - printf("FAIL: %s(\"%s\", ....) returns %d: %s\n", - retry_opname, uri1, ret, wiredtiger_strerror(ret)); - testutil_check(ret); - } - - return (ret); + WT_CURSOR *cursor; + WT_DECL_RET; + char uri1[50], uri2[50]; + const char *retry_opname; + + if (!has_schema_operation(id, op)) + return (0); + + id -= op; + retry_opname = NULL; + + switch (op) { + case 0: + /* Create a table. */ + gen_table_name(uri1, sizeof(uri1), id, threadid); + /* + fprintf(stderr, "CREATE: %s\n", uri1); + */ + testutil_check(session->create(session, uri1, "key_format=S,value_format=S")); + break; + case 1: + /* Insert a value into the table. */ + gen_table_name(uri1, sizeof(uri1), id, threadid); + /* + fprintf(stderr, "INSERT: %s\n", uri1); + */ + testutil_check(session->open_cursor(session, uri1, NULL, NULL, &cursor)); + cursor->set_key(cursor, uri1); + cursor->set_value(cursor, uri1); + testutil_check(cursor->insert(cursor)); + testutil_check(cursor->close(cursor)); + break; + case 2: + /* Rename the table. */ + if (LF_ISSET(SCHEMA_RENAME)) { + gen_table_name(uri1, sizeof(uri1), id, threadid); + gen_table2_name(uri2, sizeof(uri2), id, threadid, flags); + retry_opname = "rename"; + /* + fprintf(stderr, "RENAME: %s->%s\n", uri1, uri2); + */ + ret = session->rename(session, uri1, uri2, NULL); + } + break; + case 3: + /* Update the single value in the table. */ + gen_table_name(uri1, sizeof(uri1), id, threadid); + gen_table2_name(uri2, sizeof(uri2), id, threadid, flags); + testutil_check(session->open_cursor(session, uri2, NULL, NULL, &cursor)); + cursor->set_key(cursor, uri1); + cursor->set_value(cursor, uri2); + /* + fprintf(stderr, "UPDATE: %s\n", uri2); + */ + testutil_check(cursor->update(cursor)); + testutil_check(cursor->close(cursor)); + break; + case 4: + /* Drop the table. */ + if (LF_ISSET(SCHEMA_DROP)) { + gen_table2_name(uri1, sizeof(uri1), id, threadid, flags); + retry_opname = "drop"; + /* + fprintf(stderr, "DROP: %s\n", uri1); + */ + ret = session->drop(session, uri1, NULL); + } + } + /* + * XXX We notice occasional EBUSY errors from rename or drop, even though neither URI should be + * used by any other thread. Report it, and retry. + */ + if (retry_opname != NULL && ret == EBUSY) + printf("%s(\"%s\", ....) failed, retrying transaction\n", retry_opname, uri1); + else if (ret != 0) { + printf("FAIL: %s(\"%s\", ....) returns %d: %s\n", retry_opname, uri1, ret, + wiredtiger_strerror(ret)); + testutil_check(ret); + } + + return (ret); } /* * thread_run -- - * Run a writer thread. + * Run a writer thread. */ -static WT_THREAD_RET thread_run(void *) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static WT_THREAD_RET thread_run(void *) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static WT_THREAD_RET thread_run(void *arg) { - WT_CURSOR *cursor, *rev; - WT_DECL_RET; - WT_RAND_STATE rnd; - WT_SESSION *session; - WT_THREAD_DATA *td; - size_t lsize; - uint64_t i; - uint32_t kvsize, op; - char *buf1, *buf2; - char large[LARGE_WRITE_SIZE]; - - __wt_random_init(&rnd); - lsize = sizeof(large); - memset(large, 0, lsize); - - td = (WT_THREAD_DATA *)arg; - large_buf(large, lsize, td->id, true); - - testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); - testutil_check(session->open_cursor(session, uri_main, NULL, NULL, - &cursor)); - testutil_check(session->open_cursor(session, uri_rev, NULL, NULL, - &rev)); - - /* - * Split the allocated buffer into two parts, one for - * the key, one for the value. - */ - kvsize = td->datasize / 2; - buf1 = td->data; - buf2 = &td->data[kvsize]; - - /* - * Continuing writing until we're killed. - */ - printf("Thread %" PRIu32 "\n", td->id); - for (i = 0; ; ++i) { + WT_CURSOR *cursor, *rev; + WT_DECL_RET; + WT_RAND_STATE rnd; + WT_SESSION *session; + WT_THREAD_DATA *td; + size_t lsize; + uint64_t i; + uint32_t kvsize, op; + char *buf1, *buf2; + char large[LARGE_WRITE_SIZE]; + + __wt_random_init(&rnd); + lsize = sizeof(large); + memset(large, 0, lsize); + + td = (WT_THREAD_DATA *)arg; + large_buf(large, lsize, td->id, true); + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + testutil_check(session->open_cursor(session, uri_main, NULL, NULL, &cursor)); + testutil_check(session->open_cursor(session, uri_rev, NULL, NULL, &rev)); + + /* + * Split the allocated buffer into two parts, one for the key, one for the value. + */ + kvsize = td->datasize / 2; + buf1 = td->data; + buf2 = &td->data[kvsize]; + + /* + * Continuing writing until we're killed. + */ + printf("Thread %" PRIu32 "\n", td->id); + for (i = 0;; ++i) { again: - /* - if (i > 0 && i % 10000 == 0) - printf("Thread %" PRIu32 - " completed %" PRIu64 " entries\n", - td->id, i); - */ - - gen_kv(buf1, kvsize, i, td->id, large, true); - gen_kv(buf2, kvsize, i, td->id, large, false); - - testutil_check(session->begin_transaction(session, NULL)); - cursor->set_key(cursor, buf1); - /* - * Every 1000th record write a very large value that exceeds the - * log buffer size. This forces us to use the unbuffered path. - */ - if (i % 1000 == 0) { - cursor->set_value(cursor, large); - } else { - cursor->set_value(cursor, buf2); - } - testutil_check(cursor->insert(cursor)); - - /* - * The reverse table has no very large records. - */ - rev->set_key(rev, buf2); - rev->set_value(rev, buf1); - testutil_check(rev->insert(rev)); - - /* - * If we are not running integrated tests, then we commit the - * transaction now so that schema operations are not part of - * the transaction operations for the main table. If we are - * running 'integrated' then we'll first do the schema - * operations and commit later. - */ - if (!F_ISSET(td, SCHEMA_INTEGRATED)) - testutil_check(session->commit_transaction(session, - NULL)); - /* - * If we are doing a schema test, generate operations - * for additional tables. Each table has a 'lifetime' - * of 4 values of the id. - */ - if (F_ISSET(td, SCHEMA_ALL)) { - /* Create is implied by any schema operation. */ - testutil_assert(F_ISSET(td, SCHEMA_CREATE)); - - /* - * Any or all of the schema operations may be - * performed as part of this transaction. - * See the comment for schema operation frequency. - */ - ret = 0; - for (op = 0; op <= 4 && ret == 0; op++) - ret = schema_operation(session, td->id, i, op, - td->flags); - if (ret == EBUSY) { - /* - * Only rollback if integrated and we have - * an active transaction. - */ - if (F_ISSET(td, SCHEMA_INTEGRATED)) - testutil_check( - session->rollback_transaction( - session, NULL)); - sleep(1); - goto again; - } - } - /* - * If schema operations are integrated, commit the transaction - * now that they're complete. - */ - if (F_ISSET(td, SCHEMA_INTEGRATED)) - testutil_check(session->commit_transaction(session, - NULL)); - } - /* NOTREACHED */ + /* + if (i > 0 && i % 10000 == 0) + printf("Thread %" PRIu32 + " completed %" PRIu64 " entries\n", + td->id, i); + */ + + gen_kv(buf1, kvsize, i, td->id, large, true); + gen_kv(buf2, kvsize, i, td->id, large, false); + + testutil_check(session->begin_transaction(session, NULL)); + cursor->set_key(cursor, buf1); + /* + * Every 1000th record write a very large value that exceeds the log buffer size. This + * forces us to use the unbuffered path. + */ + if (i % 1000 == 0) { + cursor->set_value(cursor, large); + } else { + cursor->set_value(cursor, buf2); + } + testutil_check(cursor->insert(cursor)); + + /* + * The reverse table has no very large records. + */ + rev->set_key(rev, buf2); + rev->set_value(rev, buf1); + testutil_check(rev->insert(rev)); + + /* + * If we are not running integrated tests, then we commit the transaction now so that schema + * operations are not part of the transaction operations for the main table. If we are + * running 'integrated' then we'll first do the schema operations and commit later. + */ + if (!F_ISSET(td, SCHEMA_INTEGRATED)) + testutil_check(session->commit_transaction(session, NULL)); + /* + * If we are doing a schema test, generate operations for additional tables. Each table has + * a 'lifetime' of 4 values of the id. + */ + if (F_ISSET(td, SCHEMA_ALL)) { + /* Create is implied by any schema operation. */ + testutil_assert(F_ISSET(td, SCHEMA_CREATE)); + + /* + * Any or all of the schema operations may be performed as part of this transaction. See + * the comment for schema operation frequency. + */ + ret = 0; + for (op = 0; op <= 4 && ret == 0; op++) + ret = schema_operation(session, td->id, i, op, td->flags); + if (ret == EBUSY) { + /* + * Only rollback if integrated and we have an active transaction. + */ + if (F_ISSET(td, SCHEMA_INTEGRATED)) + testutil_check(session->rollback_transaction(session, NULL)); + sleep(1); + goto again; + } + } + /* + * If schema operations are integrated, commit the transaction now that they're complete. + */ + if (F_ISSET(td, SCHEMA_INTEGRATED)) + testutil_check(session->commit_transaction(session, NULL)); + } + /* NOTREACHED */ } /* * create_db -- - * Creates the database and tables so they are fully ready to be - * accessed by subordinate threads, and copied/recovered. + * Creates the database and tables so they are fully ready to be accessed by subordinate + * threads, and copied/recovered. */ static void create_db(const char *method) { - WT_CONNECTION *conn; - WT_SESSION *session; - char envconf[512]; - - testutil_check(__wt_snprintf(envconf, sizeof(envconf), - ENV_CONFIG, method)); - - testutil_check(wiredtiger_open(home, NULL, envconf, &conn)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->create( - session, uri_main, "key_format=S,value_format=S")); - testutil_check(session->create( - session, uri_rev, "key_format=S,value_format=S")); - /* - * Checkpoint to help ensure that everything gets out to disk, - * so any direct I/O copy will have at least have tables that - * can be opened. - */ - testutil_check(session->checkpoint(session, NULL)); - testutil_check(session->close(session, NULL)); - testutil_check(conn->close(conn, NULL)); + WT_CONNECTION *conn; + WT_SESSION *session; + char envconf[512]; + + testutil_check(__wt_snprintf(envconf, sizeof(envconf), ENV_CONFIG, method)); + + testutil_check(wiredtiger_open(home, NULL, envconf, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->create(session, uri_main, "key_format=S,value_format=S")); + testutil_check(session->create(session, uri_rev, "key_format=S,value_format=S")); + /* + * Checkpoint to help ensure that everything gets out to disk, so any direct I/O copy will have + * at least have tables that can be opened. + */ + testutil_check(session->checkpoint(session, NULL)); + testutil_check(session->close(session, NULL)); + testutil_check(conn->close(conn, NULL)); } /* * fill_db -- - * The child process creates worker threads to add data until it is - * killed by the parent. + * The child process creates worker threads to add data until it is killed by the parent. */ static void fill_db(uint32_t, uint32_t, const char *, uint32_t) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void fill_db(uint32_t nth, uint32_t datasize, const char *method, uint32_t flags) { - WT_CONNECTION *conn; - WT_THREAD_DATA *td; - wt_thread_t *thr; - uint32_t i; - char envconf[512]; - - thr = dcalloc(nth, sizeof(*thr)); - td = dcalloc(nth, sizeof(WT_THREAD_DATA)); - if (chdir(home) != 0) - testutil_die(errno, "Child chdir: %s", home); - testutil_check(__wt_snprintf(envconf, sizeof(envconf), - ENV_CONFIG, method)); - - testutil_check(wiredtiger_open(".", NULL, envconf, &conn)); - - datasize += 1; /* Add an extra byte for string termination */ - printf("Create %" PRIu32 " writer threads\n", nth); - for (i = 0; i < nth; ++i) { - td[i].conn = conn; - td[i].data = dcalloc(datasize, 1); - td[i].datasize = datasize; - td[i].id = i; - td[i].flags = flags; - testutil_check(__wt_thread_create( - NULL, &thr[i], thread_run, &td[i])); - } - printf("Spawned %" PRIu32 " writer threads\n", nth); - fflush(stdout); - /* - * The threads never exit, so the child will just wait here until - * it is killed. - */ - for (i = 0; i < nth; ++i) { - testutil_check(__wt_thread_join(NULL, &thr[i])); - free(td[i].data); - } - /* - * NOTREACHED - */ - free(thr); - free(td); - exit(EXIT_SUCCESS); + WT_CONNECTION *conn; + WT_THREAD_DATA *td; + wt_thread_t *thr; + uint32_t i; + char envconf[512]; + + thr = dcalloc(nth, sizeof(*thr)); + td = dcalloc(nth, sizeof(WT_THREAD_DATA)); + if (chdir(home) != 0) + testutil_die(errno, "Child chdir: %s", home); + testutil_check(__wt_snprintf(envconf, sizeof(envconf), ENV_CONFIG, method)); + + testutil_check(wiredtiger_open(".", NULL, envconf, &conn)); + + datasize += 1; /* Add an extra byte for string termination */ + printf("Create %" PRIu32 " writer threads\n", nth); + for (i = 0; i < nth; ++i) { + td[i].conn = conn; + td[i].data = dcalloc(datasize, 1); + td[i].datasize = datasize; + td[i].id = i; + td[i].flags = flags; + testutil_check(__wt_thread_create(NULL, &thr[i], thread_run, &td[i])); + } + printf("Spawned %" PRIu32 " writer threads\n", nth); + fflush(stdout); + /* + * The threads never exit, so the child will just wait here until it is killed. + */ + for (i = 0; i < nth; ++i) { + testutil_check(__wt_thread_join(NULL, &thr[i])); + free(td[i].data); + } + /* + * NOTREACHED + */ + free(thr); + free(td); + exit(EXIT_SUCCESS); } /* * check_kv -- - * Check that a key exists with a value, or does not exist. + * Check that a key exists with a value, or does not exist. */ static void check_kv(WT_CURSOR *cursor, const char *key, const char *value, bool exists) { - WT_DECL_RET; - char *got; - - cursor->set_key(cursor, key); - ret = cursor->search(cursor); - if ((ret = cursor->search(cursor)) == WT_NOTFOUND) { - if (exists) { - printf("FAIL: expected rev file to have: %s\n", key); - testutil_assert(!exists); - } - } else { - testutil_check(ret); - if (!exists) { - printf("FAIL: unexpected key in rev file: %s\n", key); - testutil_assert(exists); - } - testutil_check(cursor->get_value(cursor, &got)); - TEST_STREQ(value, got, "value"); - } + WT_DECL_RET; + char *got; + + cursor->set_key(cursor, key); + ret = cursor->search(cursor); + if ((ret = cursor->search(cursor)) == WT_NOTFOUND) { + if (exists) { + printf("FAIL: expected rev file to have: %s\n", key); + testutil_assert(!exists); + } + } else { + testutil_check(ret); + if (!exists) { + printf("FAIL: unexpected key in rev file: %s\n", key); + testutil_assert(exists); + } + testutil_check(cursor->get_value(cursor, &got)); + TEST_STREQ(value, got, "value"); + } } /* * check_dropped -- - * Check that the uri has been dropped. + * Check that the uri has been dropped. */ static void check_dropped(WT_SESSION *session, const char *uri) { - WT_CURSOR *cursor; - WT_DECL_RET; + WT_CURSOR *cursor; + WT_DECL_RET; - ret = session->open_cursor(session, uri, NULL, NULL, &cursor); - testutil_assert(ret == WT_NOTFOUND); + ret = session->open_cursor(session, uri, NULL, NULL, &cursor); + testutil_assert(ret == WT_NOTFOUND); } /* * check_empty -- - * Check that the uri exists and is empty. + * Check that the uri exists and is empty. */ static void check_empty(WT_SESSION *session, const char *uri) { - WT_CURSOR *cursor; - WT_DECL_RET; + WT_CURSOR *cursor; + WT_DECL_RET; - testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); - ret = cursor->next(cursor); - testutil_assert(ret == WT_NOTFOUND); - testutil_check(cursor->close(cursor)); + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); + ret = cursor->next(cursor); + testutil_assert(ret == WT_NOTFOUND); + testutil_check(cursor->close(cursor)); } /* * check_empty -- - * Check that the uri exists and has one entry. + * Check that the uri exists and has one entry. */ static void -check_one_entry(WT_SESSION *session, const char *uri, const char *key, - const char *value) +check_one_entry(WT_SESSION *session, const char *uri, const char *key, const char *value) { - WT_CURSOR *cursor; - WT_DECL_RET; - char *gotkey, *gotvalue; - - testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); - testutil_check(cursor->next(cursor)); - testutil_check(cursor->get_key(cursor, &gotkey)); - testutil_check(cursor->get_value(cursor, &gotvalue)); - testutil_assert(WT_STREQ(key, gotkey)); - testutil_assert(WT_STREQ(value, gotvalue)); - ret = cursor->next(cursor); - testutil_assert(ret == WT_NOTFOUND); - testutil_check(cursor->close(cursor)); + WT_CURSOR *cursor; + WT_DECL_RET; + char *gotkey, *gotvalue; + + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); + testutil_check(cursor->next(cursor)); + testutil_check(cursor->get_key(cursor, &gotkey)); + testutil_check(cursor->get_value(cursor, &gotvalue)); + testutil_assert(WT_STREQ(key, gotkey)); + testutil_assert(WT_STREQ(value, gotvalue)); + ret = cursor->next(cursor); + testutil_assert(ret == WT_NOTFOUND); + testutil_check(cursor->close(cursor)); } /* @@ -759,562 +697,519 @@ check_one_entry(WT_SESSION *session, const char *uri, const char *key, * last id seen for this thread. */ static void -check_schema(WT_SESSION *session, uint64_t lastid, uint32_t threadid, - uint32_t flags) +check_schema(WT_SESSION *session, uint64_t lastid, uint32_t threadid, uint32_t flags) { - char uri[50], uri2[50]; - - if (!LF_ISSET(SCHEMA_ALL) || !LF_ISSET(SCHEMA_INTEGRATED)) - return; - - if (LF_ISSET(SCHEMA_VERBOSE)) - fprintf(stderr, - "check_schema(%" PRIu64 ", thread=%" PRIu32 ")\n", - lastid, threadid); - if (has_schema_operation(lastid, 0)) { - /* Create table operation. */ - gen_table_name(uri, sizeof(uri), lastid, threadid); - if (LF_ISSET(SCHEMA_VERBOSE)) - fprintf(stderr, " create %s\n", uri); - if (LF_ISSET(SCHEMA_CREATE_CHECK)) - check_empty(session, uri); - } - if (has_schema_operation(lastid, 1)) { - /* Insert value operation. */ - gen_table_name(uri, sizeof(uri), lastid - 1, threadid); - if (LF_ISSET(SCHEMA_VERBOSE)) - fprintf(stderr, " insert %s\n", uri); - if (LF_ISSET(SCHEMA_DATA_CHECK)) - check_one_entry(session, uri, uri, uri); - } - if (LF_ISSET(SCHEMA_RENAME) && has_schema_operation(lastid, 2)) { - /* Table rename operation. */ - gen_table_name(uri, sizeof(uri), lastid - 2, threadid); - gen_table2_name(uri2, sizeof(uri2), lastid - 2, threadid, - flags); - if (LF_ISSET(SCHEMA_VERBOSE)) - fprintf(stderr, " rename %s,%s\n", uri, uri2); - if (LF_ISSET(SCHEMA_DROP_CHECK)) - check_dropped(session, uri); - if (LF_ISSET(SCHEMA_CREATE_CHECK)) - check_one_entry(session, uri2, uri, uri); - } - if (has_schema_operation(lastid, 3)) { - /* Value update operation. */ - gen_table_name(uri, sizeof(uri), lastid - 2, threadid); - gen_table2_name(uri2, sizeof(uri2), lastid - 2, threadid, - flags); - if (LF_ISSET(SCHEMA_VERBOSE)) - fprintf(stderr, " update %s\n", uri2); - if (LF_ISSET(SCHEMA_DATA_CHECK)) - check_one_entry(session, uri2, uri, uri2); - } - if (LF_ISSET(SCHEMA_DROP_CHECK) && has_schema_operation(lastid, 4)) { - /* Drop table operation. */ - gen_table2_name(uri2, sizeof(uri2), lastid - 2, threadid, - flags); - if (LF_ISSET(SCHEMA_VERBOSE)) - fprintf(stderr, " drop %s\n", uri2); - check_dropped(session, uri2); - } + char uri[50], uri2[50]; + + if (!LF_ISSET(SCHEMA_ALL) || !LF_ISSET(SCHEMA_INTEGRATED)) + return; + + if (LF_ISSET(SCHEMA_VERBOSE)) + fprintf(stderr, "check_schema(%" PRIu64 ", thread=%" PRIu32 ")\n", lastid, threadid); + if (has_schema_operation(lastid, 0)) { + /* Create table operation. */ + gen_table_name(uri, sizeof(uri), lastid, threadid); + if (LF_ISSET(SCHEMA_VERBOSE)) + fprintf(stderr, " create %s\n", uri); + if (LF_ISSET(SCHEMA_CREATE_CHECK)) + check_empty(session, uri); + } + if (has_schema_operation(lastid, 1)) { + /* Insert value operation. */ + gen_table_name(uri, sizeof(uri), lastid - 1, threadid); + if (LF_ISSET(SCHEMA_VERBOSE)) + fprintf(stderr, " insert %s\n", uri); + if (LF_ISSET(SCHEMA_DATA_CHECK)) + check_one_entry(session, uri, uri, uri); + } + if (LF_ISSET(SCHEMA_RENAME) && has_schema_operation(lastid, 2)) { + /* Table rename operation. */ + gen_table_name(uri, sizeof(uri), lastid - 2, threadid); + gen_table2_name(uri2, sizeof(uri2), lastid - 2, threadid, flags); + if (LF_ISSET(SCHEMA_VERBOSE)) + fprintf(stderr, " rename %s,%s\n", uri, uri2); + if (LF_ISSET(SCHEMA_DROP_CHECK)) + check_dropped(session, uri); + if (LF_ISSET(SCHEMA_CREATE_CHECK)) + check_one_entry(session, uri2, uri, uri); + } + if (has_schema_operation(lastid, 3)) { + /* Value update operation. */ + gen_table_name(uri, sizeof(uri), lastid - 2, threadid); + gen_table2_name(uri2, sizeof(uri2), lastid - 2, threadid, flags); + if (LF_ISSET(SCHEMA_VERBOSE)) + fprintf(stderr, " update %s\n", uri2); + if (LF_ISSET(SCHEMA_DATA_CHECK)) + check_one_entry(session, uri2, uri, uri2); + } + if (LF_ISSET(SCHEMA_DROP_CHECK) && has_schema_operation(lastid, 4)) { + /* Drop table operation. */ + gen_table2_name(uri2, sizeof(uri2), lastid - 2, threadid, flags); + if (LF_ISSET(SCHEMA_VERBOSE)) + fprintf(stderr, " drop %s\n", uri2); + check_dropped(session, uri2); + } } /* * check_db -- - * Make a copy of the database and verify its contents. + * Make a copy of the database and verify its contents. */ static bool check_db(uint32_t nth, uint32_t datasize, bool directio, uint32_t flags) { - WT_CONNECTION *conn; - WT_CURSOR *cursor, *meta, *rev; - WT_DECL_RET; - WT_SESSION *session; - uint64_t gotid, id; - uint64_t *lastid; - uint32_t gotth, kvsize, th, threadmap; - char checkdir[4096], savedir[4096]; - char *gotkey, *gotvalue, *keybuf, *p; - char **large_arr; - - keybuf = dcalloc(datasize, 1); - lastid = dcalloc(nth, sizeof(uint64_t)); - - large_arr = dcalloc(nth, sizeof(char *)); - for (th = 0; th < nth; th++) { - large_arr[th] = dcalloc(LARGE_WRITE_SIZE, 1); - large_buf(large_arr[th], LARGE_WRITE_SIZE, th, true); - } - testutil_check(__wt_snprintf(checkdir, sizeof(checkdir), - "%s.CHECK", home)); - testutil_check(__wt_snprintf(savedir, sizeof(savedir), - "%s.SAVE", home)); - - /* - * We make a copy of the directory (possibly using direct I/O) - * for recovery and checking, and an identical copy that - * keeps the state of all files before recovery starts. - */ - printf( - "Copy database home directory using direct I/O to run recovery,\n" - "along with a saved 'pre-recovery' copy.\n"); - copy_directory(home, checkdir, directio); - copy_directory(checkdir, savedir, false); - - printf("Open database, run recovery and verify content\n"); - testutil_check(wiredtiger_open(checkdir, NULL, ENV_CONFIG_REC, &conn)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->open_cursor(session, uri_main, NULL, NULL, - &cursor)); - testutil_check(session->open_cursor(session, uri_rev, NULL, NULL, - &rev)); - kvsize = datasize / 2; - - /* - * We're most interested in the final records on disk. - * Rather than walk all records, we do a quick scan - * to find the last complete set of written ids. - * Each thread writes each id, along with the thread id, - * so they are interleaved. Once we have the neighborhood - * where some keys may be missing, we'll back up to do a scan - * from that point. - */ -#define CHECK_INCR 1000 - for (id = 0; ; id += CHECK_INCR) { - gen_kv(keybuf, kvsize, id, 0, large_arr[0], true); - cursor->set_key(cursor, keybuf); - if ((ret = cursor->search(cursor)) == WT_NOTFOUND) - break; - testutil_check(ret); - for (th = 1; th < nth; th++) { - gen_kv(keybuf, kvsize, id, th, large_arr[th], true); - cursor->set_key(cursor, keybuf); - if ((ret = cursor->search(cursor)) == WT_NOTFOUND) - break; - testutil_check(ret); - } - if (ret == WT_NOTFOUND) - break; - } - if (id < CHECK_INCR * 2) - id = 0; - else - id -= CHECK_INCR * 2; - - printf("starting full scan at %" PRIu64 "\n", id); - gen_kv(keybuf, kvsize, id, 0, large_arr[0], true); - cursor->set_key(cursor, keybuf); - th = 0; - - /* Keep bitmap of "active" threads. */ - threadmap = (0x1U << nth) - 1; - for (ret = cursor->search(cursor); ret != WT_NOTFOUND && threadmap != 0; - ret = cursor->next(cursor)) { - testutil_check(ret); - testutil_check(cursor->get_key(cursor, &gotkey)); - gotid = (uint64_t)strtol(gotkey, &p, 10); - testutil_assert(*p == KEY_SEP[0]); - p++; - testutil_assert(isxdigit(*p)); - if (isdigit(*p)) - gotth = (uint32_t)(*p - '0'); - else if (*p >= 'a' && *p <= 'f') - gotth = (uint32_t)((*p - 'a') + 10); - else - gotth = (uint32_t)((*p - 'A') + 10); - p++; - testutil_assert(*p == KEY_SEP[0]); - p++; - - /* - * See if the expected thread has finished at this point. - * If so, remove it from the thread map. - */ - while (gotth != th) { - if ((threadmap & (0x1U << th)) != 0) { - threadmap &= ~(0x1U << th); - lastid[th] = id - 1; - /* - * Any newly removed value in the main table - * should not be present as a key in the - * reverse table, since they were - * transactionally inserted at the same time. - */ - gen_kv(keybuf, kvsize, id, th, large_arr[th], - false); - check_kv(rev, keybuf, NULL, false); - check_schema(session, id - 1, th, flags); - } - th = (th + 1) % nth; - if (th == 0) - id++; - } - testutil_assert(gotid == id); - /* - * Check that the key and value fully match. - */ - gen_kv(keybuf, kvsize, id, th, large_arr[th], true); - gen_kv(&keybuf[kvsize], kvsize, id, th, large_arr[th], false); - testutil_check(cursor->get_value(cursor, &gotvalue)); - TEST_STREQ(keybuf, gotkey, "main table key"); - - /* - * Every 1000th record is large. - */ - if (id % 1000 == 0) - TEST_STREQ(large_arr[th], gotvalue, - "main table large value"); - else - TEST_STREQ(&keybuf[kvsize], gotvalue, - "main table value"); - - /* - * Check the reverse file, with key/value reversed. - */ - check_kv(rev, &keybuf[kvsize], keybuf, true); - - check_schema(session, id, th, flags); - - /* Bump thread number and id to the next expected key. */ - th = (th + 1) % nth; - if (th == 0) - id++; - } - printf("scanned to %" PRIu64 "\n", id); - - if (LF_ISSET(SCHEMA_ALL)) { - /* - * Check metadata to see if there are any tables - * present that shouldn't be there. - */ - testutil_check(session->open_cursor(session, "metadata:", NULL, - NULL, &meta)); - while ((ret = meta->next(meta)) != WT_NOTFOUND) { - testutil_check(ret); - testutil_check(meta->get_key(meta, &gotkey)); - /* - * Names involved in schema testing are of the form: - * table:Axxx-t - * table:Bxxx-t - * xxx corresponds to the id inserted into the main - * table when the table was created, and t corresponds - * to the thread id that did this. - */ - if (WT_PREFIX_SKIP(gotkey, "table:") && - (*gotkey == 'A' || *gotkey == 'B')) { - gotid = (uint64_t)strtol(gotkey + 1, &p, 10); - testutil_assert(*p == '-'); - th = (uint32_t)strtol(p + 1, &p, 10); - testutil_assert(*p == '\0'); - /* - * If table operations are truly - * transactional, then there shouldn't - * be any extra files that unaccounted for. - */ - if (LF_ISSET(SCHEMA_DROP_CHECK)) - testutil_assert(gotid == lastid[th]); - } - } - testutil_check(meta->close(meta)); - - } - - testutil_check(cursor->close(cursor)); - testutil_check(rev->close(rev)); - testutil_check(session->close(session, NULL)); - testutil_check(conn->close(conn, NULL)); - - for (th = 0; th < nth; th++) - free(large_arr[th]); - free(large_arr); - free(keybuf); - free(lastid); - return (true); + WT_CONNECTION *conn; + WT_CURSOR *cursor, *meta, *rev; + WT_DECL_RET; + WT_SESSION *session; + uint64_t gotid, id; + uint64_t *lastid; + uint32_t gotth, kvsize, th, threadmap; + char checkdir[4096], savedir[4096]; + char *gotkey, *gotvalue, *keybuf, *p; + char **large_arr; + + keybuf = dcalloc(datasize, 1); + lastid = dcalloc(nth, sizeof(uint64_t)); + + large_arr = dcalloc(nth, sizeof(char *)); + for (th = 0; th < nth; th++) { + large_arr[th] = dcalloc(LARGE_WRITE_SIZE, 1); + large_buf(large_arr[th], LARGE_WRITE_SIZE, th, true); + } + testutil_check(__wt_snprintf(checkdir, sizeof(checkdir), "%s.CHECK", home)); + testutil_check(__wt_snprintf(savedir, sizeof(savedir), "%s.SAVE", home)); + + /* + * We make a copy of the directory (possibly using direct I/O) for recovery and checking, and an + * identical copy that keeps the state of all files before recovery starts. + */ + printf( + "Copy database home directory using direct I/O to run recovery,\n" + "along with a saved 'pre-recovery' copy.\n"); + copy_directory(home, checkdir, directio); + copy_directory(checkdir, savedir, false); + + printf("Open database, run recovery and verify content\n"); + testutil_check(wiredtiger_open(checkdir, NULL, ENV_CONFIG_REC, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->open_cursor(session, uri_main, NULL, NULL, &cursor)); + testutil_check(session->open_cursor(session, uri_rev, NULL, NULL, &rev)); + kvsize = datasize / 2; + +/* + * We're most interested in the final records on disk. Rather than walk all records, we do a quick + * scan to find the last complete set of written ids. Each thread writes each id, along with the + * thread id, so they are interleaved. Once we have the neighborhood where some keys may be missing, + * we'll back up to do a scan from that point. + */ +#define CHECK_INCR 1000 + for (id = 0;; id += CHECK_INCR) { + gen_kv(keybuf, kvsize, id, 0, large_arr[0], true); + cursor->set_key(cursor, keybuf); + if ((ret = cursor->search(cursor)) == WT_NOTFOUND) + break; + testutil_check(ret); + for (th = 1; th < nth; th++) { + gen_kv(keybuf, kvsize, id, th, large_arr[th], true); + cursor->set_key(cursor, keybuf); + if ((ret = cursor->search(cursor)) == WT_NOTFOUND) + break; + testutil_check(ret); + } + if (ret == WT_NOTFOUND) + break; + } + if (id < CHECK_INCR * 2) + id = 0; + else + id -= CHECK_INCR * 2; + + printf("starting full scan at %" PRIu64 "\n", id); + gen_kv(keybuf, kvsize, id, 0, large_arr[0], true); + cursor->set_key(cursor, keybuf); + th = 0; + + /* Keep bitmap of "active" threads. */ + threadmap = (0x1U << nth) - 1; + for (ret = cursor->search(cursor); ret != WT_NOTFOUND && threadmap != 0; + ret = cursor->next(cursor)) { + testutil_check(ret); + testutil_check(cursor->get_key(cursor, &gotkey)); + gotid = (uint64_t)strtol(gotkey, &p, 10); + testutil_assert(*p == KEY_SEP[0]); + p++; + testutil_assert(isxdigit(*p)); + if (isdigit(*p)) + gotth = (uint32_t)(*p - '0'); + else if (*p >= 'a' && *p <= 'f') + gotth = (uint32_t)((*p - 'a') + 10); + else + gotth = (uint32_t)((*p - 'A') + 10); + p++; + testutil_assert(*p == KEY_SEP[0]); + p++; + + /* + * See if the expected thread has finished at this point. If so, remove it from the thread + * map. + */ + while (gotth != th) { + if ((threadmap & (0x1U << th)) != 0) { + threadmap &= ~(0x1U << th); + lastid[th] = id - 1; + /* + * Any newly removed value in the main table should not be present as a key in the + * reverse table, since they were transactionally inserted at the same time. + */ + gen_kv(keybuf, kvsize, id, th, large_arr[th], false); + check_kv(rev, keybuf, NULL, false); + check_schema(session, id - 1, th, flags); + } + th = (th + 1) % nth; + if (th == 0) + id++; + } + testutil_assert(gotid == id); + /* + * Check that the key and value fully match. + */ + gen_kv(keybuf, kvsize, id, th, large_arr[th], true); + gen_kv(&keybuf[kvsize], kvsize, id, th, large_arr[th], false); + testutil_check(cursor->get_value(cursor, &gotvalue)); + TEST_STREQ(keybuf, gotkey, "main table key"); + + /* + * Every 1000th record is large. + */ + if (id % 1000 == 0) + TEST_STREQ(large_arr[th], gotvalue, "main table large value"); + else + TEST_STREQ(&keybuf[kvsize], gotvalue, "main table value"); + + /* + * Check the reverse file, with key/value reversed. + */ + check_kv(rev, &keybuf[kvsize], keybuf, true); + + check_schema(session, id, th, flags); + + /* Bump thread number and id to the next expected key. */ + th = (th + 1) % nth; + if (th == 0) + id++; + } + printf("scanned to %" PRIu64 "\n", id); + + if (LF_ISSET(SCHEMA_ALL)) { + /* + * Check metadata to see if there are any tables present that shouldn't be there. + */ + testutil_check(session->open_cursor(session, "metadata:", NULL, NULL, &meta)); + while ((ret = meta->next(meta)) != WT_NOTFOUND) { + testutil_check(ret); + testutil_check(meta->get_key(meta, &gotkey)); + /* + * Names involved in schema testing are of the form: + * table:Axxx-t + * table:Bxxx-t + * xxx corresponds to the id inserted into the main + * table when the table was created, and t corresponds + * to the thread id that did this. + */ + if (WT_PREFIX_SKIP(gotkey, "table:") && (*gotkey == 'A' || *gotkey == 'B')) { + gotid = (uint64_t)strtol(gotkey + 1, &p, 10); + testutil_assert(*p == '-'); + th = (uint32_t)strtol(p + 1, &p, 10); + testutil_assert(*p == '\0'); + /* + * If table operations are truly transactional, then there shouldn't be any extra + * files that unaccounted for. + */ + if (LF_ISSET(SCHEMA_DROP_CHECK)) + testutil_assert(gotid == lastid[th]); + } + } + testutil_check(meta->close(meta)); + } + + testutil_check(cursor->close(cursor)); + testutil_check(rev->close(rev)); + testutil_check(session->close(session, NULL)); + testutil_check(conn->close(conn, NULL)); + + for (th = 0; th < nth; th++) + free(large_arr[th]); + free(large_arr); + free(keybuf); + free(lastid); + return (true); } /* * handler -- - * Child signal handler + * Child signal handler */ static void handler(int sig) { - pid_t pid; - int status, termsig; - - WT_UNUSED(sig); - pid = waitpid(-1, &status, WNOHANG|WUNTRACED); - if (pid == 0) - return; /* Nothing to wait for. */ - if (WIFSTOPPED(status)) - return; - if (WIFSIGNALED(status)) { - termsig = WTERMSIG(status); - if (termsig == SIGCONT || termsig == SIGSTOP) - return; - printf("Child got signal %d (status = %d, 0x%x)\n", - termsig, status, (u_int)status); + pid_t pid; + int status, termsig; + + WT_UNUSED(sig); + pid = waitpid(-1, &status, WNOHANG | WUNTRACED); + if (pid == 0) + return; /* Nothing to wait for. */ + if (WIFSTOPPED(status)) + return; + if (WIFSIGNALED(status)) { + termsig = WTERMSIG(status); + if (termsig == SIGCONT || termsig == SIGSTOP) + return; + printf("Child got signal %d (status = %d, 0x%x)\n", termsig, status, (u_int)status); #ifdef WCOREDUMP - if (WCOREDUMP(status)) - printf( - "Child process id=%" PRIuMAX " created core file\n", - (uintmax_t)pid); + if (WCOREDUMP(status)) + printf("Child process id=%" PRIuMAX " created core file\n", (uintmax_t)pid); #endif - } - - /* - * The core file will indicate why the child exited. Choose EINVAL here. - */ - testutil_die(EINVAL, - "Child process %" PRIuMAX " abnormally exited, status=%d (0x%x)", - (uintmax_t)pid, status, (u_int)status); + } + + /* + * The core file will indicate why the child exited. Choose EINVAL here. + */ + testutil_die(EINVAL, "Child process %" PRIuMAX " abnormally exited, status=%d (0x%x)", + (uintmax_t)pid, status, (u_int)status); } /* * has_direct_io -- - * Check for direct I/O support. + * Check for direct I/O support. */ static bool has_direct_io(void) { #ifdef O_DIRECT - return (true); + return (true); #else - return (false); + return (false); #endif } /* * main -- - * Top level test. + * Top level test. */ int main(int argc, char *argv[]) { - struct sigaction sa; - WT_RAND_STATE rnd; - pid_t pid; - size_t size; - uint32_t datasize, flags, i, interval, ncycles, nth, timeout; - int ch, status; - char *arg, *p; - char args[1024], buf[1024]; - const char *method, *working_dir; - bool populate_only, rand_th, rand_time, verify_only; - - (void)testutil_set_progname(argv); - - datasize = DEFAULT_DATA_SIZE; - nth = MIN_TH; - ncycles = DEFAULT_CYCLES; - rand_th = rand_time = true; - timeout = MIN_TIME; - interval = DEFAULT_INTERVAL; - flags = 0; - populate_only = verify_only = false; - working_dir = "WT_TEST.random-directio"; - method = "none"; - pid = 0; - memset(args, 0, sizeof(args)); - - if (!has_direct_io()) { - fprintf(stderr, "**** test_random_directio: this system does " - "not support direct I/O.\n**** Skipping test.\n"); - return (EXIT_SUCCESS); - } - for (i = 0, p = args; i < (uint32_t)argc; i++) { - testutil_check(__wt_snprintf_len_set(p, - sizeof(args) - (size_t)(p - args), &size, " %s", argv[i])); - p += size; - } - while ((ch = __wt_getopt(progname, argc, argv, - "d:h:i:m:n:pS:T:t:v")) != EOF) - switch (ch) { - case 'd': - datasize = (uint32_t)atoi(__wt_optarg); - if (datasize > LARGE_WRITE_SIZE || - datasize < MIN_DATA_SIZE) { - fprintf(stderr, - "-d value is larger than maximum %" - PRId32 "\n", - LARGE_WRITE_SIZE); - return (EXIT_FAILURE); - } - break; - case 'h': - working_dir = __wt_optarg; - break; - case 'i': - interval = (uint32_t)atoi(__wt_optarg); - break; - case 'm': - method = __wt_optarg; - if (!WT_STREQ(method, "fsync") && - !WT_STREQ(method, "dsync") && - !WT_STREQ(method, "none")) { - fprintf(stderr, - "-m option requires fsync|dsync|none\n"); - return (EXIT_FAILURE); - } - break; - case 'n': - ncycles = (uint32_t)atoi(__wt_optarg); - break; - case 'p': - populate_only = true; - break; - case 'S': - p = __wt_optarg; - while ((arg = strtok_r(p, ",", &p)) != NULL) { - if (WT_STREQ(arg, "all")) - LF_SET(SCHEMA_ALL); - else if (WT_STREQ(arg, "create")) - LF_SET(SCHEMA_CREATE); - else if (WT_STREQ(arg, "create_check")) - LF_SET(SCHEMA_CREATE_CHECK); - else if (WT_STREQ(arg, "data_check")) - LF_SET(SCHEMA_DATA_CHECK); - else if (WT_STREQ(arg, "drop")) - LF_SET(SCHEMA_DROP); - else if (WT_STREQ(arg, "drop_check")) - LF_SET(SCHEMA_DROP_CHECK); - else if (WT_STREQ(arg, "integrated")) - LF_SET(SCHEMA_INTEGRATED); - else if (WT_STREQ(arg, "none")) - flags = 0; - else if (WT_STREQ(arg, "rename")) - LF_SET(SCHEMA_RENAME); - else if (WT_STREQ(arg, "verbose")) - LF_SET(SCHEMA_VERBOSE); - else { - fprintf(stderr, - "Unknown -S arg '%s'\n", arg); - usage(); - } - } - break; - case 'T': - rand_th = false; - nth = (uint32_t)atoi(__wt_optarg); - break; - case 't': - rand_time = false; - timeout = (uint32_t)atoi(__wt_optarg); - break; - case 'v': - verify_only = true; - break; - default: - usage(); - } - argc -= __wt_optind; - if (argc != 0) - usage(); - - testutil_work_dir_from_path(home, sizeof(home), working_dir); - /* - * If the user wants to verify they need to tell us how many threads - * there were so we know what records we can expect. - */ - if (verify_only && rand_th) { - fprintf(stderr, - "Verify option requires specifying number of threads\n"); - return (EXIT_FAILURE); - } - if ((LF_ISSET(SCHEMA_RENAME|SCHEMA_DROP|SCHEMA_CREATE_CHECK| - SCHEMA_DATA_CHECK) && - !LF_ISSET(SCHEMA_CREATE)) || - (LF_ISSET(SCHEMA_DROP_CHECK) && - !LF_ISSET(SCHEMA_DROP))) { - fprintf(stderr, "Schema operations incompatible\n"); - usage(); - } - if (!LF_ISSET(SCHEMA_INTEGRATED) && - LF_ISSET(SCHEMA_CREATE_CHECK|SCHEMA_DATA_CHECK|SCHEMA_DROP_CHECK)) { - fprintf(stderr, "Schema '*check' options cannot be used " - "without 'integrated'\n"); - usage(); - } - printf("CONFIG:%s\n", args); - if (!verify_only) { - testutil_check(__wt_snprintf(buf, sizeof(buf), - "rm -rf %s", home)); - if ((status = system(buf)) < 0) - testutil_die(status, "system: %s", buf); - testutil_make_work_dir(home); - - __wt_random_init_seed(NULL, &rnd); - if (rand_time) { - timeout = __wt_random(&rnd) % MAX_TIME; - if (timeout < MIN_TIME) - timeout = MIN_TIME; - } - if (rand_th) { - nth = __wt_random(&rnd) % MAX_TH; - if (nth < MIN_TH) - nth = MIN_TH; - } - printf("Parent: Create %" PRIu32 - " threads; sleep %" PRIu32 " seconds\n", nth, timeout); - - create_db(method); - if (!populate_only) { - /* - * Fork a child to insert as many items. We will - * then randomly suspend the child, run recovery and - * make sure all items we wrote exist after recovery - * runs. - */ - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = handler; - testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); - if ((pid = fork()) < 0) - testutil_die(errno, "fork"); - } - if (pid == 0) { /* child, or populate_only */ - fill_db(nth, datasize, method, flags); - return (EXIT_SUCCESS); - } - - /* parent */ - /* - * Sleep for the configured amount of time before killing - * the child. - */ - testutil_sleep_wait(timeout, pid); - - /* - * Begin our cycles of suspend, copy, recover. - */ - for (i = 0; i < ncycles; i++) { - printf("Beginning cycle %" PRIu32 "/%" PRIu32 "\n", - i + 1, ncycles); - if (i != 0) - testutil_sleep_wait(interval, pid); - printf("Suspend child\n"); - if (kill(pid, SIGSTOP) != 0) - testutil_die(errno, "kill"); - printf("Check DB\n"); - fflush(stdout); - if (!check_db(nth, datasize, true, flags)) - return (EXIT_FAILURE); - if (kill(pid, SIGCONT) != 0) - testutil_die(errno, "kill"); - printf("\n"); - } - - printf("Kill child\n"); - sa.sa_handler = SIG_DFL; - testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); - if (kill(pid, SIGKILL) != 0) - testutil_die(errno, "kill"); - if (waitpid(pid, &status, 0) == -1) - testutil_die(errno, "waitpid"); - } - if (verify_only && !check_db(nth, datasize, false, flags)) { - printf("FAIL\n"); - return (EXIT_FAILURE); - } - printf("SUCCESS\n"); - return (EXIT_SUCCESS); + struct sigaction sa; + WT_RAND_STATE rnd; + pid_t pid; + size_t size; + uint32_t datasize, flags, i, interval, ncycles, nth, timeout; + int ch, status; + char *arg, *p; + char args[1024], buf[1024]; + const char *method, *working_dir; + bool populate_only, rand_th, rand_time, verify_only; + + (void)testutil_set_progname(argv); + + datasize = DEFAULT_DATA_SIZE; + nth = MIN_TH; + ncycles = DEFAULT_CYCLES; + rand_th = rand_time = true; + timeout = MIN_TIME; + interval = DEFAULT_INTERVAL; + flags = 0; + populate_only = verify_only = false; + working_dir = "WT_TEST.random-directio"; + method = "none"; + pid = 0; + memset(args, 0, sizeof(args)); + + if (!has_direct_io()) { + fprintf(stderr, + "**** test_random_directio: this system does " + "not support direct I/O.\n**** Skipping test.\n"); + return (EXIT_SUCCESS); + } + for (i = 0, p = args; i < (uint32_t)argc; i++) { + testutil_check( + __wt_snprintf_len_set(p, sizeof(args) - (size_t)(p - args), &size, " %s", argv[i])); + p += size; + } + while ((ch = __wt_getopt(progname, argc, argv, "d:h:i:m:n:pS:T:t:v")) != EOF) + switch (ch) { + case 'd': + datasize = (uint32_t)atoi(__wt_optarg); + if (datasize > LARGE_WRITE_SIZE || datasize < MIN_DATA_SIZE) { + fprintf(stderr, "-d value is larger than maximum %" PRId32 "\n", LARGE_WRITE_SIZE); + return (EXIT_FAILURE); + } + break; + case 'h': + working_dir = __wt_optarg; + break; + case 'i': + interval = (uint32_t)atoi(__wt_optarg); + break; + case 'm': + method = __wt_optarg; + if (!WT_STREQ(method, "fsync") && !WT_STREQ(method, "dsync") && + !WT_STREQ(method, "none")) { + fprintf(stderr, "-m option requires fsync|dsync|none\n"); + return (EXIT_FAILURE); + } + break; + case 'n': + ncycles = (uint32_t)atoi(__wt_optarg); + break; + case 'p': + populate_only = true; + break; + case 'S': + p = __wt_optarg; + while ((arg = strtok_r(p, ",", &p)) != NULL) { + if (WT_STREQ(arg, "all")) + LF_SET(SCHEMA_ALL); + else if (WT_STREQ(arg, "create")) + LF_SET(SCHEMA_CREATE); + else if (WT_STREQ(arg, "create_check")) + LF_SET(SCHEMA_CREATE_CHECK); + else if (WT_STREQ(arg, "data_check")) + LF_SET(SCHEMA_DATA_CHECK); + else if (WT_STREQ(arg, "drop")) + LF_SET(SCHEMA_DROP); + else if (WT_STREQ(arg, "drop_check")) + LF_SET(SCHEMA_DROP_CHECK); + else if (WT_STREQ(arg, "integrated")) + LF_SET(SCHEMA_INTEGRATED); + else if (WT_STREQ(arg, "none")) + flags = 0; + else if (WT_STREQ(arg, "rename")) + LF_SET(SCHEMA_RENAME); + else if (WT_STREQ(arg, "verbose")) + LF_SET(SCHEMA_VERBOSE); + else { + fprintf(stderr, "Unknown -S arg '%s'\n", arg); + usage(); + } + } + break; + case 'T': + rand_th = false; + nth = (uint32_t)atoi(__wt_optarg); + break; + case 't': + rand_time = false; + timeout = (uint32_t)atoi(__wt_optarg); + break; + case 'v': + verify_only = true; + break; + default: + usage(); + } + argc -= __wt_optind; + if (argc != 0) + usage(); + + testutil_work_dir_from_path(home, sizeof(home), working_dir); + /* + * If the user wants to verify they need to tell us how many threads there were so we know what + * records we can expect. + */ + if (verify_only && rand_th) { + fprintf(stderr, "Verify option requires specifying number of threads\n"); + return (EXIT_FAILURE); + } + if ((LF_ISSET(SCHEMA_RENAME | SCHEMA_DROP | SCHEMA_CREATE_CHECK | SCHEMA_DATA_CHECK) && + !LF_ISSET(SCHEMA_CREATE)) || + (LF_ISSET(SCHEMA_DROP_CHECK) && !LF_ISSET(SCHEMA_DROP))) { + fprintf(stderr, "Schema operations incompatible\n"); + usage(); + } + if (!LF_ISSET(SCHEMA_INTEGRATED) && + LF_ISSET(SCHEMA_CREATE_CHECK | SCHEMA_DATA_CHECK | SCHEMA_DROP_CHECK)) { + fprintf(stderr, + "Schema '*check' options cannot be used " + "without 'integrated'\n"); + usage(); + } + printf("CONFIG:%s\n", args); + if (!verify_only) { + testutil_check(__wt_snprintf(buf, sizeof(buf), "rm -rf %s", home)); + if ((status = system(buf)) < 0) + testutil_die(status, "system: %s", buf); + testutil_make_work_dir(home); + + __wt_random_init_seed(NULL, &rnd); + if (rand_time) { + timeout = __wt_random(&rnd) % MAX_TIME; + if (timeout < MIN_TIME) + timeout = MIN_TIME; + } + if (rand_th) { + nth = __wt_random(&rnd) % MAX_TH; + if (nth < MIN_TH) + nth = MIN_TH; + } + printf("Parent: Create %" PRIu32 " threads; sleep %" PRIu32 " seconds\n", nth, timeout); + + create_db(method); + if (!populate_only) { + /* + * Fork a child to insert as many items. We will then randomly suspend the child, run + * recovery and make sure all items we wrote exist after recovery runs. + */ + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = handler; + testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); + if ((pid = fork()) < 0) + testutil_die(errno, "fork"); + } + if (pid == 0) { /* child, or populate_only */ + fill_db(nth, datasize, method, flags); + return (EXIT_SUCCESS); + } + + /* parent */ + /* + * Sleep for the configured amount of time before killing the child. + */ + testutil_sleep_wait(timeout, pid); + + /* + * Begin our cycles of suspend, copy, recover. + */ + for (i = 0; i < ncycles; i++) { + printf("Beginning cycle %" PRIu32 "/%" PRIu32 "\n", i + 1, ncycles); + if (i != 0) + testutil_sleep_wait(interval, pid); + printf("Suspend child\n"); + if (kill(pid, SIGSTOP) != 0) + testutil_die(errno, "kill"); + printf("Check DB\n"); + fflush(stdout); + if (!check_db(nth, datasize, true, flags)) + return (EXIT_FAILURE); + if (kill(pid, SIGCONT) != 0) + testutil_die(errno, "kill"); + printf("\n"); + } + + printf("Kill child\n"); + sa.sa_handler = SIG_DFL; + testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); + if (kill(pid, SIGKILL) != 0) + testutil_die(errno, "kill"); + if (waitpid(pid, &status, 0) == -1) + testutil_die(errno, "waitpid"); + } + if (verify_only && !check_db(nth, datasize, false, flags)) { + printf("FAIL\n"); + return (EXIT_FAILURE); + } + printf("SUCCESS\n"); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/random_directio/util.c b/src/third_party/wiredtiger/test/csuite/random_directio/util.c index 8bab68ef59c..40de5d49f36 100644 --- a/src/third_party/wiredtiger/test/csuite/random_directio/util.c +++ b/src/third_party/wiredtiger/test/csuite/random_directio/util.c @@ -30,127 +30,118 @@ #include "util.h" #include <dirent.h> -#define ALIGN_UP(p, n) ((p) % (n) == 0 ? (p) : ((p) + (n) - ((p) % (n)))) -#define ALIGN_DOWN(p, n) ((p) - ((p) % (n))) +#define ALIGN_UP(p, n) ((p) % (n) == 0 ? (p) : ((p) + (n) - ((p) % (n)))) +#define ALIGN_DOWN(p, n) ((p) - ((p) % (n))) /* * util.c * Utility functions for test that simulates system crashes. */ -#define COPY_BUF_SIZE ((size_t)(20 * 1024)) +#define COPY_BUF_SIZE ((size_t)(20 * 1024)) /* * copy_directory -- - * Copy a directory, using direct IO if indicated. + * Copy a directory, using direct IO if indicated. */ void copy_directory(const char *fromdir, const char *todir, bool directio) { - struct dirent *dp; - struct stat sb; - DIR *dirp; - size_t blksize, bufsize, readbytes, n, remaining; - ssize_t ioret; - uintptr_t bufptr; - int openflags, rfd, wfd; - u_char *buf, *orig_buf; - char fromfile[4096], tofile[4096]; + struct dirent *dp; + struct stat sb; + DIR *dirp; + size_t blksize, bufsize, readbytes, n, remaining; + ssize_t ioret; + uintptr_t bufptr; + int openflags, rfd, wfd; + u_char *buf, *orig_buf; + char fromfile[4096], tofile[4096]; #ifdef O_DIRECT - openflags = directio ? O_DIRECT : 0; + openflags = directio ? O_DIRECT : 0; #else - testutil_assert(!directio); - openflags = 0; + testutil_assert(!directio); + openflags = 0; #endif - orig_buf = dcalloc(COPY_BUF_SIZE, sizeof(u_char)); - buf = NULL; - blksize = bufsize = 0; + orig_buf = dcalloc(COPY_BUF_SIZE, sizeof(u_char)); + buf = NULL; + blksize = bufsize = 0; - dirp = opendir(todir); - if (dirp != NULL) { - while ((dp = readdir(dirp)) != NULL) { - /* - * Skip . and .. - */ - if (strcmp(dp->d_name, ".") == 0 || - strcmp(dp->d_name, "..") == 0) - continue; - testutil_check(__wt_snprintf(tofile, sizeof(tofile), - "%s/%s", todir, dp->d_name)); - testutil_check(unlink(tofile)); - } - testutil_check(closedir(dirp)); - testutil_check(rmdir(todir)); - } + dirp = opendir(todir); + if (dirp != NULL) { + while ((dp = readdir(dirp)) != NULL) { + /* + * Skip . and .. + */ + if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) + continue; + testutil_check(__wt_snprintf(tofile, sizeof(tofile), "%s/%s", todir, dp->d_name)); + testutil_check(unlink(tofile)); + } + testutil_check(closedir(dirp)); + testutil_check(rmdir(todir)); + } - testutil_check(mkdir(todir, 0777)); - dirp = opendir(fromdir); - testutil_assert(dirp != NULL); + testutil_check(mkdir(todir, 0777)); + dirp = opendir(fromdir); + testutil_assert(dirp != NULL); - while ((dp = readdir(dirp)) != NULL) { - /* - * Skip . and .. - */ - if (strcmp(dp->d_name, ".") == 0 || - strcmp(dp->d_name, "..") == 0) - continue; + while ((dp = readdir(dirp)) != NULL) { + /* + * Skip . and .. + */ + if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) + continue; - testutil_check(__wt_snprintf(fromfile, sizeof(fromfile), - "%s/%s", fromdir, dp->d_name)); - testutil_check(__wt_snprintf(tofile, sizeof(tofile), - "%s/%s", todir, dp->d_name)); - rfd = open(fromfile, O_RDONLY | openflags, 0); - testutil_assertfmt(rfd >= 0, - "Open of source %s failed with %d\n", fromfile, errno); - wfd = open(tofile, O_WRONLY | O_CREAT, 0666); - testutil_assertfmt(wfd >= 0, - "Open of dest %s failed with %d\n", tofile, errno); - testutil_check(fstat(rfd, &sb)); + testutil_check(__wt_snprintf(fromfile, sizeof(fromfile), "%s/%s", fromdir, dp->d_name)); + testutil_check(__wt_snprintf(tofile, sizeof(tofile), "%s/%s", todir, dp->d_name)); + rfd = open(fromfile, O_RDONLY | openflags, 0); + testutil_assertfmt(rfd >= 0, "Open of source %s failed with %d\n", fromfile, errno); + wfd = open(tofile, O_WRONLY | O_CREAT, 0666); + testutil_assertfmt(wfd >= 0, "Open of dest %s failed with %d\n", tofile, errno); + testutil_check(fstat(rfd, &sb)); - /* - * Do any alignment on the buffer required for direct IO. - */ - if (buf == NULL) { - if (directio) { - blksize = (size_t)sb.st_blksize; - testutil_assert(blksize < COPY_BUF_SIZE); - /* - * Make sure we have plenty of room for - * adjusting the pointer. - */ - bufsize = COPY_BUF_SIZE - blksize; - bufptr = (uintptr_t)orig_buf; - /* Align pointer up to next block boundary */ - buf = (u_char *)ALIGN_UP(bufptr, blksize); - /* Align size down to block boundary */ - testutil_assert(bufsize >= blksize); - bufsize = ALIGN_DOWN(bufsize, blksize); - } else { - buf = orig_buf; - bufsize = COPY_BUF_SIZE; - } - } else if (directio) - testutil_assert(blksize == (size_t)sb.st_blksize); - remaining = (size_t)sb.st_size; - while (remaining > 0) { - readbytes = n = WT_MIN(remaining, bufsize); - /* - * When using direct IO, read sizes must also be - * a multiple of the block size. For the last block - * of a file, we must request to read the entire block, - * and we'll get the remainder back. - */ - if (directio) - readbytes = ALIGN_UP(n, blksize); - ioret = read(rfd, buf, readbytes); - testutil_assert(ioret >= 0 && (size_t)ioret == n); - ioret = write(wfd, buf, n); - testutil_assert(ioret >= 0 && (size_t)ioret == n); - remaining -= n; - } - testutil_check(close(rfd)); - testutil_check(close(wfd)); - } - testutil_check(closedir(dirp)); - free(orig_buf); + /* + * Do any alignment on the buffer required for direct IO. + */ + if (buf == NULL) { + if (directio) { + blksize = (size_t)sb.st_blksize; + testutil_assert(blksize < COPY_BUF_SIZE); + /* + * Make sure we have plenty of room for adjusting the pointer. + */ + bufsize = COPY_BUF_SIZE - blksize; + bufptr = (uintptr_t)orig_buf; + /* Align pointer up to next block boundary */ + buf = (u_char *)ALIGN_UP(bufptr, blksize); + /* Align size down to block boundary */ + testutil_assert(bufsize >= blksize); + bufsize = ALIGN_DOWN(bufsize, blksize); + } else { + buf = orig_buf; + bufsize = COPY_BUF_SIZE; + } + } else if (directio) + testutil_assert(blksize == (size_t)sb.st_blksize); + remaining = (size_t)sb.st_size; + while (remaining > 0) { + readbytes = n = WT_MIN(remaining, bufsize); + /* + * When using direct IO, read sizes must also be a multiple of the block size. For the + * last block of a file, we must request to read the entire block, and we'll get the + * remainder back. + */ + if (directio) + readbytes = ALIGN_UP(n, blksize); + ioret = read(rfd, buf, readbytes); + testutil_assert(ioret >= 0 && (size_t)ioret == n); + ioret = write(wfd, buf, n); + testutil_assert(ioret >= 0 && (size_t)ioret == n); + remaining -= n; + } + testutil_check(close(rfd)); + testutil_check(close(wfd)); + } + testutil_check(closedir(dirp)); + free(orig_buf); } diff --git a/src/third_party/wiredtiger/test/csuite/random_directio/util.h b/src/third_party/wiredtiger/test/csuite/random_directio/util.h index 99e579b6f17..efa935d5e01 100644 --- a/src/third_party/wiredtiger/test/csuite/random_directio/util.h +++ b/src/third_party/wiredtiger/test/csuite/random_directio/util.h @@ -31,5 +31,4 @@ * Utility functions for test that simulates system crashes. */ -extern void -copy_directory(const char *, const char *, bool); +extern void copy_directory(const char *, const char *, bool); diff --git a/src/third_party/wiredtiger/test/csuite/rwlock/main.c b/src/third_party/wiredtiger/test/csuite/rwlock/main.c index 28e43be31d5..9a757eb1cab 100644 --- a/src/third_party/wiredtiger/test/csuite/rwlock/main.c +++ b/src/third_party/wiredtiger/test/csuite/rwlock/main.c @@ -28,15 +28,14 @@ #include "test_util.h" /* - * JIRA ticket reference: HELP-4355 - * Test rwlock collapse under load. + * JIRA ticket reference: HELP-4355 Test rwlock collapse under load. */ -#define MAX_THREADS 1000 -#define READS_PER_WRITE 10000 +#define MAX_THREADS 1000 +#define READS_PER_WRITE 10000 //#define READS_PER_WRITE 1000000 //#define READS_PER_WRITE 100 -#define CHECK_CORRECTNESS 1 +#define CHECK_CORRECTNESS 1 //#define USE_POSIX 1 static WT_RWLOCK rwlock; @@ -50,43 +49,42 @@ void *thread_dump(void *); int main(int argc, char *argv[]) { - struct timespec te, ts; - TEST_OPTS *opts, _opts; - pthread_t dump_id, id[MAX_THREADS]; - int i; - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - opts->nthreads = 100; - opts->nops = 1000000; /* per thread */ - testutil_check(testutil_parse_opts(argc, argv, opts)); - running = true; - - testutil_make_work_dir(opts->home); - testutil_check(wiredtiger_open(opts->home, NULL, - "create,session_max=1000,statistics=(fast)", &opts->conn)); - - testutil_check(__wt_rwlock_init(NULL, &rwlock)); - testutil_check(pthread_rwlock_init(&p_rwlock, NULL)); - - testutil_check(pthread_create(&dump_id, NULL, thread_dump, opts)); - - __wt_epoch(NULL, &ts); - for (i = 0; i < (int)opts->nthreads; ++i) - testutil_check( - pthread_create(&id[i], NULL, thread_rwlock, opts)); - - while (--i >= 0) - testutil_check(pthread_join(id[i], NULL)); - __wt_epoch(NULL, &te); - printf("%.2lf\n", WT_TIMEDIFF_MS(te, ts) / 1000.0); - - running = false; - testutil_check(pthread_join(dump_id, NULL)); - - testutil_check(pthread_rwlock_destroy(&p_rwlock)); - testutil_cleanup(opts); - return (EXIT_SUCCESS); + struct timespec te, ts; + TEST_OPTS *opts, _opts; + pthread_t dump_id, id[MAX_THREADS]; + int i; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + opts->nthreads = 100; + opts->nops = 1000000; /* per thread */ + testutil_check(testutil_parse_opts(argc, argv, opts)); + running = true; + + testutil_make_work_dir(opts->home); + testutil_check( + wiredtiger_open(opts->home, NULL, "create,session_max=1000,statistics=(fast)", &opts->conn)); + + testutil_check(__wt_rwlock_init(NULL, &rwlock)); + testutil_check(pthread_rwlock_init(&p_rwlock, NULL)); + + testutil_check(pthread_create(&dump_id, NULL, thread_dump, opts)); + + __wt_epoch(NULL, &ts); + for (i = 0; i < (int)opts->nthreads; ++i) + testutil_check(pthread_create(&id[i], NULL, thread_rwlock, opts)); + + while (--i >= 0) + testutil_check(pthread_join(id[i], NULL)); + __wt_epoch(NULL, &te); + printf("%.2lf\n", WT_TIMEDIFF_MS(te, ts) / 1000.0); + + running = false; + testutil_check(pthread_join(dump_id, NULL)); + + testutil_check(pthread_rwlock_destroy(&p_rwlock)); + testutil_cleanup(opts); + return (EXIT_SUCCESS); } /* @@ -95,91 +93,86 @@ main(int argc, char *argv[]) void * thread_rwlock(void *arg) { - TEST_OPTS *opts; - WT_SESSION *wt_session; - WT_SESSION_IMPL *session; - uint64_t i, counter; - bool writelock; - - opts = (TEST_OPTS *)arg; - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &wt_session)); - session = (WT_SESSION_IMPL *)wt_session; - - if (opts->verbose) - printf("Running rwlock thread\n"); - for (i = 1; i <= opts->nops; ++i) { - writelock = (i % READS_PER_WRITE == 0); + TEST_OPTS *opts; + WT_SESSION *wt_session; + WT_SESSION_IMPL *session; + uint64_t i, counter; + bool writelock; + + opts = (TEST_OPTS *)arg; + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &wt_session)); + session = (WT_SESSION_IMPL *)wt_session; + + if (opts->verbose) + printf("Running rwlock thread\n"); + for (i = 1; i <= opts->nops; ++i) { + writelock = (i % READS_PER_WRITE == 0); #ifdef USE_POSIX - if (writelock) - testutil_check(pthread_rwlock_wrlock(&p_rwlock)); - else - testutil_check(pthread_rwlock_rdlock(&p_rwlock)); + if (writelock) + testutil_check(pthread_rwlock_wrlock(&p_rwlock)); + else + testutil_check(pthread_rwlock_rdlock(&p_rwlock)); #else - if (writelock) - __wt_writelock(session, &rwlock); - else - __wt_readlock(session, &rwlock); + if (writelock) + __wt_writelock(session, &rwlock); + else + __wt_readlock(session, &rwlock); #endif - /* - * Do a tiny amount of work inside the lock so the compiler - * can't optimize everything away. - */ - (void)__wt_atomic_add64(&counter, 1); + /* + * Do a tiny amount of work inside the lock so the compiler can't optimize everything away. + */ + (void)__wt_atomic_add64(&counter, 1); #ifdef CHECK_CORRECTNESS - if (writelock) - counter = ++shared_counter; - else - counter = shared_counter; + if (writelock) + counter = ++shared_counter; + else + counter = shared_counter; - __wt_yield(); + __wt_yield(); - testutil_assert(counter == shared_counter); + testutil_assert(counter == shared_counter); #endif #ifdef USE_POSIX - testutil_check(pthread_rwlock_unlock(&p_rwlock)); + testutil_check(pthread_rwlock_unlock(&p_rwlock)); #else - if (writelock) - __wt_writeunlock(session, &rwlock); - else - __wt_readunlock(session, &rwlock); + if (writelock) + __wt_writeunlock(session, &rwlock); + else + __wt_readunlock(session, &rwlock); #endif - if (opts->verbose && i % 10000 == 0) { - printf("%s", session->id == 20 ? ".\n" : "."); - fflush(stdout); - } - } + if (opts->verbose && i % 10000 == 0) { + printf("%s", session->id == 20 ? ".\n" : "."); + fflush(stdout); + } + } - opts->running = false; + opts->running = false; - return (NULL); + return (NULL); } void * thread_dump(void *arg) { - TEST_OPTS *opts; - - opts = arg; - - while (running) { - sleep(1); - if (opts->verbose) - printf("\n" - "rwlock { current %" PRIu8 ", next %" PRIu8 - ", reader %" PRIu8 ", readers_active %" PRIu32 - ", readers_queued %" PRIu8 " }\n", - rwlock.u.s.current, - rwlock.u.s.next, - rwlock.u.s.reader, - rwlock.u.s.readers_active, - rwlock.u.s.readers_queued); - } - - return (NULL); + TEST_OPTS *opts; + + opts = arg; + + while (running) { + sleep(1); + if (opts->verbose) + printf( + "\n" + "rwlock { current %" PRIu8 ", next %" PRIu8 ", reader %" PRIu8 + ", readers_active %" PRIu32 ", readers_queued %" PRIu8 " }\n", + rwlock.u.s.current, rwlock.u.s.next, rwlock.u.s.reader, rwlock.u.s.readers_active, + rwlock.u.s.readers_queued); + } + + return (NULL); } diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c index e7f22571cc6..bd127d8a686 100644 --- a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c @@ -31,7 +31,7 @@ #include <sys/wait.h> #include <signal.h> -static char home[1024]; /* Program working dir */ +static char home[1024]; /* Program working dir */ /* * Create three tables that we will write the same data to and verify that @@ -54,107 +54,102 @@ static char home[1024]; /* Program working dir */ * Each worker thread creates its own records file that records the data it * inserted and it records the timestamp that was used for that insertion. */ -#define INVALID_KEY UINT64_MAX -#define MAX_CKPT_INVL 2 /* Maximum interval between checkpoints */ +#define INVALID_KEY UINT64_MAX +#define MAX_CKPT_INVL 2 /* Maximum interval between checkpoints */ /* Set large, some slow I/O systems take tens of seconds to fsync. */ -#define MAX_STARTUP 30 /* Seconds to start up and set stable */ -#define MAX_TH 12 -#define MAX_TIME 40 -#define MAX_VAL 1024 -#define MIN_TH 5 -#define MIN_TIME 10 -#define PREPARE_FREQ 5 -#define PREPARE_YIELD (PREPARE_FREQ * 10) -#define RECORDS_FILE "records-%" PRIu32 -#define STABLE_PERIOD 100 - -static const char * const uri = "table:wt"; -static const char * const uri_local = "table:local"; -static const char * const uri_oplog = "table:oplog"; -static const char * const uri_collection = "table:collection"; - -static const char * const ckpt_file = "checkpoint_done"; +#define MAX_STARTUP 30 /* Seconds to start up and set stable */ +#define MAX_TH 12 +#define MAX_TIME 40 +#define MAX_VAL 1024 +#define MIN_TH 5 +#define MIN_TIME 10 +#define PREPARE_FREQ 5 +#define PREPARE_YIELD (PREPARE_FREQ * 10) +#define RECORDS_FILE "records-%" PRIu32 +#define STABLE_PERIOD 100 + +static const char *const uri = "table:wt"; +static const char *const uri_local = "table:local"; +static const char *const uri_oplog = "table:oplog"; +static const char *const uri_collection = "table:collection"; + +static const char *const ckpt_file = "checkpoint_done"; static bool compat, inmem, stable_set, use_ts, use_txn; static volatile uint64_t global_ts = 1; static volatile uint64_t uid = 1; typedef struct { - uint64_t ts; - const char *op; + uint64_t ts; + const char *op; } THREAD_TS; static volatile THREAD_TS th_ts[MAX_TH]; -#define ENV_CONFIG_COMPAT ",compatibility=(release=\"2.9\")" -#define ENV_CONFIG_DEF \ - "create,log=(archive=false,file_max=10M,enabled)" -#define ENV_CONFIG_TXNSYNC \ - "create,log=(archive=false,file_max=10M,enabled)," \ +#define ENV_CONFIG_COMPAT ",compatibility=(release=\"2.9\")" +#define ENV_CONFIG_DEF "create,log=(archive=false,file_max=10M,enabled)" +#define ENV_CONFIG_TXNSYNC \ + "create,log=(archive=false,file_max=10M,enabled)," \ "transaction_sync=(enabled,method=none)" -#define ENV_CONFIG_REC "log=(archive=false,recover=on)" +#define ENV_CONFIG_REC "log=(archive=false,recover=on)" typedef struct { - uint64_t absent_key; /* Last absent key */ - uint64_t exist_key; /* First existing key after miss */ - uint64_t first_key; /* First key in range */ - uint64_t first_miss; /* First missing key */ - uint64_t last_key; /* Last key in range */ + uint64_t absent_key; /* Last absent key */ + uint64_t exist_key; /* First existing key after miss */ + uint64_t first_key; /* First key in range */ + uint64_t first_miss; /* First missing key */ + uint64_t last_key; /* Last key in range */ } REPORT; typedef struct { - WT_CONNECTION *conn; - uint64_t start; - uint32_t info; - const char *op; + WT_CONNECTION *conn; + uint64_t start; + uint32_t info; + const char *op; } THREAD_DATA; -#define NOOP "noop" -#define BULK "bulk" -#define BULK_UNQ "bulk_unique" -#define CREATE "create" -#define CREATE_UNQ "create_unique" -#define CURSOR "cursor" -#define DROP "drop" -#define REBALANCE "rebalance" -#define UPGRADE "upgrade" -#define VERIFY "verify" - -static void sig_handler(int) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); -static void usage(void) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +#define NOOP "noop" +#define BULK "bulk" +#define BULK_UNQ "bulk_unique" +#define CREATE "create" +#define CREATE_UNQ "create_unique" +#define CURSOR "cursor" +#define DROP "drop" +#define REBALANCE "rebalance" +#define UPGRADE "upgrade" +#define VERIFY "verify" + +static void sig_handler(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void usage(void) { - fprintf(stderr, - "usage: %s [-h dir] [-T threads] [-t time] [-Cmvxz]\n", progname); - exit(EXIT_FAILURE); + fprintf(stderr, "usage: %s [-h dir] [-T threads] [-t time] [-Cmvxz]\n", progname); + exit(EXIT_FAILURE); } -static const char * const config = NULL; +static const char *const config = NULL; /* * subtest_error_handler -- * Error event handler. */ static int -subtest_error_handler(WT_EVENT_HANDLER *handler, - WT_SESSION *session, int error, const char *message) +subtest_error_handler( + WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const char *message) { - (void)(handler); - (void)(session); - (void)(error); - - /* Filter out errors about bulk load usage - they are annoying */ - if (strstr(message, "bulk-load is only supported on newly") == NULL) - fprintf(stderr, "%s", message); - return (0); + (void)(handler); + (void)(session); + (void)(error); + + /* Filter out errors about bulk load usage - they are annoying */ + if (strstr(message, "bulk-load is only supported on newly") == NULL) + fprintf(stderr, "%s", message); + return (0); } static WT_EVENT_HANDLER event_handler = { - subtest_error_handler, - NULL, /* Message handler */ - NULL, /* Progress handler */ - NULL /* Close handler */ + subtest_error_handler, NULL, /* Message handler */ + NULL, /* Progress handler */ + NULL /* Close handler */ }; /* @@ -170,796 +165,715 @@ static WT_EVENT_HANDLER event_handler = { static void dump_ts(uint64_t nth) { - uint64_t i; + uint64_t i; - for (i = 0; i < nth; ++i) - fprintf(stderr, "THREAD %" PRIu64 ": ts: %" PRIu64 - " op %s\n", i, th_ts[i].ts, th_ts[i].op); + for (i = 0; i < nth; ++i) + fprintf(stderr, "THREAD %" PRIu64 ": ts: %" PRIu64 " op %s\n", i, th_ts[i].ts, th_ts[i].op); } /* * test_bulk -- - * Test creating a bulk cursor. + * Test creating a bulk cursor. */ static void test_bulk(THREAD_DATA *td) { - WT_CURSOR *c; - WT_DECL_RET; - WT_SESSION *session; - bool create; - - testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); - - if (use_txn) - testutil_check(session->begin_transaction(session, NULL)); - create = false; - if ((ret = session->create(session, uri, config)) != 0) - if (ret != EEXIST && ret != EBUSY) - testutil_die(ret, "session.create"); - - if (ret == 0) { - create = true; - if ((ret = session->open_cursor( - session, uri, NULL, "bulk", &c)) == 0) { - __wt_yield(); - testutil_check(c->close(c)); - } else if (ret != ENOENT && ret != EBUSY && ret != EINVAL) - testutil_die(ret, "session.open_cursor bulk"); - } - - if (use_txn) { - /* If create fails, rollback else will commit.*/ - if (!create) - ret = session->rollback_transaction(session, NULL); - else - ret = session->commit_transaction(session, NULL); - - if (ret == EINVAL) { - fprintf(stderr, "BULK: EINVAL on %s. ABORT\n", - create ? "commit" : "rollback"); - testutil_die(ret, "session.commit bulk"); - } - } - testutil_check(session->close(session, NULL)); + WT_CURSOR *c; + WT_DECL_RET; + WT_SESSION *session; + bool create; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + create = false; + if ((ret = session->create(session, uri, config)) != 0) + if (ret != EEXIST && ret != EBUSY) + testutil_die(ret, "session.create"); + + if (ret == 0) { + create = true; + if ((ret = session->open_cursor(session, uri, NULL, "bulk", &c)) == 0) { + __wt_yield(); + testutil_check(c->close(c)); + } else if (ret != ENOENT && ret != EBUSY && ret != EINVAL) + testutil_die(ret, "session.open_cursor bulk"); + } + + if (use_txn) { + /* If create fails, rollback else will commit.*/ + if (!create) + ret = session->rollback_transaction(session, NULL); + else + ret = session->commit_transaction(session, NULL); + + if (ret == EINVAL) { + fprintf(stderr, "BULK: EINVAL on %s. ABORT\n", create ? "commit" : "rollback"); + testutil_die(ret, "session.commit bulk"); + } + } + testutil_check(session->close(session, NULL)); } /* * test_bulk_unique -- - * Test creating a bulk cursor with a unique name. + * Test creating a bulk cursor with a unique name. */ static void test_bulk_unique(THREAD_DATA *td, int force) { - WT_CURSOR *c; - WT_DECL_RET; - WT_SESSION *session; - uint64_t my_uid; - char new_uri[64]; - - testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); - - /* Generate a unique object name. */ - my_uid = __wt_atomic_addv64(&uid, 1); - testutil_check(__wt_snprintf( - new_uri, sizeof(new_uri), "%s.%" PRIu64, uri, my_uid)); - - if (use_txn) - testutil_check(session->begin_transaction(session, NULL)); - testutil_check(session->create(session, new_uri, config)); - - __wt_yield(); - /* - * Opening a bulk cursor may have raced with a forced checkpoint - * which created a checkpoint of the empty file, and triggers an EINVAL. - */ - if ((ret = session->open_cursor( - session, new_uri, NULL, "bulk", &c)) == 0) - testutil_check(c->close(c)); - else if (ret != EINVAL) - testutil_die(ret, - "session.open_cursor bulk unique: %s, new_uri"); - - while ((ret = session->drop( - session, new_uri, force ? "force" : NULL)) != 0) - if (ret != EBUSY) - testutil_die(ret, "session.drop: %s", new_uri); - - if (use_txn && - (ret = session->commit_transaction(session, NULL)) != 0 && - ret != EINVAL) - testutil_die(ret, "session.commit bulk unique"); - testutil_check(session->close(session, NULL)); + WT_CURSOR *c; + WT_DECL_RET; + WT_SESSION *session; + uint64_t my_uid; + char new_uri[64]; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + + /* Generate a unique object name. */ + my_uid = __wt_atomic_addv64(&uid, 1); + testutil_check(__wt_snprintf(new_uri, sizeof(new_uri), "%s.%" PRIu64, uri, my_uid)); + + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + testutil_check(session->create(session, new_uri, config)); + + __wt_yield(); + /* + * Opening a bulk cursor may have raced with a forced checkpoint which created a checkpoint of + * the empty file, and triggers an EINVAL. + */ + if ((ret = session->open_cursor(session, new_uri, NULL, "bulk", &c)) == 0) + testutil_check(c->close(c)); + else if (ret != EINVAL) + testutil_die(ret, "session.open_cursor bulk unique: %s, new_uri"); + + while ((ret = session->drop(session, new_uri, force ? "force" : NULL)) != 0) + if (ret != EBUSY) + testutil_die(ret, "session.drop: %s", new_uri); + + if (use_txn && (ret = session->commit_transaction(session, NULL)) != 0 && ret != EINVAL) + testutil_die(ret, "session.commit bulk unique"); + testutil_check(session->close(session, NULL)); } /* * test_cursor -- - * Open a cursor on a data source. + * Open a cursor on a data source. */ static void test_cursor(THREAD_DATA *td) { - WT_CURSOR *cursor; - WT_DECL_RET; - WT_SESSION *session; - - testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); - - if (use_txn) - testutil_check(session->begin_transaction(session, NULL)); - if ((ret = - session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0) { - if (ret != ENOENT && ret != EBUSY) - testutil_die(ret, "session.open_cursor"); - } else { - __wt_yield(); - testutil_check(cursor->close(cursor)); - } - - if (use_txn && - (ret = session->commit_transaction(session, NULL)) != 0 && - ret != EINVAL) - testutil_die(ret, "session.commit cursor"); - testutil_check(session->close(session, NULL)); + WT_CURSOR *cursor; + WT_DECL_RET; + WT_SESSION *session; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + if ((ret = session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0) { + if (ret != ENOENT && ret != EBUSY) + testutil_die(ret, "session.open_cursor"); + } else { + __wt_yield(); + testutil_check(cursor->close(cursor)); + } + + if (use_txn && (ret = session->commit_transaction(session, NULL)) != 0 && ret != EINVAL) + testutil_die(ret, "session.commit cursor"); + testutil_check(session->close(session, NULL)); } /* * test_create -- - * Create a table. + * Create a table. */ static void test_create(THREAD_DATA *td) { - WT_DECL_RET; - WT_SESSION *session; - - testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); - - if (use_txn) - testutil_check(session->begin_transaction(session, NULL)); - if ((ret = session->create(session, uri, config)) != 0) - if (ret != EEXIST && ret != EBUSY) - testutil_die(ret, "session.create"); - __wt_yield(); - if (use_txn && - (ret = session->commit_transaction(session, NULL)) != 0 && - ret != EINVAL) - testutil_die(ret, "session.commit create"); - testutil_check(session->close(session, NULL)); + WT_DECL_RET; + WT_SESSION *session; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + if ((ret = session->create(session, uri, config)) != 0) + if (ret != EEXIST && ret != EBUSY) + testutil_die(ret, "session.create"); + __wt_yield(); + if (use_txn && (ret = session->commit_transaction(session, NULL)) != 0 && ret != EINVAL) + testutil_die(ret, "session.commit create"); + testutil_check(session->close(session, NULL)); } /* * test_create_unique -- - * Create a uniquely named table. + * Create a uniquely named table. */ static void test_create_unique(THREAD_DATA *td, int force) { - WT_DECL_RET; - WT_SESSION *session; - uint64_t my_uid; - char new_uri[64]; - - testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); - - /* Generate a unique object name. */ - my_uid = __wt_atomic_addv64(&uid, 1); - testutil_check(__wt_snprintf( - new_uri, sizeof(new_uri), "%s.%" PRIu64, uri, my_uid)); - - if (use_txn) - testutil_check(session->begin_transaction(session, NULL)); - testutil_check(session->create(session, new_uri, config)); - if (use_txn && - (ret = session->commit_transaction(session, NULL)) != 0 && - ret != EINVAL) - testutil_die(ret, "session.commit create unique"); - - __wt_yield(); - if (use_txn) - testutil_check(session->begin_transaction(session, NULL)); - while ((ret = session->drop( - session, new_uri, force ? "force" : NULL)) != 0) - if (ret != EBUSY) - testutil_die(ret, "session.drop: %s", new_uri); - if (use_txn && - (ret = session->commit_transaction(session, NULL)) != 0 && - ret != EINVAL) - testutil_die(ret, "session.commit create unique"); - - testutil_check(session->close(session, NULL)); + WT_DECL_RET; + WT_SESSION *session; + uint64_t my_uid; + char new_uri[64]; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + + /* Generate a unique object name. */ + my_uid = __wt_atomic_addv64(&uid, 1); + testutil_check(__wt_snprintf(new_uri, sizeof(new_uri), "%s.%" PRIu64, uri, my_uid)); + + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + testutil_check(session->create(session, new_uri, config)); + if (use_txn && (ret = session->commit_transaction(session, NULL)) != 0 && ret != EINVAL) + testutil_die(ret, "session.commit create unique"); + + __wt_yield(); + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + while ((ret = session->drop(session, new_uri, force ? "force" : NULL)) != 0) + if (ret != EBUSY) + testutil_die(ret, "session.drop: %s", new_uri); + if (use_txn && (ret = session->commit_transaction(session, NULL)) != 0 && ret != EINVAL) + testutil_die(ret, "session.commit create unique"); + + testutil_check(session->close(session, NULL)); } /* * test_drop -- - * Test dropping a table. + * Test dropping a table. */ static void test_drop(THREAD_DATA *td, int force) { - WT_DECL_RET; - WT_SESSION *session; - - testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); - - if (use_txn) - testutil_check(session->begin_transaction(session, NULL)); - if ((ret = session->drop(session, uri, force ? "force" : NULL)) != 0) - if (ret != ENOENT && ret != EBUSY) - testutil_die(ret, "session.drop"); - - if (use_txn) { - /* - * As the operations are being performed concurrently, - * return value can be ENOENT or EBUSY will set - * error to transaction opened by session. In these - * cases the transaction has to be aborted. - */ - if (ret != ENOENT && ret != EBUSY) - ret = session->commit_transaction(session, NULL); - else - ret = session->rollback_transaction(session, NULL); - if (ret == EINVAL) - testutil_die(ret, "session.commit drop"); - } - testutil_check(session->close(session, NULL)); + WT_DECL_RET; + WT_SESSION *session; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + if ((ret = session->drop(session, uri, force ? "force" : NULL)) != 0) + if (ret != ENOENT && ret != EBUSY) + testutil_die(ret, "session.drop"); + + if (use_txn) { + /* + * As the operations are being performed concurrently, return value can be ENOENT or EBUSY + * will set error to transaction opened by session. In these cases the transaction has to be + * aborted. + */ + if (ret != ENOENT && ret != EBUSY) + ret = session->commit_transaction(session, NULL); + else + ret = session->rollback_transaction(session, NULL); + if (ret == EINVAL) + testutil_die(ret, "session.commit drop"); + } + testutil_check(session->close(session, NULL)); } /* * test_rebalance -- - * Rebalance a tree. + * Rebalance a tree. */ static void test_rebalance(THREAD_DATA *td) { - WT_DECL_RET; - WT_SESSION *session; + WT_DECL_RET; + WT_SESSION *session; - testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); - if ((ret = session->rebalance(session, uri, NULL)) != 0) - if (ret != ENOENT && ret != EBUSY) - testutil_die(ret, "session.rebalance"); + if ((ret = session->rebalance(session, uri, NULL)) != 0) + if (ret != ENOENT && ret != EBUSY) + testutil_die(ret, "session.rebalance"); - testutil_check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); } /* * test_upgrade -- - * Upgrade a tree. + * Upgrade a tree. */ static void test_upgrade(THREAD_DATA *td) { - WT_DECL_RET; - WT_SESSION *session; + WT_DECL_RET; + WT_SESSION *session; - testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); - if ((ret = session->upgrade(session, uri, NULL)) != 0) - if (ret != ENOENT && ret != EBUSY) - testutil_die(ret, "session.upgrade"); + if ((ret = session->upgrade(session, uri, NULL)) != 0) + if (ret != ENOENT && ret != EBUSY) + testutil_die(ret, "session.upgrade"); - testutil_check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); } /* * test_verify -- - * Verify a tree. + * Verify a tree. */ static void test_verify(THREAD_DATA *td) { - WT_DECL_RET; - WT_SESSION *session; + WT_DECL_RET; + WT_SESSION *session; - testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); - if ((ret = session->verify(session, uri, NULL)) != 0) - if (ret != ENOENT && ret != EBUSY) - testutil_die(ret, "session.verify"); + if ((ret = session->verify(session, uri, NULL)) != 0) + if (ret != ENOENT && ret != EBUSY) + testutil_die(ret, "session.verify"); - testutil_check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); } /* * thread_ts_run -- - * Runner function for a timestamp thread. + * Runner function for a timestamp thread. */ static WT_THREAD_RET thread_ts_run(void *arg) { - WT_SESSION *session; - THREAD_DATA *td; - uint64_t i, last_ts, oldest_ts, this_ts; - char tscfg[64]; - - td = (THREAD_DATA *)arg; - last_ts = 0; - - testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); - /* - * Every N records we will record our stable timestamp into the stable - * table. That will define our threshold where we expect to find records - * after recovery. - */ - for (;;) { - oldest_ts = UINT64_MAX; - /* - * For the timestamp thread, the info field contains the number - * of worker threads. - */ - for (i = 0; i < td->info; ++i) { - /* - * We need to let all threads get started, so if we find - * any thread still with a zero timestamp we go to - * sleep. - */ - this_ts = th_ts[i].ts; - if (this_ts == 0) - goto ts_wait; - else if (this_ts < oldest_ts) - oldest_ts = this_ts; - } - - if (oldest_ts != UINT64_MAX && - oldest_ts - last_ts > STABLE_PERIOD) { - /* - * Set both the oldest and stable timestamp so that we - * don't need to maintain read availability at older - * timestamps. - */ - testutil_check(__wt_snprintf( - tscfg, sizeof(tscfg), - "oldest_timestamp=%" PRIx64 - ",stable_timestamp=%" PRIx64, - oldest_ts, oldest_ts)); - testutil_check( - td->conn->set_timestamp(td->conn, tscfg)); - last_ts = oldest_ts; - if (!stable_set) { - stable_set = true; - printf("SET STABLE: %" PRIx64 " %" PRIu64 "\n", - oldest_ts, oldest_ts); - } - } else -ts_wait: __wt_sleep(0, 1000); - } - /* NOTREACHED */ + WT_SESSION *session; + THREAD_DATA *td; + uint64_t i, last_ts, oldest_ts, this_ts; + char tscfg[64]; + + td = (THREAD_DATA *)arg; + last_ts = 0; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + /* + * Every N records we will record our stable timestamp into the stable table. That will define + * our threshold where we expect to find records after recovery. + */ + for (;;) { + oldest_ts = UINT64_MAX; + /* + * For the timestamp thread, the info field contains the number of worker threads. + */ + for (i = 0; i < td->info; ++i) { + /* + * We need to let all threads get started, so if we find any thread still with a zero + * timestamp we go to sleep. + */ + this_ts = th_ts[i].ts; + if (this_ts == 0) + goto ts_wait; + else if (this_ts < oldest_ts) + oldest_ts = this_ts; + } + + if (oldest_ts != UINT64_MAX && oldest_ts - last_ts > STABLE_PERIOD) { + /* + * Set both the oldest and stable timestamp so that we don't need to maintain read + * availability at older timestamps. + */ + testutil_check(__wt_snprintf(tscfg, sizeof(tscfg), + "oldest_timestamp=%" PRIx64 ",stable_timestamp=%" PRIx64, oldest_ts, oldest_ts)); + testutil_check(td->conn->set_timestamp(td->conn, tscfg)); + last_ts = oldest_ts; + if (!stable_set) { + stable_set = true; + printf("SET STABLE: %" PRIx64 " %" PRIu64 "\n", oldest_ts, oldest_ts); + } + } else + ts_wait: + __wt_sleep(0, 1000); + } + /* NOTREACHED */ } /* * thread_ckpt_run -- - * Runner function for the checkpoint thread. + * Runner function for the checkpoint thread. */ static WT_THREAD_RET thread_ckpt_run(void *arg) { - struct timespec now, start; - FILE *fp; - WT_RAND_STATE rnd; - WT_SESSION *session; - THREAD_DATA *td; - uint64_t ts; - uint32_t sleep_time; - int i; - bool first_ckpt; - - __wt_random_init(&rnd); - - td = (THREAD_DATA *)arg; - /* - * Keep a separate file with the records we wrote for checking. - */ - (void)unlink(ckpt_file); - testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); - first_ckpt = true; - ts = 0; - /* - * Keep writing checkpoints until killed by parent. - */ - __wt_epoch(NULL, &start); - for (i = 0;;) { - sleep_time = __wt_random(&rnd) % MAX_CKPT_INVL; - sleep(sleep_time); - if (use_ts) { - ts = global_ts; - /* - * If we're using timestamps wait for the stable - * timestamp to get set the first time. - */ - if (!stable_set) { - __wt_epoch(NULL, &now); - if (WT_TIMEDIFF_SEC(now, start) >= 1) - printf("CKPT: !stable_set time %" - PRIu64 "\n", - WT_TIMEDIFF_SEC(now, start)); - if (WT_TIMEDIFF_SEC(now, start) > MAX_STARTUP) { - fprintf(stderr, - "After %d seconds stable still not " - "set. Aborting.\n", MAX_STARTUP); - /* - * For the checkpoint thread the info - * contains the number of threads. - */ - dump_ts(td->info); - abort(); - } - continue; - } - } - /* - * Since this is the default, send in this string even if - * running without timestamps. - */ - testutil_check(session->checkpoint( - session, "use_timestamp=true")); - printf("Checkpoint %d complete. Minimum ts %" PRIu64 "\n", - ++i, ts); - fflush(stdout); - /* - * Create the checkpoint file so that the parent process knows - * at least one checkpoint has finished and can start its - * timer. Start the timer for stable after the first checkpoint - * completes because a slow I/O lag during the checkpoint can - * cause a false positive for a timeout. - */ - if (first_ckpt) { - testutil_checksys((fp = fopen(ckpt_file, "w")) == NULL); - first_ckpt = false; - testutil_checksys(fclose(fp) != 0); - } - } - /* NOTREACHED */ + struct timespec now, start; + FILE *fp; + WT_RAND_STATE rnd; + WT_SESSION *session; + THREAD_DATA *td; + uint64_t ts; + uint32_t sleep_time; + int i; + bool first_ckpt; + + __wt_random_init(&rnd); + + td = (THREAD_DATA *)arg; + /* + * Keep a separate file with the records we wrote for checking. + */ + (void)unlink(ckpt_file); + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + first_ckpt = true; + ts = 0; + /* + * Keep writing checkpoints until killed by parent. + */ + __wt_epoch(NULL, &start); + for (i = 0;;) { + sleep_time = __wt_random(&rnd) % MAX_CKPT_INVL; + sleep(sleep_time); + if (use_ts) { + ts = global_ts; + /* + * If we're using timestamps wait for the stable timestamp to get set the first time. + */ + if (!stable_set) { + __wt_epoch(NULL, &now); + if (WT_TIMEDIFF_SEC(now, start) >= 1) + printf("CKPT: !stable_set time %" PRIu64 "\n", WT_TIMEDIFF_SEC(now, start)); + if (WT_TIMEDIFF_SEC(now, start) > MAX_STARTUP) { + fprintf(stderr, + "After %d seconds stable still not " + "set. Aborting.\n", + MAX_STARTUP); + /* + * For the checkpoint thread the info contains the number of threads. + */ + dump_ts(td->info); + abort(); + } + continue; + } + } + /* + * Since this is the default, send in this string even if running without timestamps. + */ + testutil_check(session->checkpoint(session, "use_timestamp=true")); + printf("Checkpoint %d complete. Minimum ts %" PRIu64 "\n", ++i, ts); + fflush(stdout); + /* + * Create the checkpoint file so that the parent process knows at least one checkpoint has + * finished and can start its timer. Start the timer for stable after the first checkpoint + * completes because a slow I/O lag during the checkpoint can cause a false positive for a + * timeout. + */ + if (first_ckpt) { + testutil_checksys((fp = fopen(ckpt_file, "w")) == NULL); + first_ckpt = false; + testutil_checksys(fclose(fp) != 0); + } + } + /* NOTREACHED */ } /* * thread_run -- - * Runner function for the worker threads. + * Runner function for the worker threads. */ static WT_THREAD_RET thread_run(void *arg) { - FILE *fp; - WT_CURSOR *cur_coll, *cur_local, *cur_oplog; - WT_ITEM data; - WT_RAND_STATE rnd; - WT_SESSION *oplog_session, *session; - THREAD_DATA *td; - uint64_t i, stable_ts; - char cbuf[MAX_VAL], lbuf[MAX_VAL], obuf[MAX_VAL]; - char kname[64], tscfg[64]; - bool use_prep; - - __wt_random_init(&rnd); - memset(cbuf, 0, sizeof(cbuf)); - memset(lbuf, 0, sizeof(lbuf)); - memset(obuf, 0, sizeof(obuf)); - memset(kname, 0, sizeof(kname)); - - td = (THREAD_DATA *)arg; - /* - * Set up the separate file for checking. - */ - testutil_check(__wt_snprintf( - cbuf, sizeof(cbuf), RECORDS_FILE, td->info)); - (void)unlink(cbuf); - testutil_checksys((fp = fopen(cbuf, "w")) == NULL); - /* - * Set to line buffering. But that is advisory only. We've seen - * cases where the result files end up with partial lines. - */ - __wt_stream_set_line_buffer(fp); - - /* - * Have half the threads use prepared transactions if timestamps - * are in use. - */ - use_prep = (use_ts && td->info % 2 == 0) ? true : false; - /* - * We may have two sessions so that the oplog session can have its own - * transaction in parallel with the collection session for threads - * that are going to be using prepared transactions. We need this - * because prepared transactions cannot have any operations that modify - * a table that is logged. But we also want to test mixed logged and - * not-logged transactions. - */ - testutil_check(td->conn->open_session( - td->conn, NULL, "isolation=snapshot", &session)); - /* - * Open a cursor to each table. - */ - testutil_check(session->open_cursor(session, - uri_collection, NULL, NULL, &cur_coll)); - testutil_check(session->open_cursor(session, - uri_local, NULL, NULL, &cur_local)); - oplog_session = NULL; - if (use_prep) { - testutil_check(td->conn->open_session( - td->conn, NULL, "isolation=snapshot", &oplog_session)); - testutil_check(session->open_cursor(oplog_session, - uri_oplog, NULL, NULL, &cur_oplog)); - } else - testutil_check(session->open_cursor(session, - uri_oplog, NULL, NULL, &cur_oplog)); - - /* - * Write our portion of the key space until we're killed. - */ - printf("Thread %" PRIu32 " starts at %" PRIu64 "\n", - td->info, td->start); - stable_ts = 0; - for (i = td->start;; ++i) { - /* - * Allow some threads to skip schema operations so that they - * are generating sufficient dirty data. - */ - WT_PUBLISH(th_ts[td->info].op, NOOP); - if (td->info != 0 && td->info != 1) - /* - * Do a schema operation about 50% of the time by having - * a case for only about half the possible mod values. - */ - switch (__wt_random(&rnd) % 20) { - case 0: - WT_PUBLISH(th_ts[td->info].op, BULK); - test_bulk(td); - break; - case 1: - WT_PUBLISH(th_ts[td->info].op, BULK_UNQ); - test_bulk_unique(td, __wt_random(&rnd) & 1); - break; - case 2: - WT_PUBLISH(th_ts[td->info].op, CREATE); - test_create(td); - break; - case 3: - WT_PUBLISH(th_ts[td->info].op, CREATE_UNQ); - test_create_unique(td, __wt_random(&rnd) & 1); - break; - case 4: - WT_PUBLISH(th_ts[td->info].op, CURSOR); - test_cursor(td); - break; - case 5: - WT_PUBLISH(th_ts[td->info].op, DROP); - test_drop(td, __wt_random(&rnd) & 1); - break; - case 6: - WT_PUBLISH(th_ts[td->info].op, REBALANCE); - test_rebalance(td); - break; - case 7: - WT_PUBLISH(th_ts[td->info].op, UPGRADE); - test_upgrade(td); - break; - case 8: - WT_PUBLISH(th_ts[td->info].op, VERIFY); - test_verify(td); - break; - } - if (use_ts) - stable_ts = __wt_atomic_addv64(&global_ts, 1); - testutil_check(__wt_snprintf( - kname, sizeof(kname), "%" PRIu64, i)); - - testutil_check(session->begin_transaction(session, NULL)); - if (use_prep) - testutil_check(oplog_session->begin_transaction( - oplog_session, NULL)); - cur_coll->set_key(cur_coll, kname); - cur_local->set_key(cur_local, kname); - cur_oplog->set_key(cur_oplog, kname); - /* - * Put an informative string into the value so that it - * can be viewed well in a binary dump. - */ - testutil_check(__wt_snprintf(cbuf, sizeof(cbuf), - "COLL: thread:%" PRIu32 " ts:%" PRIu64 " key: %" PRIu64, - td->info, stable_ts, i)); - testutil_check(__wt_snprintf(lbuf, sizeof(lbuf), - "LOCAL: thread:%" PRIu32 " ts:%" PRIu64 " key: %" PRIu64, - td->info, stable_ts, i)); - testutil_check(__wt_snprintf(obuf, sizeof(obuf), - "OPLOG: thread:%" PRIu32 " ts:%" PRIu64 " key: %" PRIu64, - td->info, stable_ts, i)); - data.size = __wt_random(&rnd) % MAX_VAL; - data.data = cbuf; - cur_coll->set_value(cur_coll, &data); - testutil_check(cur_coll->insert(cur_coll)); - data.size = __wt_random(&rnd) % MAX_VAL; - data.data = obuf; - cur_oplog->set_value(cur_oplog, &data); - testutil_check(cur_oplog->insert(cur_oplog)); - if (use_ts) { - /* - * Run with prepare every once in a while. And also - * yield after prepare sometimes too. This is only done - * on the regular session. - */ - if (use_prep && i % PREPARE_FREQ == 0) { - testutil_check(__wt_snprintf( - tscfg, sizeof(tscfg), - "prepare_timestamp=%" PRIx64, stable_ts)); - testutil_check(session->prepare_transaction( - session, tscfg)); - if (i % PREPARE_YIELD == 0) - __wt_yield(); - - testutil_check(__wt_snprintf( - tscfg, sizeof(tscfg), - "commit_timestamp=%" PRIx64 - ",durable_timestamp=%" PRIx64, - stable_ts, stable_ts)); - } else - testutil_check(__wt_snprintf( - tscfg, sizeof(tscfg), - "commit_timestamp=%" PRIx64, stable_ts)); - - testutil_check( - session->commit_transaction(session, tscfg)); - if (use_prep) { - /* - * Durable timestamp should not be passed as - * oplog transaction is a non-prepared - * transaction. - */ - testutil_check(__wt_snprintf( - tscfg, sizeof(tscfg), - "commit_timestamp=%" PRIx64, stable_ts)); - testutil_check( - oplog_session->commit_transaction( - oplog_session, tscfg)); - } - /* - * Update the thread's last-committed timestamp. - * Don't let the compiler re-order this statement, - * if we were to race with the timestamp thread, it - * might see our thread update before the commit. - */ - WT_PUBLISH(th_ts[td->info].ts, stable_ts); - } else { - testutil_check( - session->commit_transaction(session, NULL)); - if (use_prep) - testutil_check( - oplog_session->commit_transaction( - oplog_session, NULL)); - } - /* - * Insert into the local table outside the timestamp txn. - */ - data.size = __wt_random(&rnd) % MAX_VAL; - data.data = lbuf; - cur_local->set_value(cur_local, &data); - testutil_check(cur_local->insert(cur_local)); - - /* - * Save the timestamp and key separately for checking later. - */ - if (fprintf(fp, - "%" PRIu64 " %" PRIu64 "\n", stable_ts, i) < 0) - testutil_die(EIO, "fprintf"); - } - /* NOTREACHED */ + FILE *fp; + WT_CURSOR *cur_coll, *cur_local, *cur_oplog; + WT_ITEM data; + WT_RAND_STATE rnd; + WT_SESSION *oplog_session, *session; + THREAD_DATA *td; + uint64_t i, stable_ts; + char cbuf[MAX_VAL], lbuf[MAX_VAL], obuf[MAX_VAL]; + char kname[64], tscfg[64]; + bool use_prep; + + __wt_random_init(&rnd); + memset(cbuf, 0, sizeof(cbuf)); + memset(lbuf, 0, sizeof(lbuf)); + memset(obuf, 0, sizeof(obuf)); + memset(kname, 0, sizeof(kname)); + + td = (THREAD_DATA *)arg; + /* + * Set up the separate file for checking. + */ + testutil_check(__wt_snprintf(cbuf, sizeof(cbuf), RECORDS_FILE, td->info)); + (void)unlink(cbuf); + testutil_checksys((fp = fopen(cbuf, "w")) == NULL); + /* + * Set to line buffering. But that is advisory only. We've seen cases where the result files end + * up with partial lines. + */ + __wt_stream_set_line_buffer(fp); + + /* + * Have half the threads use prepared transactions if timestamps are in use. + */ + use_prep = (use_ts && td->info % 2 == 0) ? true : false; + /* + * We may have two sessions so that the oplog session can have its own transaction in parallel + * with the collection session for threads that are going to be using prepared transactions. We + * need this because prepared transactions cannot have any operations that modify a table that + * is logged. But we also want to test mixed logged and not-logged transactions. + */ + testutil_check(td->conn->open_session(td->conn, NULL, "isolation=snapshot", &session)); + /* + * Open a cursor to each table. + */ + testutil_check(session->open_cursor(session, uri_collection, NULL, NULL, &cur_coll)); + testutil_check(session->open_cursor(session, uri_local, NULL, NULL, &cur_local)); + oplog_session = NULL; + if (use_prep) { + testutil_check( + td->conn->open_session(td->conn, NULL, "isolation=snapshot", &oplog_session)); + testutil_check(session->open_cursor(oplog_session, uri_oplog, NULL, NULL, &cur_oplog)); + } else + testutil_check(session->open_cursor(session, uri_oplog, NULL, NULL, &cur_oplog)); + + /* + * Write our portion of the key space until we're killed. + */ + printf("Thread %" PRIu32 " starts at %" PRIu64 "\n", td->info, td->start); + stable_ts = 0; + for (i = td->start;; ++i) { + /* + * Allow some threads to skip schema operations so that they are generating sufficient dirty + * data. + */ + WT_PUBLISH(th_ts[td->info].op, NOOP); + if (td->info != 0 && td->info != 1) + /* + * Do a schema operation about 50% of the time by having a case for only about half the + * possible mod values. + */ + switch (__wt_random(&rnd) % 20) { + case 0: + WT_PUBLISH(th_ts[td->info].op, BULK); + test_bulk(td); + break; + case 1: + WT_PUBLISH(th_ts[td->info].op, BULK_UNQ); + test_bulk_unique(td, __wt_random(&rnd) & 1); + break; + case 2: + WT_PUBLISH(th_ts[td->info].op, CREATE); + test_create(td); + break; + case 3: + WT_PUBLISH(th_ts[td->info].op, CREATE_UNQ); + test_create_unique(td, __wt_random(&rnd) & 1); + break; + case 4: + WT_PUBLISH(th_ts[td->info].op, CURSOR); + test_cursor(td); + break; + case 5: + WT_PUBLISH(th_ts[td->info].op, DROP); + test_drop(td, __wt_random(&rnd) & 1); + break; + case 6: + WT_PUBLISH(th_ts[td->info].op, REBALANCE); + test_rebalance(td); + break; + case 7: + WT_PUBLISH(th_ts[td->info].op, UPGRADE); + test_upgrade(td); + break; + case 8: + WT_PUBLISH(th_ts[td->info].op, VERIFY); + test_verify(td); + break; + } + if (use_ts) + stable_ts = __wt_atomic_addv64(&global_ts, 1); + testutil_check(__wt_snprintf(kname, sizeof(kname), "%" PRIu64, i)); + + testutil_check(session->begin_transaction(session, NULL)); + if (use_prep) + testutil_check(oplog_session->begin_transaction(oplog_session, NULL)); + cur_coll->set_key(cur_coll, kname); + cur_local->set_key(cur_local, kname); + cur_oplog->set_key(cur_oplog, kname); + /* + * Put an informative string into the value so that it can be viewed well in a binary dump. + */ + testutil_check(__wt_snprintf(cbuf, sizeof(cbuf), + "COLL: thread:%" PRIu32 " ts:%" PRIu64 " key: %" PRIu64, td->info, stable_ts, i)); + testutil_check(__wt_snprintf(lbuf, sizeof(lbuf), + "LOCAL: thread:%" PRIu32 " ts:%" PRIu64 " key: %" PRIu64, td->info, stable_ts, i)); + testutil_check(__wt_snprintf(obuf, sizeof(obuf), + "OPLOG: thread:%" PRIu32 " ts:%" PRIu64 " key: %" PRIu64, td->info, stable_ts, i)); + data.size = __wt_random(&rnd) % MAX_VAL; + data.data = cbuf; + cur_coll->set_value(cur_coll, &data); + testutil_check(cur_coll->insert(cur_coll)); + data.size = __wt_random(&rnd) % MAX_VAL; + data.data = obuf; + cur_oplog->set_value(cur_oplog, &data); + testutil_check(cur_oplog->insert(cur_oplog)); + if (use_ts) { + /* + * Run with prepare every once in a while. And also yield after prepare sometimes too. + * This is only done on the regular session. + */ + if (use_prep && i % PREPARE_FREQ == 0) { + testutil_check( + __wt_snprintf(tscfg, sizeof(tscfg), "prepare_timestamp=%" PRIx64, stable_ts)); + testutil_check(session->prepare_transaction(session, tscfg)); + if (i % PREPARE_YIELD == 0) + __wt_yield(); + + testutil_check(__wt_snprintf(tscfg, sizeof(tscfg), + "commit_timestamp=%" PRIx64 ",durable_timestamp=%" PRIx64, stable_ts, stable_ts)); + } else + testutil_check( + __wt_snprintf(tscfg, sizeof(tscfg), "commit_timestamp=%" PRIx64, stable_ts)); + + testutil_check(session->commit_transaction(session, tscfg)); + if (use_prep) { + /* + * Durable timestamp should not be passed as oplog transaction is a non-prepared + * transaction. + */ + testutil_check( + __wt_snprintf(tscfg, sizeof(tscfg), "commit_timestamp=%" PRIx64, stable_ts)); + testutil_check(oplog_session->commit_transaction(oplog_session, tscfg)); + } + /* + * Update the thread's last-committed timestamp. Don't let the compiler re-order this + * statement, if we were to race with the timestamp thread, it might see our thread + * update before the commit. + */ + WT_PUBLISH(th_ts[td->info].ts, stable_ts); + } else { + testutil_check(session->commit_transaction(session, NULL)); + if (use_prep) + testutil_check(oplog_session->commit_transaction(oplog_session, NULL)); + } + /* + * Insert into the local table outside the timestamp txn. + */ + data.size = __wt_random(&rnd) % MAX_VAL; + data.data = lbuf; + cur_local->set_value(cur_local, &data); + testutil_check(cur_local->insert(cur_local)); + + /* + * Save the timestamp and key separately for checking later. + */ + if (fprintf(fp, "%" PRIu64 " %" PRIu64 "\n", stable_ts, i) < 0) + testutil_die(EIO, "fprintf"); + } + /* NOTREACHED */ } /* - * Child process creates the database and table, and then creates worker - * threads to add data until it is killed by the parent. + * Child process creates the database and table, and then creates worker threads to add data until + * it is killed by the parent. */ -static void run_workload(uint32_t) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void run_workload(uint32_t) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void run_workload(uint32_t nth) { - WT_CONNECTION *conn; - WT_SESSION *session; - THREAD_DATA *td; - wt_thread_t *thr; - uint32_t ckpt_id, i, ts_id; - char envconf[512]; - - thr = dcalloc(nth+2, sizeof(*thr)); - td = dcalloc(nth+2, sizeof(THREAD_DATA)); - stable_set = false; - if (chdir(home) != 0) - testutil_die(errno, "Child chdir: %s", home); - if (inmem) - strcpy(envconf, ENV_CONFIG_DEF); - else - strcpy(envconf, ENV_CONFIG_TXNSYNC); - if (compat) - strcat(envconf, ENV_CONFIG_COMPAT); - - testutil_check(wiredtiger_open(NULL, &event_handler, envconf, &conn)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - /* - * Create all the tables. - */ - testutil_check(session->create(session, uri_collection, - "key_format=S,value_format=u,log=(enabled=false)")); - testutil_check(session->create(session, - uri_local, "key_format=S,value_format=u")); - testutil_check(session->create(session, - uri_oplog, "key_format=S,value_format=u")); - /* - * Don't log the stable timestamp table so that we know what timestamp - * was stored at the checkpoint. - */ - testutil_check(session->close(session, NULL)); - - /* - * The checkpoint thread and the timestamp threads are added at the end. - */ - ckpt_id = nth; - td[ckpt_id].conn = conn; - td[ckpt_id].info = nth; - printf("Create checkpoint thread\n"); - testutil_check(__wt_thread_create( - NULL, &thr[ckpt_id], thread_ckpt_run, &td[ckpt_id])); - ts_id = nth + 1; - if (use_ts) { - td[ts_id].conn = conn; - td[ts_id].info = nth; - printf("Create timestamp thread\n"); - testutil_check(__wt_thread_create( - NULL, &thr[ts_id], thread_ts_run, &td[ts_id])); - } - printf("Create %" PRIu32 " writer threads\n", nth); - for (i = 0; i < nth; ++i) { - td[i].conn = conn; - td[i].start = WT_BILLION * (uint64_t)i; - td[i].info = i; - testutil_check(__wt_thread_create( - NULL, &thr[i], thread_run, &td[i])); - } - /* - * The threads never exit, so the child will just wait here until - * it is killed. - */ - fflush(stdout); - for (i = 0; i <= ts_id; ++i) - testutil_check(__wt_thread_join(NULL, &thr[i])); - /* - * NOTREACHED - */ - free(thr); - free(td); - exit(EXIT_SUCCESS); + WT_CONNECTION *conn; + WT_SESSION *session; + THREAD_DATA *td; + wt_thread_t *thr; + uint32_t ckpt_id, i, ts_id; + char envconf[512]; + + thr = dcalloc(nth + 2, sizeof(*thr)); + td = dcalloc(nth + 2, sizeof(THREAD_DATA)); + stable_set = false; + if (chdir(home) != 0) + testutil_die(errno, "Child chdir: %s", home); + if (inmem) + strcpy(envconf, ENV_CONFIG_DEF); + else + strcpy(envconf, ENV_CONFIG_TXNSYNC); + if (compat) + strcat(envconf, ENV_CONFIG_COMPAT); + + testutil_check(wiredtiger_open(NULL, &event_handler, envconf, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + /* + * Create all the tables. + */ + testutil_check( + session->create(session, uri_collection, "key_format=S,value_format=u,log=(enabled=false)")); + testutil_check(session->create(session, uri_local, "key_format=S,value_format=u")); + testutil_check(session->create(session, uri_oplog, "key_format=S,value_format=u")); + /* + * Don't log the stable timestamp table so that we know what timestamp was stored at the + * checkpoint. + */ + testutil_check(session->close(session, NULL)); + + /* + * The checkpoint thread and the timestamp threads are added at the end. + */ + ckpt_id = nth; + td[ckpt_id].conn = conn; + td[ckpt_id].info = nth; + printf("Create checkpoint thread\n"); + testutil_check(__wt_thread_create(NULL, &thr[ckpt_id], thread_ckpt_run, &td[ckpt_id])); + ts_id = nth + 1; + if (use_ts) { + td[ts_id].conn = conn; + td[ts_id].info = nth; + printf("Create timestamp thread\n"); + testutil_check(__wt_thread_create(NULL, &thr[ts_id], thread_ts_run, &td[ts_id])); + } + printf("Create %" PRIu32 " writer threads\n", nth); + for (i = 0; i < nth; ++i) { + td[i].conn = conn; + td[i].start = WT_BILLION * (uint64_t)i; + td[i].info = i; + testutil_check(__wt_thread_create(NULL, &thr[i], thread_run, &td[i])); + } + /* + * The threads never exit, so the child will just wait here until it is killed. + */ + fflush(stdout); + for (i = 0; i <= ts_id; ++i) + testutil_check(__wt_thread_join(NULL, &thr[i])); + /* + * NOTREACHED + */ + free(thr); + free(td); + exit(EXIT_SUCCESS); } extern int __wt_optind; extern char *__wt_optarg; /* - * Initialize a report structure. Since zero is a valid key we - * cannot just clear it. + * Initialize a report structure. Since zero is a valid key we cannot just clear it. */ static void initialize_rep(REPORT *r) { - r->first_key = r->first_miss = INVALID_KEY; - r->absent_key = r->exist_key = r->last_key = INVALID_KEY; + r->first_key = r->first_miss = INVALID_KEY; + r->absent_key = r->exist_key = r->last_key = INVALID_KEY; } /* - * Print out information if we detect missing records in the - * middle of the data of a report structure. + * Print out information if we detect missing records in the middle of the data of a report + * structure. */ static void print_missing(REPORT *r, const char *fname, const char *msg) { - if (r->exist_key != INVALID_KEY) - printf("%s: %s error %" PRIu64 - " absent records %" PRIu64 "-%" PRIu64 - ". Then keys %" PRIu64 "-%" PRIu64 " exist." - " Key range %" PRIu64 "-%" PRIu64 "\n", - fname, msg, - (r->exist_key - r->first_miss) - 1, - r->first_miss, r->exist_key - 1, - r->exist_key, r->last_key, - r->first_key, r->last_key); + if (r->exist_key != INVALID_KEY) + printf("%s: %s error %" PRIu64 " absent records %" PRIu64 "-%" PRIu64 ". Then keys %" PRIu64 + "-%" PRIu64 + " exist." + " Key range %" PRIu64 "-%" PRIu64 "\n", + fname, msg, (r->exist_key - r->first_miss) - 1, r->first_miss, r->exist_key - 1, + r->exist_key, r->last_key, r->first_key, r->last_key); } /* @@ -968,385 +882,345 @@ print_missing(REPORT *r, const char *fname, const char *msg) static void sig_handler(int sig) { - pid_t pid; - - WT_UNUSED(sig); - pid = wait(NULL); - /* - * The core file will indicate why the child exited. Choose EINVAL here. - */ - testutil_die(EINVAL, - "Child process %" PRIu64 " abnormally exited", (uint64_t)pid); + pid_t pid; + + WT_UNUSED(sig); + pid = wait(NULL); + /* + * The core file will indicate why the child exited. Choose EINVAL here. + */ + testutil_die(EINVAL, "Child process %" PRIu64 " abnormally exited", (uint64_t)pid); } int main(int argc, char *argv[]) { - struct sigaction sa; - struct stat sb; - FILE *fp; - REPORT c_rep[MAX_TH], l_rep[MAX_TH], o_rep[MAX_TH]; - WT_CONNECTION *conn; - WT_CURSOR *cur_coll, *cur_local, *cur_oplog; - WT_DECL_RET; - WT_RAND_STATE rnd; - WT_SESSION *session; - pid_t pid; - uint64_t absent_coll, absent_local, absent_oplog, count, key, last_key; - uint64_t stable_fp, stable_val; - uint32_t i, nth, timeout; - int ch, status; - char buf[512], statname[1024]; - char fname[64], kname[64]; - const char *working_dir; - bool fatal, rand_th, rand_time, verify_only; - - (void)testutil_set_progname(argv); - - compat = inmem = false; - use_ts = true; - /* - * Setting this to false forces us to use internal library code. - * Allow an override but default to using that code. - */ - use_txn = false; - nth = MIN_TH; - rand_th = rand_time = true; - timeout = MIN_TIME; - verify_only = false; - working_dir = "WT_TEST.schema-abort"; - - while ((ch = __wt_getopt(progname, argc, argv, "Ch:mT:t:vxz")) != EOF) - switch (ch) { - case 'C': - compat = true; - break; - case 'h': - working_dir = __wt_optarg; - break; - case 'm': - inmem = true; - break; - case 'T': - rand_th = false; - nth = (uint32_t)atoi(__wt_optarg); - break; - case 't': - rand_time = false; - timeout = (uint32_t)atoi(__wt_optarg); - break; - case 'v': - verify_only = true; - break; - case 'x': - use_txn = true; - break; - case 'z': - use_ts = false; - break; - default: - usage(); - } - argc -= __wt_optind; - if (argc != 0) - usage(); - - testutil_work_dir_from_path(home, sizeof(home), working_dir); - /* - * If the user wants to verify they need to tell us how many threads - * there were so we can find the old record files. - */ - if (verify_only && rand_th) { - fprintf(stderr, - "Verify option requires specifying number of threads\n"); - exit (EXIT_FAILURE); - } - if (!verify_only) { - testutil_make_work_dir(home); - - __wt_random_init_seed(NULL, &rnd); - if (rand_time) { - timeout = __wt_random(&rnd) % MAX_TIME; - if (timeout < MIN_TIME) - timeout = MIN_TIME; - } - if (rand_th) { - nth = __wt_random(&rnd) % MAX_TH; - if (nth < MIN_TH) - nth = MIN_TH; - } - - printf("Parent: compatibility: %s, " - "in-mem log sync: %s, timestamp in use: %s\n", - compat ? "true" : "false", - inmem ? "true" : "false", - use_ts ? "true" : "false"); - printf("Parent: Create %" PRIu32 - " threads; sleep %" PRIu32 " seconds\n", nth, timeout); - printf("CONFIG: %s%s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n", - progname, - compat ? " -C" : "", - inmem ? " -m" : "", - !use_ts ? " -z" : "", - working_dir, nth, timeout); - /* - * Fork a child to insert as many items. We will then randomly - * kill the child, run recovery and make sure all items we wrote - * exist after recovery runs. - */ - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = sig_handler; - testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); - testutil_checksys((pid = fork()) < 0); - - if (pid == 0) { /* child */ - run_workload(nth); - return (EXIT_SUCCESS); - } - - /* parent */ - /* - * Sleep for the configured amount of time before killing - * the child. Start the timeout from the time we notice that - * the file has been created. That allows the test to run - * correctly on really slow machines. - */ - testutil_check(__wt_snprintf( - statname, sizeof(statname), "%s/%s", home, ckpt_file)); - while (stat(statname, &sb) != 0) - testutil_sleep_wait(1, pid); - sleep(timeout); - sa.sa_handler = SIG_DFL; - testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); - - /* - * !!! It should be plenty long enough to make sure more than - * one log file exists. If wanted, that check would be added - * here. - */ - printf("Kill child\n"); - testutil_checksys(kill(pid, SIGKILL) != 0); - testutil_checksys(waitpid(pid, &status, 0) == -1); - } - /* - * !!! If we wanted to take a copy of the directory before recovery, - * this is the place to do it. Don't do it all the time because - * it can use a lot of disk space, which can cause test machine - * issues. - */ - if (chdir(home) != 0) - testutil_die(errno, "parent chdir: %s", home); - /* - * The tables can get very large, so while we'd ideally like to - * copy the entire database, we only copy the log files for now. - * Otherwise it can take far too long to run the test, particularly - * in automated testing. - */ - testutil_check(__wt_snprintf(buf, sizeof(buf), - "rm -rf ../%s.SAVE && mkdir ../%s.SAVE && " - "cp -p * ../%s.SAVE", - home, home, home)); - if ((status = system(buf)) < 0) - testutil_die(status, "system: %s", buf); - printf("Open database, run recovery and verify content\n"); - - /* - * Open the connection which forces recovery to be run. - */ - testutil_check(wiredtiger_open( - NULL, &event_handler, ENV_CONFIG_REC, &conn)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - /* - * Open a cursor on all the tables. - */ - testutil_check(session->open_cursor(session, - uri_collection, NULL, NULL, &cur_coll)); - testutil_check(session->open_cursor(session, - uri_local, NULL, NULL, &cur_local)); - testutil_check(session->open_cursor(session, - uri_oplog, NULL, NULL, &cur_oplog)); - - /* - * Find the biggest stable timestamp value that was saved. - */ - stable_val = 0; - if (use_ts) { - testutil_check( - conn->query_timestamp(conn, buf, "get=recovery")); - sscanf(buf, "%" SCNx64, &stable_val); - printf("Got stable_val %" PRIu64 "\n", stable_val); - } - - count = 0; - absent_coll = absent_local = absent_oplog = 0; - fatal = false; - for (i = 0; i < nth; ++i) { - initialize_rep(&c_rep[i]); - initialize_rep(&l_rep[i]); - initialize_rep(&o_rep[i]); - testutil_check(__wt_snprintf( - fname, sizeof(fname), RECORDS_FILE, i)); - if ((fp = fopen(fname, "r")) == NULL) - testutil_die(errno, "fopen: %s", fname); - - /* - * For every key in the saved file, verify that the key exists - * in the table after recovery. If we're doing in-memory - * log buffering we never expect a record missing in the middle, - * but records may be missing at the end. If we did - * write-no-sync, we expect every key to have been recovered. - */ - for (last_key = INVALID_KEY;; ++count, last_key = key) { - ret = fscanf(fp, "%" SCNu64 "%" SCNu64 "\n", - &stable_fp, &key); - if (last_key == INVALID_KEY) { - c_rep[i].first_key = key; - l_rep[i].first_key = key; - o_rep[i].first_key = key; - } - if (ret != EOF && ret != 2) { - /* - * If we find a partial line, consider it - * like an EOF. - */ - if (ret == 1 || ret == 0) - break; - testutil_die(errno, "fscanf"); - } - if (ret == EOF) - break; - /* - * If we're unlucky, the last line may be a partially - * written key at the end that can result in a false - * negative error for a missing record. Detect it. - */ - if (last_key != INVALID_KEY && key != last_key + 1) { - printf("%s: Ignore partial record %" PRIu64 - " last valid key %" PRIu64 "\n", - fname, key, last_key); - break; - } - testutil_check(__wt_snprintf( - kname, sizeof(kname), "%" PRIu64, key)); - cur_coll->set_key(cur_coll, kname); - cur_local->set_key(cur_local, kname); - cur_oplog->set_key(cur_oplog, kname); - /* - * The collection table should always only have the - * data as of the checkpoint. - */ - if ((ret = cur_coll->search(cur_coll)) != 0) { - if (ret != WT_NOTFOUND) - testutil_die(ret, "search"); - /* - * If we don't find a record, the stable - * timestamp written to our file better be - * larger than the saved one. - */ - if (!inmem && - stable_fp != 0 && stable_fp <= stable_val) { - printf("%s: COLLECTION no record with " - "key %" PRIu64 " record ts %" PRIu64 - " <= stable ts %" PRIu64 "\n", - fname, key, stable_fp, stable_val); - absent_coll++; - } - if (c_rep[i].first_miss == INVALID_KEY) - c_rep[i].first_miss = key; - c_rep[i].absent_key = key; - } else if (c_rep[i].absent_key != INVALID_KEY && - c_rep[i].exist_key == INVALID_KEY) { - /* - * If we get here we found a record that exists - * after absent records, a hole in our data. - */ - c_rep[i].exist_key = key; - fatal = true; - } else if (!inmem && - stable_fp != 0 && stable_fp > stable_val) { - /* - * If we found a record, the stable timestamp - * written to our file better be no larger - * than the checkpoint one. - */ - printf("%s: COLLECTION record with " - "key %" PRIu64 " record ts %" PRIu64 - " > stable ts %" PRIu64 "\n", - fname, key, stable_fp, stable_val); - fatal = true; - } - /* - * The local table should always have all data. - */ - if ((ret = cur_local->search(cur_local)) != 0) { - if (ret != WT_NOTFOUND) - testutil_die(ret, "search"); - if (!inmem) - printf("%s: LOCAL no record with key %" - PRIu64 "\n", fname, key); - absent_local++; - if (l_rep[i].first_miss == INVALID_KEY) - l_rep[i].first_miss = key; - l_rep[i].absent_key = key; - } else if (l_rep[i].absent_key != INVALID_KEY && - l_rep[i].exist_key == INVALID_KEY) { - /* - * We should never find an existing key after - * we have detected one missing. - */ - l_rep[i].exist_key = key; - fatal = true; - } - /* - * The oplog table should always have all data. - */ - if ((ret = cur_oplog->search(cur_oplog)) != 0) { - if (ret != WT_NOTFOUND) - testutil_die(ret, "search"); - if (!inmem) - printf("%s: OPLOG no record with key %" - PRIu64 "\n", fname, key); - absent_oplog++; - if (o_rep[i].first_miss == INVALID_KEY) - o_rep[i].first_miss = key; - o_rep[i].absent_key = key; - } else if (o_rep[i].absent_key != INVALID_KEY && - o_rep[i].exist_key == INVALID_KEY) { - /* - * We should never find an existing key after - * we have detected one missing. - */ - o_rep[i].exist_key = key; - fatal = true; - } - } - c_rep[i].last_key = last_key; - l_rep[i].last_key = last_key; - o_rep[i].last_key = last_key; - testutil_checksys(fclose(fp) != 0); - print_missing(&c_rep[i], fname, "COLLECTION"); - print_missing(&l_rep[i], fname, "LOCAL"); - print_missing(&o_rep[i], fname, "OPLOG"); - } - testutil_check(conn->close(conn, NULL)); - if (!inmem && absent_coll) { - printf("COLLECTION: %" PRIu64 - " record(s) absent from %" PRIu64 "\n", - absent_coll, count); - fatal = true; - } - if (!inmem && absent_local) { - printf("LOCAL: %" PRIu64 " record(s) absent from %" PRIu64 "\n", - absent_local, count); - fatal = true; - } - if (!inmem && absent_oplog) { - printf("OPLOG: %" PRIu64 " record(s) absent from %" PRIu64 "\n", - absent_oplog, count); - fatal = true; - } - if (fatal) - return (EXIT_FAILURE); - printf("%" PRIu64 " records verified\n", count); - return (EXIT_SUCCESS); + struct sigaction sa; + struct stat sb; + FILE *fp; + REPORT c_rep[MAX_TH], l_rep[MAX_TH], o_rep[MAX_TH]; + WT_CONNECTION *conn; + WT_CURSOR *cur_coll, *cur_local, *cur_oplog; + WT_DECL_RET; + WT_RAND_STATE rnd; + WT_SESSION *session; + pid_t pid; + uint64_t absent_coll, absent_local, absent_oplog, count, key, last_key; + uint64_t stable_fp, stable_val; + uint32_t i, nth, timeout; + int ch, status; + char buf[512], statname[1024]; + char fname[64], kname[64]; + const char *working_dir; + bool fatal, rand_th, rand_time, verify_only; + + (void)testutil_set_progname(argv); + + compat = inmem = false; + use_ts = true; + /* + * Setting this to false forces us to use internal library code. Allow an override but default + * to using that code. + */ + use_txn = false; + nth = MIN_TH; + rand_th = rand_time = true; + timeout = MIN_TIME; + verify_only = false; + working_dir = "WT_TEST.schema-abort"; + + while ((ch = __wt_getopt(progname, argc, argv, "Ch:mT:t:vxz")) != EOF) + switch (ch) { + case 'C': + compat = true; + break; + case 'h': + working_dir = __wt_optarg; + break; + case 'm': + inmem = true; + break; + case 'T': + rand_th = false; + nth = (uint32_t)atoi(__wt_optarg); + break; + case 't': + rand_time = false; + timeout = (uint32_t)atoi(__wt_optarg); + break; + case 'v': + verify_only = true; + break; + case 'x': + use_txn = true; + break; + case 'z': + use_ts = false; + break; + default: + usage(); + } + argc -= __wt_optind; + if (argc != 0) + usage(); + + testutil_work_dir_from_path(home, sizeof(home), working_dir); + /* + * If the user wants to verify they need to tell us how many threads there were so we can find + * the old record files. + */ + if (verify_only && rand_th) { + fprintf(stderr, "Verify option requires specifying number of threads\n"); + exit(EXIT_FAILURE); + } + if (!verify_only) { + testutil_make_work_dir(home); + + __wt_random_init_seed(NULL, &rnd); + if (rand_time) { + timeout = __wt_random(&rnd) % MAX_TIME; + if (timeout < MIN_TIME) + timeout = MIN_TIME; + } + if (rand_th) { + nth = __wt_random(&rnd) % MAX_TH; + if (nth < MIN_TH) + nth = MIN_TH; + } + + printf( + "Parent: compatibility: %s, " + "in-mem log sync: %s, timestamp in use: %s\n", + compat ? "true" : "false", inmem ? "true" : "false", use_ts ? "true" : "false"); + printf("Parent: Create %" PRIu32 " threads; sleep %" PRIu32 " seconds\n", nth, timeout); + printf("CONFIG: %s%s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n", progname, + compat ? " -C" : "", inmem ? " -m" : "", !use_ts ? " -z" : "", working_dir, nth, timeout); + /* + * Fork a child to insert as many items. We will then randomly kill the child, run recovery + * and make sure all items we wrote exist after recovery runs. + */ + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = sig_handler; + testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); + testutil_checksys((pid = fork()) < 0); + + if (pid == 0) { /* child */ + run_workload(nth); + return (EXIT_SUCCESS); + } + + /* parent */ + /* + * Sleep for the configured amount of time before killing the child. Start the timeout from + * the time we notice that the file has been created. That allows the test to run correctly + * on really slow machines. + */ + testutil_check(__wt_snprintf(statname, sizeof(statname), "%s/%s", home, ckpt_file)); + while (stat(statname, &sb) != 0) + testutil_sleep_wait(1, pid); + sleep(timeout); + sa.sa_handler = SIG_DFL; + testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); + + /* + * !!! It should be plenty long enough to make sure more than + * one log file exists. If wanted, that check would be added + * here. + */ + printf("Kill child\n"); + testutil_checksys(kill(pid, SIGKILL) != 0); + testutil_checksys(waitpid(pid, &status, 0) == -1); + } + /* + * !!! If we wanted to take a copy of the directory before recovery, + * this is the place to do it. Don't do it all the time because + * it can use a lot of disk space, which can cause test machine + * issues. + */ + if (chdir(home) != 0) + testutil_die(errno, "parent chdir: %s", home); + /* + * The tables can get very large, so while we'd ideally like to copy the entire database, we + * only copy the log files for now. Otherwise it can take far too long to run the test, + * particularly in automated testing. + */ + testutil_check(__wt_snprintf(buf, sizeof(buf), + "rm -rf ../%s.SAVE && mkdir ../%s.SAVE && " + "cp -p * ../%s.SAVE", + home, home, home)); + if ((status = system(buf)) < 0) + testutil_die(status, "system: %s", buf); + printf("Open database, run recovery and verify content\n"); + + /* + * Open the connection which forces recovery to be run. + */ + testutil_check(wiredtiger_open(NULL, &event_handler, ENV_CONFIG_REC, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + /* + * Open a cursor on all the tables. + */ + testutil_check(session->open_cursor(session, uri_collection, NULL, NULL, &cur_coll)); + testutil_check(session->open_cursor(session, uri_local, NULL, NULL, &cur_local)); + testutil_check(session->open_cursor(session, uri_oplog, NULL, NULL, &cur_oplog)); + + /* + * Find the biggest stable timestamp value that was saved. + */ + stable_val = 0; + if (use_ts) { + testutil_check(conn->query_timestamp(conn, buf, "get=recovery")); + sscanf(buf, "%" SCNx64, &stable_val); + printf("Got stable_val %" PRIu64 "\n", stable_val); + } + + count = 0; + absent_coll = absent_local = absent_oplog = 0; + fatal = false; + for (i = 0; i < nth; ++i) { + initialize_rep(&c_rep[i]); + initialize_rep(&l_rep[i]); + initialize_rep(&o_rep[i]); + testutil_check(__wt_snprintf(fname, sizeof(fname), RECORDS_FILE, i)); + if ((fp = fopen(fname, "r")) == NULL) + testutil_die(errno, "fopen: %s", fname); + + /* + * For every key in the saved file, verify that the key exists in the table after recovery. + * If we're doing in-memory log buffering we never expect a record missing in the middle, + * but records may be missing at the end. If we did write-no-sync, we expect every key to + * have been recovered. + */ + for (last_key = INVALID_KEY;; ++count, last_key = key) { + ret = fscanf(fp, "%" SCNu64 "%" SCNu64 "\n", &stable_fp, &key); + if (last_key == INVALID_KEY) { + c_rep[i].first_key = key; + l_rep[i].first_key = key; + o_rep[i].first_key = key; + } + if (ret != EOF && ret != 2) { + /* + * If we find a partial line, consider it like an EOF. + */ + if (ret == 1 || ret == 0) + break; + testutil_die(errno, "fscanf"); + } + if (ret == EOF) + break; + /* + * If we're unlucky, the last line may be a partially written key at the end that can + * result in a false negative error for a missing record. Detect it. + */ + if (last_key != INVALID_KEY && key != last_key + 1) { + printf("%s: Ignore partial record %" PRIu64 " last valid key %" PRIu64 "\n", fname, + key, last_key); + break; + } + testutil_check(__wt_snprintf(kname, sizeof(kname), "%" PRIu64, key)); + cur_coll->set_key(cur_coll, kname); + cur_local->set_key(cur_local, kname); + cur_oplog->set_key(cur_oplog, kname); + /* + * The collection table should always only have the data as of the checkpoint. + */ + if ((ret = cur_coll->search(cur_coll)) != 0) { + if (ret != WT_NOTFOUND) + testutil_die(ret, "search"); + /* + * If we don't find a record, the stable timestamp written to our file better be + * larger than the saved one. + */ + if (!inmem && stable_fp != 0 && stable_fp <= stable_val) { + printf( + "%s: COLLECTION no record with " + "key %" PRIu64 " record ts %" PRIu64 " <= stable ts %" PRIu64 "\n", + fname, key, stable_fp, stable_val); + absent_coll++; + } + if (c_rep[i].first_miss == INVALID_KEY) + c_rep[i].first_miss = key; + c_rep[i].absent_key = key; + } else if (c_rep[i].absent_key != INVALID_KEY && c_rep[i].exist_key == INVALID_KEY) { + /* + * If we get here we found a record that exists after absent records, a hole in our + * data. + */ + c_rep[i].exist_key = key; + fatal = true; + } else if (!inmem && stable_fp != 0 && stable_fp > stable_val) { + /* + * If we found a record, the stable timestamp written to our file better be no + * larger than the checkpoint one. + */ + printf( + "%s: COLLECTION record with " + "key %" PRIu64 " record ts %" PRIu64 " > stable ts %" PRIu64 "\n", + fname, key, stable_fp, stable_val); + fatal = true; + } + /* + * The local table should always have all data. + */ + if ((ret = cur_local->search(cur_local)) != 0) { + if (ret != WT_NOTFOUND) + testutil_die(ret, "search"); + if (!inmem) + printf("%s: LOCAL no record with key %" PRIu64 "\n", fname, key); + absent_local++; + if (l_rep[i].first_miss == INVALID_KEY) + l_rep[i].first_miss = key; + l_rep[i].absent_key = key; + } else if (l_rep[i].absent_key != INVALID_KEY && l_rep[i].exist_key == INVALID_KEY) { + /* + * We should never find an existing key after we have detected one missing. + */ + l_rep[i].exist_key = key; + fatal = true; + } + /* + * The oplog table should always have all data. + */ + if ((ret = cur_oplog->search(cur_oplog)) != 0) { + if (ret != WT_NOTFOUND) + testutil_die(ret, "search"); + if (!inmem) + printf("%s: OPLOG no record with key %" PRIu64 "\n", fname, key); + absent_oplog++; + if (o_rep[i].first_miss == INVALID_KEY) + o_rep[i].first_miss = key; + o_rep[i].absent_key = key; + } else if (o_rep[i].absent_key != INVALID_KEY && o_rep[i].exist_key == INVALID_KEY) { + /* + * We should never find an existing key after we have detected one missing. + */ + o_rep[i].exist_key = key; + fatal = true; + } + } + c_rep[i].last_key = last_key; + l_rep[i].last_key = last_key; + o_rep[i].last_key = last_key; + testutil_checksys(fclose(fp) != 0); + print_missing(&c_rep[i], fname, "COLLECTION"); + print_missing(&l_rep[i], fname, "LOCAL"); + print_missing(&o_rep[i], fname, "OPLOG"); + } + testutil_check(conn->close(conn, NULL)); + if (!inmem && absent_coll) { + printf("COLLECTION: %" PRIu64 " record(s) absent from %" PRIu64 "\n", absent_coll, count); + fatal = true; + } + if (!inmem && absent_local) { + printf("LOCAL: %" PRIu64 " record(s) absent from %" PRIu64 "\n", absent_local, count); + fatal = true; + } + if (!inmem && absent_oplog) { + printf("OPLOG: %" PRIu64 " record(s) absent from %" PRIu64 "\n", absent_oplog, count); + fatal = true; + } + if (fatal) + return (EXIT_FAILURE); + printf("%" PRIu64 " records verified\n", count); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/scope/main.c b/src/third_party/wiredtiger/test/csuite/scope/main.c index 3a98fbc8fde..dc7b312e5c8 100644 --- a/src/third_party/wiredtiger/test/csuite/scope/main.c +++ b/src/third_party/wiredtiger/test/csuite/scope/main.c @@ -27,326 +27,308 @@ */ #include "test_util.h" -#define KEY "key" -#define VALUE "value,value,value" +#define KEY "key" +#define VALUE "value,value,value" static int ignore_errors; static int -handle_error(WT_EVENT_HANDLER *handler, - WT_SESSION *session, int error, const char *message) +handle_error(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const char *message) { - (void)(handler); + (void)(handler); - /* Skip the error messages we're expecting to see. */ - if (ignore_errors > 0 && - (strstr(message, "requires key be set") != NULL || - strstr(message, "requires value be set") != NULL)) { - --ignore_errors; - return (0); - } + /* Skip the error messages we're expecting to see. */ + if (ignore_errors > 0 && (strstr(message, "requires key be set") != NULL || + strstr(message, "requires value be set") != NULL)) { + --ignore_errors; + return (0); + } - (void)fprintf(stderr, "%s: %s\n", - message, session->strerror(session, error)); - return (0); + (void)fprintf(stderr, "%s: %s\n", message, session->strerror(session, error)); + return (0); } -static WT_EVENT_HANDLER event_handler = { - handle_error, - NULL, - NULL, - NULL -}; +static WT_EVENT_HANDLER event_handler = {handle_error, NULL, NULL, NULL}; static void cursor_scope_ops(WT_SESSION *session, const char *uri) { - struct { - const char *op; - enum { INSERT, MODIFY, SEARCH, SEARCH_NEAR, - REMOVE, REMOVE_POS, RESERVE, UPDATE } func; - const char *config; - } *op, ops[] = { - /* - * The ops order is specific: insert has to happen first so - * other operations are possible, and remove has to be last. - */ - { "insert", INSERT, NULL, }, - { "search", SEARCH, NULL, }, - { "search", SEARCH_NEAR, NULL, }, - { "reserve", RESERVE, NULL, }, - { "insert", MODIFY, NULL, }, - { "update", UPDATE, NULL, }, - { "remove", REMOVE, NULL, }, - { "remove", REMOVE_POS, NULL, }, - { NULL, INSERT, NULL } - }; - WT_CURSOR *cursor; -#define MODIFY_ENTRIES 2 - WT_MODIFY entries[MODIFY_ENTRIES]; - WT_ITEM vu; - uint64_t keyr; - const char *key, *vs; - char keybuf[100], valuebuf[100]; - int exact; - bool recno, vstring; + struct { + const char *op; + enum { INSERT, MODIFY, SEARCH, SEARCH_NEAR, REMOVE, REMOVE_POS, RESERVE, UPDATE } func; + const char *config; + } * op, ops[] = {/* + * The ops order is specific: insert has to happen first so + * other operations are possible, and remove has to be last. + */ + { + "insert", INSERT, NULL, + }, + { + "search", SEARCH, NULL, + }, + { + "search", SEARCH_NEAR, NULL, + }, + { + "reserve", RESERVE, NULL, + }, + { + "insert", MODIFY, NULL, + }, + { + "update", UPDATE, NULL, + }, + { + "remove", REMOVE, NULL, + }, + { + "remove", REMOVE_POS, NULL, + }, + {NULL, INSERT, NULL}}; + WT_CURSOR *cursor; +#define MODIFY_ENTRIES 2 + WT_MODIFY entries[MODIFY_ENTRIES]; + WT_ITEM vu; + uint64_t keyr; + const char *key, *vs; + char keybuf[100], valuebuf[100]; + int exact; + bool recno, vstring; - /* - * Modify and reserve require a transaction, modify requires snapshot - * isolation. - */ - testutil_check( - session->begin_transaction(session, "isolation=snapshot")); + /* + * Modify and reserve require a transaction, modify requires snapshot isolation. + */ + testutil_check(session->begin_transaction(session, "isolation=snapshot")); - cursor = NULL; - for (op = ops; op->op != NULL; op++) { - key = vs = NULL; - memset(&vu, 0, sizeof(vu)); + cursor = NULL; + for (op = ops; op->op != NULL; op++) { + key = vs = NULL; + memset(&vu, 0, sizeof(vu)); - /* Open a cursor. */ - if (cursor != NULL) - testutil_check(cursor->close(cursor)); - testutil_check(session->open_cursor( - session, uri, NULL, op->config, &cursor)); + /* Open a cursor. */ + if (cursor != NULL) + testutil_check(cursor->close(cursor)); + testutil_check(session->open_cursor(session, uri, NULL, op->config, &cursor)); - /* Operations change based on the key/value formats. */ - recno = strcmp(cursor->key_format, "r") == 0; - vstring = strcmp(cursor->value_format, "S") == 0; + /* Operations change based on the key/value formats. */ + recno = strcmp(cursor->key_format, "r") == 0; + vstring = strcmp(cursor->value_format, "S") == 0; - /* Modify is only possible with "item" values. */ - if (vstring && op->func == MODIFY) - continue; + /* Modify is only possible with "item" values. */ + if (vstring && op->func == MODIFY) + continue; - /* - * Set up application buffers so we can detect overwrites - * or failure to copy application information into library - * memory. - */ - if (recno) - cursor->set_key(cursor, (uint64_t)1); - else { - strcpy(keybuf, KEY); - cursor->set_key(cursor, keybuf); - } - strcpy(valuebuf, VALUE); - if (vstring) - cursor->set_value(cursor, valuebuf); - else { - vu.size = strlen(vu.data = valuebuf); - cursor->set_value(cursor, &vu); - } + /* + * Set up application buffers so we can detect overwrites or failure to copy application + * information into library memory. + */ + if (recno) + cursor->set_key(cursor, (uint64_t)1); + else { + strcpy(keybuf, KEY); + cursor->set_key(cursor, keybuf); + } + strcpy(valuebuf, VALUE); + if (vstring) + cursor->set_value(cursor, valuebuf); + else { + vu.size = strlen(vu.data = valuebuf); + cursor->set_value(cursor, &vu); + } - /* - * The application must keep key and value memory valid until - * the next operation that positions the cursor, modifies the - * data, or resets or closes the cursor. - * - * Modifying either the key or value buffers is not permitted. - */ - switch (op->func) { - case INSERT: - testutil_check(cursor->insert(cursor)); - break; - case MODIFY: - /* Modify, but don't really change anything. */ - entries[0].data.data = &VALUE[0]; - entries[0].data.size = 2; - entries[0].offset = 0; - entries[0].size = 2; - entries[1].data.data = &VALUE[3]; - entries[1].data.size = 5; - entries[1].offset = 3; - entries[1].size = 5; + /* + * The application must keep key and value memory valid until + * the next operation that positions the cursor, modifies the + * data, or resets or closes the cursor. + * + * Modifying either the key or value buffers is not permitted. + */ + switch (op->func) { + case INSERT: + testutil_check(cursor->insert(cursor)); + break; + case MODIFY: + /* Modify, but don't really change anything. */ + entries[0].data.data = &VALUE[0]; + entries[0].data.size = 2; + entries[0].offset = 0; + entries[0].size = 2; + entries[1].data.data = &VALUE[3]; + entries[1].data.size = 5; + entries[1].offset = 3; + entries[1].size = 5; - testutil_check( - cursor->modify(cursor, entries, MODIFY_ENTRIES)); - break; - case SEARCH: - testutil_check(cursor->search(cursor)); - break; - case SEARCH_NEAR: - testutil_check(cursor->search_near(cursor, &exact)); - break; - case REMOVE_POS: - /* - * Remove has two modes, one where the remove is based - * on a cursor position, the other where it's based on - * a set key. The results are different, so test them - * separately. - */ - testutil_check(cursor->search(cursor)); - /* FALLTHROUGH */ - case REMOVE: - testutil_check(cursor->remove(cursor)); - break; - case RESERVE: - testutil_check(cursor->reserve(cursor)); - break; - case UPDATE: - testutil_check(cursor->update(cursor)); - break; - } + testutil_check(cursor->modify(cursor, entries, MODIFY_ENTRIES)); + break; + case SEARCH: + testutil_check(cursor->search(cursor)); + break; + case SEARCH_NEAR: + testutil_check(cursor->search_near(cursor, &exact)); + break; + case REMOVE_POS: + /* + * Remove has two modes, one where the remove is based on a cursor position, the other + * where it's based on a set key. The results are different, so test them separately. + */ + testutil_check(cursor->search(cursor)); + /* FALLTHROUGH */ + case REMOVE: + testutil_check(cursor->remove(cursor)); + break; + case RESERVE: + testutil_check(cursor->reserve(cursor)); + break; + case UPDATE: + testutil_check(cursor->update(cursor)); + break; + } - /* - * The cursor should no longer reference application memory, - * and application buffers can be safely overwritten. - */ - memset(keybuf, 'K', sizeof(keybuf)); - memset(valuebuf, 'V', sizeof(valuebuf)); + /* + * The cursor should no longer reference application memory, and application buffers can be + * safely overwritten. + */ + memset(keybuf, 'K', sizeof(keybuf)); + memset(valuebuf, 'V', sizeof(valuebuf)); - /* - * Check that get_key/get_value behave as expected after the - * operation. - */ - switch (op->func) { - case INSERT: - case REMOVE: - /* - * Insert and remove configured with a search key do - * not position the cursor and have no key or value. - * - * There should be two error messages, ignore them. - */ - ignore_errors = 2; - if (recno) - testutil_assert( - cursor->get_key(cursor, &keyr) != 0); - else - testutil_assert( - cursor->get_key(cursor, &key) != 0); - if (vstring) - testutil_assert( - cursor->get_value(cursor, &vs) != 0); - else - testutil_assert( - cursor->get_value(cursor, &vu) != 0); - testutil_assert(ignore_errors == 0); - break; - case REMOVE_POS: - /* - * Remove configured with a cursor position has a key, - * but no value. - * - * There should be one error message, ignore it. - */ - if (recno) { - testutil_assert( - cursor->get_key(cursor, &keyr) == 0); - testutil_assert(keyr == 1); - } else { - testutil_assert( - cursor->get_key(cursor, &key) == 0); - testutil_assert(key != keybuf); - testutil_assert(strcmp(key, KEY) == 0); - } - ignore_errors = 1; - if (vstring) - testutil_assert( - cursor->get_value(cursor, &vs) != 0); - else - testutil_assert( - cursor->get_value(cursor, &vu) != 0); - testutil_assert(ignore_errors == 0); - break; - case MODIFY: - case RESERVE: - case SEARCH: - case SEARCH_NEAR: - case UPDATE: - /* - * Modify, reserve, search, search-near and update all - * position the cursor and have both a key and value. - * - * Any key/value should not reference application - * memory. - */ - if (recno) { - testutil_assert( - cursor->get_key(cursor, &keyr) == 0); - testutil_assert(keyr == 1); - } else { - testutil_assert( - cursor->get_key(cursor, &key) == 0); - testutil_assert(key != keybuf); - testutil_assert(strcmp(key, KEY) == 0); - } - if (vstring) { - testutil_assert( - cursor->get_value(cursor, &vs) == 0); - testutil_assert(vs != valuebuf); - testutil_assert(strcmp(vs, VALUE) == 0); - } else { - testutil_assert( - cursor->get_value(cursor, &vu) == 0); - testutil_assert(vu.data != valuebuf); - testutil_assert(vu.size == strlen(VALUE)); - testutil_assert( - memcmp(vu.data, VALUE, strlen(VALUE)) == 0); - } - break; - } + /* + * Check that get_key/get_value behave as expected after the operation. + */ + switch (op->func) { + case INSERT: + case REMOVE: + /* + * Insert and remove configured with a search key do + * not position the cursor and have no key or value. + * + * There should be two error messages, ignore them. + */ + ignore_errors = 2; + if (recno) + testutil_assert(cursor->get_key(cursor, &keyr) != 0); + else + testutil_assert(cursor->get_key(cursor, &key) != 0); + if (vstring) + testutil_assert(cursor->get_value(cursor, &vs) != 0); + else + testutil_assert(cursor->get_value(cursor, &vu) != 0); + testutil_assert(ignore_errors == 0); + break; + case REMOVE_POS: + /* + * Remove configured with a cursor position has a key, + * but no value. + * + * There should be one error message, ignore it. + */ + if (recno) { + testutil_assert(cursor->get_key(cursor, &keyr) == 0); + testutil_assert(keyr == 1); + } else { + testutil_assert(cursor->get_key(cursor, &key) == 0); + testutil_assert(key != keybuf); + testutil_assert(strcmp(key, KEY) == 0); + } + ignore_errors = 1; + if (vstring) + testutil_assert(cursor->get_value(cursor, &vs) != 0); + else + testutil_assert(cursor->get_value(cursor, &vu) != 0); + testutil_assert(ignore_errors == 0); + break; + case MODIFY: + case RESERVE: + case SEARCH: + case SEARCH_NEAR: + case UPDATE: + /* + * Modify, reserve, search, search-near and update all + * position the cursor and have both a key and value. + * + * Any key/value should not reference application + * memory. + */ + if (recno) { + testutil_assert(cursor->get_key(cursor, &keyr) == 0); + testutil_assert(keyr == 1); + } else { + testutil_assert(cursor->get_key(cursor, &key) == 0); + testutil_assert(key != keybuf); + testutil_assert(strcmp(key, KEY) == 0); + } + if (vstring) { + testutil_assert(cursor->get_value(cursor, &vs) == 0); + testutil_assert(vs != valuebuf); + testutil_assert(strcmp(vs, VALUE) == 0); + } else { + testutil_assert(cursor->get_value(cursor, &vu) == 0); + testutil_assert(vu.data != valuebuf); + testutil_assert(vu.size == strlen(VALUE)); + testutil_assert(memcmp(vu.data, VALUE, strlen(VALUE)) == 0); + } + break; + } - /* - * We have more than one remove operation, add the key back - * in. - */ - if (op->func == REMOVE || op->func == REMOVE_POS) { - if (recno) - cursor->set_key(cursor, (uint64_t)1); - else { - strcpy(keybuf, KEY); - cursor->set_key(cursor, keybuf); - } - strcpy(valuebuf, VALUE); - if (vstring) - cursor->set_value(cursor, valuebuf); - else { - vu.size = strlen(vu.data = valuebuf); - cursor->set_value(cursor, &vu); - } - testutil_check(cursor->insert(cursor)); - } - } + /* + * We have more than one remove operation, add the key back in. + */ + if (op->func == REMOVE || op->func == REMOVE_POS) { + if (recno) + cursor->set_key(cursor, (uint64_t)1); + else { + strcpy(keybuf, KEY); + cursor->set_key(cursor, keybuf); + } + strcpy(valuebuf, VALUE); + if (vstring) + cursor->set_value(cursor, valuebuf); + else { + vu.size = strlen(vu.data = valuebuf); + cursor->set_value(cursor, &vu); + } + testutil_check(cursor->insert(cursor)); + } + } } static void run(WT_CONNECTION *conn, const char *uri, const char *config) { - WT_SESSION *session; + WT_SESSION *session; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->create(session, uri, config)); - cursor_scope_ops(session, uri); - testutil_check(session->close(session, NULL)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->create(session, uri, config)); + cursor_scope_ops(session, uri); + testutil_check(session->close(session, NULL)); } int main(int argc, char *argv[]) { - TEST_OPTS *opts, _opts; + TEST_OPTS *opts, _opts; - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); - testutil_check( - wiredtiger_open(opts->home, &event_handler, "create", &opts->conn)); + testutil_check(wiredtiger_open(opts->home, &event_handler, "create", &opts->conn)); - run(opts->conn, "file:file.SS", "key_format=S,value_format=S"); - run(opts->conn, "file:file.Su", "key_format=S,value_format=u"); - run(opts->conn, "file:file.rS", "key_format=r,value_format=S"); - run(opts->conn, "file:file.ru", "key_format=r,value_format=u"); + run(opts->conn, "file:file.SS", "key_format=S,value_format=S"); + run(opts->conn, "file:file.Su", "key_format=S,value_format=u"); + run(opts->conn, "file:file.rS", "key_format=r,value_format=S"); + run(opts->conn, "file:file.ru", "key_format=r,value_format=u"); - run(opts->conn, "lsm:lsm.SS", "key_format=S,value_format=S"); - run(opts->conn, "lsm:lsm.Su", "key_format=S,value_format=u"); + run(opts->conn, "lsm:lsm.SS", "key_format=S,value_format=S"); + run(opts->conn, "lsm:lsm.Su", "key_format=S,value_format=u"); - run(opts->conn, "table:table.SS", "key_format=S,value_format=S"); - run(opts->conn, "table:table.Su", "key_format=S,value_format=u"); - run(opts->conn, "table:table.rS", "key_format=r,value_format=S"); - run(opts->conn, "table:table.ru", "key_format=r,value_format=u"); + run(opts->conn, "table:table.SS", "key_format=S,value_format=S"); + run(opts->conn, "table:table.Su", "key_format=S,value_format=u"); + run(opts->conn, "table:table.rS", "key_format=r,value_format=S"); + run(opts->conn, "table:table.ru", "key_format=r,value_format=u"); - testutil_cleanup(opts); + testutil_cleanup(opts); - return (EXIT_SUCCESS); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c index 2645dfefe23..1b69427d9f2 100644 --- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c @@ -31,7 +31,7 @@ #include <sys/wait.h> #include <signal.h> -static char home[1024]; /* Program working dir */ +static char home[1024]; /* Program working dir */ /* * Create three tables that we will write the same data to and verify that @@ -56,511 +56,443 @@ static char home[1024]; /* Program working dir */ * Each worker thread creates its own records file that records the data it * inserted and it records the timestamp that was used for that insertion. */ -#define INVALID_KEY UINT64_MAX -#define MAX_CKPT_INVL 5 /* Maximum interval between checkpoints */ -#define MAX_TH 200 /* Maximum configurable threads */ -#define MAX_TIME 40 -#define MAX_VAL 1024 -#define MIN_TH 5 -#define MIN_TIME 10 -#define PREPARE_FREQ 5 -#define PREPARE_PCT 10 -#define PREPARE_YIELD (PREPARE_FREQ * 10) -#define RECORDS_FILE "records-%" PRIu32 +#define INVALID_KEY UINT64_MAX +#define MAX_CKPT_INVL 5 /* Maximum interval between checkpoints */ +#define MAX_TH 200 /* Maximum configurable threads */ +#define MAX_TIME 40 +#define MAX_VAL 1024 +#define MIN_TH 5 +#define MIN_TIME 10 +#define PREPARE_FREQ 5 +#define PREPARE_PCT 10 +#define PREPARE_YIELD (PREPARE_FREQ * 10) +#define RECORDS_FILE "records-%" PRIu32 /* Include worker threads and prepare extra sessions */ -#define SESSION_MAX (MAX_TH + 3 + MAX_TH * PREPARE_PCT) +#define SESSION_MAX (MAX_TH + 3 + MAX_TH * PREPARE_PCT) -static const char * table_pfx = "table"; -static const char * const uri_collection = "collection"; -static const char * const uri_local = "local"; -static const char * const uri_oplog = "oplog"; -static const char * const uri_shadow = "shadow"; +static const char *table_pfx = "table"; +static const char *const uri_collection = "collection"; +static const char *const uri_local = "local"; +static const char *const uri_oplog = "oplog"; +static const char *const uri_shadow = "shadow"; -static const char * const ckpt_file = "checkpoint_done"; +static const char *const ckpt_file = "checkpoint_done"; static bool compat, inmem, use_ts; static volatile uint64_t global_ts = 1; -#define ENV_CONFIG_COMPAT ",compatibility=(release=\"2.9\")" -#define ENV_CONFIG_DEF \ - "cache_size=20M,create,log=(archive=true,file_max=10M,enabled)," \ - "debug_mode=(table_logging=true,checkpoint_retention=5)," \ +#define ENV_CONFIG_COMPAT ",compatibility=(release=\"2.9\")" +#define ENV_CONFIG_DEF \ + "cache_size=20M,create,log=(archive=true,file_max=10M,enabled)," \ + "debug_mode=(table_logging=true,checkpoint_retention=5)," \ "statistics=(fast),statistics_log=(wait=1,json=true),session_max=%d" -#define ENV_CONFIG_TXNSYNC \ - "cache_size=20M,create,log=(archive=true,file_max=10M,enabled)," \ - "debug_mode=(table_logging=true,checkpoint_retention=5)," \ - "statistics=(fast),statistics_log=(wait=1,json=true)," \ +#define ENV_CONFIG_TXNSYNC \ + "cache_size=20M,create,log=(archive=true,file_max=10M,enabled)," \ + "debug_mode=(table_logging=true,checkpoint_retention=5)," \ + "statistics=(fast),statistics_log=(wait=1,json=true)," \ "transaction_sync=(enabled,method=none),session_max=%d" -#define ENV_CONFIG_REC "log=(archive=false,recover=on)" +#define ENV_CONFIG_REC "log=(archive=false,recover=on)" typedef struct { - uint64_t absent_key; /* Last absent key */ - uint64_t exist_key; /* First existing key after miss */ - uint64_t first_key; /* First key in range */ - uint64_t first_miss; /* First missing key */ - uint64_t last_key; /* Last key in range */ + uint64_t absent_key; /* Last absent key */ + uint64_t exist_key; /* First existing key after miss */ + uint64_t first_key; /* First key in range */ + uint64_t first_miss; /* First missing key */ + uint64_t last_key; /* Last key in range */ } REPORT; typedef struct { - WT_CONNECTION *conn; - uint64_t start; - uint32_t info; + WT_CONNECTION *conn; + uint64_t start; + uint32_t info; } THREAD_DATA; /* Lock for transactional ops that set or query a timestamp. */ static pthread_rwlock_t ts_lock; -static void handler(int) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); -static void usage(void) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void handler(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void usage(void) { - fprintf(stderr, - "usage: %s [-h dir] [-T threads] [-t time] [-Cmvz]\n", progname); - exit(EXIT_FAILURE); + fprintf(stderr, "usage: %s [-h dir] [-T threads] [-t time] [-Cmvz]\n", progname); + exit(EXIT_FAILURE); } /* * thread_ts_run -- - * Runner function for a timestamp thread. + * Runner function for a timestamp thread. */ static WT_THREAD_RET thread_ts_run(void *arg) { - WT_DECL_RET; - WT_SESSION *session; - THREAD_DATA *td; - char tscfg[64], ts_string[WT_TS_HEX_STRING_SIZE]; - - td = (THREAD_DATA *)arg; - - testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); - /* Update the oldest timestamp every 1 millisecond. */ - for (;;) { - /* - * We get the last committed timestamp periodically in order to - * update the oldest timestamp, that requires locking out - * transactional ops that set or query a timestamp. - */ - testutil_check(pthread_rwlock_wrlock(&ts_lock)); - ret = td->conn->query_timestamp( - td->conn, ts_string, "get=all_committed"); - testutil_check(pthread_rwlock_unlock(&ts_lock)); - testutil_assert(ret == 0 || ret == WT_NOTFOUND); - if (ret == 0) { - /* - * Set both the oldest and stable timestamp so that we - * don't need to maintain read availability at older - * timestamps. - */ - testutil_check(__wt_snprintf( - tscfg, sizeof(tscfg), - "oldest_timestamp=%s,stable_timestamp=%s", - ts_string, ts_string)); - testutil_check( - td->conn->set_timestamp(td->conn, tscfg)); - } - __wt_sleep(0, 1000); - } - /* NOTREACHED */ + WT_DECL_RET; + WT_SESSION *session; + THREAD_DATA *td; + char tscfg[64], ts_string[WT_TS_HEX_STRING_SIZE]; + + td = (THREAD_DATA *)arg; + + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + /* Update the oldest timestamp every 1 millisecond. */ + for (;;) { + /* + * We get the last committed timestamp periodically in order to update the oldest timestamp, + * that requires locking out transactional ops that set or query a timestamp. + */ + testutil_check(pthread_rwlock_wrlock(&ts_lock)); + ret = td->conn->query_timestamp(td->conn, ts_string, "get=all_committed"); + testutil_check(pthread_rwlock_unlock(&ts_lock)); + testutil_assert(ret == 0 || ret == WT_NOTFOUND); + if (ret == 0) { + /* + * Set both the oldest and stable timestamp so that we don't need to maintain read + * availability at older timestamps. + */ + testutil_check(__wt_snprintf(tscfg, sizeof(tscfg), + "oldest_timestamp=%s,stable_timestamp=%s", ts_string, ts_string)); + testutil_check(td->conn->set_timestamp(td->conn, tscfg)); + } + __wt_sleep(0, 1000); + } + /* NOTREACHED */ } /* * thread_ckpt_run -- - * Runner function for the checkpoint thread. + * Runner function for the checkpoint thread. */ static WT_THREAD_RET thread_ckpt_run(void *arg) { - FILE *fp; - WT_RAND_STATE rnd; - WT_SESSION *session; - THREAD_DATA *td; - uint64_t stable; - uint32_t sleep_time; - int i; - bool first_ckpt; - char ts_string[WT_TS_HEX_STRING_SIZE]; - - __wt_random_init(&rnd); - - td = (THREAD_DATA *)arg; - /* - * Keep a separate file with the records we wrote for checking. - */ - (void)unlink(ckpt_file); - testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); - first_ckpt = true; - for (i = 0; ;++i) { - sleep_time = __wt_random(&rnd) % MAX_CKPT_INVL; - sleep(sleep_time); - /* - * Since this is the default, send in this string even if - * running without timestamps. - */ - testutil_check(session->checkpoint( - session, "use_timestamp=true")); - testutil_check(td->conn->query_timestamp( - td->conn, ts_string, "get=last_checkpoint")); - testutil_assert(sscanf(ts_string, "%" SCNx64, &stable) == 1); - printf("Checkpoint %d complete at stable %" - PRIu64 ".\n", i, stable); - fflush(stdout); - /* - * Create the checkpoint file so that the parent process knows - * at least one checkpoint has finished and can start its - * timer. - */ - if (first_ckpt) { - testutil_checksys((fp = fopen(ckpt_file, "w")) == NULL); - first_ckpt = false; - testutil_checksys(fclose(fp) != 0); - } - } - /* NOTREACHED */ + FILE *fp; + WT_RAND_STATE rnd; + WT_SESSION *session; + THREAD_DATA *td; + uint64_t stable; + uint32_t sleep_time; + int i; + bool first_ckpt; + char ts_string[WT_TS_HEX_STRING_SIZE]; + + __wt_random_init(&rnd); + + td = (THREAD_DATA *)arg; + /* + * Keep a separate file with the records we wrote for checking. + */ + (void)unlink(ckpt_file); + testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session)); + first_ckpt = true; + for (i = 0;; ++i) { + sleep_time = __wt_random(&rnd) % MAX_CKPT_INVL; + sleep(sleep_time); + /* + * Since this is the default, send in this string even if running without timestamps. + */ + testutil_check(session->checkpoint(session, "use_timestamp=true")); + testutil_check(td->conn->query_timestamp(td->conn, ts_string, "get=last_checkpoint")); + testutil_assert(sscanf(ts_string, "%" SCNx64, &stable) == 1); + printf("Checkpoint %d complete at stable %" PRIu64 ".\n", i, stable); + fflush(stdout); + /* + * Create the checkpoint file so that the parent process knows at least one checkpoint has + * finished and can start its timer. + */ + if (first_ckpt) { + testutil_checksys((fp = fopen(ckpt_file, "w")) == NULL); + first_ckpt = false; + testutil_checksys(fclose(fp) != 0); + } + } + /* NOTREACHED */ } /* * thread_run -- - * Runner function for the worker threads. + * Runner function for the worker threads. */ static WT_THREAD_RET thread_run(void *arg) { - FILE *fp; - WT_CURSOR *cur_coll, *cur_local, *cur_oplog, *cur_shadow; - WT_ITEM data; - WT_RAND_STATE rnd; - WT_SESSION *prepared_session, *session; - THREAD_DATA *td; - uint64_t i, active_ts; - char cbuf[MAX_VAL], lbuf[MAX_VAL], obuf[MAX_VAL]; - char kname[64], tscfg[64], uri[128]; - bool use_prep; - - __wt_random_init(&rnd); - memset(cbuf, 0, sizeof(cbuf)); - memset(lbuf, 0, sizeof(lbuf)); - memset(obuf, 0, sizeof(obuf)); - memset(kname, 0, sizeof(kname)); - - prepared_session = NULL; - td = (THREAD_DATA *)arg; - /* - * Set up the separate file for checking. - */ - testutil_check(__wt_snprintf( - cbuf, sizeof(cbuf), RECORDS_FILE, td->info)); - (void)unlink(cbuf); - testutil_checksys((fp = fopen(cbuf, "w")) == NULL); - /* - * Set to line buffering. But that is advisory only. We've seen - * cases where the result files end up with partial lines. - */ - __wt_stream_set_line_buffer(fp); - - /* - * Have 10% of the threads use prepared transactions if timestamps - * are in use. Thread numbers start at 0 so we're always guaranteed - * that at least one thread is using prepared transactions. - */ - use_prep = (use_ts && td->info % PREPARE_PCT == 0) ? true : false; - - /* - * For the prepared case we have two sessions so that the oplog session - * can have its own transaction in parallel with the collection session - * We need this because prepared transactions cannot have any operations - * that modify a table that is logged. But we also want to test mixed - * logged and not-logged transactions. - */ - testutil_check(td->conn->open_session( - td->conn, NULL, "isolation=snapshot", &session)); - if (use_prep) - testutil_check(td->conn->open_session( - td->conn, NULL, "isolation=snapshot", &prepared_session)); - /* - * Open a cursor to each table. - */ - testutil_check(__wt_snprintf( - uri, sizeof(uri), "%s:%s", table_pfx, uri_collection)); - if (use_prep) - testutil_check(prepared_session->open_cursor(prepared_session, - uri, NULL, NULL, &cur_coll)); - else - testutil_check(session->open_cursor(session, - uri, NULL, NULL, &cur_coll)); - testutil_check(__wt_snprintf( - uri, sizeof(uri), "%s:%s", table_pfx, uri_shadow)); - if (use_prep) - testutil_check(prepared_session->open_cursor(prepared_session, - uri, NULL, NULL, &cur_shadow)); - else - testutil_check(session->open_cursor(session, - uri, NULL, NULL, &cur_shadow)); - - testutil_check(__wt_snprintf( - uri, sizeof(uri), "%s:%s", table_pfx, uri_local)); - if (use_prep) - testutil_check(prepared_session->open_cursor(prepared_session, - uri, NULL, NULL, &cur_local)); - else - testutil_check(session->open_cursor(session, - uri, NULL, NULL, &cur_local)); - testutil_check(__wt_snprintf( - uri, sizeof(uri), "%s:%s", table_pfx, uri_oplog)); - testutil_check(session->open_cursor(session, - uri, NULL, NULL, &cur_oplog)); - - /* - * Write our portion of the key space until we're killed. - */ - printf("Thread %" PRIu32 " starts at %" PRIu64 "\n", - td->info, td->start); - active_ts = 0; - for (i = td->start;; ++i) { - testutil_check(__wt_snprintf( - kname, sizeof(kname), "%" PRIu64, i)); - - testutil_check(session->begin_transaction(session, NULL)); - if (use_prep) - testutil_check(prepared_session->begin_transaction( - prepared_session, NULL)); - - if (use_ts) { - testutil_check(pthread_rwlock_rdlock(&ts_lock)); - active_ts = __wt_atomic_addv64(&global_ts, 2); - testutil_check(__wt_snprintf(tscfg, - sizeof(tscfg), "commit_timestamp=%" PRIx64, - active_ts)); - /* - * Set the transaction's timestamp now before performing - * the operation. If we are using prepared transactions, - * set the timestamp for the session used for oplog. The - * collection session in that case would continue to use - * this timestamp. - */ - testutil_check(session->timestamp_transaction( - session, tscfg)); - testutil_check(pthread_rwlock_unlock(&ts_lock)); - } - - cur_coll->set_key(cur_coll, kname); - cur_local->set_key(cur_local, kname); - cur_oplog->set_key(cur_oplog, kname); - cur_shadow->set_key(cur_shadow, kname); - /* - * Put an informative string into the value so that it - * can be viewed well in a binary dump. - */ - testutil_check(__wt_snprintf(cbuf, sizeof(cbuf), - "COLL: thread:%" PRIu32 " ts:%" PRIu64 " key: %" PRIu64, - td->info, active_ts, i)); - testutil_check(__wt_snprintf(lbuf, sizeof(lbuf), - "LOCAL: thread:%" PRIu32 " ts:%" PRIu64 " key: %" PRIu64, - td->info, active_ts, i)); - testutil_check(__wt_snprintf(obuf, sizeof(obuf), - "OPLOG: thread:%" PRIu32 " ts:%" PRIu64 " key: %" PRIu64, - td->info, active_ts, i)); - data.size = __wt_random(&rnd) % MAX_VAL; - data.data = cbuf; - cur_coll->set_value(cur_coll, &data); - testutil_check(cur_coll->insert(cur_coll)); - cur_shadow->set_value(cur_shadow, &data); - if (use_ts) { - /* - * Change the timestamp in the middle of the - * transaction so that we simulate a secondary. - */ - ++active_ts; - testutil_check(__wt_snprintf(tscfg, - sizeof(tscfg), "commit_timestamp=%" PRIx64, - active_ts)); - testutil_check(session->timestamp_transaction( - session, tscfg)); - } - testutil_check(cur_shadow->insert(cur_shadow)); - data.size = __wt_random(&rnd) % MAX_VAL; - data.data = obuf; - cur_oplog->set_value(cur_oplog, &data); - testutil_check(cur_oplog->insert(cur_oplog)); - if (use_prep) { - /* - * Run with prepare every once in a while. And also - * yield after prepare sometimes too. This is only done - * on the collection session. - */ - if (i % PREPARE_FREQ == 0) { - testutil_check(__wt_snprintf(tscfg, - sizeof(tscfg), "prepare_timestamp=%" - PRIx64, active_ts)); - testutil_check( - prepared_session->prepare_transaction( - prepared_session, tscfg)); - if (i % PREPARE_YIELD == 0) - __wt_yield(); - testutil_check( - __wt_snprintf(tscfg, sizeof(tscfg), - "commit_timestamp=%" PRIx64 - ",durable_timestamp=%" PRIx64, - active_ts, active_ts)); - } else - testutil_check( - __wt_snprintf(tscfg, sizeof(tscfg), - "commit_timestamp=%" PRIx64, active_ts)); - - testutil_check( - prepared_session->commit_transaction( - prepared_session, tscfg)); - } - testutil_check( - session->commit_transaction(session, NULL)); - /* - * Insert into the local table outside the timestamp txn. - */ - data.size = __wt_random(&rnd) % MAX_VAL; - data.data = lbuf; - cur_local->set_value(cur_local, &data); - testutil_check(cur_local->insert(cur_local)); - - /* - * Save the timestamp and key separately for checking later. - */ - if (fprintf(fp, - "%" PRIu64 " %" PRIu64 "\n", active_ts, i) < 0) - testutil_die(EIO, "fprintf"); - } - /* NOTREACHED */ + FILE *fp; + WT_CURSOR *cur_coll, *cur_local, *cur_oplog, *cur_shadow; + WT_ITEM data; + WT_RAND_STATE rnd; + WT_SESSION *prepared_session, *session; + THREAD_DATA *td; + uint64_t i, active_ts; + char cbuf[MAX_VAL], lbuf[MAX_VAL], obuf[MAX_VAL]; + char kname[64], tscfg[64], uri[128]; + bool use_prep; + + __wt_random_init(&rnd); + memset(cbuf, 0, sizeof(cbuf)); + memset(lbuf, 0, sizeof(lbuf)); + memset(obuf, 0, sizeof(obuf)); + memset(kname, 0, sizeof(kname)); + + prepared_session = NULL; + td = (THREAD_DATA *)arg; + /* + * Set up the separate file for checking. + */ + testutil_check(__wt_snprintf(cbuf, sizeof(cbuf), RECORDS_FILE, td->info)); + (void)unlink(cbuf); + testutil_checksys((fp = fopen(cbuf, "w")) == NULL); + /* + * Set to line buffering. But that is advisory only. We've seen cases where the result files end + * up with partial lines. + */ + __wt_stream_set_line_buffer(fp); + + /* + * Have 10% of the threads use prepared transactions if timestamps are in use. Thread numbers + * start at 0 so we're always guaranteed that at least one thread is using prepared + * transactions. + */ + use_prep = (use_ts && td->info % PREPARE_PCT == 0) ? true : false; + + /* + * For the prepared case we have two sessions so that the oplog session can have its own + * transaction in parallel with the collection session We need this because prepared + * transactions cannot have any operations that modify a table that is logged. But we also want + * to test mixed logged and not-logged transactions. + */ + testutil_check(td->conn->open_session(td->conn, NULL, "isolation=snapshot", &session)); + if (use_prep) + testutil_check( + td->conn->open_session(td->conn, NULL, "isolation=snapshot", &prepared_session)); + /* + * Open a cursor to each table. + */ + testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_collection)); + if (use_prep) + testutil_check(prepared_session->open_cursor(prepared_session, uri, NULL, NULL, &cur_coll)); + else + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cur_coll)); + testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_shadow)); + if (use_prep) + testutil_check( + prepared_session->open_cursor(prepared_session, uri, NULL, NULL, &cur_shadow)); + else + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cur_shadow)); + + testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_local)); + if (use_prep) + testutil_check( + prepared_session->open_cursor(prepared_session, uri, NULL, NULL, &cur_local)); + else + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cur_local)); + testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_oplog)); + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cur_oplog)); + + /* + * Write our portion of the key space until we're killed. + */ + printf("Thread %" PRIu32 " starts at %" PRIu64 "\n", td->info, td->start); + active_ts = 0; + for (i = td->start;; ++i) { + testutil_check(__wt_snprintf(kname, sizeof(kname), "%" PRIu64, i)); + + testutil_check(session->begin_transaction(session, NULL)); + if (use_prep) + testutil_check(prepared_session->begin_transaction(prepared_session, NULL)); + + if (use_ts) { + testutil_check(pthread_rwlock_rdlock(&ts_lock)); + active_ts = __wt_atomic_addv64(&global_ts, 2); + testutil_check( + __wt_snprintf(tscfg, sizeof(tscfg), "commit_timestamp=%" PRIx64, active_ts)); + /* + * Set the transaction's timestamp now before performing the operation. If we are using + * prepared transactions, set the timestamp for the session used for oplog. The + * collection session in that case would continue to use this timestamp. + */ + testutil_check(session->timestamp_transaction(session, tscfg)); + testutil_check(pthread_rwlock_unlock(&ts_lock)); + } + + cur_coll->set_key(cur_coll, kname); + cur_local->set_key(cur_local, kname); + cur_oplog->set_key(cur_oplog, kname); + cur_shadow->set_key(cur_shadow, kname); + /* + * Put an informative string into the value so that it can be viewed well in a binary dump. + */ + testutil_check(__wt_snprintf(cbuf, sizeof(cbuf), + "COLL: thread:%" PRIu32 " ts:%" PRIu64 " key: %" PRIu64, td->info, active_ts, i)); + testutil_check(__wt_snprintf(lbuf, sizeof(lbuf), + "LOCAL: thread:%" PRIu32 " ts:%" PRIu64 " key: %" PRIu64, td->info, active_ts, i)); + testutil_check(__wt_snprintf(obuf, sizeof(obuf), + "OPLOG: thread:%" PRIu32 " ts:%" PRIu64 " key: %" PRIu64, td->info, active_ts, i)); + data.size = __wt_random(&rnd) % MAX_VAL; + data.data = cbuf; + cur_coll->set_value(cur_coll, &data); + testutil_check(cur_coll->insert(cur_coll)); + cur_shadow->set_value(cur_shadow, &data); + if (use_ts) { + /* + * Change the timestamp in the middle of the transaction so that we simulate a + * secondary. + */ + ++active_ts; + testutil_check( + __wt_snprintf(tscfg, sizeof(tscfg), "commit_timestamp=%" PRIx64, active_ts)); + testutil_check(session->timestamp_transaction(session, tscfg)); + } + testutil_check(cur_shadow->insert(cur_shadow)); + data.size = __wt_random(&rnd) % MAX_VAL; + data.data = obuf; + cur_oplog->set_value(cur_oplog, &data); + testutil_check(cur_oplog->insert(cur_oplog)); + if (use_prep) { + /* + * Run with prepare every once in a while. And also yield after prepare sometimes too. + * This is only done on the collection session. + */ + if (i % PREPARE_FREQ == 0) { + testutil_check( + __wt_snprintf(tscfg, sizeof(tscfg), "prepare_timestamp=%" PRIx64, active_ts)); + testutil_check(prepared_session->prepare_transaction(prepared_session, tscfg)); + if (i % PREPARE_YIELD == 0) + __wt_yield(); + testutil_check(__wt_snprintf(tscfg, sizeof(tscfg), + "commit_timestamp=%" PRIx64 ",durable_timestamp=%" PRIx64, active_ts, active_ts)); + } else + testutil_check( + __wt_snprintf(tscfg, sizeof(tscfg), "commit_timestamp=%" PRIx64, active_ts)); + + testutil_check(prepared_session->commit_transaction(prepared_session, tscfg)); + } + testutil_check(session->commit_transaction(session, NULL)); + /* + * Insert into the local table outside the timestamp txn. + */ + data.size = __wt_random(&rnd) % MAX_VAL; + data.data = lbuf; + cur_local->set_value(cur_local, &data); + testutil_check(cur_local->insert(cur_local)); + + /* + * Save the timestamp and key separately for checking later. + */ + if (fprintf(fp, "%" PRIu64 " %" PRIu64 "\n", active_ts, i) < 0) + testutil_die(EIO, "fprintf"); + } + /* NOTREACHED */ } /* - * Child process creates the database and table, and then creates worker - * threads to add data until it is killed by the parent. + * Child process creates the database and table, and then creates worker threads to add data until + * it is killed by the parent. */ -static void run_workload(uint32_t) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void run_workload(uint32_t) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void run_workload(uint32_t nth) { - WT_CONNECTION *conn; - WT_SESSION *session; - THREAD_DATA *td; - wt_thread_t *thr; - uint32_t ckpt_id, i, ts_id; - char envconf[512], uri[128]; - - thr = dcalloc(nth+2, sizeof(*thr)); - td = dcalloc(nth+2, sizeof(THREAD_DATA)); - if (chdir(home) != 0) - testutil_die(errno, "Child chdir: %s", home); - if (inmem) - testutil_check(__wt_snprintf(envconf, sizeof(envconf), - ENV_CONFIG_DEF, SESSION_MAX)); - else - testutil_check(__wt_snprintf(envconf, sizeof(envconf), - ENV_CONFIG_TXNSYNC, SESSION_MAX)); - if (compat) - strcat(envconf, ENV_CONFIG_COMPAT); - - testutil_check(wiredtiger_open(NULL, NULL, envconf, &conn)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - /* - * Create all the tables. - */ - testutil_check(__wt_snprintf( - uri, sizeof(uri), "%s:%s", table_pfx, uri_collection)); - testutil_check(session->create(session, uri, - "key_format=S,value_format=u,log=(enabled=false)")); - testutil_check(__wt_snprintf( - uri, sizeof(uri), "%s:%s", table_pfx, uri_shadow)); - testutil_check(session->create(session, uri, - "key_format=S,value_format=u,log=(enabled=false)")); - testutil_check(__wt_snprintf( - uri, sizeof(uri), "%s:%s", table_pfx, uri_local)); - testutil_check(session->create(session, - uri, "key_format=S,value_format=u")); - testutil_check(__wt_snprintf( - uri, sizeof(uri), "%s:%s", table_pfx, uri_oplog)); - testutil_check(session->create(session, - uri, "key_format=S,value_format=u")); - /* - * Don't log the stable timestamp table so that we know what timestamp - * was stored at the checkpoint. - */ - testutil_check(session->close(session, NULL)); - - /* - * The checkpoint thread and the timestamp threads are added at the end. - */ - ckpt_id = nth; - td[ckpt_id].conn = conn; - td[ckpt_id].info = nth; - printf("Create checkpoint thread\n"); - testutil_check(__wt_thread_create( - NULL, &thr[ckpt_id], thread_ckpt_run, &td[ckpt_id])); - ts_id = nth + 1; - if (use_ts) { - td[ts_id].conn = conn; - td[ts_id].info = nth; - printf("Create timestamp thread\n"); - testutil_check(__wt_thread_create( - NULL, &thr[ts_id], thread_ts_run, &td[ts_id])); - } - printf("Create %" PRIu32 " writer threads\n", nth); - for (i = 0; i < nth; ++i) { - td[i].conn = conn; - td[i].start = WT_BILLION * (uint64_t)i; - td[i].info = i; - testutil_check(__wt_thread_create( - NULL, &thr[i], thread_run, &td[i])); - } - /* - * The threads never exit, so the child will just wait here until - * it is killed. - */ - fflush(stdout); - for (i = 0; i <= ts_id; ++i) - testutil_check(__wt_thread_join(NULL, &thr[i])); - /* - * NOTREACHED - */ - free(thr); - free(td); - exit(EXIT_SUCCESS); + WT_CONNECTION *conn; + WT_SESSION *session; + THREAD_DATA *td; + wt_thread_t *thr; + uint32_t ckpt_id, i, ts_id; + char envconf[512], uri[128]; + + thr = dcalloc(nth + 2, sizeof(*thr)); + td = dcalloc(nth + 2, sizeof(THREAD_DATA)); + if (chdir(home) != 0) + testutil_die(errno, "Child chdir: %s", home); + if (inmem) + testutil_check(__wt_snprintf(envconf, sizeof(envconf), ENV_CONFIG_DEF, SESSION_MAX)); + else + testutil_check(__wt_snprintf(envconf, sizeof(envconf), ENV_CONFIG_TXNSYNC, SESSION_MAX)); + if (compat) + strcat(envconf, ENV_CONFIG_COMPAT); + + testutil_check(wiredtiger_open(NULL, NULL, envconf, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + /* + * Create all the tables. + */ + testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_collection)); + testutil_check( + session->create(session, uri, "key_format=S,value_format=u,log=(enabled=false)")); + testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_shadow)); + testutil_check( + session->create(session, uri, "key_format=S,value_format=u,log=(enabled=false)")); + testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_local)); + testutil_check(session->create(session, uri, "key_format=S,value_format=u")); + testutil_check(__wt_snprintf(uri, sizeof(uri), "%s:%s", table_pfx, uri_oplog)); + testutil_check(session->create(session, uri, "key_format=S,value_format=u")); + /* + * Don't log the stable timestamp table so that we know what timestamp was stored at the + * checkpoint. + */ + testutil_check(session->close(session, NULL)); + + /* + * The checkpoint thread and the timestamp threads are added at the end. + */ + ckpt_id = nth; + td[ckpt_id].conn = conn; + td[ckpt_id].info = nth; + printf("Create checkpoint thread\n"); + testutil_check(__wt_thread_create(NULL, &thr[ckpt_id], thread_ckpt_run, &td[ckpt_id])); + ts_id = nth + 1; + if (use_ts) { + td[ts_id].conn = conn; + td[ts_id].info = nth; + printf("Create timestamp thread\n"); + testutil_check(__wt_thread_create(NULL, &thr[ts_id], thread_ts_run, &td[ts_id])); + } + printf("Create %" PRIu32 " writer threads\n", nth); + for (i = 0; i < nth; ++i) { + td[i].conn = conn; + td[i].start = WT_BILLION * (uint64_t)i; + td[i].info = i; + testutil_check(__wt_thread_create(NULL, &thr[i], thread_run, &td[i])); + } + /* + * The threads never exit, so the child will just wait here until it is killed. + */ + fflush(stdout); + for (i = 0; i <= ts_id; ++i) + testutil_check(__wt_thread_join(NULL, &thr[i])); + /* + * NOTREACHED + */ + free(thr); + free(td); + exit(EXIT_SUCCESS); } extern int __wt_optind; extern char *__wt_optarg; /* - * Initialize a report structure. Since zero is a valid key we - * cannot just clear it. + * Initialize a report structure. Since zero is a valid key we cannot just clear it. */ static void initialize_rep(REPORT *r) { - r->first_key = r->first_miss = INVALID_KEY; - r->absent_key = r->exist_key = r->last_key = INVALID_KEY; + r->first_key = r->first_miss = INVALID_KEY; + r->absent_key = r->exist_key = r->last_key = INVALID_KEY; } /* - * Print out information if we detect missing records in the - * middle of the data of a report structure. + * Print out information if we detect missing records in the middle of the data of a report + * structure. */ static void print_missing(REPORT *r, const char *fname, const char *msg) { - if (r->exist_key != INVALID_KEY) - printf("%s: %s error %" PRIu64 - " absent records %" PRIu64 "-%" PRIu64 - ". Then keys %" PRIu64 "-%" PRIu64 " exist." - " Key range %" PRIu64 "-%" PRIu64 "\n", - fname, msg, - (r->exist_key - r->first_miss) - 1, - r->first_miss, r->exist_key - 1, - r->exist_key, r->last_key, - r->first_key, r->last_key); + if (r->exist_key != INVALID_KEY) + printf("%s: %s error %" PRIu64 " absent records %" PRIu64 "-%" PRIu64 ". Then keys %" PRIu64 + "-%" PRIu64 + " exist." + " Key range %" PRIu64 "-%" PRIu64 "\n", + fname, msg, (r->exist_key - r->first_miss) - 1, r->first_miss, r->exist_key - 1, + r->exist_key, r->last_key, r->first_key, r->last_key); } /* @@ -569,408 +501,362 @@ print_missing(REPORT *r, const char *fname, const char *msg) static void handler(int sig) { - pid_t pid; - - WT_UNUSED(sig); - pid = wait(NULL); - /* - * The core file will indicate why the child exited. Choose EINVAL here. - */ - testutil_die(EINVAL, - "Child process %" PRIu64 " abnormally exited", (uint64_t)pid); + pid_t pid; + + WT_UNUSED(sig); + pid = wait(NULL); + /* + * The core file will indicate why the child exited. Choose EINVAL here. + */ + testutil_die(EINVAL, "Child process %" PRIu64 " abnormally exited", (uint64_t)pid); } int main(int argc, char *argv[]) { - struct sigaction sa; - struct stat sb; - FILE *fp; - REPORT c_rep[MAX_TH], l_rep[MAX_TH], o_rep[MAX_TH]; - WT_CONNECTION *conn; - WT_CURSOR *cur_coll, *cur_local, *cur_oplog, *cur_shadow; - WT_RAND_STATE rnd; - WT_SESSION *session; - pid_t pid; - uint64_t absent_coll, absent_local, absent_oplog, count, key, last_key; - uint64_t stable_fp, stable_val; - uint32_t i, nth, timeout; - int ch, status, ret; - const char *working_dir; - char buf[512], fname[64], kname[64], statname[1024]; - char ts_string[WT_TS_HEX_STRING_SIZE]; - bool fatal, rand_th, rand_time, verify_only; - - (void)testutil_set_progname(argv); - - compat = inmem = false; - use_ts = true; - nth = MIN_TH; - rand_th = rand_time = true; - timeout = MIN_TIME; - verify_only = false; - working_dir = "WT_TEST.timestamp-abort"; - - while ((ch = __wt_getopt(progname, argc, argv, "Ch:LmT:t:vz")) != EOF) - switch (ch) { - case 'C': - compat = true; - break; - case 'h': - working_dir = __wt_optarg; - break; - case 'L': - table_pfx = "lsm"; - break; - case 'm': - inmem = true; - break; - case 'T': - rand_th = false; - nth = (uint32_t)atoi(__wt_optarg); - if (nth > MAX_TH) { - fprintf(stderr, - "Number of threads is larger than the" - " maximum %" PRId32 "\n", MAX_TH); - return (EXIT_FAILURE); - } - break; - case 't': - rand_time = false; - timeout = (uint32_t)atoi(__wt_optarg); - break; - case 'v': - verify_only = true; - break; - case 'z': - use_ts = false; - break; - default: - usage(); - } - argc -= __wt_optind; - if (argc != 0) - usage(); - - testutil_work_dir_from_path(home, sizeof(home), working_dir); - testutil_check(pthread_rwlock_init(&ts_lock, NULL)); - - /* - * If the user wants to verify they need to tell us how many threads - * there were so we can find the old record files. - */ - if (verify_only && rand_th) { - fprintf(stderr, - "Verify option requires specifying number of threads\n"); - exit (EXIT_FAILURE); - } - if (!verify_only) { - testutil_make_work_dir(home); - - __wt_random_init_seed(NULL, &rnd); - if (rand_time) { - timeout = __wt_random(&rnd) % MAX_TIME; - if (timeout < MIN_TIME) - timeout = MIN_TIME; - } - if (rand_th) { - nth = __wt_random(&rnd) % MAX_TH; - if (nth < MIN_TH) - nth = MIN_TH; - } - - printf("Parent: compatibility: %s, " - "in-mem log sync: %s, timestamp in use: %s\n", - compat ? "true" : "false", - inmem ? "true" : "false", - use_ts ? "true" : "false"); - printf("Parent: Create %" PRIu32 - " threads; sleep %" PRIu32 " seconds\n", nth, timeout); - printf("CONFIG: %s%s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n", - progname, - compat ? " -C" : "", - inmem ? " -m" : "", - !use_ts ? " -z" : "", - working_dir, nth, timeout); - /* - * Fork a child to insert as many items. We will then randomly - * kill the child, run recovery and make sure all items we wrote - * exist after recovery runs. - */ - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = handler; - testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); - testutil_checksys((pid = fork()) < 0); - - if (pid == 0) { /* child */ - run_workload(nth); - return (EXIT_SUCCESS); - } - - /* parent */ - /* - * Sleep for the configured amount of time before killing - * the child. Start the timeout from the time we notice that - * the file has been created. That allows the test to run - * correctly on really slow machines. - */ - testutil_check(__wt_snprintf( - statname, sizeof(statname), "%s/%s", home, ckpt_file)); - while (stat(statname, &sb) != 0) - testutil_sleep_wait(1, pid); - sleep(timeout); - sa.sa_handler = SIG_DFL; - testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); - - /* - * !!! It should be plenty long enough to make sure more than - * one log file exists. If wanted, that check would be added - * here. - */ - printf("Kill child\n"); - testutil_checksys(kill(pid, SIGKILL) != 0); - testutil_checksys(waitpid(pid, &status, 0) == -1); - } - /* - * !!! If we wanted to take a copy of the directory before recovery, - * this is the place to do it. Don't do it all the time because - * it can use a lot of disk space, which can cause test machine - * issues. - */ - if (chdir(home) != 0) - testutil_die(errno, "parent chdir: %s", home); - /* - * The tables can get very large, so while we'd ideally like to - * copy the entire database, we only copy the log files for now. - * Otherwise it can take far too long to run the test, particularly - * in automated testing. - */ - testutil_check(__wt_snprintf(buf, sizeof(buf), - "rm -rf ../%s.SAVE && mkdir ../%s.SAVE && " - "cp -p * ../%s.SAVE", - home, home, home)); - if ((status = system(buf)) < 0) - testutil_die(status, "system: %s", buf); - printf("Open database, run recovery and verify content\n"); - - /* - * Open the connection which forces recovery to be run. - */ - testutil_check(wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - /* - * Open a cursor on all the tables. - */ - testutil_check(__wt_snprintf( - buf, sizeof(buf), "%s:%s", table_pfx, uri_collection)); - testutil_check(session->open_cursor(session, - buf, NULL, NULL, &cur_coll)); - testutil_check(__wt_snprintf( - buf, sizeof(buf), "%s:%s", table_pfx, uri_shadow)); - testutil_check(session->open_cursor(session, - buf, NULL, NULL, &cur_shadow)); - testutil_check(__wt_snprintf( - buf, sizeof(buf), "%s:%s", table_pfx, uri_local)); - testutil_check(session->open_cursor(session, - buf, NULL, NULL, &cur_local)); - testutil_check(__wt_snprintf( - buf, sizeof(buf), "%s:%s", table_pfx, uri_oplog)); - testutil_check(session->open_cursor(session, - buf, NULL, NULL, &cur_oplog)); - - /* - * Find the biggest stable timestamp value that was saved. - */ - stable_val = 0; - if (use_ts) { - testutil_check( - conn->query_timestamp(conn, ts_string, "get=recovery")); - testutil_assert( - sscanf(ts_string, "%" SCNx64, &stable_val) == 1); - printf("Got stable_val %" PRIu64 "\n", stable_val); - } - - count = 0; - absent_coll = absent_local = absent_oplog = 0; - fatal = false; - for (i = 0; i < nth; ++i) { - initialize_rep(&c_rep[i]); - initialize_rep(&l_rep[i]); - initialize_rep(&o_rep[i]); - testutil_check(__wt_snprintf( - fname, sizeof(fname), RECORDS_FILE, i)); - if ((fp = fopen(fname, "r")) == NULL) - testutil_die(errno, "fopen: %s", fname); - - /* - * For every key in the saved file, verify that the key exists - * in the table after recovery. If we're doing in-memory - * log buffering we never expect a record missing in the middle, - * but records may be missing at the end. If we did - * write-no-sync, we expect every key to have been recovered. - */ - for (last_key = INVALID_KEY;; ++count, last_key = key) { - ret = fscanf(fp, "%" SCNu64 "%" SCNu64 "\n", - &stable_fp, &key); - if (last_key == INVALID_KEY) { - c_rep[i].first_key = key; - l_rep[i].first_key = key; - o_rep[i].first_key = key; - } - if (ret != EOF && ret != 2) { - /* - * If we find a partial line, consider it - * like an EOF. - */ - if (ret == 1 || ret == 0) - break; - testutil_die(errno, "fscanf"); - } - if (ret == EOF) - break; - /* - * If we're unlucky, the last line may be a partially - * written key at the end that can result in a false - * negative error for a missing record. Detect it. - */ - if (last_key != INVALID_KEY && key != last_key + 1) { - printf("%s: Ignore partial record %" PRIu64 - " last valid key %" PRIu64 "\n", - fname, key, last_key); - break; - } - testutil_check(__wt_snprintf( - kname, sizeof(kname), "%" PRIu64, key)); - cur_coll->set_key(cur_coll, kname); - cur_local->set_key(cur_local, kname); - cur_oplog->set_key(cur_oplog, kname); - cur_shadow->set_key(cur_shadow, kname); - /* - * The collection table should always only have the - * data as of the checkpoint. The shadow table should - * always have the exact same data (or not) as the - * collection table. - */ - if ((ret = cur_coll->search(cur_coll)) != 0) { - if (ret != WT_NOTFOUND) - testutil_die(ret, "search"); - if ((ret = cur_shadow->search(cur_shadow)) == 0) - testutil_die(ret, - "shadow search success"); - - /* - * If we don't find a record, the stable - * timestamp written to our file better be - * larger than the saved one. - */ - if (!inmem && - stable_fp != 0 && stable_fp <= stable_val) { - printf("%s: COLLECTION no record with " - "key %" PRIu64 " record ts %" PRIu64 - " <= stable ts %" PRIu64 "\n", - fname, key, stable_fp, stable_val); - absent_coll++; - } - if (c_rep[i].first_miss == INVALID_KEY) - c_rep[i].first_miss = key; - c_rep[i].absent_key = key; - } else if (c_rep[i].absent_key != INVALID_KEY && - c_rep[i].exist_key == INVALID_KEY) { - /* - * If we get here we found a record that exists - * after absent records, a hole in our data. - */ - c_rep[i].exist_key = key; - fatal = true; - } else if (!inmem && - stable_fp != 0 && stable_fp > stable_val) { - /* - * If we found a record, the stable timestamp - * written to our file better be no larger - * than the checkpoint one. - */ - printf("%s: COLLECTION record with " - "key %" PRIu64 " record ts %" PRIu64 - " > stable ts %" PRIu64 "\n", - fname, key, stable_fp, stable_val); - fatal = true; - } else if ((ret = cur_shadow->search(cur_shadow)) != 0) - /* Collection and shadow both have the data. */ - testutil_die(ret, "shadow search failure"); - - /* - * The local table should always have all data. - */ - if ((ret = cur_local->search(cur_local)) != 0) { - if (ret != WT_NOTFOUND) - testutil_die(ret, "search"); - if (!inmem) - printf("%s: LOCAL no record with key %" - PRIu64 "\n", fname, key); - absent_local++; - if (l_rep[i].first_miss == INVALID_KEY) - l_rep[i].first_miss = key; - l_rep[i].absent_key = key; - } else if (l_rep[i].absent_key != INVALID_KEY && - l_rep[i].exist_key == INVALID_KEY) { - /* - * We should never find an existing key after - * we have detected one missing. - */ - l_rep[i].exist_key = key; - fatal = true; - } - /* - * The oplog table should always have all data. - */ - if ((ret = cur_oplog->search(cur_oplog)) != 0) { - if (ret != WT_NOTFOUND) - testutil_die(ret, "search"); - if (!inmem) - printf("%s: OPLOG no record with key %" - PRIu64 "\n", fname, key); - absent_oplog++; - if (o_rep[i].first_miss == INVALID_KEY) - o_rep[i].first_miss = key; - o_rep[i].absent_key = key; - } else if (o_rep[i].absent_key != INVALID_KEY && - o_rep[i].exist_key == INVALID_KEY) { - /* - * We should never find an existing key after - * we have detected one missing. - */ - o_rep[i].exist_key = key; - fatal = true; - } - } - c_rep[i].last_key = last_key; - l_rep[i].last_key = last_key; - o_rep[i].last_key = last_key; - testutil_checksys(fclose(fp) != 0); - print_missing(&c_rep[i], fname, "COLLECTION"); - print_missing(&l_rep[i], fname, "LOCAL"); - print_missing(&o_rep[i], fname, "OPLOG"); - } - testutil_check(conn->close(conn, NULL)); - if (!inmem && absent_coll) { - printf("COLLECTION: %" PRIu64 - " record(s) absent from %" PRIu64 "\n", - absent_coll, count); - fatal = true; - } - if (!inmem && absent_local) { - printf("LOCAL: %" PRIu64 " record(s) absent from %" PRIu64 "\n", - absent_local, count); - fatal = true; - } - if (!inmem && absent_oplog) { - printf("OPLOG: %" PRIu64 " record(s) absent from %" PRIu64 "\n", - absent_oplog, count); - fatal = true; - } - testutil_check(pthread_rwlock_destroy(&ts_lock)); - if (fatal) - return (EXIT_FAILURE); - printf("%" PRIu64 " records verified\n", count); - return (EXIT_SUCCESS); + struct sigaction sa; + struct stat sb; + FILE *fp; + REPORT c_rep[MAX_TH], l_rep[MAX_TH], o_rep[MAX_TH]; + WT_CONNECTION *conn; + WT_CURSOR *cur_coll, *cur_local, *cur_oplog, *cur_shadow; + WT_RAND_STATE rnd; + WT_SESSION *session; + pid_t pid; + uint64_t absent_coll, absent_local, absent_oplog, count, key, last_key; + uint64_t stable_fp, stable_val; + uint32_t i, nth, timeout; + int ch, status, ret; + const char *working_dir; + char buf[512], fname[64], kname[64], statname[1024]; + char ts_string[WT_TS_HEX_STRING_SIZE]; + bool fatal, rand_th, rand_time, verify_only; + + (void)testutil_set_progname(argv); + + compat = inmem = false; + use_ts = true; + nth = MIN_TH; + rand_th = rand_time = true; + timeout = MIN_TIME; + verify_only = false; + working_dir = "WT_TEST.timestamp-abort"; + + while ((ch = __wt_getopt(progname, argc, argv, "Ch:LmT:t:vz")) != EOF) + switch (ch) { + case 'C': + compat = true; + break; + case 'h': + working_dir = __wt_optarg; + break; + case 'L': + table_pfx = "lsm"; + break; + case 'm': + inmem = true; + break; + case 'T': + rand_th = false; + nth = (uint32_t)atoi(__wt_optarg); + if (nth > MAX_TH) { + fprintf(stderr, + "Number of threads is larger than the" + " maximum %" PRId32 "\n", + MAX_TH); + return (EXIT_FAILURE); + } + break; + case 't': + rand_time = false; + timeout = (uint32_t)atoi(__wt_optarg); + break; + case 'v': + verify_only = true; + break; + case 'z': + use_ts = false; + break; + default: + usage(); + } + argc -= __wt_optind; + if (argc != 0) + usage(); + + testutil_work_dir_from_path(home, sizeof(home), working_dir); + testutil_check(pthread_rwlock_init(&ts_lock, NULL)); + + /* + * If the user wants to verify they need to tell us how many threads there were so we can find + * the old record files. + */ + if (verify_only && rand_th) { + fprintf(stderr, "Verify option requires specifying number of threads\n"); + exit(EXIT_FAILURE); + } + if (!verify_only) { + testutil_make_work_dir(home); + + __wt_random_init_seed(NULL, &rnd); + if (rand_time) { + timeout = __wt_random(&rnd) % MAX_TIME; + if (timeout < MIN_TIME) + timeout = MIN_TIME; + } + if (rand_th) { + nth = __wt_random(&rnd) % MAX_TH; + if (nth < MIN_TH) + nth = MIN_TH; + } + + printf( + "Parent: compatibility: %s, " + "in-mem log sync: %s, timestamp in use: %s\n", + compat ? "true" : "false", inmem ? "true" : "false", use_ts ? "true" : "false"); + printf("Parent: Create %" PRIu32 " threads; sleep %" PRIu32 " seconds\n", nth, timeout); + printf("CONFIG: %s%s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n", progname, + compat ? " -C" : "", inmem ? " -m" : "", !use_ts ? " -z" : "", working_dir, nth, timeout); + /* + * Fork a child to insert as many items. We will then randomly kill the child, run recovery + * and make sure all items we wrote exist after recovery runs. + */ + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = handler; + testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); + testutil_checksys((pid = fork()) < 0); + + if (pid == 0) { /* child */ + run_workload(nth); + return (EXIT_SUCCESS); + } + + /* parent */ + /* + * Sleep for the configured amount of time before killing the child. Start the timeout from + * the time we notice that the file has been created. That allows the test to run correctly + * on really slow machines. + */ + testutil_check(__wt_snprintf(statname, sizeof(statname), "%s/%s", home, ckpt_file)); + while (stat(statname, &sb) != 0) + testutil_sleep_wait(1, pid); + sleep(timeout); + sa.sa_handler = SIG_DFL; + testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); + + /* + * !!! It should be plenty long enough to make sure more than + * one log file exists. If wanted, that check would be added + * here. + */ + printf("Kill child\n"); + testutil_checksys(kill(pid, SIGKILL) != 0); + testutil_checksys(waitpid(pid, &status, 0) == -1); + } + /* + * !!! If we wanted to take a copy of the directory before recovery, + * this is the place to do it. Don't do it all the time because + * it can use a lot of disk space, which can cause test machine + * issues. + */ + if (chdir(home) != 0) + testutil_die(errno, "parent chdir: %s", home); + /* + * The tables can get very large, so while we'd ideally like to copy the entire database, we + * only copy the log files for now. Otherwise it can take far too long to run the test, + * particularly in automated testing. + */ + testutil_check(__wt_snprintf(buf, sizeof(buf), + "rm -rf ../%s.SAVE && mkdir ../%s.SAVE && " + "cp -p * ../%s.SAVE", + home, home, home)); + if ((status = system(buf)) < 0) + testutil_die(status, "system: %s", buf); + printf("Open database, run recovery and verify content\n"); + + /* + * Open the connection which forces recovery to be run. + */ + testutil_check(wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + /* + * Open a cursor on all the tables. + */ + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s:%s", table_pfx, uri_collection)); + testutil_check(session->open_cursor(session, buf, NULL, NULL, &cur_coll)); + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s:%s", table_pfx, uri_shadow)); + testutil_check(session->open_cursor(session, buf, NULL, NULL, &cur_shadow)); + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s:%s", table_pfx, uri_local)); + testutil_check(session->open_cursor(session, buf, NULL, NULL, &cur_local)); + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s:%s", table_pfx, uri_oplog)); + testutil_check(session->open_cursor(session, buf, NULL, NULL, &cur_oplog)); + + /* + * Find the biggest stable timestamp value that was saved. + */ + stable_val = 0; + if (use_ts) { + testutil_check(conn->query_timestamp(conn, ts_string, "get=recovery")); + testutil_assert(sscanf(ts_string, "%" SCNx64, &stable_val) == 1); + printf("Got stable_val %" PRIu64 "\n", stable_val); + } + + count = 0; + absent_coll = absent_local = absent_oplog = 0; + fatal = false; + for (i = 0; i < nth; ++i) { + initialize_rep(&c_rep[i]); + initialize_rep(&l_rep[i]); + initialize_rep(&o_rep[i]); + testutil_check(__wt_snprintf(fname, sizeof(fname), RECORDS_FILE, i)); + if ((fp = fopen(fname, "r")) == NULL) + testutil_die(errno, "fopen: %s", fname); + + /* + * For every key in the saved file, verify that the key exists in the table after recovery. + * If we're doing in-memory log buffering we never expect a record missing in the middle, + * but records may be missing at the end. If we did write-no-sync, we expect every key to + * have been recovered. + */ + for (last_key = INVALID_KEY;; ++count, last_key = key) { + ret = fscanf(fp, "%" SCNu64 "%" SCNu64 "\n", &stable_fp, &key); + if (last_key == INVALID_KEY) { + c_rep[i].first_key = key; + l_rep[i].first_key = key; + o_rep[i].first_key = key; + } + if (ret != EOF && ret != 2) { + /* + * If we find a partial line, consider it like an EOF. + */ + if (ret == 1 || ret == 0) + break; + testutil_die(errno, "fscanf"); + } + if (ret == EOF) + break; + /* + * If we're unlucky, the last line may be a partially written key at the end that can + * result in a false negative error for a missing record. Detect it. + */ + if (last_key != INVALID_KEY && key != last_key + 1) { + printf("%s: Ignore partial record %" PRIu64 " last valid key %" PRIu64 "\n", fname, + key, last_key); + break; + } + testutil_check(__wt_snprintf(kname, sizeof(kname), "%" PRIu64, key)); + cur_coll->set_key(cur_coll, kname); + cur_local->set_key(cur_local, kname); + cur_oplog->set_key(cur_oplog, kname); + cur_shadow->set_key(cur_shadow, kname); + /* + * The collection table should always only have the data as of the checkpoint. The + * shadow table should always have the exact same data (or not) as the collection table. + */ + if ((ret = cur_coll->search(cur_coll)) != 0) { + if (ret != WT_NOTFOUND) + testutil_die(ret, "search"); + if ((ret = cur_shadow->search(cur_shadow)) == 0) + testutil_die(ret, "shadow search success"); + + /* + * If we don't find a record, the stable timestamp written to our file better be + * larger than the saved one. + */ + if (!inmem && stable_fp != 0 && stable_fp <= stable_val) { + printf( + "%s: COLLECTION no record with " + "key %" PRIu64 " record ts %" PRIu64 " <= stable ts %" PRIu64 "\n", + fname, key, stable_fp, stable_val); + absent_coll++; + } + if (c_rep[i].first_miss == INVALID_KEY) + c_rep[i].first_miss = key; + c_rep[i].absent_key = key; + } else if (c_rep[i].absent_key != INVALID_KEY && c_rep[i].exist_key == INVALID_KEY) { + /* + * If we get here we found a record that exists after absent records, a hole in our + * data. + */ + c_rep[i].exist_key = key; + fatal = true; + } else if (!inmem && stable_fp != 0 && stable_fp > stable_val) { + /* + * If we found a record, the stable timestamp written to our file better be no + * larger than the checkpoint one. + */ + printf( + "%s: COLLECTION record with " + "key %" PRIu64 " record ts %" PRIu64 " > stable ts %" PRIu64 "\n", + fname, key, stable_fp, stable_val); + fatal = true; + } else if ((ret = cur_shadow->search(cur_shadow)) != 0) + /* Collection and shadow both have the data. */ + testutil_die(ret, "shadow search failure"); + + /* + * The local table should always have all data. + */ + if ((ret = cur_local->search(cur_local)) != 0) { + if (ret != WT_NOTFOUND) + testutil_die(ret, "search"); + if (!inmem) + printf("%s: LOCAL no record with key %" PRIu64 "\n", fname, key); + absent_local++; + if (l_rep[i].first_miss == INVALID_KEY) + l_rep[i].first_miss = key; + l_rep[i].absent_key = key; + } else if (l_rep[i].absent_key != INVALID_KEY && l_rep[i].exist_key == INVALID_KEY) { + /* + * We should never find an existing key after we have detected one missing. + */ + l_rep[i].exist_key = key; + fatal = true; + } + /* + * The oplog table should always have all data. + */ + if ((ret = cur_oplog->search(cur_oplog)) != 0) { + if (ret != WT_NOTFOUND) + testutil_die(ret, "search"); + if (!inmem) + printf("%s: OPLOG no record with key %" PRIu64 "\n", fname, key); + absent_oplog++; + if (o_rep[i].first_miss == INVALID_KEY) + o_rep[i].first_miss = key; + o_rep[i].absent_key = key; + } else if (o_rep[i].absent_key != INVALID_KEY && o_rep[i].exist_key == INVALID_KEY) { + /* + * We should never find an existing key after we have detected one missing. + */ + o_rep[i].exist_key = key; + fatal = true; + } + } + c_rep[i].last_key = last_key; + l_rep[i].last_key = last_key; + o_rep[i].last_key = last_key; + testutil_checksys(fclose(fp) != 0); + print_missing(&c_rep[i], fname, "COLLECTION"); + print_missing(&l_rep[i], fname, "LOCAL"); + print_missing(&o_rep[i], fname, "OPLOG"); + } + testutil_check(conn->close(conn, NULL)); + if (!inmem && absent_coll) { + printf("COLLECTION: %" PRIu64 " record(s) absent from %" PRIu64 "\n", absent_coll, count); + fatal = true; + } + if (!inmem && absent_local) { + printf("LOCAL: %" PRIu64 " record(s) absent from %" PRIu64 "\n", absent_local, count); + fatal = true; + } + if (!inmem && absent_oplog) { + printf("OPLOG: %" PRIu64 " record(s) absent from %" PRIu64 "\n", absent_oplog, count); + fatal = true; + } + testutil_check(pthread_rwlock_destroy(&ts_lock)); + if (fatal) + return (EXIT_FAILURE); + printf("%" PRIu64 " records verified\n", count); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/truncated_log/main.c b/src/third_party/wiredtiger/test/csuite/truncated_log/main.c index 9e13ee01e2b..befc30eab61 100644 --- a/src/third_party/wiredtiger/test/csuite/truncated_log/main.c +++ b/src/third_party/wiredtiger/test/csuite/truncated_log/main.c @@ -30,71 +30,64 @@ #include <sys/wait.h> -static char home[1024]; /* Program working dir */ -static const char * const uri = "table:main"; +static char home[1024]; /* Program working dir */ +static const char *const uri = "table:main"; -#define RECORDS_FILE "records" +#define RECORDS_FILE "records" -#define ENV_CONFIG \ - "create,log=(file_max=100K,archive=false,enabled)," \ +#define ENV_CONFIG \ + "create,log=(file_max=100K,archive=false,enabled)," \ "transaction_sync=(enabled,method=none)" -#define ENV_CONFIG_REC "log=(recover=on)" +#define ENV_CONFIG_REC "log=(recover=on)" -#define LOG_FILE_1 "WiredTigerLog.0000000001" +#define LOG_FILE_1 "WiredTigerLog.0000000001" -#define K_SIZE 16 -#define V_SIZE 256 +#define K_SIZE 16 +#define V_SIZE 256 /* - * Write a new log record into the log via log print, then open up a log - * cursor and walk the log to make sure we can read it. The reason for this - * test is that if there is a partial log record at the end of the previous - * log file and truncate does not exist, this tests that we can still read + * Write a new log record into the log via log print, then open up a log cursor and walk the log to + * make sure we can read it. The reason for this test is that if there is a partial log record at + * the end of the previous log file and truncate does not exist, this tests that we can still read * past that record. */ static void write_and_read_new(WT_SESSION *); static void write_and_read_new(WT_SESSION *session) { - WT_CURSOR *logc; - WT_ITEM logrec_key, logrec_value; - uint64_t txnid; - uint32_t fileid, log_file, log_offset, opcount, optype, rectype; - bool saw_msg; + WT_CURSOR *logc; + WT_ITEM logrec_key, logrec_value; + uint64_t txnid; + uint32_t fileid, log_file, log_offset, opcount, optype, rectype; + bool saw_msg; - /* - * Write a log record and force it to disk so we can read it. - */ - printf("Write log_printf record and verify.\n"); - testutil_check(session->log_printf(session, "Test Log Record")); - testutil_check(session->log_flush(session, "sync=on")); - testutil_check( - session->open_cursor(session, "log:", NULL, NULL, &logc)); - testutil_check( - session->open_cursor(session, "log:", NULL, NULL, &logc)); - saw_msg = false; - while (logc->next(logc) == 0) { - /* - * We don't really need to get the key, but in case we want - * the LSN for some message, get it. - */ - testutil_check(logc->get_key( - logc, &log_file, &log_offset, &opcount)); - testutil_check(logc->get_value(logc, &txnid, - &rectype, &optype, &fileid, &logrec_key, &logrec_value)); - /* - * We should never see a record from us in log file 2. We wrote - * a record there, but then the record in log file 1 was - * truncated to be a partial record, ending the log there. - * So everything after that, including everything in log - * file 2, is invalid until we get to log file 3 which is where - * the post-recovery records will be written. - * The one exception in log file two is the system record for - * the previous log file's LSN. Although it is written by the - * system, we do walk it when using a cursor. - */ - if (log_file == 2 && rectype != WT_LOGREC_SYSTEM) - testutil_die(EINVAL, "Found LSN in Log 2"); + /* + * Write a log record and force it to disk so we can read it. + */ + printf("Write log_printf record and verify.\n"); + testutil_check(session->log_printf(session, "Test Log Record")); + testutil_check(session->log_flush(session, "sync=on")); + testutil_check(session->open_cursor(session, "log:", NULL, NULL, &logc)); + testutil_check(session->open_cursor(session, "log:", NULL, NULL, &logc)); + saw_msg = false; + while (logc->next(logc) == 0) { + /* + * We don't really need to get the key, but in case we want the LSN for some message, get + * it. + */ + testutil_check(logc->get_key(logc, &log_file, &log_offset, &opcount)); + testutil_check( + logc->get_value(logc, &txnid, &rectype, &optype, &fileid, &logrec_key, &logrec_value)); + /* + * We should never see a record from us in log file 2. We wrote a record there, but then the + * record in log file 1 was truncated to be a partial record, ending the log there. So + * everything after that, including everything in log file 2, is invalid until we get to log + * file 3 which is where the post-recovery records will be written. The one exception in log + * file two is the system record for the previous log file's LSN. Although it is written by + * the system, we do walk it when using a cursor. + */ + if (log_file == 2 && rectype != WT_LOGREC_SYSTEM) + testutil_die(EINVAL, "Found LSN in Log 2"); #if 0 printf("LSN [%" PRIu32 "][%" PRIu32 "].%" PRIu32 ": record type %" PRIu32 " optype %" PRIu32 @@ -102,136 +95,123 @@ write_and_read_new(WT_SESSION *session) log_file, log_offset, opcount, rectype, optype, txnid, fileid); #endif - if (rectype == WT_LOGREC_MESSAGE) { - saw_msg = true; - printf("Application Record: %s\n", - (char *)logrec_value.data); - break; - } - } - testutil_check(logc->close(logc)); - if (!saw_msg) - testutil_die(EINVAL, "Did not traverse log printf record"); + if (rectype == WT_LOGREC_MESSAGE) { + saw_msg = true; + printf("Application Record: %s\n", (char *)logrec_value.data); + break; + } + } + testutil_check(logc->close(logc)); + if (!saw_msg) + testutil_die(EINVAL, "Did not traverse log printf record"); } -static void usage(void) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void usage(void) { - fprintf(stderr, "usage: %s [-h dir]\n", progname); - exit(EXIT_FAILURE); + fprintf(stderr, "usage: %s [-h dir]\n", progname); + exit(EXIT_FAILURE); } /* - * Child process creates the database and table, and then writes data into - * the table until it switches into log file 2. + * Child process creates the database and table, and then writes data into the table until it + * switches into log file 2. */ -static void fill_db(void) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void fill_db(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void fill_db(void) { - FILE *fp; - WT_CONNECTION *conn; - WT_CURSOR *cursor, *logc; - WT_LSN lsn, save_lsn; - WT_SESSION *session; - uint32_t i, max_key, min_key, units, unused; - char k[K_SIZE], v[V_SIZE]; - bool first; + FILE *fp; + WT_CONNECTION *conn; + WT_CURSOR *cursor, *logc; + WT_LSN lsn, save_lsn; + WT_SESSION *session; + uint32_t i, max_key, min_key, units, unused; + char k[K_SIZE], v[V_SIZE]; + bool first; - /* - * Run in the home directory so that the records file is in there too. - */ - if (chdir(home) != 0) - testutil_die(errno, "chdir: %s", home); - testutil_check(wiredtiger_open(NULL, NULL, ENV_CONFIG, &conn)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check( - session->create(session, uri, "key_format=S,value_format=S")); - testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); + /* + * Run in the home directory so that the records file is in there too. + */ + if (chdir(home) != 0) + testutil_die(errno, "chdir: %s", home); + testutil_check(wiredtiger_open(NULL, NULL, ENV_CONFIG, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->create(session, uri, "key_format=S,value_format=S")); + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); - /* - * Keep a separate file with the records we wrote for checking. - */ - (void)unlink(RECORDS_FILE); - if ((fp = fopen(RECORDS_FILE, "w")) == NULL) - testutil_die(errno, "fopen"); - /* - * Set to no buffering. - */ - __wt_stream_set_no_buffer(fp); - save_lsn.l.file = 0; + /* + * Keep a separate file with the records we wrote for checking. + */ + (void)unlink(RECORDS_FILE); + if ((fp = fopen(RECORDS_FILE, "w")) == NULL) + testutil_die(errno, "fopen"); + /* + * Set to no buffering. + */ + __wt_stream_set_no_buffer(fp); + save_lsn.l.file = 0; - /* - * Write data into the table until we move to log file 2. - * We do the calculation below so that we don't have to walk the - * log for every record. - * - * Calculate about how many records should fit in the log file. - * Subtract a bunch for metadata and file creation records. - * Then subtract out a few more records to be conservative. - */ - units = (K_SIZE + V_SIZE) / 128 + 1; - min_key = 90000 / (units * 128) - 15; - max_key = min_key * 2; - first = true; - for (i = 0; i < max_key; ++i) { - testutil_check(__wt_snprintf(k, sizeof(k), "key%03d", (int)i)); - testutil_check(__wt_snprintf(v, sizeof(v), "value%0*d", - (int)(V_SIZE - (strlen("value") + 1)), (int)i)); - cursor->set_key(cursor, k); - cursor->set_value(cursor, v); - testutil_check(cursor->insert(cursor)); + /* + * Write data into the table until we move to log file 2. + * We do the calculation below so that we don't have to walk the + * log for every record. + * + * Calculate about how many records should fit in the log file. + * Subtract a bunch for metadata and file creation records. + * Then subtract out a few more records to be conservative. + */ + units = (K_SIZE + V_SIZE) / 128 + 1; + min_key = 90000 / (units * 128) - 15; + max_key = min_key * 2; + first = true; + for (i = 0; i < max_key; ++i) { + testutil_check(__wt_snprintf(k, sizeof(k), "key%03d", (int)i)); + testutil_check( + __wt_snprintf(v, sizeof(v), "value%0*d", (int)(V_SIZE - (strlen("value") + 1)), (int)i)); + cursor->set_key(cursor, k); + cursor->set_value(cursor, v); + testutil_check(cursor->insert(cursor)); - /* - * Walking the ever growing log can be slow, so only start - * looking for the cross into log file 2 after a minimum. - */ - if (i > min_key) { - testutil_check(session->open_cursor( - session, "log:", NULL, NULL, &logc)); - if (save_lsn.l.file != 0) { - logc->set_key(logc, - save_lsn.l.file, save_lsn.l.offset, 0); - testutil_check(logc->search(logc)); - } - while (logc->next(logc) == 0) { - testutil_check(logc->get_key( - logc, &lsn.l.file, &lsn.l.offset, &unused)); - /* - * Save the LSN so that we know the offset - * of the last LSN in log file 1 later. - */ - if (lsn.l.file < 2) - save_lsn = lsn; - else { - /* - * If this is the first time through - * that the key is larger than the - * minimum key and we're already in - * log file 2 then we did not calculate - * correctly and the test should fail. - */ - if (first) - testutil_die(EINVAL, - "min_key too high"); - if (fprintf(fp, - "%" PRIu32 " %" PRIu32 "\n", - save_lsn.l.offset, i - 1) == -1) - testutil_die(errno, "fprintf"); - break; - } - } - first = false; - testutil_check(logc->close(logc)); - } - } - if (fclose(fp) != 0) - testutil_die(errno, "fclose"); - exit(0); - /* NOTREACHED */ + /* + * Walking the ever growing log can be slow, so only start looking for the cross into log + * file 2 after a minimum. + */ + if (i > min_key) { + testutil_check(session->open_cursor(session, "log:", NULL, NULL, &logc)); + if (save_lsn.l.file != 0) { + logc->set_key(logc, save_lsn.l.file, save_lsn.l.offset, 0); + testutil_check(logc->search(logc)); + } + while (logc->next(logc) == 0) { + testutil_check(logc->get_key(logc, &lsn.l.file, &lsn.l.offset, &unused)); + /* + * Save the LSN so that we know the offset of the last LSN in log file 1 later. + */ + if (lsn.l.file < 2) + save_lsn = lsn; + else { + /* + * If this is the first time through that the key is larger than the minimum key + * and we're already in log file 2 then we did not calculate correctly and the + * test should fail. + */ + if (first) + testutil_die(EINVAL, "min_key too high"); + if (fprintf(fp, "%" PRIu32 " %" PRIu32 "\n", save_lsn.l.offset, i - 1) == -1) + testutil_die(errno, "fprintf"); + break; + } + } + first = false; + testutil_check(logc->close(logc)); + } + } + if (fclose(fp) != 0) + testutil_die(errno, "fclose"); + exit(0); + /* NOTREACHED */ } extern int __wt_optind; @@ -240,108 +220,104 @@ extern char *__wt_optarg; int main(int argc, char *argv[]) { - FILE *fp; - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_SESSION *session; - pid_t pid; - uint64_t new_offset, offset; - uint32_t count, max_key; - int ch, ret, status; - const char *working_dir; + FILE *fp; + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_SESSION *session; + pid_t pid; + uint64_t new_offset, offset; + uint32_t count, max_key; + int ch, ret, status; + const char *working_dir; - (void)testutil_set_progname(argv); + (void)testutil_set_progname(argv); - working_dir = "WT_TEST.truncated-log"; - while ((ch = __wt_getopt(progname, argc, argv, "h:")) != EOF) - switch (ch) { - case 'h': - working_dir = __wt_optarg; - break; - default: - usage(); - } - argc -= __wt_optind; - if (argc != 0) - usage(); + working_dir = "WT_TEST.truncated-log"; + while ((ch = __wt_getopt(progname, argc, argv, "h:")) != EOF) + switch (ch) { + case 'h': + working_dir = __wt_optarg; + break; + default: + usage(); + } + argc -= __wt_optind; + if (argc != 0) + usage(); - testutil_work_dir_from_path(home, sizeof(home), working_dir); - testutil_make_work_dir(home); + testutil_work_dir_from_path(home, sizeof(home), working_dir); + testutil_make_work_dir(home); - /* - * Fork a child to do its work. Wait for it to exit. - */ - if ((pid = fork()) < 0) - testutil_die(errno, "fork"); + /* + * Fork a child to do its work. Wait for it to exit. + */ + if ((pid = fork()) < 0) + testutil_die(errno, "fork"); - if (pid == 0) { /* child */ - fill_db(); - return (EXIT_SUCCESS); - } + if (pid == 0) { /* child */ + fill_db(); + return (EXIT_SUCCESS); + } - /* parent */ - /* Wait for child to kill itself. */ - if (waitpid(pid, &status, 0) == -1) - testutil_die(errno, "waitpid"); + /* parent */ + /* Wait for child to kill itself. */ + if (waitpid(pid, &status, 0) == -1) + testutil_die(errno, "waitpid"); - /* - * !!! If we wanted to take a copy of the directory before recovery, - * this is the place to do it. - */ - if (chdir(home) != 0) - testutil_die(errno, "chdir: %s", home); + /* + * !!! If we wanted to take a copy of the directory before recovery, + * this is the place to do it. + */ + if (chdir(home) != 0) + testutil_die(errno, "chdir: %s", home); - printf("Open database, run recovery and verify content\n"); - if ((fp = fopen(RECORDS_FILE, "r")) == NULL) - testutil_die(errno, "fopen"); - ret = fscanf(fp, "%" SCNu64 " %" SCNu32 "\n", &offset, &max_key); - if (ret != 2) - testutil_die(errno, "fscanf"); - if (fclose(fp) != 0) - testutil_die(errno, "fclose"); - /* - * The offset is the beginning of the last record. Truncate to - * the middle of that last record (i.e. ahead of that offset). - */ - if (offset > UINT64_MAX - V_SIZE) - testutil_die(ERANGE, "offset"); - new_offset = offset + V_SIZE; - printf("Parent: Log file 1: Key %" PRIu32 " at %" PRIu64 "\n", - max_key, offset); - printf("Parent: Truncate mid-record to %" PRIu64 "\n", new_offset); - if (truncate(LOG_FILE_1, (wt_off_t)new_offset) != 0) - testutil_die(errno, "truncate"); + printf("Open database, run recovery and verify content\n"); + if ((fp = fopen(RECORDS_FILE, "r")) == NULL) + testutil_die(errno, "fopen"); + ret = fscanf(fp, "%" SCNu64 " %" SCNu32 "\n", &offset, &max_key); + if (ret != 2) + testutil_die(errno, "fscanf"); + if (fclose(fp) != 0) + testutil_die(errno, "fclose"); + /* + * The offset is the beginning of the last record. Truncate to the middle of that last record + * (i.e. ahead of that offset). + */ + if (offset > UINT64_MAX - V_SIZE) + testutil_die(ERANGE, "offset"); + new_offset = offset + V_SIZE; + printf("Parent: Log file 1: Key %" PRIu32 " at %" PRIu64 "\n", max_key, offset); + printf("Parent: Truncate mid-record to %" PRIu64 "\n", new_offset); + if (truncate(LOG_FILE_1, (wt_off_t)new_offset) != 0) + testutil_die(errno, "truncate"); - testutil_check(wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); + testutil_check(wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); - /* - * For every key in the saved file, verify that the key exists - * in the table after recovery. Since we did write-no-sync, we - * expect every key to have been recovered. - */ - count = 0; - while (cursor->next(cursor) == 0) - ++count; - /* - * The max key in the saved file is the key we truncated, but the - * key space starts at 0 and we're counting the records here, so we - * expect the max key number of records. Add one for the system - * record for the previous LSN that the cursor will see too. - */ - if (count > (max_key + 1)) { - printf("expected %" PRIu32 " records found %" PRIu32 "\n", - max_key, count); - return (EXIT_FAILURE); - } - printf("%" PRIu32 " records verified\n", count); + /* + * For every key in the saved file, verify that the key exists in the table after recovery. + * Since we did write-no-sync, we expect every key to have been recovered. + */ + count = 0; + while (cursor->next(cursor) == 0) + ++count; + /* + * The max key in the saved file is the key we truncated, but the key space starts at 0 and + * we're counting the records here, so we expect the max key number of records. Add one for the + * system record for the previous LSN that the cursor will see too. + */ + if (count > (max_key + 1)) { + printf("expected %" PRIu32 " records found %" PRIu32 "\n", max_key, count); + return (EXIT_FAILURE); + } + printf("%" PRIu32 " records verified\n", count); - /* - * Write a log record and then walk the log to make sure we can - * read that log record that is beyond the truncated record. - */ - write_and_read_new(session); - testutil_check(conn->close(conn, NULL)); - return (EXIT_SUCCESS); + /* + * Write a log record and then walk the log to make sure we can read that log record that is + * beyond the truncated record. + */ + write_and_read_new(session); + testutil_check(conn->close(conn, NULL)); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c b/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c index 057d216d042..efa477f98e1 100644 --- a/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c @@ -28,157 +28,139 @@ #include "test_util.h" /* - * JIRA ticket reference: WT-1965 - * Test case description: The reported issue was that column store tables - * exhibit high CPU usage when populated with sparse record IDs. - * Failure mode: It isn't simple to make this test case failure explicit since - * it is demonstrating an inefficiency rather than a correctness bug. + * JIRA ticket reference: WT-1965 Test case description: The reported issue was that column store + * tables exhibit high CPU usage when populated with sparse record IDs. Failure mode: It isn't + * simple to make this test case failure explicit since it is demonstrating an inefficiency rather + * than a correctness bug. */ /* If changing field count also need to change set_value and get_value calls */ -#define NR_FIELDS 8 -#define NR_OBJECTS 100 -#define NR_THREADS 4 +#define NR_FIELDS 8 +#define NR_OBJECTS 100 +#define NR_THREADS 4 static uint64_t g_ts = 0; /* - * Each thread inserts a set of keys into the record store database. The keys - * are generated in such a way that there are large gaps in the key range. + * Each thread inserts a set of keys into the record store database. The keys are generated in such + * a way that there are large gaps in the key range. */ static void * thread_func(void *arg) { - TEST_OPTS *opts; - WT_CURSOR *cursor, *idx_cursor; - WT_SESSION *session; - uint64_t i, ins_rotor, ins_thr_idx, thr_idx, ts; - uint64_t *obj_data; - - opts = (TEST_OPTS *)arg; - thr_idx = __wt_atomic_fetch_addv64(&opts->next_threadid, 1); - ts = g_ts; - obj_data = dcalloc( - (NR_OBJECTS/NR_THREADS + 1) * NR_FIELDS, sizeof(*obj_data)); - - testutil_check(opts->conn->open_session( - opts->conn, NULL, NULL, &session)); - - testutil_check(session->open_cursor( - session, opts->uri, NULL, NULL, &cursor)); - testutil_check(session->open_cursor( - session, "table:index", NULL, NULL, &idx_cursor)); - - for (ins_rotor = 1; ins_rotor < 10; ++ins_rotor) { - for (ins_thr_idx = thr_idx, i = 0; ins_thr_idx < NR_OBJECTS; - ins_thr_idx += NR_THREADS, i += NR_FIELDS) { - - testutil_check( - session->begin_transaction(session, "sync=false")); - - cursor->set_key(cursor, ins_thr_idx << 40 | ins_rotor); - cursor->set_value(cursor, ts, - obj_data[i+0], obj_data[i+1], obj_data[i+2], - obj_data[i+3], obj_data[i+4], obj_data[i+5], - obj_data[i+6], obj_data[i+7]); - testutil_check(cursor->insert(cursor)); - - idx_cursor->set_key( - idx_cursor, ins_thr_idx << 40 | ts); - idx_cursor->set_value(idx_cursor, ins_rotor); - testutil_check(idx_cursor->insert(idx_cursor)); - - testutil_check( - session->commit_transaction(session, NULL)); - - /* change object fields */ - ++obj_data[i + ((ins_thr_idx + ins_rotor) % NR_FIELDS)]; - ++obj_data[i + - ((ins_thr_idx + ins_rotor + 1) % NR_FIELDS)]; - - ++g_ts; - /* 5K updates/sec */ - (void)usleep(1000000ULL * NR_THREADS / 5000); - } - } - - testutil_check(session->close(session, NULL)); - free(obj_data); - return (NULL); + TEST_OPTS *opts; + WT_CURSOR *cursor, *idx_cursor; + WT_SESSION *session; + uint64_t i, ins_rotor, ins_thr_idx, thr_idx, ts; + uint64_t *obj_data; + + opts = (TEST_OPTS *)arg; + thr_idx = __wt_atomic_fetch_addv64(&opts->next_threadid, 1); + ts = g_ts; + obj_data = dcalloc((NR_OBJECTS / NR_THREADS + 1) * NR_FIELDS, sizeof(*obj_data)); + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &cursor)); + testutil_check(session->open_cursor(session, "table:index", NULL, NULL, &idx_cursor)); + + for (ins_rotor = 1; ins_rotor < 10; ++ins_rotor) { + for (ins_thr_idx = thr_idx, i = 0; ins_thr_idx < NR_OBJECTS; + ins_thr_idx += NR_THREADS, i += NR_FIELDS) { + + testutil_check(session->begin_transaction(session, "sync=false")); + + cursor->set_key(cursor, ins_thr_idx << 40 | ins_rotor); + cursor->set_value(cursor, ts, obj_data[i + 0], obj_data[i + 1], obj_data[i + 2], + obj_data[i + 3], obj_data[i + 4], obj_data[i + 5], obj_data[i + 6], obj_data[i + 7]); + testutil_check(cursor->insert(cursor)); + + idx_cursor->set_key(idx_cursor, ins_thr_idx << 40 | ts); + idx_cursor->set_value(idx_cursor, ins_rotor); + testutil_check(idx_cursor->insert(idx_cursor)); + + testutil_check(session->commit_transaction(session, NULL)); + + /* change object fields */ + ++obj_data[i + ((ins_thr_idx + ins_rotor) % NR_FIELDS)]; + ++obj_data[i + ((ins_thr_idx + ins_rotor + 1) % NR_FIELDS)]; + + ++g_ts; + /* 5K updates/sec */ + (void)usleep(1000000ULL * NR_THREADS / 5000); + } + } + + testutil_check(session->close(session, NULL)); + free(obj_data); + return (NULL); } int main(int argc, char *argv[]) { - TEST_OPTS *opts, _opts; - WT_CURSOR *cursor; - WT_SESSION *session; - pthread_t thr[NR_THREADS]; - size_t t; - uint64_t f[NR_FIELDS], r, ts; - int i, ret; - char table_format[256]; - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - - testutil_check(wiredtiger_open(opts->home, NULL, - "create,cache_size=1G,checkpoint=(wait=30)," - "eviction_trigger=80,eviction_target=64,eviction_dirty_target=65," - "log=(enabled,file_max=10M)," - "transaction_sync=(enabled=true,method=none)", &opts->conn)); - testutil_check(opts->conn->open_session( - opts->conn, NULL, NULL, &session)); - - testutil_check(__wt_snprintf( - table_format, sizeof(table_format), "key_format=r,value_format=")); - for (i = 0; i < NR_FIELDS; i++) - strcat(table_format, "Q"); - - /* recno -> timestamp + NR_FIELDS * Q */ - testutil_check(session->create( - session, opts->uri, table_format)); - /* timestamp -> recno */ - testutil_check(session->create(session, - "table:index", "key_format=Q,value_format=Q")); - - testutil_check(session->close(session, NULL)); - - for (t = 0; t < NR_THREADS; ++t) - testutil_check( - pthread_create(&thr[t], NULL, thread_func, opts)); - - for (t = 0; t < NR_THREADS; ++t) - (void)pthread_join(thr[t], NULL); - - testutil_check(opts->conn->open_session( - opts->conn, NULL, NULL, &session)); - - /* recno -> timestamp + NR_FIELDS * Q */ - testutil_check(session->create(session, opts->uri, table_format)); - - testutil_check(session->open_cursor( - session, opts->uri, NULL, NULL, &cursor)); - - while ((ret = cursor->next(cursor)) == 0) { - testutil_check(cursor->get_key(cursor, &r)); - testutil_check(cursor->get_value(cursor, &ts, - &f[0], &f[1], &f[2], &f[3], &f[4], &f[5], &f[6], &f[7])); - - if (!opts->verbose) - continue; - - printf("(%" PRIu64 ",%llu)\t\t%" PRIu64, - (r >> 40), r & ((1ULL << 40) - 1), ts); - - for (i = 0; i < NR_FIELDS; i++) - printf("\t%" PRIu64, f[i]); - printf("\n"); - } - testutil_assert(ret == WT_NOTFOUND); - - testutil_cleanup(opts); - return (EXIT_SUCCESS); + TEST_OPTS *opts, _opts; + WT_CURSOR *cursor; + WT_SESSION *session; + pthread_t thr[NR_THREADS]; + size_t t; + uint64_t f[NR_FIELDS], r, ts; + int i, ret; + char table_format[256]; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(wiredtiger_open(opts->home, NULL, + "create,cache_size=1G,checkpoint=(wait=30)," + "eviction_trigger=80,eviction_target=64,eviction_dirty_target=65," + "log=(enabled,file_max=10M)," + "transaction_sync=(enabled=true,method=none)", + &opts->conn)); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(__wt_snprintf(table_format, sizeof(table_format), "key_format=r,value_format=")); + for (i = 0; i < NR_FIELDS; i++) + strcat(table_format, "Q"); + + /* recno -> timestamp + NR_FIELDS * Q */ + testutil_check(session->create(session, opts->uri, table_format)); + /* timestamp -> recno */ + testutil_check(session->create(session, "table:index", "key_format=Q,value_format=Q")); + + testutil_check(session->close(session, NULL)); + + for (t = 0; t < NR_THREADS; ++t) + testutil_check(pthread_create(&thr[t], NULL, thread_func, opts)); + + for (t = 0; t < NR_THREADS; ++t) + (void)pthread_join(thr[t], NULL); + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + /* recno -> timestamp + NR_FIELDS * Q */ + testutil_check(session->create(session, opts->uri, table_format)); + + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &cursor)); + + while ((ret = cursor->next(cursor)) == 0) { + testutil_check(cursor->get_key(cursor, &r)); + testutil_check( + cursor->get_value(cursor, &ts, &f[0], &f[1], &f[2], &f[3], &f[4], &f[5], &f[6], &f[7])); + + if (!opts->verbose) + continue; + + printf("(%" PRIu64 ",%llu)\t\t%" PRIu64, (r >> 40), r & ((1ULL << 40) - 1), ts); + + for (i = 0; i < NR_FIELDS; i++) + printf("\t%" PRIu64, f[i]); + printf("\n"); + } + testutil_assert(ret == WT_NOTFOUND); + + testutil_cleanup(opts); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c b/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c index 3eb1162fc0c..882b9867557 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c @@ -28,19 +28,17 @@ #include "test_util.h" /* - * JIRA ticket reference: WT-2246 - * Test case description: The column-store search routine used to search the - * target leaf page even when the cursor is configured with append and we're - * allocating a record number. That was inefficient, this test case - * demonstrates the inefficiency. - * Failure mode: It isn't simple to make this test case failure explicit since - * it is demonstrating an inefficiency rather than a correctness bug. + * JIRA ticket reference: WT-2246 Test case description: The column-store search routine used to + * search the target leaf page even when the cursor is configured with append and we're allocating a + * record number. That was inefficient, this test case demonstrates the inefficiency. Failure mode: + * It isn't simple to make this test case failure explicit since it is demonstrating an inefficiency + * rather than a correctness bug. */ /* Don't move into shared function there is a cross platform solution */ #include <signal.h> -#define MILLION 1000000 +#define MILLION 1000000 /* Needs to be global for signal handling. */ static TEST_OPTS *opts, _opts; @@ -48,108 +46,103 @@ static TEST_OPTS *opts, _opts; static void page_init(uint64_t n) { - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_SESSION *session; - uint64_t recno, vrecno; - char buf[64]; - - conn = opts->conn; - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check( - session->open_cursor(session, opts->uri, NULL, "append", &cursor)); - - vrecno = 0; - buf[0] = '\2'; - for (recno = 1;; ++recno) { - if (opts->table_type == TABLE_FIX) - cursor->set_value(cursor, buf[0]); - else { - if (recno % 3 == 0) - ++vrecno; - testutil_check(__wt_snprintf(buf, - sizeof(buf), "%" PRIu64 " VALUE ------", vrecno)); - cursor->set_value(cursor, buf); - } - testutil_check(cursor->insert(cursor)); - testutil_check(cursor->get_key(cursor, &opts->max_inserted_id)); - if (opts->max_inserted_id >= n) - break; - } + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_SESSION *session; + uint64_t recno, vrecno; + char buf[64]; + + conn = opts->conn; + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->open_cursor(session, opts->uri, NULL, "append", &cursor)); + + vrecno = 0; + buf[0] = '\2'; + for (recno = 1;; ++recno) { + if (opts->table_type == TABLE_FIX) + cursor->set_value(cursor, buf[0]); + else { + if (recno % 3 == 0) + ++vrecno; + testutil_check(__wt_snprintf(buf, sizeof(buf), "%" PRIu64 " VALUE ------", vrecno)); + cursor->set_value(cursor, buf); + } + testutil_check(cursor->insert(cursor)); + testutil_check(cursor->get_key(cursor, &opts->max_inserted_id)); + if (opts->max_inserted_id >= n) + break; + } } static void onsig(int signo) { - WT_UNUSED(signo); - opts->running = false; + WT_UNUSED(signo); + opts->running = false; } -#define N_APPEND_THREADS 6 -#define N_RECORDS (20 * WT_MILLION) +#define N_APPEND_THREADS 6 +#define N_RECORDS (20 * WT_MILLION) int main(int argc, char *argv[]) { - WT_SESSION *session; - clock_t ce, cs; - pthread_t idlist[100]; - uint64_t i, id; - char buf[100]; - - /* Bypass this test for valgrind */ - if (testutil_is_flag_set("TESTUTIL_BYPASS_VALGRIND")) - return (EXIT_SUCCESS); - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - opts->table_type = TABLE_ROW; - opts->n_append_threads = N_APPEND_THREADS; - opts->nrecords = N_RECORDS; - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - - testutil_check(__wt_snprintf(buf, sizeof(buf), - "create," - "cache_size=%s," - "eviction=(threads_max=5)," - "statistics=(fast)", - opts->table_type == TABLE_FIX ? "500MB" : "2GB")); - testutil_check(wiredtiger_open(opts->home, NULL, buf, &opts->conn)); - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check(__wt_snprintf(buf, sizeof(buf), - "key_format=r,value_format=%s," - "allocation_size=4K,leaf_page_max=64K", - opts->table_type == TABLE_FIX ? "8t" : "S")); - testutil_check(session->create(session, opts->uri, buf)); - testutil_check(session->close(session, NULL)); - - page_init(5000); - - /* Force to disk and re-open. */ - testutil_check(opts->conn->close(opts->conn, NULL)); - testutil_check(wiredtiger_open(opts->home, NULL, NULL, &opts->conn)); - - (void)signal(SIGINT, onsig); - - cs = clock(); - id = 0; - for (i = 0; i < opts->n_append_threads; ++i, ++id) { - printf("append: %" PRIu64 "\n", id); - testutil_check( - pthread_create(&idlist[id], NULL, thread_append, opts)); - } - - for (i = 0; i < id; ++i) - testutil_check(pthread_join(idlist[i], NULL)); - - ce = clock(); - printf("%" PRIu64 "M records: %.2lf processor seconds\n", - opts->max_inserted_id / MILLION, - (ce - cs) / (double)CLOCKS_PER_SEC); - - testutil_cleanup(opts); - return (EXIT_SUCCESS); + WT_SESSION *session; + clock_t ce, cs; + pthread_t idlist[100]; + uint64_t i, id; + char buf[100]; + + /* Bypass this test for valgrind */ + if (testutil_is_flag_set("TESTUTIL_BYPASS_VALGRIND")) + return (EXIT_SUCCESS); + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + opts->table_type = TABLE_ROW; + opts->n_append_threads = N_APPEND_THREADS; + opts->nrecords = N_RECORDS; + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(__wt_snprintf(buf, sizeof(buf), + "create," + "cache_size=%s," + "eviction=(threads_max=5)," + "statistics=(fast)", + opts->table_type == TABLE_FIX ? "500MB" : "2GB")); + testutil_check(wiredtiger_open(opts->home, NULL, buf, &opts->conn)); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(__wt_snprintf(buf, sizeof(buf), + "key_format=r,value_format=%s," + "allocation_size=4K,leaf_page_max=64K", + opts->table_type == TABLE_FIX ? "8t" : "S")); + testutil_check(session->create(session, opts->uri, buf)); + testutil_check(session->close(session, NULL)); + + page_init(5000); + + /* Force to disk and re-open. */ + testutil_check(opts->conn->close(opts->conn, NULL)); + testutil_check(wiredtiger_open(opts->home, NULL, NULL, &opts->conn)); + + (void)signal(SIGINT, onsig); + + cs = clock(); + id = 0; + for (i = 0; i < opts->n_append_threads; ++i, ++id) { + printf("append: %" PRIu64 "\n", id); + testutil_check(pthread_create(&idlist[id], NULL, thread_append, opts)); + } + + for (i = 0; i < id; ++i) + testutil_check(pthread_join(idlist[i], NULL)); + + ce = clock(); + printf("%" PRIu64 "M records: %.2lf processor seconds\n", opts->max_inserted_id / MILLION, + (ce - cs) / (double)CLOCKS_PER_SEC); + + testutil_cleanup(opts); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c index bdfed982bbc..388b079f842 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c @@ -52,33 +52,33 @@ * of inserts set low as a default. */ -#define N_RECORDS 10000 -#define N_INSERT 500000 -#define N_INSERT_THREAD 2 -#define N_JOIN_THREAD 2 -#define S64 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789::" -#define S1024 (S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64) +#define N_RECORDS 10000 +#define N_INSERT 500000 +#define N_INSERT_THREAD 2 +#define N_JOIN_THREAD 2 +#define S64 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789::" +#define S1024 (S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64) typedef struct { - char posturi[256]; - char baluri[256]; - char flaguri[256]; - char joinuri[256]; - bool bloom; - bool remove; + char posturi[256]; + char baluri[256]; + char flaguri[256]; + char joinuri[256]; + bool bloom; + bool remove; } SHARED_OPTS; typedef struct { - TEST_OPTS *testopts; - SHARED_OPTS *sharedopts; - int threadnum; - int nthread; - int done; - int joins; - int removes; - int inserts; - int notfounds; - int rollbacks; + TEST_OPTS *testopts; + SHARED_OPTS *sharedopts; + int threadnum; + int nthread; + int done; + int joins; + int removes; + int inserts; + int notfounds; + int rollbacks; } THREAD_ARGS; static void *thread_insert(void *); @@ -88,325 +88,288 @@ static void test_join(TEST_OPTS *, SHARED_OPTS *, bool, bool); int main(int argc, char *argv[]) { - SHARED_OPTS *sharedopts, _sharedopts; - TEST_OPTS *opts, _opts; - const char *tablename; - - /* Bypass this test for valgrind */ - if (testutil_is_flag_set("TESTUTIL_BYPASS_VALGRIND")) - return (EXIT_SUCCESS); - - opts = &_opts; - sharedopts = &_sharedopts; - memset(opts, 0, sizeof(*opts)); - memset(sharedopts, 0, sizeof(*sharedopts)); - - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - - tablename = strchr(opts->uri, ':'); - testutil_assert(tablename != NULL); - tablename++; - testutil_check(__wt_snprintf( - sharedopts->posturi, sizeof(sharedopts->posturi), - "index:%s:post", tablename)); - testutil_check(__wt_snprintf( - sharedopts->baluri, sizeof(sharedopts->baluri), - "index:%s:bal", tablename)); - testutil_check(__wt_snprintf( - sharedopts->flaguri, sizeof(sharedopts->flaguri), - "index:%s:flag", tablename)); - testutil_check(__wt_snprintf( - sharedopts->joinuri, sizeof(sharedopts->joinuri), - "join:%s", opts->uri)); - - testutil_check(wiredtiger_open(opts->home, NULL, - "create,cache_size=1G", &opts->conn)); - - test_join(opts, sharedopts, true, true); - test_join(opts, sharedopts, true, false); - test_join(opts, sharedopts, false, true); - test_join(opts, sharedopts, false, false); - - testutil_cleanup(opts); - - return (0); + SHARED_OPTS *sharedopts, _sharedopts; + TEST_OPTS *opts, _opts; + const char *tablename; + + /* Bypass this test for valgrind */ + if (testutil_is_flag_set("TESTUTIL_BYPASS_VALGRIND")) + return (EXIT_SUCCESS); + + opts = &_opts; + sharedopts = &_sharedopts; + memset(opts, 0, sizeof(*opts)); + memset(sharedopts, 0, sizeof(*sharedopts)); + + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + tablename = strchr(opts->uri, ':'); + testutil_assert(tablename != NULL); + tablename++; + testutil_check( + __wt_snprintf(sharedopts->posturi, sizeof(sharedopts->posturi), "index:%s:post", tablename)); + testutil_check( + __wt_snprintf(sharedopts->baluri, sizeof(sharedopts->baluri), "index:%s:bal", tablename)); + testutil_check( + __wt_snprintf(sharedopts->flaguri, sizeof(sharedopts->flaguri), "index:%s:flag", tablename)); + testutil_check( + __wt_snprintf(sharedopts->joinuri, sizeof(sharedopts->joinuri), "join:%s", opts->uri)); + + testutil_check(wiredtiger_open(opts->home, NULL, "create,cache_size=1G", &opts->conn)); + + test_join(opts, sharedopts, true, true); + test_join(opts, sharedopts, true, false); + test_join(opts, sharedopts, false, true); + test_join(opts, sharedopts, false, false); + + testutil_cleanup(opts); + + return (0); } static void -test_join(TEST_OPTS *opts, SHARED_OPTS *sharedopts, bool bloom, - bool sometimes_remove) +test_join(TEST_OPTS *opts, SHARED_OPTS *sharedopts, bool bloom, bool sometimes_remove) { - THREAD_ARGS insert_args[N_INSERT_THREAD], join_args[N_JOIN_THREAD]; - WT_CURSOR *maincur; - WT_SESSION *session; - pthread_t insert_tid[N_INSERT_THREAD], join_tid[N_JOIN_THREAD]; - int i; - - memset(insert_args, 0, sizeof(insert_args)); - memset(join_args, 0, sizeof(join_args)); - - sharedopts->bloom = bloom; - sharedopts->remove = sometimes_remove; - - fprintf(stderr, "Running with bloom=%d, remove=%d\n", - (int)bloom, (int)sometimes_remove); - - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - - /* - * Note: id is repeated as id2. This makes it easier to - * identify the primary key in dumps of the index files. - */ - testutil_check(session->create(session, opts->uri, - "key_format=i,value_format=iiSii," - "columns=(id,post,bal,extra,flag,id2)")); - - testutil_check(session->create(session, sharedopts->posturi, - "columns=(post)")); - testutil_check(session->create(session, sharedopts->baluri, - "columns=(bal)")); - testutil_check(session->create(session, sharedopts->flaguri, - "columns=(flag)")); - - /* - * Insert a single record with all items we need to - * call search() on, this makes our join logic easier. - */ - testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, - &maincur)); - maincur->set_key(maincur, N_RECORDS); - maincur->set_value(maincur, 54321, 0, "", 0, N_RECORDS); - testutil_check(maincur->insert(maincur)); - testutil_check(maincur->close(maincur)); - - for (i = 0; i < N_INSERT_THREAD; ++i) { - insert_args[i].threadnum = i; - insert_args[i].nthread = N_INSERT_THREAD; - insert_args[i].testopts = opts; - insert_args[i].sharedopts = sharedopts; - testutil_check(pthread_create( - &insert_tid[i], NULL, thread_insert, &insert_args[i])); - } - - for (i = 0; i < N_JOIN_THREAD; ++i) { - join_args[i].threadnum = i; - join_args[i].nthread = N_JOIN_THREAD; - join_args[i].testopts = opts; - join_args[i].sharedopts = sharedopts; - testutil_check(pthread_create( - &join_tid[i], NULL, thread_join, &join_args[i])); - } - - /* - * Wait for insert threads to finish. When they - * are done, signal join threads to complete. - */ - for (i = 0; i < N_INSERT_THREAD; ++i) - testutil_check(pthread_join(insert_tid[i], NULL)); - - for (i = 0; i < N_JOIN_THREAD; ++i) - join_args[i].done = 1; - - for (i = 0; i < N_JOIN_THREAD; ++i) - testutil_check(pthread_join(join_tid[i], NULL)); - - fprintf(stderr, "\n"); - for (i = 0; i < N_JOIN_THREAD; ++i) { - fprintf(stderr, " join thread %d did %d joins\n", - i, join_args[i].joins); - } - for (i = 0; i < N_INSERT_THREAD; ++i) - fprintf(stderr, - " insert thread %d did " - "%d inserts, %d removes, %d notfound, %d rollbacks\n", - i, insert_args[i].inserts, insert_args[i].removes, - insert_args[i].notfounds, insert_args[i].rollbacks); - - testutil_check(session->drop(session, sharedopts->posturi, NULL)); - testutil_check(session->drop(session, sharedopts->baluri, NULL)); - testutil_check(session->drop(session, sharedopts->flaguri, NULL)); - testutil_check(session->drop(session, opts->uri, NULL)); - testutil_check(session->close(session, NULL)); + THREAD_ARGS insert_args[N_INSERT_THREAD], join_args[N_JOIN_THREAD]; + WT_CURSOR *maincur; + WT_SESSION *session; + pthread_t insert_tid[N_INSERT_THREAD], join_tid[N_JOIN_THREAD]; + int i; + + memset(insert_args, 0, sizeof(insert_args)); + memset(join_args, 0, sizeof(join_args)); + + sharedopts->bloom = bloom; + sharedopts->remove = sometimes_remove; + + fprintf(stderr, "Running with bloom=%d, remove=%d\n", (int)bloom, (int)sometimes_remove); + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + /* + * Note: id is repeated as id2. This makes it easier to identify the primary key in dumps of the + * index files. + */ + testutil_check(session->create(session, opts->uri, + "key_format=i,value_format=iiSii," + "columns=(id,post,bal,extra,flag,id2)")); + + testutil_check(session->create(session, sharedopts->posturi, "columns=(post)")); + testutil_check(session->create(session, sharedopts->baluri, "columns=(bal)")); + testutil_check(session->create(session, sharedopts->flaguri, "columns=(flag)")); + + /* + * Insert a single record with all items we need to call search() on, this makes our join logic + * easier. + */ + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &maincur)); + maincur->set_key(maincur, N_RECORDS); + maincur->set_value(maincur, 54321, 0, "", 0, N_RECORDS); + testutil_check(maincur->insert(maincur)); + testutil_check(maincur->close(maincur)); + + for (i = 0; i < N_INSERT_THREAD; ++i) { + insert_args[i].threadnum = i; + insert_args[i].nthread = N_INSERT_THREAD; + insert_args[i].testopts = opts; + insert_args[i].sharedopts = sharedopts; + testutil_check(pthread_create(&insert_tid[i], NULL, thread_insert, &insert_args[i])); + } + + for (i = 0; i < N_JOIN_THREAD; ++i) { + join_args[i].threadnum = i; + join_args[i].nthread = N_JOIN_THREAD; + join_args[i].testopts = opts; + join_args[i].sharedopts = sharedopts; + testutil_check(pthread_create(&join_tid[i], NULL, thread_join, &join_args[i])); + } + + /* + * Wait for insert threads to finish. When they are done, signal join threads to complete. + */ + for (i = 0; i < N_INSERT_THREAD; ++i) + testutil_check(pthread_join(insert_tid[i], NULL)); + + for (i = 0; i < N_JOIN_THREAD; ++i) + join_args[i].done = 1; + + for (i = 0; i < N_JOIN_THREAD; ++i) + testutil_check(pthread_join(join_tid[i], NULL)); + + fprintf(stderr, "\n"); + for (i = 0; i < N_JOIN_THREAD; ++i) { + fprintf(stderr, " join thread %d did %d joins\n", i, join_args[i].joins); + } + for (i = 0; i < N_INSERT_THREAD; ++i) + fprintf(stderr, + " insert thread %d did " + "%d inserts, %d removes, %d notfound, %d rollbacks\n", + i, insert_args[i].inserts, insert_args[i].removes, insert_args[i].notfounds, + insert_args[i].rollbacks); + + testutil_check(session->drop(session, sharedopts->posturi, NULL)); + testutil_check(session->drop(session, sharedopts->baluri, NULL)); + testutil_check(session->drop(session, sharedopts->flaguri, NULL)); + testutil_check(session->drop(session, opts->uri, NULL)); + testutil_check(session->close(session, NULL)); } static void * thread_insert(void *arg) { - SHARED_OPTS *sharedopts; - TEST_OPTS *opts; - THREAD_ARGS *threadargs; - WT_CURSOR *maincur; - WT_RAND_STATE rnd; - WT_SESSION *session; - int bal, i, flag, key, post, ret; - const char *extra = S1024; - - threadargs = (THREAD_ARGS *)arg; - opts = threadargs->testopts; - sharedopts = threadargs->sharedopts; - __wt_random_init_seed(NULL, &rnd); - - testutil_check(opts->conn->open_session( - opts->conn, NULL, NULL, &session)); - - testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, - &maincur)); - - for (i = 0; i < N_INSERT; i++) { - /* - * Insert threads may stomp on each other's records; - * that's okay. - */ - key = (int)(__wt_random(&rnd) % N_RECORDS); - maincur->set_key(maincur, key); - if (sharedopts->remove) - testutil_check(session->begin_transaction(session, - "isolation=snapshot")); - if (sharedopts->remove && __wt_random(&rnd) % 5 == 0 && - maincur->search(maincur) == 0) { - /* - * Another thread can be removing at the - * same time. - */ - ret = maincur->remove(maincur); - testutil_assert(ret == 0 || - (N_INSERT_THREAD > 1 && - (ret == WT_NOTFOUND || ret == WT_ROLLBACK))); - if (ret == 0) - threadargs->removes++; - else if (ret == WT_NOTFOUND) - threadargs->notfounds++; - else if (ret == WT_ROLLBACK) - threadargs->rollbacks++; - } else { - if (__wt_random(&rnd) % 2 == 0) - post = 54321; - else - post = i % 100000; - if (__wt_random(&rnd) % 2 == 0) { - bal = -100; - flag = 1; - } else { - bal = 1 + (i % 1000) * 100; - flag = 0; - } - maincur->set_value(maincur, post, bal, extra, flag, - key); - ret = maincur->insert(maincur); - testutil_assert(ret == 0 || - (N_INSERT_THREAD > 1 && ret == WT_ROLLBACK)); - testutil_check(maincur->reset(maincur)); - if (ret == 0) - threadargs->inserts++; - else if (ret == WT_ROLLBACK) - threadargs->rollbacks++; - } - if (sharedopts->remove) - testutil_check(session->commit_transaction(session, - NULL)); - if (i % 1000 == 0 && i != 0) { - if (i % 10000 == 0) - fprintf(stderr, "*"); - else - fprintf(stderr, "."); - } - } - testutil_check(maincur->close(maincur)); - testutil_check(session->close(session, NULL)); - return (NULL); + SHARED_OPTS *sharedopts; + TEST_OPTS *opts; + THREAD_ARGS *threadargs; + WT_CURSOR *maincur; + WT_RAND_STATE rnd; + WT_SESSION *session; + int bal, i, flag, key, post, ret; + const char *extra = S1024; + + threadargs = (THREAD_ARGS *)arg; + opts = threadargs->testopts; + sharedopts = threadargs->sharedopts; + __wt_random_init_seed(NULL, &rnd); + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &maincur)); + + for (i = 0; i < N_INSERT; i++) { + /* + * Insert threads may stomp on each other's records; that's okay. + */ + key = (int)(__wt_random(&rnd) % N_RECORDS); + maincur->set_key(maincur, key); + if (sharedopts->remove) + testutil_check(session->begin_transaction(session, "isolation=snapshot")); + if (sharedopts->remove && __wt_random(&rnd) % 5 == 0 && maincur->search(maincur) == 0) { + /* + * Another thread can be removing at the same time. + */ + ret = maincur->remove(maincur); + testutil_assert( + ret == 0 || (N_INSERT_THREAD > 1 && (ret == WT_NOTFOUND || ret == WT_ROLLBACK))); + if (ret == 0) + threadargs->removes++; + else if (ret == WT_NOTFOUND) + threadargs->notfounds++; + else if (ret == WT_ROLLBACK) + threadargs->rollbacks++; + } else { + if (__wt_random(&rnd) % 2 == 0) + post = 54321; + else + post = i % 100000; + if (__wt_random(&rnd) % 2 == 0) { + bal = -100; + flag = 1; + } else { + bal = 1 + (i % 1000) * 100; + flag = 0; + } + maincur->set_value(maincur, post, bal, extra, flag, key); + ret = maincur->insert(maincur); + testutil_assert(ret == 0 || (N_INSERT_THREAD > 1 && ret == WT_ROLLBACK)); + testutil_check(maincur->reset(maincur)); + if (ret == 0) + threadargs->inserts++; + else if (ret == WT_ROLLBACK) + threadargs->rollbacks++; + } + if (sharedopts->remove) + testutil_check(session->commit_transaction(session, NULL)); + if (i % 1000 == 0 && i != 0) { + if (i % 10000 == 0) + fprintf(stderr, "*"); + else + fprintf(stderr, "."); + } + } + testutil_check(maincur->close(maincur)); + testutil_check(session->close(session, NULL)); + return (NULL); } static void * thread_join(void *arg) { - SHARED_OPTS *sharedopts; - TEST_OPTS *opts; - THREAD_ARGS *threadargs; - WT_CURSOR *balcur, *flagcur, *joincur, *postcur; - WT_SESSION *session; - int bal, flag, key, key2, post, ret; - char cfg[128]; - char *extra; - - threadargs = (THREAD_ARGS *)arg; - opts = threadargs->testopts; - sharedopts = threadargs->sharedopts; - - testutil_check(opts->conn->open_session( - opts->conn, NULL, NULL, &session)); - - testutil_check(session->open_cursor( - session, sharedopts->posturi, NULL, NULL, &postcur)); - testutil_check(session->open_cursor( - session, sharedopts->baluri, NULL, NULL, &balcur)); - testutil_check(session->open_cursor( - session, sharedopts->flaguri, NULL, NULL, &flagcur)); - - for (threadargs->joins = 0; threadargs->done == 0; - threadargs->joins++) { - testutil_check(session->open_cursor( - session, sharedopts->joinuri, NULL, NULL, &joincur)); - postcur->set_key(postcur, 54321); - testutil_check(postcur->search(postcur)); - testutil_check(session->join(session, joincur, postcur, - "compare=eq")); - - balcur->set_key(balcur, 0); - testutil_check(balcur->search(balcur)); - if (sharedopts->bloom) - testutil_check(__wt_snprintf(cfg, sizeof(cfg), - "compare=lt,strategy=bloom,count=%d", N_RECORDS)); - else - testutil_check(__wt_snprintf( - cfg, sizeof(cfg), "compare=lt")); - testutil_check(session->join(session, joincur, balcur, cfg)); - - flagcur->set_key(flagcur, 0); - testutil_check(flagcur->search(flagcur)); - if (sharedopts->bloom) - testutil_check(__wt_snprintf(cfg, sizeof(cfg), - "compare=eq,strategy=bloom,count=%d", N_RECORDS)); - else - testutil_check(__wt_snprintf( - cfg, sizeof(cfg), "compare=eq")); - testutil_check(session->join(session, joincur, flagcur, cfg)); - - /* Expect no values returned */ - ret = joincur->next(joincur); - if (ret == 0) { - /* - * The values may already have been changed, but - * print them for informational purposes. - */ - testutil_check(joincur->get_key(joincur, &key)); - testutil_check(joincur->get_value(joincur, &post, - &bal, &extra, &flag, &key2)); - fprintf(stderr, "FAIL: iteration %d: " - "key=%d/%d, postal_code=%d, balance=%d, flag=%d\n", - threadargs->joins, key, key2, post, bal, flag); - /* Save the results. */ - testutil_check(opts->conn->close(opts->conn, NULL)); - opts->conn = NULL; - return (NULL); - } - testutil_assert(ret == WT_NOTFOUND); - testutil_check(joincur->close(joincur)); - - /* - * Reset the cursors, potentially allowing the insert - * threads to proceed. - */ - testutil_check(postcur->reset(postcur)); - testutil_check(balcur->reset(balcur)); - testutil_check(flagcur->reset(flagcur)); - if (threadargs->joins % 100 == 0) - fprintf(stderr, "J"); - } - testutil_check(postcur->close(postcur)); - testutil_check(balcur->close(balcur)); - testutil_check(flagcur->close(flagcur)); - testutil_check(session->close(session, NULL)); - return (NULL); + SHARED_OPTS *sharedopts; + TEST_OPTS *opts; + THREAD_ARGS *threadargs; + WT_CURSOR *balcur, *flagcur, *joincur, *postcur; + WT_SESSION *session; + int bal, flag, key, key2, post, ret; + char cfg[128]; + char *extra; + + threadargs = (THREAD_ARGS *)arg; + opts = threadargs->testopts; + sharedopts = threadargs->sharedopts; + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(session->open_cursor(session, sharedopts->posturi, NULL, NULL, &postcur)); + testutil_check(session->open_cursor(session, sharedopts->baluri, NULL, NULL, &balcur)); + testutil_check(session->open_cursor(session, sharedopts->flaguri, NULL, NULL, &flagcur)); + + for (threadargs->joins = 0; threadargs->done == 0; threadargs->joins++) { + testutil_check(session->open_cursor(session, sharedopts->joinuri, NULL, NULL, &joincur)); + postcur->set_key(postcur, 54321); + testutil_check(postcur->search(postcur)); + testutil_check(session->join(session, joincur, postcur, "compare=eq")); + + balcur->set_key(balcur, 0); + testutil_check(balcur->search(balcur)); + if (sharedopts->bloom) + testutil_check( + __wt_snprintf(cfg, sizeof(cfg), "compare=lt,strategy=bloom,count=%d", N_RECORDS)); + else + testutil_check(__wt_snprintf(cfg, sizeof(cfg), "compare=lt")); + testutil_check(session->join(session, joincur, balcur, cfg)); + + flagcur->set_key(flagcur, 0); + testutil_check(flagcur->search(flagcur)); + if (sharedopts->bloom) + testutil_check( + __wt_snprintf(cfg, sizeof(cfg), "compare=eq,strategy=bloom,count=%d", N_RECORDS)); + else + testutil_check(__wt_snprintf(cfg, sizeof(cfg), "compare=eq")); + testutil_check(session->join(session, joincur, flagcur, cfg)); + + /* Expect no values returned */ + ret = joincur->next(joincur); + if (ret == 0) { + /* + * The values may already have been changed, but print them for informational purposes. + */ + testutil_check(joincur->get_key(joincur, &key)); + testutil_check(joincur->get_value(joincur, &post, &bal, &extra, &flag, &key2)); + fprintf(stderr, + "FAIL: iteration %d: " + "key=%d/%d, postal_code=%d, balance=%d, flag=%d\n", + threadargs->joins, key, key2, post, bal, flag); + /* Save the results. */ + testutil_check(opts->conn->close(opts->conn, NULL)); + opts->conn = NULL; + return (NULL); + } + testutil_assert(ret == WT_NOTFOUND); + testutil_check(joincur->close(joincur)); + + /* + * Reset the cursors, potentially allowing the insert threads to proceed. + */ + testutil_check(postcur->reset(postcur)); + testutil_check(balcur->reset(balcur)); + testutil_check(flagcur->reset(flagcur)); + if (threadargs->joins % 100 == 0) + fprintf(stderr, "J"); + } + testutil_check(postcur->close(postcur)); + testutil_check(balcur->close(balcur)); + testutil_check(flagcur->close(flagcur)); + testutil_check(session->close(session, NULL)); + return (NULL); } diff --git a/src/third_party/wiredtiger/test/csuite/wt2403_lsm_workload/main.c b/src/third_party/wiredtiger/test/csuite/wt2403_lsm_workload/main.c index b8a99b68db2..870304f8252 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2403_lsm_workload/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2403_lsm_workload/main.c @@ -29,213 +29,193 @@ #include "test_util.h" static const char name[] = "lsm:test"; -#define NUM_DOCS 100000 -#define NUM_QUERIES (NUM_DOCS/100) +#define NUM_DOCS 100000 +#define NUM_QUERIES (NUM_DOCS / 100) static void rand_str(uint64_t i, char *str) { - uint64_t x, y; + uint64_t x, y; - y = strlen(str); - for (x = y; x > y - 8; x--) { - str[x - 1] = (char)(i % 10) + 48; - i = i / 10; - } + y = strlen(str); + for (x = y; x > y - 8; x--) { + str[x - 1] = (char)(i % 10) + 48; + i = i / 10; + } } static void check_str(uint64_t i, char *str, bool mod) { - char str2[] = "0000000000000000"; + char str2[] = "0000000000000000"; - rand_str(i, str2); - if (mod) - str2[0] = 'A'; - testutil_checkfmt(strcmp(str, str2), - "strcmp failed, got %s, expected %s", str, str2); + rand_str(i, str2); + if (mod) + str2[0] = 'A'; + testutil_checkfmt(strcmp(str, str2), "strcmp failed, got %s, expected %s", str, str2); } static void query_docs(WT_CURSOR *cursor, bool mod) { - WT_ITEM key, value; - int i; - - for (i = 0; i < NUM_QUERIES; i++) { - testutil_check(cursor->next(cursor)); - testutil_check(cursor->get_key(cursor, &key)); - testutil_check(cursor->get_value(cursor, &value)); - check_str((uint64_t)key.data, (char *)value.data, mod); - } - printf("%d documents read\n", NUM_QUERIES); + WT_ITEM key, value; + int i; + + for (i = 0; i < NUM_QUERIES; i++) { + testutil_check(cursor->next(cursor)); + testutil_check(cursor->get_key(cursor, &key)); + testutil_check(cursor->get_value(cursor, &value)); + check_str((uint64_t)key.data, (char *)value.data, mod); + } + printf("%d documents read\n", NUM_QUERIES); } static void * compact_thread(void *args) { - WT_SESSION *session; + WT_SESSION *session; - session = (WT_SESSION *)args; - testutil_check(session->compact(session, name, NULL)); - return (NULL); + session = (WT_SESSION *)args; + testutil_check(session->compact(session, name, NULL)); + return (NULL); } int main(int argc, char *argv[]) { - TEST_OPTS *opts, _opts; - WT_CURSOR *rcursor, *wcursor; - WT_ITEM key, value; - WT_SESSION *session, *session2; - pthread_t thread; - uint64_t i; - - char str[] = "0000000000000000"; - - /* - * Create a clean test directory for this run of the test program if the - * environment variable isn't already set (as is done by make check). - */ - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - testutil_check(wiredtiger_open(opts->home, - NULL, "create,cache_size=200M", &opts->conn)); - - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session2)); - - testutil_check(session->create(session, name, - "key_format=Q,value_format=S")); - - /* Populate the table with some data. */ - testutil_check(session->open_cursor( - session, name, NULL, "overwrite", &wcursor)); - for (i = 0; i < NUM_DOCS; i++) { - wcursor->set_key(wcursor, i); - rand_str(i, str); - wcursor->set_value(wcursor, str); - testutil_check(wcursor->insert(wcursor)); - } - testutil_check(wcursor->close(wcursor)); - printf("%d documents inserted\n", NUM_DOCS); - - /* Perform some random reads */ - testutil_check(session->open_cursor( - session, name, NULL, "next_random=true", &rcursor)); - query_docs(rcursor, false); - testutil_check(rcursor->close(rcursor)); - - /* Setup Transaction to pin the current values */ - testutil_check( - session2->begin_transaction(session2, "isolation=snapshot")); - testutil_check(session2->open_cursor( - session2, name, NULL, "next_random=true", &rcursor)); - - /* Perform updates in a txn to confirm that we see only the original. */ - testutil_check(session->open_cursor( - session, name, NULL, "overwrite", &wcursor)); - for (i = 0; i < NUM_DOCS; i++) { - rand_str(i, str); - str[0] = 'A'; - wcursor->set_key(wcursor, i); - wcursor->set_value(wcursor, str); - testutil_check(wcursor->update(wcursor)); - } - testutil_check(wcursor->close(wcursor)); - printf("%d documents set to update\n", NUM_DOCS); - - /* Random reads, which should see the original values */ - query_docs(rcursor, false); - testutil_check(rcursor->close(rcursor)); - - /* Finish the txn */ - testutil_check(session2->rollback_transaction(session2, NULL)); - - /* Random reads, which should see the updated values */ - testutil_check(session2->open_cursor( - session2, name, NULL, "next_random=true", &rcursor)); - query_docs(rcursor, true); - testutil_check(rcursor->close(rcursor)); - - /* Setup a pre-delete txn */ - testutil_check( - session2->begin_transaction(session2, "isolation=snapshot")); - testutil_check(session2->open_cursor( - session2, name, NULL, "next_random=true", &rcursor)); - - /* Delete all but one document */ - testutil_check(session->open_cursor( - session, name, NULL, "overwrite", &wcursor)); - for (i = 0; i < NUM_DOCS - 1; i++) { - wcursor->set_key(wcursor, i); - testutil_check(wcursor->remove(wcursor)); - } - testutil_check(wcursor->close(wcursor)); - printf("%d documents deleted\n", NUM_DOCS - 1); - - /* Random reads, which should not see the deletes */ - query_docs(rcursor, true); - testutil_check(rcursor->close(rcursor)); - - /* Rollback the txn so we can see the deletes */ - testutil_check(session2->rollback_transaction(session2, NULL)); - - /* Find the one remaining document 3 times */ - testutil_check(session2->open_cursor( - session2, name, NULL, "next_random=true", &rcursor)); - for (i = 0; i < 3; i++) { - testutil_check(rcursor->next(rcursor)); - testutil_check(rcursor->get_key(rcursor, &key)); - testutil_check(rcursor->get_value(rcursor, &value)); - /* There should only be one value available to us */ - testutil_assertfmt((uint64_t)key.data == NUM_DOCS - 1, - "expected %d and got %" PRIu64, - NUM_DOCS - 1, (uint64_t)key.data); - check_str((uint64_t)key.data, (char *)value.data, true); - } - printf("Found the deleted doc 3 times\n"); - testutil_check(rcursor->close(rcursor)); - - /* Repopulate the table for compact. */ - testutil_check(session->open_cursor( - session, name, NULL, "overwrite", &wcursor)); - for (i = 0; i < NUM_DOCS - 1; i++) { - wcursor->set_key(wcursor, i); - rand_str(i, str); - str[0] = 'A'; - wcursor->set_value(wcursor, str); - testutil_check(wcursor->insert(wcursor)); - } - testutil_check(wcursor->close(wcursor)); - - /* Run random cursor queries while compact is running */ - testutil_check(session2->open_cursor( - session2, name, NULL, "next_random=true", &rcursor)); - testutil_check(pthread_create(&thread, NULL, compact_thread, session)); - query_docs(rcursor, true); - testutil_check(rcursor->close(rcursor)); - testutil_check(pthread_join(thread, NULL)); - - /* Delete everything. Check for infinite loops */ - testutil_check(session->open_cursor( - session, name, NULL, "overwrite", &wcursor)); - for (i = 0; i < NUM_DOCS; i++) { - wcursor->set_key(wcursor, i); - testutil_check(wcursor->remove(wcursor)); - } - testutil_check(wcursor->close(wcursor)); - - testutil_check(session2->open_cursor( - session2, name, NULL, "next_random=true", &rcursor)); - for (i = 0; i < 3; i++) - testutil_assert(rcursor->next(rcursor) == WT_NOTFOUND); - printf("Successfully got WT_NOTFOUND\n"); - - testutil_cleanup(opts); - return (EXIT_SUCCESS); + TEST_OPTS *opts, _opts; + WT_CURSOR *rcursor, *wcursor; + WT_ITEM key, value; + WT_SESSION *session, *session2; + pthread_t thread; + uint64_t i; + + char str[] = "0000000000000000"; + + /* + * Create a clean test directory for this run of the test program if the environment variable + * isn't already set (as is done by make check). + */ + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + testutil_check(wiredtiger_open(opts->home, NULL, "create,cache_size=200M", &opts->conn)); + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session2)); + + testutil_check(session->create(session, name, "key_format=Q,value_format=S")); + + /* Populate the table with some data. */ + testutil_check(session->open_cursor(session, name, NULL, "overwrite", &wcursor)); + for (i = 0; i < NUM_DOCS; i++) { + wcursor->set_key(wcursor, i); + rand_str(i, str); + wcursor->set_value(wcursor, str); + testutil_check(wcursor->insert(wcursor)); + } + testutil_check(wcursor->close(wcursor)); + printf("%d documents inserted\n", NUM_DOCS); + + /* Perform some random reads */ + testutil_check(session->open_cursor(session, name, NULL, "next_random=true", &rcursor)); + query_docs(rcursor, false); + testutil_check(rcursor->close(rcursor)); + + /* Setup Transaction to pin the current values */ + testutil_check(session2->begin_transaction(session2, "isolation=snapshot")); + testutil_check(session2->open_cursor(session2, name, NULL, "next_random=true", &rcursor)); + + /* Perform updates in a txn to confirm that we see only the original. */ + testutil_check(session->open_cursor(session, name, NULL, "overwrite", &wcursor)); + for (i = 0; i < NUM_DOCS; i++) { + rand_str(i, str); + str[0] = 'A'; + wcursor->set_key(wcursor, i); + wcursor->set_value(wcursor, str); + testutil_check(wcursor->update(wcursor)); + } + testutil_check(wcursor->close(wcursor)); + printf("%d documents set to update\n", NUM_DOCS); + + /* Random reads, which should see the original values */ + query_docs(rcursor, false); + testutil_check(rcursor->close(rcursor)); + + /* Finish the txn */ + testutil_check(session2->rollback_transaction(session2, NULL)); + + /* Random reads, which should see the updated values */ + testutil_check(session2->open_cursor(session2, name, NULL, "next_random=true", &rcursor)); + query_docs(rcursor, true); + testutil_check(rcursor->close(rcursor)); + + /* Setup a pre-delete txn */ + testutil_check(session2->begin_transaction(session2, "isolation=snapshot")); + testutil_check(session2->open_cursor(session2, name, NULL, "next_random=true", &rcursor)); + + /* Delete all but one document */ + testutil_check(session->open_cursor(session, name, NULL, "overwrite", &wcursor)); + for (i = 0; i < NUM_DOCS - 1; i++) { + wcursor->set_key(wcursor, i); + testutil_check(wcursor->remove(wcursor)); + } + testutil_check(wcursor->close(wcursor)); + printf("%d documents deleted\n", NUM_DOCS - 1); + + /* Random reads, which should not see the deletes */ + query_docs(rcursor, true); + testutil_check(rcursor->close(rcursor)); + + /* Rollback the txn so we can see the deletes */ + testutil_check(session2->rollback_transaction(session2, NULL)); + + /* Find the one remaining document 3 times */ + testutil_check(session2->open_cursor(session2, name, NULL, "next_random=true", &rcursor)); + for (i = 0; i < 3; i++) { + testutil_check(rcursor->next(rcursor)); + testutil_check(rcursor->get_key(rcursor, &key)); + testutil_check(rcursor->get_value(rcursor, &value)); + /* There should only be one value available to us */ + testutil_assertfmt((uint64_t)key.data == NUM_DOCS - 1, "expected %d and got %" PRIu64, + NUM_DOCS - 1, (uint64_t)key.data); + check_str((uint64_t)key.data, (char *)value.data, true); + } + printf("Found the deleted doc 3 times\n"); + testutil_check(rcursor->close(rcursor)); + + /* Repopulate the table for compact. */ + testutil_check(session->open_cursor(session, name, NULL, "overwrite", &wcursor)); + for (i = 0; i < NUM_DOCS - 1; i++) { + wcursor->set_key(wcursor, i); + rand_str(i, str); + str[0] = 'A'; + wcursor->set_value(wcursor, str); + testutil_check(wcursor->insert(wcursor)); + } + testutil_check(wcursor->close(wcursor)); + + /* Run random cursor queries while compact is running */ + testutil_check(session2->open_cursor(session2, name, NULL, "next_random=true", &rcursor)); + testutil_check(pthread_create(&thread, NULL, compact_thread, session)); + query_docs(rcursor, true); + testutil_check(rcursor->close(rcursor)); + testutil_check(pthread_join(thread, NULL)); + + /* Delete everything. Check for infinite loops */ + testutil_check(session->open_cursor(session, name, NULL, "overwrite", &wcursor)); + for (i = 0; i < NUM_DOCS; i++) { + wcursor->set_key(wcursor, i); + testutil_check(wcursor->remove(wcursor)); + } + testutil_check(wcursor->close(wcursor)); + + testutil_check(session2->open_cursor(session2, name, NULL, "next_random=true", &rcursor)); + for (i = 0; i < 3; i++) + testutil_assert(rcursor->next(rcursor) == WT_NOTFOUND); + printf("Successfully got WT_NOTFOUND\n"); + + testutil_cleanup(opts); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/main.c b/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/main.c index 89c186501f5..f1b01e4e977 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/main.c @@ -49,142 +49,126 @@ * table. */ -#define N_RECORDS 10000 +#define N_RECORDS 10000 static void -get_stat_total(WT_SESSION *session, WT_CURSOR *jcursor, const char *descmatch, - uint64_t *pval) +get_stat_total(WT_SESSION *session, WT_CURSOR *jcursor, const char *descmatch, uint64_t *pval) { - WT_CURSOR *statcursor; - WT_DECL_RET; - uint64_t val; - char *desc, *valstr; - bool match; - - match = false; - *pval = 0; - testutil_check(session->open_cursor(session, "statistics:join", jcursor, - NULL, &statcursor)); - - while ((ret = statcursor->next(statcursor)) == 0) { - testutil_assert(statcursor->get_value( - statcursor, &desc, &valstr, &val) == 0); - - printf("statistics: %s: %s: %" PRIu64 "\n", desc, valstr, val); - - if (strstr(desc, descmatch) != NULL) { - *pval += val; - match = true; - } - } - testutil_assert(ret == WT_NOTFOUND); - testutil_check(statcursor->close(statcursor)); - testutil_assert(match); + WT_CURSOR *statcursor; + WT_DECL_RET; + uint64_t val; + char *desc, *valstr; + bool match; + + match = false; + *pval = 0; + testutil_check(session->open_cursor(session, "statistics:join", jcursor, NULL, &statcursor)); + + while ((ret = statcursor->next(statcursor)) == 0) { + testutil_assert(statcursor->get_value(statcursor, &desc, &valstr, &val) == 0); + + printf("statistics: %s: %s: %" PRIu64 "\n", desc, valstr, val); + + if (strstr(desc, descmatch) != NULL) { + *pval += val; + match = true; + } + } + testutil_assert(ret == WT_NOTFOUND); + testutil_check(statcursor->close(statcursor)); + testutil_assert(match); } int main(int argc, char *argv[]) { - TEST_OPTS *opts, _opts; - WT_CURSOR *cursor1, *cursor2, *jcursor; - WT_ITEM d; - WT_SESSION *session; - uint64_t maincount; - int half, i, j; - char bloom_cfg[128], index1uri[256], index2uri[256], joinuri[256]; - const char *tablename; - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - - tablename = strchr(opts->uri, ':'); - testutil_assert(tablename != NULL); - tablename++; - testutil_check(__wt_snprintf( - index1uri, sizeof(index1uri), "index:%s:index1", tablename)); - testutil_check(__wt_snprintf( - index2uri, sizeof(index2uri), "index:%s:index2", tablename)); - testutil_check(__wt_snprintf( - joinuri, sizeof(joinuri), "join:%s", opts->uri)); - - testutil_check(wiredtiger_open(opts->home, NULL, - "statistics=(all),create", &opts->conn)); - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - - testutil_check(session->create(session, opts->uri, - "key_format=i,value_format=iiu,columns=(k,v1,v2,d)")); - testutil_check(session->create(session, index1uri, "columns=(v1)")); - testutil_check(session->create(session, index2uri, "columns=(v2)")); - - testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, - &cursor1)); - - d.size = 4100; - d.data = dmalloc(d.size); - memset((char *)d.data, 7, d.size); - - for (i = 0; i < N_RECORDS; ++i) - { - cursor1->set_key(cursor1, i); - cursor1->set_value(cursor1, i, i, &d); - testutil_check(cursor1->insert(cursor1)); - } - - free((void*)d.data); - - testutil_check(opts->conn->close(opts->conn, NULL)); - testutil_check(wiredtiger_open(opts->home, NULL, - "statistics=(all),create,cache_size=1GB", &opts->conn)); - testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, - &session)); - - testutil_check(session->open_cursor(session, index1uri, NULL, NULL, - &cursor1)); - testutil_check(session->open_cursor(session, index2uri, NULL, NULL, - &cursor2)); - - half = N_RECORDS / 2; - cursor1->set_key(cursor1, half); - testutil_check(cursor1->search(cursor1)); - - cursor2->set_key(cursor2, half + 1); - testutil_check(cursor2->search(cursor2)); - - testutil_check(__wt_snprintf(bloom_cfg, sizeof(bloom_cfg), - "compare=lt,strategy=bloom,count=%d", half)); - - testutil_check(session->open_cursor(session, joinuri, NULL, NULL, - &jcursor)); - testutil_check(session->join(session, jcursor, cursor1, "compare=ge")); - testutil_check(session->join(session, jcursor, cursor2, bloom_cfg)); - - /* Expect one value returned */ - testutil_assert(jcursor->next(jcursor) == 0); - i = 0; - testutil_assert(jcursor->get_key(jcursor, &i) == 0); - testutil_assert(i == (int)half); - i = j = 0; - memset(&d, 0, sizeof(d)); - testutil_assert(jcursor->get_value(jcursor, &i, &j, &d) == 0); - testutil_assert(i == (int)half); - testutil_assert(j == (int)half); - testutil_assert(d.size == 4100); - for (i = 0; i < 4100; i++) - testutil_assert(((char *)d.data)[i] == 7); - - testutil_assert(jcursor->next(jcursor) == WT_NOTFOUND); - - /* - * Make sure there have been 2 accesses to the main table, - * explained in the discussion above. - */ - get_stat_total(session, jcursor, "accesses to the main table", - &maincount); - testutil_assert(maincount == 2); - - testutil_cleanup(opts); - return (EXIT_SUCCESS); + TEST_OPTS *opts, _opts; + WT_CURSOR *cursor1, *cursor2, *jcursor; + WT_ITEM d; + WT_SESSION *session; + uint64_t maincount; + int half, i, j; + char bloom_cfg[128], index1uri[256], index2uri[256], joinuri[256]; + const char *tablename; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + tablename = strchr(opts->uri, ':'); + testutil_assert(tablename != NULL); + tablename++; + testutil_check(__wt_snprintf(index1uri, sizeof(index1uri), "index:%s:index1", tablename)); + testutil_check(__wt_snprintf(index2uri, sizeof(index2uri), "index:%s:index2", tablename)); + testutil_check(__wt_snprintf(joinuri, sizeof(joinuri), "join:%s", opts->uri)); + + testutil_check(wiredtiger_open(opts->home, NULL, "statistics=(all),create", &opts->conn)); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check( + session->create(session, opts->uri, "key_format=i,value_format=iiu,columns=(k,v1,v2,d)")); + testutil_check(session->create(session, index1uri, "columns=(v1)")); + testutil_check(session->create(session, index2uri, "columns=(v2)")); + + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &cursor1)); + + d.size = 4100; + d.data = dmalloc(d.size); + memset((char *)d.data, 7, d.size); + + for (i = 0; i < N_RECORDS; ++i) { + cursor1->set_key(cursor1, i); + cursor1->set_value(cursor1, i, i, &d); + testutil_check(cursor1->insert(cursor1)); + } + + free((void *)d.data); + + testutil_check(opts->conn->close(opts->conn, NULL)); + testutil_check( + wiredtiger_open(opts->home, NULL, "statistics=(all),create,cache_size=1GB", &opts->conn)); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(session->open_cursor(session, index1uri, NULL, NULL, &cursor1)); + testutil_check(session->open_cursor(session, index2uri, NULL, NULL, &cursor2)); + + half = N_RECORDS / 2; + cursor1->set_key(cursor1, half); + testutil_check(cursor1->search(cursor1)); + + cursor2->set_key(cursor2, half + 1); + testutil_check(cursor2->search(cursor2)); + + testutil_check( + __wt_snprintf(bloom_cfg, sizeof(bloom_cfg), "compare=lt,strategy=bloom,count=%d", half)); + + testutil_check(session->open_cursor(session, joinuri, NULL, NULL, &jcursor)); + testutil_check(session->join(session, jcursor, cursor1, "compare=ge")); + testutil_check(session->join(session, jcursor, cursor2, bloom_cfg)); + + /* Expect one value returned */ + testutil_assert(jcursor->next(jcursor) == 0); + i = 0; + testutil_assert(jcursor->get_key(jcursor, &i) == 0); + testutil_assert(i == (int)half); + i = j = 0; + memset(&d, 0, sizeof(d)); + testutil_assert(jcursor->get_value(jcursor, &i, &j, &d) == 0); + testutil_assert(i == (int)half); + testutil_assert(j == (int)half); + testutil_assert(d.size == 4100); + for (i = 0; i < 4100; i++) + testutil_assert(((char *)d.data)[i] == 7); + + testutil_assert(jcursor->next(jcursor) == WT_NOTFOUND); + + /* + * Make sure there have been 2 accesses to the main table, explained in the discussion above. + */ + get_stat_total(session, jcursor, "accesses to the main table", &maincount); + testutil_assert(maincount == 2); + + testutil_cleanup(opts); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c b/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c index 142d794e5d8..376dc5f81ef 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c @@ -28,11 +28,9 @@ #include "test_util.h" /* - * JIRA ticket reference: WT-2535 - * Test case description: This is a test case that looks for lost updates to - * a single record. That is multiple threads each do the same number of read - * modify write operations on a single record. At the end verify that the - * data contains the expected value. + * JIRA ticket reference: WT-2535 Test case description: This is a test case that looks for lost + * updates to a single record. That is multiple threads each do the same number of read modify write + * operations on a single record. At the end verify that the data contains the expected value. * Failure mode: Check that the data is correct at the end of the run. */ @@ -43,126 +41,118 @@ static uint64_t ready_counter; int main(int argc, char *argv[]) { - TEST_OPTS *opts, _opts; - WT_CURSOR *c; - WT_SESSION *session; - clock_t ce, cs; - pthread_t id[100]; - uint64_t current_value; - int i; - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - opts->nthreads = 20; - opts->nrecords = 100000; - opts->table_type = TABLE_ROW; - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - - testutil_check(wiredtiger_open(opts->home, NULL, - "create," - "cache_size=2G," - "eviction=(threads_max=5)," - "statistics=(fast)", &opts->conn)); - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check(session->create(session, opts->uri, - "key_format=Q,value_format=Q," - "leaf_page_max=32k,")); - - /* Create the single record. */ - testutil_check( - session->open_cursor(session, opts->uri, NULL, NULL, &c)); - c->set_key(c, 1); - c->set_value(c, 0); - testutil_check(c->insert(c)); - testutil_check(c->close(c)); - cs = clock(); - for (i = 0; i < (int)opts->nthreads; ++i) { - testutil_check( - pthread_create(&id[i], NULL, thread_insert_race, opts)); - } - while (--i >= 0) - testutil_check(pthread_join(id[i], NULL)); - testutil_check( - session->open_cursor(session, opts->uri, NULL, NULL, &c)); - c->set_key(c, 1); - testutil_check(c->search(c)); - testutil_check(c->get_value(c, ¤t_value)); - if (current_value != opts->nthreads * opts->nrecords) { - fprintf(stderr, - "ERROR: didn't get expected number of changes\n"); - fprintf(stderr, "got: %" PRIu64 ", expected: %" PRIu64 "\n", - current_value, opts->nthreads * opts->nrecords); - return (EXIT_FAILURE); - } - testutil_check(session->close(session, NULL)); - ce = clock(); - printf("%" PRIu64 ": %.2lf\n", - opts->nrecords, (ce - cs) / (double)CLOCKS_PER_SEC); - - testutil_cleanup(opts); - return (EXIT_SUCCESS); + TEST_OPTS *opts, _opts; + WT_CURSOR *c; + WT_SESSION *session; + clock_t ce, cs; + pthread_t id[100]; + uint64_t current_value; + int i; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + opts->nthreads = 20; + opts->nrecords = 100000; + opts->table_type = TABLE_ROW; + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(wiredtiger_open(opts->home, NULL, + "create," + "cache_size=2G," + "eviction=(threads_max=5)," + "statistics=(fast)", + &opts->conn)); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(session->create(session, opts->uri, + "key_format=Q,value_format=Q," + "leaf_page_max=32k,")); + + /* Create the single record. */ + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &c)); + c->set_key(c, 1); + c->set_value(c, 0); + testutil_check(c->insert(c)); + testutil_check(c->close(c)); + cs = clock(); + for (i = 0; i < (int)opts->nthreads; ++i) { + testutil_check(pthread_create(&id[i], NULL, thread_insert_race, opts)); + } + while (--i >= 0) + testutil_check(pthread_join(id[i], NULL)); + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &c)); + c->set_key(c, 1); + testutil_check(c->search(c)); + testutil_check(c->get_value(c, ¤t_value)); + if (current_value != opts->nthreads * opts->nrecords) { + fprintf(stderr, "ERROR: didn't get expected number of changes\n"); + fprintf(stderr, "got: %" PRIu64 ", expected: %" PRIu64 "\n", current_value, + opts->nthreads * opts->nrecords); + return (EXIT_FAILURE); + } + testutil_check(session->close(session, NULL)); + ce = clock(); + printf("%" PRIu64 ": %.2lf\n", opts->nrecords, (ce - cs) / (double)CLOCKS_PER_SEC); + + testutil_cleanup(opts); + return (EXIT_SUCCESS); } /* - * Append to a table in a "racy" fashion - that is attempt to insert the - * same record another thread is likely to also be inserting. + * Append to a table in a "racy" fashion - that is attempt to insert the same record another thread + * is likely to also be inserting. */ void * thread_insert_race(void *arg) { - TEST_OPTS *opts; - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_DECL_RET; - WT_SESSION *session; - uint64_t i, value, ready_counter_local; - - opts = (TEST_OPTS *)arg; - conn = opts->conn; - - printf("Running insert thread\n"); - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->open_cursor( - session, opts->uri, NULL, NULL, &cursor)); - - /* Wait until all the threads are ready to go. */ - (void)__wt_atomic_add64(&ready_counter, 1); - for (;; __wt_yield()) { - WT_ORDERED_READ(ready_counter_local, ready_counter); - if (ready_counter_local >= opts->nthreads) - break; - } - - for (i = 0; i < opts->nrecords; ++i) { - testutil_check( - session->begin_transaction(session, "isolation=snapshot")); - cursor->set_key(cursor, 1); - testutil_check(cursor->search(cursor)); - testutil_check(cursor->get_value(cursor, &value)); - cursor->set_key(cursor, 1); - cursor->set_value(cursor, value + 1); - if ((ret = cursor->update(cursor)) != 0) { - if (ret == WT_ROLLBACK) { - testutil_check(session->rollback_transaction( - session, NULL)); - i--; - continue; - } - printf("Error in update: %d\n", ret); - } - testutil_check(session->commit_transaction(session, NULL)); - if (i % 10000 == 0) { - printf("insert: %" PRIu64 "\r", i); - fflush(stdout); - } - } - if (i > 10000) - printf("\n"); - - opts->running = false; - - return (NULL); + TEST_OPTS *opts; + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_DECL_RET; + WT_SESSION *session; + uint64_t i, value, ready_counter_local; + + opts = (TEST_OPTS *)arg; + conn = opts->conn; + + printf("Running insert thread\n"); + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &cursor)); + + /* Wait until all the threads are ready to go. */ + (void)__wt_atomic_add64(&ready_counter, 1); + for (;; __wt_yield()) { + WT_ORDERED_READ(ready_counter_local, ready_counter); + if (ready_counter_local >= opts->nthreads) + break; + } + + for (i = 0; i < opts->nrecords; ++i) { + testutil_check(session->begin_transaction(session, "isolation=snapshot")); + cursor->set_key(cursor, 1); + testutil_check(cursor->search(cursor)); + testutil_check(cursor->get_value(cursor, &value)); + cursor->set_key(cursor, 1); + cursor->set_value(cursor, value + 1); + if ((ret = cursor->update(cursor)) != 0) { + if (ret == WT_ROLLBACK) { + testutil_check(session->rollback_transaction(session, NULL)); + i--; + continue; + } + printf("Error in update: %d\n", ret); + } + testutil_check(session->commit_transaction(session, NULL)); + if (i % 10000 == 0) { + printf("insert: %" PRIu64 "\r", i); + fflush(stdout); + } + } + if (i > 10000) + printf("\n"); + + opts->running = false; + + return (NULL); } diff --git a/src/third_party/wiredtiger/test/csuite/wt2592_join_schema/main.c b/src/third_party/wiredtiger/test/csuite/wt2592_join_schema/main.c index 0d165df2b45..60cfbadb034 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2592_join_schema/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2592_join_schema/main.c @@ -28,190 +28,154 @@ #include "test_util.h" /* - * JIRA ticket reference: WT-2592 - * Test case description: This is an adaptation of the join parts of - * ex_schema.c, but written as a test. Though we have join tests in the - * Python test suite, the Python API uses raw mode for cursors, so errors - * that are specific to non-raw mode are undetected in Python. - * Failure mode: The failure seen in WT-2592 was that no items were returned - * by a join. + * JIRA ticket reference: WT-2592 Test case description: This is an adaptation of the join parts of + * ex_schema.c, but written as a test. Though we have join tests in the Python test suite, the + * Python API uses raw mode for cursors, so errors that are specific to non-raw mode are undetected + * in Python. Failure mode: The failure seen in WT-2592 was that no items were returned by a join. */ /* The C struct for the data we are storing in a WiredTiger table. */ typedef struct { - char country[5]; - uint16_t year; - uint64_t population; + char country[5]; + uint16_t year; + uint64_t population; } POP_RECORD; -static POP_RECORD pop_data[] = { - { "AU", 1900, 4000000 }, - { "AU", 1950, 8267337 }, - { "AU", 2000, 19053186 }, - { "CAN", 1900, 5500000 }, - { "CAN", 1950, 14011422 }, - { "CAN", 2000, 31099561 }, - { "UK", 1900, 369000000 }, - { "UK", 1950, 50127000 }, - { "UK", 2000, 59522468 }, - { "USA", 1900, 76212168 }, - { "USA", 1950, 150697361 }, - { "USA", 2000, 301279593 }, - { "", 0, 0 } -}; +static POP_RECORD pop_data[] = {{"AU", 1900, 4000000}, {"AU", 1950, 8267337}, + {"AU", 2000, 19053186}, {"CAN", 1900, 5500000}, {"CAN", 1950, 14011422}, {"CAN", 2000, 31099561}, + {"UK", 1900, 369000000}, {"UK", 1950, 50127000}, {"UK", 2000, 59522468}, {"USA", 1900, 76212168}, + {"USA", 1950, 150697361}, {"USA", 2000, 301279593}, {"", 0, 0}}; int main(int argc, char *argv[]) { - POP_RECORD *p; - TEST_OPTS *opts, _opts; - WT_CURSOR *country_cursor, *country_cursor2, *cursor, *join_cursor, - *subjoin_cursor, *year_cursor; - WT_SESSION *session; - const char *country, *tablename; - char countryuri[256], joinuri[256], yearuri[256]; - uint64_t population, recno; - uint16_t year; - int count, ret; - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - - tablename = strchr(opts->uri, ':'); - testutil_assert(tablename != NULL); - tablename++; - testutil_check(__wt_snprintf( - countryuri, sizeof(countryuri), "index:%s:country", tablename)); - testutil_check(__wt_snprintf( - yearuri, sizeof(yearuri), "index:%s:year", tablename)); - testutil_check(__wt_snprintf( - joinuri, sizeof(joinuri), "join:%s", opts->uri)); - - testutil_check(wiredtiger_open(opts->home, NULL, - "create,cache_size=200M", &opts->conn)); - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check(session->create(session, opts->uri, - "key_format=r," - "value_format=5sHQ," - "columns=(id,country,year,population)")); - - /* Create an index with a simple key. */ - testutil_check(session->create(session, - countryuri, "columns=(country)")); - - /* Create an immutable index. */ - testutil_check(session->create(session, - yearuri, "columns=(year),immutable")); - - /* Insert the records into the table. */ - testutil_check(session->open_cursor( - session, opts->uri, NULL, "append", &cursor)); - count = 1; - for (p = pop_data; p->year != 0; p++) { - cursor->set_key(cursor, count); - cursor->set_value(cursor, p->country, p->year, p->population); - testutil_check(cursor->insert(cursor)); - count++; - } - testutil_check(cursor->close(cursor)); - - /* Open cursors needed by the join. */ - testutil_check(session->open_cursor(session, - joinuri, NULL, NULL, &join_cursor)); - testutil_check(session->open_cursor(session, - countryuri, NULL, NULL, &country_cursor)); - testutil_check(session->open_cursor(session, - yearuri, NULL, NULL, &year_cursor)); - - /* select values WHERE country == "AU" AND year > 1900 */ - country_cursor->set_key(country_cursor, "AU\0\0\0"); - testutil_check(country_cursor->search(country_cursor)); - testutil_check(session->join(session, join_cursor, country_cursor, - "compare=eq,count=10")); - year_cursor->set_key(year_cursor, (uint16_t)1900); - testutil_check(year_cursor->search(year_cursor)); - testutil_check(session->join(session, join_cursor, year_cursor, - "compare=gt,count=10,strategy=bloom")); - - count = 0; - /* List the values that are joined */ - while ((ret = join_cursor->next(join_cursor)) == 0) { - testutil_check(join_cursor->get_key(join_cursor, &recno)); - testutil_check(join_cursor->get_value(join_cursor, &country, - &year, &population)); - printf("ID %" PRIu64, recno); - printf( - ": country %s, year %" PRIu16 ", population %" PRIu64 "\n", - country, year, population); - count++; - } - testutil_assert(ret == WT_NOTFOUND); - testutil_assert(count == 2); - - testutil_check(join_cursor->close(join_cursor)); - testutil_check(year_cursor->close(year_cursor)); - testutil_check(country_cursor->close(country_cursor)); - - /* Open cursors needed by the join. */ - testutil_check(session->open_cursor(session, - joinuri, NULL, NULL, &join_cursor)); - testutil_check(session->open_cursor(session, - joinuri, NULL, NULL, &subjoin_cursor)); - testutil_check(session->open_cursor(session, - countryuri, NULL, NULL, &country_cursor)); - testutil_check(session->open_cursor(session, - countryuri, NULL, NULL, &country_cursor2)); - testutil_check(session->open_cursor(session, - yearuri, NULL, NULL, &year_cursor)); - - /* - * select values WHERE (country == "AU" OR country == "UK") - * AND year > 1900 - * - * First, set up the join representing the country clause. - */ - country_cursor->set_key(country_cursor, "AU\0\0\0"); - testutil_check(country_cursor->search(country_cursor)); - testutil_check(session->join(session, subjoin_cursor, country_cursor, - "operation=or,compare=eq,count=10")); - country_cursor2->set_key(country_cursor2, "UK\0\0\0"); - testutil_check(country_cursor2->search(country_cursor2)); - testutil_check(session->join(session, subjoin_cursor, country_cursor2, - "operation=or,compare=eq,count=10")); - - /* Join that to the top join, and add the year clause */ - testutil_check(session->join(session, join_cursor, subjoin_cursor, - NULL)); - year_cursor->set_key(year_cursor, (uint16_t)1900); - testutil_check(year_cursor->search(year_cursor)); - testutil_check(session->join(session, join_cursor, year_cursor, - "compare=gt,count=10,strategy=bloom")); - - count = 0; - /* List the values that are joined */ - while ((ret = join_cursor->next(join_cursor)) == 0) { - testutil_check(join_cursor->get_key(join_cursor, &recno)); - testutil_check(join_cursor->get_value(join_cursor, &country, - &year, &population)); - printf("ID %" PRIu64, recno); - printf( - ": country %s, year %" PRIu16 ", population %" PRIu64 "\n", - country, year, population); - count++; - } - testutil_assert(ret == WT_NOTFOUND); - testutil_assert(count == 4); - - testutil_check(join_cursor->close(join_cursor)); - testutil_check(subjoin_cursor->close(subjoin_cursor)); - testutil_check(country_cursor->close(country_cursor)); - testutil_check(country_cursor2->close(country_cursor2)); - testutil_check(year_cursor->close(year_cursor)); - testutil_check(session->close(session, NULL)); - - testutil_cleanup(opts); - return (EXIT_SUCCESS); + POP_RECORD *p; + TEST_OPTS *opts, _opts; + WT_CURSOR *country_cursor, *country_cursor2, *cursor, *join_cursor, *subjoin_cursor, + *year_cursor; + WT_SESSION *session; + const char *country, *tablename; + char countryuri[256], joinuri[256], yearuri[256]; + uint64_t population, recno; + uint16_t year; + int count, ret; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + tablename = strchr(opts->uri, ':'); + testutil_assert(tablename != NULL); + tablename++; + testutil_check(__wt_snprintf(countryuri, sizeof(countryuri), "index:%s:country", tablename)); + testutil_check(__wt_snprintf(yearuri, sizeof(yearuri), "index:%s:year", tablename)); + testutil_check(__wt_snprintf(joinuri, sizeof(joinuri), "join:%s", opts->uri)); + + testutil_check(wiredtiger_open(opts->home, NULL, "create,cache_size=200M", &opts->conn)); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(session->create(session, opts->uri, + "key_format=r," + "value_format=5sHQ," + "columns=(id,country,year,population)")); + + /* Create an index with a simple key. */ + testutil_check(session->create(session, countryuri, "columns=(country)")); + + /* Create an immutable index. */ + testutil_check(session->create(session, yearuri, "columns=(year),immutable")); + + /* Insert the records into the table. */ + testutil_check(session->open_cursor(session, opts->uri, NULL, "append", &cursor)); + count = 1; + for (p = pop_data; p->year != 0; p++) { + cursor->set_key(cursor, count); + cursor->set_value(cursor, p->country, p->year, p->population); + testutil_check(cursor->insert(cursor)); + count++; + } + testutil_check(cursor->close(cursor)); + + /* Open cursors needed by the join. */ + testutil_check(session->open_cursor(session, joinuri, NULL, NULL, &join_cursor)); + testutil_check(session->open_cursor(session, countryuri, NULL, NULL, &country_cursor)); + testutil_check(session->open_cursor(session, yearuri, NULL, NULL, &year_cursor)); + + /* select values WHERE country == "AU" AND year > 1900 */ + country_cursor->set_key(country_cursor, "AU\0\0\0"); + testutil_check(country_cursor->search(country_cursor)); + testutil_check(session->join(session, join_cursor, country_cursor, "compare=eq,count=10")); + year_cursor->set_key(year_cursor, (uint16_t)1900); + testutil_check(year_cursor->search(year_cursor)); + testutil_check( + session->join(session, join_cursor, year_cursor, "compare=gt,count=10,strategy=bloom")); + + count = 0; + /* List the values that are joined */ + while ((ret = join_cursor->next(join_cursor)) == 0) { + testutil_check(join_cursor->get_key(join_cursor, &recno)); + testutil_check(join_cursor->get_value(join_cursor, &country, &year, &population)); + printf("ID %" PRIu64, recno); + printf( + ": country %s, year %" PRIu16 ", population %" PRIu64 "\n", country, year, population); + count++; + } + testutil_assert(ret == WT_NOTFOUND); + testutil_assert(count == 2); + + testutil_check(join_cursor->close(join_cursor)); + testutil_check(year_cursor->close(year_cursor)); + testutil_check(country_cursor->close(country_cursor)); + + /* Open cursors needed by the join. */ + testutil_check(session->open_cursor(session, joinuri, NULL, NULL, &join_cursor)); + testutil_check(session->open_cursor(session, joinuri, NULL, NULL, &subjoin_cursor)); + testutil_check(session->open_cursor(session, countryuri, NULL, NULL, &country_cursor)); + testutil_check(session->open_cursor(session, countryuri, NULL, NULL, &country_cursor2)); + testutil_check(session->open_cursor(session, yearuri, NULL, NULL, &year_cursor)); + + /* + * select values WHERE (country == "AU" OR country == "UK") + * AND year > 1900 + * + * First, set up the join representing the country clause. + */ + country_cursor->set_key(country_cursor, "AU\0\0\0"); + testutil_check(country_cursor->search(country_cursor)); + testutil_check( + session->join(session, subjoin_cursor, country_cursor, "operation=or,compare=eq,count=10")); + country_cursor2->set_key(country_cursor2, "UK\0\0\0"); + testutil_check(country_cursor2->search(country_cursor2)); + testutil_check( + session->join(session, subjoin_cursor, country_cursor2, "operation=or,compare=eq,count=10")); + + /* Join that to the top join, and add the year clause */ + testutil_check(session->join(session, join_cursor, subjoin_cursor, NULL)); + year_cursor->set_key(year_cursor, (uint16_t)1900); + testutil_check(year_cursor->search(year_cursor)); + testutil_check( + session->join(session, join_cursor, year_cursor, "compare=gt,count=10,strategy=bloom")); + + count = 0; + /* List the values that are joined */ + while ((ret = join_cursor->next(join_cursor)) == 0) { + testutil_check(join_cursor->get_key(join_cursor, &recno)); + testutil_check(join_cursor->get_value(join_cursor, &country, &year, &population)); + printf("ID %" PRIu64, recno); + printf( + ": country %s, year %" PRIu16 ", population %" PRIu64 "\n", country, year, population); + count++; + } + testutil_assert(ret == WT_NOTFOUND); + testutil_assert(count == 4); + + testutil_check(join_cursor->close(join_cursor)); + testutil_check(subjoin_cursor->close(subjoin_cursor)); + testutil_check(country_cursor->close(country_cursor)); + testutil_check(country_cursor2->close(country_cursor2)); + testutil_check(year_cursor->close(year_cursor)); + testutil_check(session->close(session, NULL)); + + testutil_cleanup(opts); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt2695_checksum/main.c b/src/third_party/wiredtiger/test/csuite/wt2695_checksum/main.c index 6231677b8df..646c1cbc894 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2695_checksum/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2695_checksum/main.c @@ -28,120 +28,116 @@ #include "test_util.h" /* - * JIRA ticket reference: WT-2695 - * Test case description: Smoke-test the CRC. + * JIRA ticket reference: WT-2695 Test case description: Smoke-test the CRC. */ static inline void check(uint32_t hw, uint32_t sw, size_t len, const char *msg) { - testutil_checkfmt(hw == sw ? 0 : 1, - "%s checksum mismatch of %" WT_SIZET_FMT " bytes: %#08x != %#08x\n", - msg, len, hw, sw); + testutil_checkfmt(hw == sw ? 0 : 1, + "%s checksum mismatch of %" WT_SIZET_FMT " bytes: %#08x != %#08x\n", msg, len, hw, sw); } -#define DATASIZE (128 * 1024) +#define DATASIZE (128 * 1024) int main(int argc, char *argv[]) { - TEST_OPTS *opts, _opts; - WT_RAND_STATE rnd; - size_t len; - uint32_t hw, sw; - uint8_t *data; - u_int i, j; - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - testutil_check( - wiredtiger_open(opts->home, NULL, "create", &opts->conn)); - - /* Initialize the RNG. */ - __wt_random_init_seed(NULL, &rnd); - - /* Allocate aligned memory for the data. */ - data = dcalloc(DATASIZE, sizeof(uint8_t)); - - /* - * Some simple known checksums. - */ - len = 1; - hw = __wt_checksum(data, len); - check(hw, (uint32_t)0x527d5351, len, "nul x1: hardware"); - sw = __wt_checksum_sw(data, len); - check(sw, (uint32_t)0x527d5351, len, "nul x1: software"); - - len = 2; - hw = __wt_checksum(data, len); - check(hw, (uint32_t)0xf16177d2, len, "nul x2: hardware"); - sw = __wt_checksum_sw(data, len); - check(sw, (uint32_t)0xf16177d2, len, "nul x2: software"); - - len = 3; - hw = __wt_checksum(data, len); - check(hw, (uint32_t)0x6064a37a, len, "nul x3: hardware"); - sw = __wt_checksum_sw(data, len); - check(sw, (uint32_t)0x6064a37a, len, "nul x3: software"); - - len = 4; - hw = __wt_checksum(data, len); - check(hw, (uint32_t)0x48674bc7, len, "nul x4: hardware"); - sw = __wt_checksum_sw(data, len); - check(sw, (uint32_t)0x48674bc7, len, "nul x4: software"); - - len = strlen("123456789"); - memcpy(data, "123456789", len); - hw = __wt_checksum(data, len); - check(hw, (uint32_t)0xe3069283, len, "known string #1: hardware"); - sw = __wt_checksum_sw(data, len); - check(sw, (uint32_t)0xe3069283, len, "known string #1: software"); - - len = strlen("The quick brown fox jumps over the lazy dog"); - memcpy(data, "The quick brown fox jumps over the lazy dog", len); - hw = __wt_checksum(data, len); - check(hw, (uint32_t)0x22620404, len, "known string #2: hardware"); - sw = __wt_checksum_sw(data, len); - check(sw, (uint32_t)0x22620404, len, "known string #2: software"); - - /* - * Offset the string by 1 to ensure the hardware code handles unaligned - * reads. - */ - hw = __wt_checksum(data + 1, len - 1); - check(hw, (uint32_t)0xae11f7f5, len, "known string #2: hardware"); - sw = __wt_checksum_sw(data + 1, len - 1); - check(sw, (uint32_t)0xae11f7f5, len, "known string #2: software"); - - /* - * Checksums of power-of-two data chunks. - */ - for (i = 0, len = 512; i < 1000; ++i) { - for (j = 0; j < len; ++j) - data[j] = __wt_random(&rnd) & 0xff; - hw = __wt_checksum(data, len); - sw = __wt_checksum_sw(data, len); - check(hw, sw, len, "random power-of-two"); - - len *= 2; - if (len > DATASIZE) - len = 512; - } - - /* - * Checksums of random data chunks. - */ - for (i = 0; i < 1000; ++i) { - len = __wt_random(&rnd) % DATASIZE; - for (j = 0; j < len; ++j) - data[j] = __wt_random(&rnd) & 0xff; - hw = __wt_checksum(data, len); - sw = __wt_checksum_sw(data, len); - check(hw, sw, len, "random"); - } - - free(data); - testutil_cleanup(opts); - return (EXIT_SUCCESS); + TEST_OPTS *opts, _opts; + WT_RAND_STATE rnd; + size_t len; + uint32_t hw, sw; + uint8_t *data; + u_int i, j; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + testutil_check(wiredtiger_open(opts->home, NULL, "create", &opts->conn)); + + /* Initialize the RNG. */ + __wt_random_init_seed(NULL, &rnd); + + /* Allocate aligned memory for the data. */ + data = dcalloc(DATASIZE, sizeof(uint8_t)); + + /* + * Some simple known checksums. + */ + len = 1; + hw = __wt_checksum(data, len); + check(hw, (uint32_t)0x527d5351, len, "nul x1: hardware"); + sw = __wt_checksum_sw(data, len); + check(sw, (uint32_t)0x527d5351, len, "nul x1: software"); + + len = 2; + hw = __wt_checksum(data, len); + check(hw, (uint32_t)0xf16177d2, len, "nul x2: hardware"); + sw = __wt_checksum_sw(data, len); + check(sw, (uint32_t)0xf16177d2, len, "nul x2: software"); + + len = 3; + hw = __wt_checksum(data, len); + check(hw, (uint32_t)0x6064a37a, len, "nul x3: hardware"); + sw = __wt_checksum_sw(data, len); + check(sw, (uint32_t)0x6064a37a, len, "nul x3: software"); + + len = 4; + hw = __wt_checksum(data, len); + check(hw, (uint32_t)0x48674bc7, len, "nul x4: hardware"); + sw = __wt_checksum_sw(data, len); + check(sw, (uint32_t)0x48674bc7, len, "nul x4: software"); + + len = strlen("123456789"); + memcpy(data, "123456789", len); + hw = __wt_checksum(data, len); + check(hw, (uint32_t)0xe3069283, len, "known string #1: hardware"); + sw = __wt_checksum_sw(data, len); + check(sw, (uint32_t)0xe3069283, len, "known string #1: software"); + + len = strlen("The quick brown fox jumps over the lazy dog"); + memcpy(data, "The quick brown fox jumps over the lazy dog", len); + hw = __wt_checksum(data, len); + check(hw, (uint32_t)0x22620404, len, "known string #2: hardware"); + sw = __wt_checksum_sw(data, len); + check(sw, (uint32_t)0x22620404, len, "known string #2: software"); + + /* + * Offset the string by 1 to ensure the hardware code handles unaligned reads. + */ + hw = __wt_checksum(data + 1, len - 1); + check(hw, (uint32_t)0xae11f7f5, len, "known string #2: hardware"); + sw = __wt_checksum_sw(data + 1, len - 1); + check(sw, (uint32_t)0xae11f7f5, len, "known string #2: software"); + + /* + * Checksums of power-of-two data chunks. + */ + for (i = 0, len = 512; i < 1000; ++i) { + for (j = 0; j < len; ++j) + data[j] = __wt_random(&rnd) & 0xff; + hw = __wt_checksum(data, len); + sw = __wt_checksum_sw(data, len); + check(hw, sw, len, "random power-of-two"); + + len *= 2; + if (len > DATASIZE) + len = 512; + } + + /* + * Checksums of random data chunks. + */ + for (i = 0; i < 1000; ++i) { + len = __wt_random(&rnd) % DATASIZE; + for (j = 0; j < len; ++j) + data[j] = __wt_random(&rnd) & 0xff; + hw = __wt_checksum(data, len); + sw = __wt_checksum_sw(data, len); + check(hw, sw, len, "random"); + } + + free(data); + testutil_cleanup(opts); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c b/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c index 1e513dc4d53..6321584e9f1 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c @@ -30,284 +30,199 @@ #include <signal.h> /* - * JIRA ticket reference: WT-2719 - * Test case description: Fuzz testing for WiredTiger reconfiguration. + * JIRA ticket reference: WT-2719 Test case description: Fuzz testing for WiredTiger + * reconfiguration. */ -static const char * const list[] = { - ",async=(enabled=0)", - ",async=(enabled=1)", - ",async=(ops_max=2048)", - ",async=(ops_max=2348)", - ",async=(ops_max=1790)", - ",async=(threads=10)", - ",async=(threads=7)", - ",async=(threads=17)", - - ",cache_overhead=13", - ",cache_overhead=27", - ",cache_overhead=8", - - ",cache_size=75MB", - ",cache_size=214MB", - ",cache_size=37MB", - - ",checkpoint=(log_size=104857600)", /* 100MB */ - ",checkpoint=(log_size=1073741824)", /* 1GB */ - ",checkpoint=(log_size=2)", - ",checkpoint=(log_size=0)", - ",checkpoint=(wait=100)", - ",checkpoint=(wait=10000)", - ",checkpoint=(wait=2)", - ",checkpoint=(wait=0)", - - ",compatibility=(release=2.6)", - ",compatibility=(release=3.0)", - - ",error_prefix=\"prefix\"", - - ",eviction=(threads_min=7,threads_max=10)", - ",eviction=(threads_min=17,threads_max=18)", - ",eviction=(threads_min=3,threads_max=7)", - ",eviction=(threads_max=12,threads_min=10)", - ",eviction=(threads_max=18,threads_min=16)", - ",eviction=(threads_max=10,threads_min=9)", - - ",eviction_dirty_target=45", - ",eviction_dirty_target=87", - ",eviction_dirty_target=8", - - ",eviction_dirty_trigger=37", - ",eviction_dirty_trigger=98", - ",eviction_dirty_trigger=7", - - ",eviction_target=22", - ",eviction_target=84", - ",eviction_target=30", - - ",eviction_trigger=75", - ",eviction_trigger=95", - ",eviction_trigger=66", - - ",file_manager=(close_handle_minimum=200)", - ",file_manager=(close_handle_minimum=137)", - ",file_manager=(close_handle_minimum=226)", - ",file_manager=(close_idle_time=10000)", - ",file_manager=(close_idle_time=12000)", - ",file_manager=(close_idle_time=7)", - ",file_manager=(close_idle_time=0)", - ",file_manager=(close_scan_interval=50000)", - ",file_manager=(close_scan_interval=59000)", - ",file_manager=(close_scan_interval=3)", - - ",log=(archive=0)", - ",log=(archive=1)", - ",log=(prealloc=0)", - ",log=(prealloc=1)", - ",log=(zero_fill=0)", - ",log=(zero_fill=1)", - - ",lsm_manager=(merge=0)", - ",lsm_manager=(merge=1)", - ",lsm_manager=(worker_thread_max=5)", - ",lsm_manager=(worker_thread_max=18)", - ",lsm_manager=(worker_thread_max=3)", - - ",shared_cache=(chunk=20MB)", - ",shared_cache=(chunk=30MB)", - ",shared_cache=(chunk=5MB)", - ",shared_cache=(name=\"shared\")", - ",shared_cache=(name=\"none\")", - ",shared_cache=(quota=20MB)", - ",shared_cache=(quota=30MB)", - ",shared_cache=(quota=5MB)", - ",shared_cache=(quota=0)", - ",shared_cache=(reserve=20MB)", - ",shared_cache=(reserve=30MB)", - ",shared_cache=(reserve=5MB)", - ",shared_cache=(reserve=0)", - ",shared_cache=(size=100MB)", - ",shared_cache=(size=1GB)", - ",shared_cache=(size=75MB)", - - ",statistics=(\"all\")", - ",statistics=(\"fast\")", - ",statistics=(\"none\")", - ",statistics=(\"all\",\"clear\")", - ",statistics=(\"fast\",\"clear\")", - - ",statistics_log=(json=0)", - ",statistics_log=(json=1)", - ",statistics_log=(on_close=0)", - ",statistics_log=(on_close=1)", - ",statistics_log=(sources=(\"file:\"))", - ",statistics_log=(sources=())", - ",statistics_log=(timestamp=\"%b:%S\")", - ",statistics_log=(timestamp=\"%H:%M\")", - ",statistics_log=(wait=60)", - ",statistics_log=(wait=76)", - ",statistics_log=(wait=37)", - ",statistics_log=(wait=0)", - - ",verbose=(\"api\")", - ",verbose=(\"block\")", - ",verbose=(\"checkpoint\")", - ",verbose=(\"compact\")", - ",verbose=(\"evict\")", - ",verbose=(\"evictserver\")", - ",verbose=(\"fileops\")", - ",verbose=(\"handleops\")", - ",verbose=(\"log\")", - ",verbose=(\"lsm\")", - ",verbose=(\"lsm_manager\")", - ",verbose=(\"metadata\")", - ",verbose=(\"mutex\")", - ",verbose=(\"overflow\")", - ",verbose=(\"read\")", - ",verbose=(\"rebalance\")", - ",verbose=(\"reconcile\")", - ",verbose=(\"recovery\")", - ",verbose=(\"salvage\")", - ",verbose=(\"shared_cache\")", - ",verbose=(\"split\")", - ",verbose=(\"transaction\")", - ",verbose=(\"verify\")", - ",verbose=(\"version\")", - ",verbose=(\"write\")", - ",verbose=()" -}; +static const char *const list[] = {",async=(enabled=0)", ",async=(enabled=1)", + ",async=(ops_max=2048)", ",async=(ops_max=2348)", ",async=(ops_max=1790)", ",async=(threads=10)", + ",async=(threads=7)", ",async=(threads=17)", + + ",cache_overhead=13", ",cache_overhead=27", ",cache_overhead=8", + + ",cache_size=75MB", ",cache_size=214MB", ",cache_size=37MB", + + ",checkpoint=(log_size=104857600)", /* 100MB */ + ",checkpoint=(log_size=1073741824)", /* 1GB */ + ",checkpoint=(log_size=2)", ",checkpoint=(log_size=0)", ",checkpoint=(wait=100)", + ",checkpoint=(wait=10000)", ",checkpoint=(wait=2)", ",checkpoint=(wait=0)", + + ",compatibility=(release=2.6)", ",compatibility=(release=3.0)", + + ",error_prefix=\"prefix\"", + + ",eviction=(threads_min=7,threads_max=10)", ",eviction=(threads_min=17,threads_max=18)", + ",eviction=(threads_min=3,threads_max=7)", ",eviction=(threads_max=12,threads_min=10)", + ",eviction=(threads_max=18,threads_min=16)", ",eviction=(threads_max=10,threads_min=9)", + + ",eviction_dirty_target=45", ",eviction_dirty_target=87", ",eviction_dirty_target=8", + + ",eviction_dirty_trigger=37", ",eviction_dirty_trigger=98", ",eviction_dirty_trigger=7", + + ",eviction_target=22", ",eviction_target=84", ",eviction_target=30", + + ",eviction_trigger=75", ",eviction_trigger=95", ",eviction_trigger=66", + + ",file_manager=(close_handle_minimum=200)", ",file_manager=(close_handle_minimum=137)", + ",file_manager=(close_handle_minimum=226)", ",file_manager=(close_idle_time=10000)", + ",file_manager=(close_idle_time=12000)", ",file_manager=(close_idle_time=7)", + ",file_manager=(close_idle_time=0)", ",file_manager=(close_scan_interval=50000)", + ",file_manager=(close_scan_interval=59000)", ",file_manager=(close_scan_interval=3)", + + ",log=(archive=0)", ",log=(archive=1)", ",log=(prealloc=0)", ",log=(prealloc=1)", + ",log=(zero_fill=0)", ",log=(zero_fill=1)", + + ",lsm_manager=(merge=0)", ",lsm_manager=(merge=1)", ",lsm_manager=(worker_thread_max=5)", + ",lsm_manager=(worker_thread_max=18)", ",lsm_manager=(worker_thread_max=3)", + + ",shared_cache=(chunk=20MB)", ",shared_cache=(chunk=30MB)", ",shared_cache=(chunk=5MB)", + ",shared_cache=(name=\"shared\")", ",shared_cache=(name=\"none\")", ",shared_cache=(quota=20MB)", + ",shared_cache=(quota=30MB)", ",shared_cache=(quota=5MB)", ",shared_cache=(quota=0)", + ",shared_cache=(reserve=20MB)", ",shared_cache=(reserve=30MB)", ",shared_cache=(reserve=5MB)", + ",shared_cache=(reserve=0)", ",shared_cache=(size=100MB)", ",shared_cache=(size=1GB)", + ",shared_cache=(size=75MB)", + + ",statistics=(\"all\")", ",statistics=(\"fast\")", ",statistics=(\"none\")", + ",statistics=(\"all\",\"clear\")", ",statistics=(\"fast\",\"clear\")", + + ",statistics_log=(json=0)", ",statistics_log=(json=1)", ",statistics_log=(on_close=0)", + ",statistics_log=(on_close=1)", ",statistics_log=(sources=(\"file:\"))", + ",statistics_log=(sources=())", ",statistics_log=(timestamp=\"%b:%S\")", + ",statistics_log=(timestamp=\"%H:%M\")", ",statistics_log=(wait=60)", ",statistics_log=(wait=76)", + ",statistics_log=(wait=37)", ",statistics_log=(wait=0)", + + ",verbose=(\"api\")", ",verbose=(\"block\")", ",verbose=(\"checkpoint\")", + ",verbose=(\"compact\")", ",verbose=(\"evict\")", ",verbose=(\"evictserver\")", + ",verbose=(\"fileops\")", ",verbose=(\"handleops\")", ",verbose=(\"log\")", ",verbose=(\"lsm\")", + ",verbose=(\"lsm_manager\")", ",verbose=(\"metadata\")", ",verbose=(\"mutex\")", + ",verbose=(\"overflow\")", ",verbose=(\"read\")", ",verbose=(\"rebalance\")", + ",verbose=(\"reconcile\")", ",verbose=(\"recovery\")", ",verbose=(\"salvage\")", + ",verbose=(\"shared_cache\")", ",verbose=(\"split\")", ",verbose=(\"transaction\")", + ",verbose=(\"verify\")", ",verbose=(\"version\")", ",verbose=(\"write\")", ",verbose=()"}; static int -handle_message(WT_EVENT_HANDLER *handler, - WT_SESSION *session, const char *message) +handle_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message) { - (void)(handler); - (void)(session); - (void)(message); + (void)(handler); + (void)(session); + (void)(message); - /* We configure verbose output, so just ignore. */ - return (0); + /* We configure verbose output, so just ignore. */ + return (0); } -static WT_EVENT_HANDLER event_handler = { NULL, handle_message, NULL, NULL }; +static WT_EVENT_HANDLER event_handler = {NULL, handle_message, NULL, NULL}; -static const char *current; /* Current test configuration */ +static const char *current; /* Current test configuration */ static void on_alarm(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void on_alarm(int signo) { - (void)signo; /* Unused parameter */ + (void)signo; /* Unused parameter */ - fprintf(stderr, "configuration timed out: %s\n", current); - abort(); + fprintf(stderr, "configuration timed out: %s\n", current); + abort(); - /* NOTREACHED */ + /* NOTREACHED */ } static void reconfig(TEST_OPTS *opts, WT_SESSION *session, const char *config) { - WT_DECL_RET; - - current = config; - - /* - * Reconfiguration starts and stops servers, so hangs are more likely - * here than in other tests. Don't let the test run too long and get - * a core dump when it happens. - */ - (void)alarm(60); - if ((ret = opts->conn->reconfigure(opts->conn, config)) != 0) { - fprintf(stderr, "%s: %s\n", - config, session->strerror(session, ret)); - exit (EXIT_FAILURE); - } - (void)alarm(0); + WT_DECL_RET; + + current = config; + + /* + * Reconfiguration starts and stops servers, so hangs are more likely here than in other tests. + * Don't let the test run too long and get a core dump when it happens. + */ + (void)alarm(60); + if ((ret = opts->conn->reconfigure(opts->conn, config)) != 0) { + fprintf(stderr, "%s: %s\n", config, session->strerror(session, ret)); + exit(EXIT_FAILURE); + } + (void)alarm(0); } int main(int argc, char *argv[]) { - enum { CACHE_SHARED, CACHE_SET, CACHE_NONE } cache; - TEST_OPTS *opts, _opts; - WT_RAND_STATE rnd; - WT_SESSION *session; - size_t len; - u_int i, j; - const char *p; - char *config; - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - opts->table_type = TABLE_ROW; - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - - testutil_check( - wiredtiger_open(opts->home, &event_handler, "create", &opts->conn)); - - /* Open an LSM file so the LSM reconfiguration options make sense. */ - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check(session->create( - session, opts->uri, "type=lsm,key_format=S,value_format=S")); - - /* Initialize the RNG. */ - __wt_random_init_seed(NULL, &rnd); - - /* Allocate memory for the config. */ - len = WT_ELEMENTS(list) * 64; - config = dmalloc(len); - - /* Set an alarm so we can debug hangs. */ - (void)signal(SIGALRM, on_alarm); - - /* A linear pass through the list. */ - for (i = 0; i < WT_ELEMENTS(list); ++i) - reconfig(opts, session, list[i]); - - /* - * A linear pass through the list, adding random elements. - * - * WiredTiger configurations are usually "the last one set wins", but - * "shared_cache" and "cache_set" options aren't allowed in the same - * configuration string. - */ - for (i = 0; i < WT_ELEMENTS(list); ++i) { - p = list[i]; - cache = CACHE_NONE; - if (WT_PREFIX_MATCH(p, ",shared_cache")) - cache = CACHE_SHARED; - else if (WT_PREFIX_MATCH(p, ",cache_size")) - cache = CACHE_SET; - strcpy(config, p); - - for (j = - (__wt_random(&rnd) % WT_ELEMENTS(list)) + 1; j > 0; --j) { - p = list[__wt_random(&rnd) % WT_ELEMENTS(list)]; - if (WT_PREFIX_MATCH(p, ",shared_cache")) { - if (cache == CACHE_SET) - continue; - cache = CACHE_SHARED; - } else if (WT_PREFIX_MATCH(p, ",cache_size")) { - if (cache == CACHE_SHARED) - continue; - cache = CACHE_SET; - } - strcat(config, p); - } - reconfig(opts, session, config); - } - - /* - * Turn on-close statistics off, if on-close is on and statistics were - * randomly turned off during the run, close would fail. - */ - testutil_check(opts->conn->reconfigure( - opts->conn, "statistics_log=(on_close=0)")); - - free(config); - testutil_cleanup(opts); - return (EXIT_SUCCESS); + enum { CACHE_SHARED, CACHE_SET, CACHE_NONE } cache; + TEST_OPTS *opts, _opts; + WT_RAND_STATE rnd; + WT_SESSION *session; + size_t len; + u_int i, j; + const char *p; + char *config; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + opts->table_type = TABLE_ROW; + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(wiredtiger_open(opts->home, &event_handler, "create", &opts->conn)); + + /* Open an LSM file so the LSM reconfiguration options make sense. */ + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(session->create(session, opts->uri, "type=lsm,key_format=S,value_format=S")); + + /* Initialize the RNG. */ + __wt_random_init_seed(NULL, &rnd); + + /* Allocate memory for the config. */ + len = WT_ELEMENTS(list) * 64; + config = dmalloc(len); + + /* Set an alarm so we can debug hangs. */ + (void)signal(SIGALRM, on_alarm); + + /* A linear pass through the list. */ + for (i = 0; i < WT_ELEMENTS(list); ++i) + reconfig(opts, session, list[i]); + + /* + * A linear pass through the list, adding random elements. + * + * WiredTiger configurations are usually "the last one set wins", but + * "shared_cache" and "cache_set" options aren't allowed in the same + * configuration string. + */ + for (i = 0; i < WT_ELEMENTS(list); ++i) { + p = list[i]; + cache = CACHE_NONE; + if (WT_PREFIX_MATCH(p, ",shared_cache")) + cache = CACHE_SHARED; + else if (WT_PREFIX_MATCH(p, ",cache_size")) + cache = CACHE_SET; + strcpy(config, p); + + for (j = (__wt_random(&rnd) % WT_ELEMENTS(list)) + 1; j > 0; --j) { + p = list[__wt_random(&rnd) % WT_ELEMENTS(list)]; + if (WT_PREFIX_MATCH(p, ",shared_cache")) { + if (cache == CACHE_SET) + continue; + cache = CACHE_SHARED; + } else if (WT_PREFIX_MATCH(p, ",cache_size")) { + if (cache == CACHE_SHARED) + continue; + cache = CACHE_SET; + } + strcat(config, p); + } + reconfig(opts, session, config); + } + + /* + * Turn on-close statistics off, if on-close is on and statistics were randomly turned off + * during the run, close would fail. + */ + testutil_check(opts->conn->reconfigure(opts->conn, "statistics_log=(on_close=0)")); + + free(config); + testutil_cleanup(opts); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c b/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c index dd3ba07cc66..29381d7d0a4 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c @@ -39,170 +39,151 @@ * * Failure mode: We get results back from our join. */ -#define N_RECORDS 100000 -#define N_INSERT 1000000 +#define N_RECORDS 100000 +#define N_INSERT 1000000 void populate(TEST_OPTS *opts); int main(int argc, char *argv[]) { - TEST_OPTS *opts, _opts; - WT_CURSOR *balancecur, *flagcur, *joincur, *postcur; - WT_CURSOR *maincur; - WT_SESSION *session; - int balance, count, flag, key, key2, post, ret; - char balanceuri[256]; - char cfg[128]; - char flaguri[256]; - char joinuri[256]; - char posturi[256]; - const char *tablename; - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - testutil_progress(opts, "start"); - - testutil_check(wiredtiger_open(opts->home, NULL, - "create,cache_size=250M", &opts->conn)); - testutil_progress(opts, "wiredtiger_open"); - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_progress(opts, "sessions opened"); - - /* - * Note: repeated primary key 'id' as 'id2'. This makes - * it easier to dump an index and know which record we're - * looking at. - */ - testutil_check(session->create(session, opts->uri, - "key_format=i,value_format=iiii," - "columns=(id,post,balance,flag,id2)")); - - tablename = strchr(opts->uri, ':'); - testutil_assert(tablename != NULL); - tablename++; - testutil_check(__wt_snprintf( - posturi, sizeof(posturi), "index:%s:post", tablename)); - testutil_check(__wt_snprintf( - balanceuri, sizeof(balanceuri), "index:%s:balance", tablename)); - testutil_check(__wt_snprintf( - flaguri, sizeof(flaguri), "index:%s:flag", tablename)); - testutil_check(__wt_snprintf( - joinuri, sizeof(joinuri), "join:%s", opts->uri)); - - testutil_check(session->create(session, posturi, "columns=(post)")); - testutil_check(session->create(session, balanceuri, - "columns=(balance)")); - testutil_check(session->create(session, flaguri, "columns=(flag)")); - testutil_progress(opts, "setup complete"); - - /* - * Insert a single record with all items we are search for, - * this makes our logic easier. - */ - testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, - &maincur)); - maincur->set_key(maincur, N_RECORDS); - maincur->set_value(maincur, 54321, 0, "", 0, N_RECORDS); - testutil_check(maincur->insert(maincur)); - testutil_check(maincur->close(maincur)); - testutil_check(session->close(session, NULL)); - - testutil_progress(opts, "populate start"); - populate(opts); - testutil_progress(opts, "populate end"); - - testutil_check(opts->conn->open_session( - opts->conn, NULL, NULL, &session)); - - testutil_check(session->open_cursor(session, - posturi, NULL, NULL, &postcur)); - testutil_check(session->open_cursor(session, - balanceuri, NULL, NULL, &balancecur)); - testutil_check(session->open_cursor(session, - flaguri, NULL, NULL, &flagcur)); - testutil_check(session->open_cursor(session, - joinuri, NULL, NULL, &joincur)); - - postcur->set_key(postcur, 54321); - testutil_check(postcur->search(postcur)); - testutil_check(session->join(session, joincur, postcur, - "compare=eq")); - - balancecur->set_key(balancecur, 0); - testutil_check(balancecur->search(balancecur)); - testutil_check(__wt_snprintf(cfg, sizeof(cfg), - "compare=lt,strategy=bloom,count=%d", N_RECORDS / 100)); - testutil_check(session->join(session, joincur, balancecur, cfg)); - - flagcur->set_key(flagcur, 0); - testutil_check(flagcur->search(flagcur)); - testutil_check(__wt_snprintf(cfg, sizeof(cfg), - "compare=eq,strategy=bloom,count=%d", N_RECORDS / 100)); - testutil_check(session->join(session, joincur, flagcur, cfg)); - - /* Expect no values returned */ - count = 0; - while ((ret = joincur->next(joincur)) == 0) { - /* - * The values may already have been changed, but - * print them for informational purposes. - */ - testutil_check(joincur->get_key(joincur, &key)); - testutil_check(joincur->get_value(joincur, &post, - &balance, &flag, &key2)); - fprintf(stderr, "FAIL: " - "key=%d/%d, postal_code=%d, balance=%d, flag=%d\n", - key, key2, post, balance, flag); - count++; - } - testutil_assert(ret == WT_NOTFOUND); - testutil_assert(count == 0); - - testutil_progress(opts, "cleanup starting"); - testutil_cleanup(opts); - return (EXIT_SUCCESS); + TEST_OPTS *opts, _opts; + WT_CURSOR *balancecur, *flagcur, *joincur, *postcur; + WT_CURSOR *maincur; + WT_SESSION *session; + int balance, count, flag, key, key2, post, ret; + char balanceuri[256]; + char cfg[128]; + char flaguri[256]; + char joinuri[256]; + char posturi[256]; + const char *tablename; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + testutil_progress(opts, "start"); + + testutil_check(wiredtiger_open(opts->home, NULL, "create,cache_size=250M", &opts->conn)); + testutil_progress(opts, "wiredtiger_open"); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_progress(opts, "sessions opened"); + + /* + * Note: repeated primary key 'id' as 'id2'. This makes it easier to dump an index and know + * which record we're looking at. + */ + testutil_check(session->create(session, opts->uri, + "key_format=i,value_format=iiii," + "columns=(id,post,balance,flag,id2)")); + + tablename = strchr(opts->uri, ':'); + testutil_assert(tablename != NULL); + tablename++; + testutil_check(__wt_snprintf(posturi, sizeof(posturi), "index:%s:post", tablename)); + testutil_check(__wt_snprintf(balanceuri, sizeof(balanceuri), "index:%s:balance", tablename)); + testutil_check(__wt_snprintf(flaguri, sizeof(flaguri), "index:%s:flag", tablename)); + testutil_check(__wt_snprintf(joinuri, sizeof(joinuri), "join:%s", opts->uri)); + + testutil_check(session->create(session, posturi, "columns=(post)")); + testutil_check(session->create(session, balanceuri, "columns=(balance)")); + testutil_check(session->create(session, flaguri, "columns=(flag)")); + testutil_progress(opts, "setup complete"); + + /* + * Insert a single record with all items we are search for, this makes our logic easier. + */ + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &maincur)); + maincur->set_key(maincur, N_RECORDS); + maincur->set_value(maincur, 54321, 0, "", 0, N_RECORDS); + testutil_check(maincur->insert(maincur)); + testutil_check(maincur->close(maincur)); + testutil_check(session->close(session, NULL)); + + testutil_progress(opts, "populate start"); + populate(opts); + testutil_progress(opts, "populate end"); + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(session->open_cursor(session, posturi, NULL, NULL, &postcur)); + testutil_check(session->open_cursor(session, balanceuri, NULL, NULL, &balancecur)); + testutil_check(session->open_cursor(session, flaguri, NULL, NULL, &flagcur)); + testutil_check(session->open_cursor(session, joinuri, NULL, NULL, &joincur)); + + postcur->set_key(postcur, 54321); + testutil_check(postcur->search(postcur)); + testutil_check(session->join(session, joincur, postcur, "compare=eq")); + + balancecur->set_key(balancecur, 0); + testutil_check(balancecur->search(balancecur)); + testutil_check( + __wt_snprintf(cfg, sizeof(cfg), "compare=lt,strategy=bloom,count=%d", N_RECORDS / 100)); + testutil_check(session->join(session, joincur, balancecur, cfg)); + + flagcur->set_key(flagcur, 0); + testutil_check(flagcur->search(flagcur)); + testutil_check( + __wt_snprintf(cfg, sizeof(cfg), "compare=eq,strategy=bloom,count=%d", N_RECORDS / 100)); + testutil_check(session->join(session, joincur, flagcur, cfg)); + + /* Expect no values returned */ + count = 0; + while ((ret = joincur->next(joincur)) == 0) { + /* + * The values may already have been changed, but print them for informational purposes. + */ + testutil_check(joincur->get_key(joincur, &key)); + testutil_check(joincur->get_value(joincur, &post, &balance, &flag, &key2)); + fprintf(stderr, + "FAIL: " + "key=%d/%d, postal_code=%d, balance=%d, flag=%d\n", + key, key2, post, balance, flag); + count++; + } + testutil_assert(ret == WT_NOTFOUND); + testutil_assert(count == 0); + + testutil_progress(opts, "cleanup starting"); + testutil_cleanup(opts); + return (EXIT_SUCCESS); } void populate(TEST_OPTS *opts) { - WT_CURSOR *maincur; - WT_RAND_STATE rnd; - WT_SESSION *session; - uint32_t key; - int balance, i, flag, post; - - __wt_random_init_seed(NULL, &rnd); - - testutil_check(opts->conn->open_session( - opts->conn, NULL, NULL, &session)); - - testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, - &maincur)); - - for (i = 0; i < N_INSERT; i++) { - testutil_check(session->begin_transaction(session, NULL)); - key = (__wt_random(&rnd) % (N_RECORDS)); - maincur->set_key(maincur, key); - if (__wt_random(&rnd) % 11 == 0) - post = 54321; - else - post = i % 100000; - if (__wt_random(&rnd) % 4 == 0) { - balance = -100; - flag = 1; - } else { - balance = 100 * (i + 1); - flag = 0; - } - maincur->set_value(maincur, post, balance, flag, key); - testutil_check(maincur->insert(maincur)); - testutil_check(session->commit_transaction(session, NULL)); - } - testutil_check(maincur->close(maincur)); - testutil_check(session->close(session, NULL)); + WT_CURSOR *maincur; + WT_RAND_STATE rnd; + WT_SESSION *session; + uint32_t key; + int balance, i, flag, post; + + __wt_random_init_seed(NULL, &rnd); + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &maincur)); + + for (i = 0; i < N_INSERT; i++) { + testutil_check(session->begin_transaction(session, NULL)); + key = (__wt_random(&rnd) % (N_RECORDS)); + maincur->set_key(maincur, key); + if (__wt_random(&rnd) % 11 == 0) + post = 54321; + else + post = i % 100000; + if (__wt_random(&rnd) % 4 == 0) { + balance = -100; + flag = 1; + } else { + balance = 100 * (i + 1); + flag = 0; + } + maincur->set_value(maincur, post, balance, flag, key); + testutil_check(maincur->insert(maincur)); + testutil_check(session->commit_transaction(session, NULL)); + } + testutil_check(maincur->close(maincur)); + testutil_check(session->close(session, NULL)); } diff --git a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c index be6811b38af..3a39ffa4c57 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c @@ -45,309 +45,286 @@ static void *thread_insert(void *); static void *thread_get(void *); -#define BLOOM false -#define MAX_GAP 7 -#define N_RECORDS 10000 -#define N_INSERT 1000000 -#define N_INSERT_THREAD 1 -#define N_GET_THREAD 1 -#define S64 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789::" -#define S1024 (S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64) +#define BLOOM false +#define MAX_GAP 7 +#define N_RECORDS 10000 +#define N_INSERT 1000000 +#define N_INSERT_THREAD 1 +#define N_GET_THREAD 1 +#define S64 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789::" +#define S1024 (S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64 S64) typedef struct { - char posturi[256]; - char baluri[256]; - char flaguri[256]; - bool bloom; + char posturi[256]; + char baluri[256]; + char flaguri[256]; + bool bloom; } SHARED_OPTS; typedef struct { - TEST_OPTS *testopts; - SHARED_OPTS *sharedopts; - int threadnum; - int nthread; - int done; - int njoins; - int nfail; + TEST_OPTS *testopts; + SHARED_OPTS *sharedopts; + int threadnum; + int nthread; + int done; + int njoins; + int nfail; } THREAD_ARGS; int main(int argc, char *argv[]) { - SHARED_OPTS *sharedopts, _sharedopts; - TEST_OPTS *opts, _opts; - THREAD_ARGS get_args[N_GET_THREAD], insert_args[N_INSERT_THREAD]; - WT_CURSOR *maincur; - WT_SESSION *session; - pthread_t get_tid[N_GET_THREAD], insert_tid[N_INSERT_THREAD]; - int i, nfail; - const char *tablename; - - /* - * Bypass this test for valgrind or slow test machines. This - * test is timing sensitive. - */ - if (testutil_is_flag_set("TESTUTIL_BYPASS_VALGRIND") || - testutil_is_flag_set("TESTUTIL_SLOW_MACHINE")) - return (EXIT_SUCCESS); - - opts = &_opts; - sharedopts = &_sharedopts; - memset(opts, 0, sizeof(*opts)); - memset(sharedopts, 0, sizeof(*sharedopts)); - memset(insert_args, 0, sizeof(insert_args)); - memset(get_args, 0, sizeof(get_args)); - nfail = 0; - - sharedopts->bloom = BLOOM; - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - testutil_progress(opts, "start"); - - testutil_check(wiredtiger_open(opts->home, NULL, - "create,cache_size=1G", &opts->conn)); - testutil_progress(opts, "wiredtiger_open"); - - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_progress(opts, "sessions opened"); - - /* - * Note: id is repeated as id2. This makes it easier to - * identify the primary key in dumps of the index files. - */ - testutil_check(session->create(session, opts->uri, - "key_format=i,value_format=iiSii," - "columns=(id,post,bal,extra,flag,id2)")); - - tablename = strchr(opts->uri, ':'); - testutil_assert(tablename != NULL); - tablename++; - testutil_check(__wt_snprintf( - sharedopts->posturi, sizeof(sharedopts->posturi), - "index:%s:post", tablename)); - testutil_check(__wt_snprintf( - sharedopts->baluri, sizeof(sharedopts->baluri), - "index:%s:bal", tablename)); - testutil_check(__wt_snprintf( - sharedopts->flaguri, sizeof(sharedopts->flaguri), - "index:%s:flag", tablename)); - - testutil_check(session->create(session, sharedopts->posturi, - "columns=(post)")); - testutil_check(session->create(session, sharedopts->baluri, - "columns=(bal)")); - testutil_check(session->create(session, sharedopts->flaguri, - "columns=(flag)")); - - /* - * Insert a single record with all items we need to - * call search() on, this makes our join logic easier. - */ - testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, - &maincur)); - maincur->set_key(maincur, N_RECORDS); - maincur->set_value(maincur, 54321, 0, "", 0, N_RECORDS); - testutil_check(maincur->insert(maincur)); - testutil_check(maincur->close(maincur)); - testutil_check(session->close(session, NULL)); - testutil_progress(opts, "setup complete"); - - for (i = 0; i < N_INSERT_THREAD; ++i) { - insert_args[i].threadnum = i; - insert_args[i].nthread = N_INSERT_THREAD; - insert_args[i].testopts = opts; - insert_args[i].sharedopts = sharedopts; - testutil_check(pthread_create( - &insert_tid[i], NULL, thread_insert, &insert_args[i])); - } - - for (i = 0; i < N_GET_THREAD; ++i) { - get_args[i].threadnum = i; - get_args[i].nthread = N_GET_THREAD; - get_args[i].testopts = opts; - get_args[i].sharedopts = sharedopts; - testutil_check(pthread_create( - &get_tid[i], NULL, thread_get, &get_args[i])); - } - testutil_progress(opts, "threads started"); - - /* - * Wait for insert threads to finish. When they - * are done, signal get threads to complete. - */ - for (i = 0; i < N_INSERT_THREAD; ++i) - testutil_check(pthread_join(insert_tid[i], NULL)); - - for (i = 0; i < N_GET_THREAD; ++i) - get_args[i].done = 1; - - for (i = 0; i < N_GET_THREAD; ++i) - testutil_check(pthread_join(get_tid[i], NULL)); - - testutil_progress(opts, "threads joined"); - fprintf(stderr, "\n"); - for (i = 0; i < N_GET_THREAD; ++i) { - fprintf(stderr, " thread %d did %d joins (%d fails)\n", i, - get_args[i].njoins, get_args[i].nfail); - nfail += get_args[i].nfail; - } - - /* - * Note that slow machines can be skipped for this test. - * See the bypass code earlier. - */ - if (nfail != 0) - fprintf(stderr, - "ERROR: %d failures when a single commit" - " took more than %d seconds.\n" - "This may indicate a real problem or a" - " particularly slow machine.\n", nfail, MAX_GAP); - testutil_assert(nfail == 0); - testutil_progress(opts, "cleanup starting"); - testutil_cleanup(opts); - return (0); + SHARED_OPTS *sharedopts, _sharedopts; + TEST_OPTS *opts, _opts; + THREAD_ARGS get_args[N_GET_THREAD], insert_args[N_INSERT_THREAD]; + WT_CURSOR *maincur; + WT_SESSION *session; + pthread_t get_tid[N_GET_THREAD], insert_tid[N_INSERT_THREAD]; + int i, nfail; + const char *tablename; + + /* + * Bypass this test for valgrind or slow test machines. This test is timing sensitive. + */ + if (testutil_is_flag_set("TESTUTIL_BYPASS_VALGRIND") || + testutil_is_flag_set("TESTUTIL_SLOW_MACHINE")) + return (EXIT_SUCCESS); + + opts = &_opts; + sharedopts = &_sharedopts; + memset(opts, 0, sizeof(*opts)); + memset(sharedopts, 0, sizeof(*sharedopts)); + memset(insert_args, 0, sizeof(insert_args)); + memset(get_args, 0, sizeof(get_args)); + nfail = 0; + + sharedopts->bloom = BLOOM; + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + testutil_progress(opts, "start"); + + testutil_check(wiredtiger_open(opts->home, NULL, "create,cache_size=1G", &opts->conn)); + testutil_progress(opts, "wiredtiger_open"); + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_progress(opts, "sessions opened"); + + /* + * Note: id is repeated as id2. This makes it easier to identify the primary key in dumps of the + * index files. + */ + testutil_check(session->create(session, opts->uri, + "key_format=i,value_format=iiSii," + "columns=(id,post,bal,extra,flag,id2)")); + + tablename = strchr(opts->uri, ':'); + testutil_assert(tablename != NULL); + tablename++; + testutil_check( + __wt_snprintf(sharedopts->posturi, sizeof(sharedopts->posturi), "index:%s:post", tablename)); + testutil_check( + __wt_snprintf(sharedopts->baluri, sizeof(sharedopts->baluri), "index:%s:bal", tablename)); + testutil_check( + __wt_snprintf(sharedopts->flaguri, sizeof(sharedopts->flaguri), "index:%s:flag", tablename)); + + testutil_check(session->create(session, sharedopts->posturi, "columns=(post)")); + testutil_check(session->create(session, sharedopts->baluri, "columns=(bal)")); + testutil_check(session->create(session, sharedopts->flaguri, "columns=(flag)")); + + /* + * Insert a single record with all items we need to call search() on, this makes our join logic + * easier. + */ + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &maincur)); + maincur->set_key(maincur, N_RECORDS); + maincur->set_value(maincur, 54321, 0, "", 0, N_RECORDS); + testutil_check(maincur->insert(maincur)); + testutil_check(maincur->close(maincur)); + testutil_check(session->close(session, NULL)); + testutil_progress(opts, "setup complete"); + + for (i = 0; i < N_INSERT_THREAD; ++i) { + insert_args[i].threadnum = i; + insert_args[i].nthread = N_INSERT_THREAD; + insert_args[i].testopts = opts; + insert_args[i].sharedopts = sharedopts; + testutil_check(pthread_create(&insert_tid[i], NULL, thread_insert, &insert_args[i])); + } + + for (i = 0; i < N_GET_THREAD; ++i) { + get_args[i].threadnum = i; + get_args[i].nthread = N_GET_THREAD; + get_args[i].testopts = opts; + get_args[i].sharedopts = sharedopts; + testutil_check(pthread_create(&get_tid[i], NULL, thread_get, &get_args[i])); + } + testutil_progress(opts, "threads started"); + + /* + * Wait for insert threads to finish. When they are done, signal get threads to complete. + */ + for (i = 0; i < N_INSERT_THREAD; ++i) + testutil_check(pthread_join(insert_tid[i], NULL)); + + for (i = 0; i < N_GET_THREAD; ++i) + get_args[i].done = 1; + + for (i = 0; i < N_GET_THREAD; ++i) + testutil_check(pthread_join(get_tid[i], NULL)); + + testutil_progress(opts, "threads joined"); + fprintf(stderr, "\n"); + for (i = 0; i < N_GET_THREAD; ++i) { + fprintf(stderr, " thread %d did %d joins (%d fails)\n", i, get_args[i].njoins, + get_args[i].nfail); + nfail += get_args[i].nfail; + } + + /* + * Note that slow machines can be skipped for this test. See the bypass code earlier. + */ + if (nfail != 0) + fprintf(stderr, + "ERROR: %d failures when a single commit" + " took more than %d seconds.\n" + "This may indicate a real problem or a" + " particularly slow machine.\n", + nfail, MAX_GAP); + testutil_assert(nfail == 0); + testutil_progress(opts, "cleanup starting"); + testutil_cleanup(opts); + return (0); } static void * thread_insert(void *arg) { - TEST_OPTS *opts; - THREAD_ARGS *threadargs; - WT_CURSOR *maincur; - WT_RAND_STATE rnd; - WT_SESSION *session; - uint64_t curtime, elapsed, prevtime; /* 1 second resolution enough */ - int bal, i, flag, key, post; - const char *extra = S1024; - - threadargs = (THREAD_ARGS *)arg; - opts = threadargs->testopts; - - testutil_check(opts->conn->open_session( - opts->conn, NULL, NULL, &session)); - - __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); - __wt_seconds((WT_SESSION_IMPL *)session, &prevtime); - - testutil_check(session->open_cursor( - session, opts->uri, NULL, NULL, &maincur)); - - testutil_progress(opts, "insert start"); - for (i = 0; i < N_INSERT; i++) { - /* - * Insert threads may stomp on each other's records; - * that's okay. - */ - key = (int)(__wt_random(&rnd) % N_RECORDS); - testutil_check(session->begin_transaction(session, NULL)); - maincur->set_key(maincur, key); - if (__wt_random(&rnd) % 2 == 0) - post = 54321; - else - post = i % 100000; - if (__wt_random(&rnd) % 2 == 0) { - bal = -100; - flag = 1; - } else { - bal = 100 * (i + 1); - flag = 0; - } - maincur->set_value(maincur, post, bal, extra, flag, key); - testutil_check(maincur->insert(maincur)); - testutil_check(maincur->reset(maincur)); - testutil_check(session->commit_transaction(session, NULL)); - if (i % 1000 == 0 && i != 0) { - if (i % 10000 == 0) - fprintf(stderr, "*"); - else - fprintf(stderr, "."); - __wt_seconds((WT_SESSION_IMPL *)session, &curtime); - if ((elapsed = curtime - prevtime) > MAX_GAP) { - testutil_progress(opts, "insert time gap"); - fprintf(stderr, "\n" - "GAP: %" PRIu64 " secs after %d inserts\n", - elapsed, i); - threadargs->nfail++; - } - prevtime = curtime; - } - } - testutil_progress(opts, "insert end"); - testutil_check(maincur->close(maincur)); - testutil_check(session->close(session, NULL)); - return (NULL); + TEST_OPTS *opts; + THREAD_ARGS *threadargs; + WT_CURSOR *maincur; + WT_RAND_STATE rnd; + WT_SESSION *session; + uint64_t curtime, elapsed, prevtime; /* 1 second resolution enough */ + int bal, i, flag, key, post; + const char *extra = S1024; + + threadargs = (THREAD_ARGS *)arg; + opts = threadargs->testopts; + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); + __wt_seconds((WT_SESSION_IMPL *)session, &prevtime); + + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &maincur)); + + testutil_progress(opts, "insert start"); + for (i = 0; i < N_INSERT; i++) { + /* + * Insert threads may stomp on each other's records; that's okay. + */ + key = (int)(__wt_random(&rnd) % N_RECORDS); + testutil_check(session->begin_transaction(session, NULL)); + maincur->set_key(maincur, key); + if (__wt_random(&rnd) % 2 == 0) + post = 54321; + else + post = i % 100000; + if (__wt_random(&rnd) % 2 == 0) { + bal = -100; + flag = 1; + } else { + bal = 100 * (i + 1); + flag = 0; + } + maincur->set_value(maincur, post, bal, extra, flag, key); + testutil_check(maincur->insert(maincur)); + testutil_check(maincur->reset(maincur)); + testutil_check(session->commit_transaction(session, NULL)); + if (i % 1000 == 0 && i != 0) { + if (i % 10000 == 0) + fprintf(stderr, "*"); + else + fprintf(stderr, "."); + __wt_seconds((WT_SESSION_IMPL *)session, &curtime); + if ((elapsed = curtime - prevtime) > MAX_GAP) { + testutil_progress(opts, "insert time gap"); + fprintf(stderr, + "\n" + "GAP: %" PRIu64 " secs after %d inserts\n", + elapsed, i); + threadargs->nfail++; + } + prevtime = curtime; + } + } + testutil_progress(opts, "insert end"); + testutil_check(maincur->close(maincur)); + testutil_check(session->close(session, NULL)); + return (NULL); } static void * thread_get(void *arg) { - SHARED_OPTS *sharedopts; - TEST_OPTS *opts; - THREAD_ARGS *threadargs; - WT_CURSOR *maincur, *postcur; - WT_SESSION *session; - uint64_t curtime, elapsed, prevtime; /* 1 second resolution enough */ - int bal, bal2, flag, flag2, key, key2, post, post2; - char *extra; - - threadargs = (THREAD_ARGS *)arg; - opts = threadargs->testopts; - sharedopts = threadargs->sharedopts; - - testutil_check(opts->conn->open_session( - opts->conn, NULL, NULL, &session)); - - __wt_seconds((WT_SESSION_IMPL *)session, &prevtime); - - testutil_check(session->open_cursor( - session, opts->uri, NULL, NULL, &maincur)); - testutil_check(session->open_cursor( - session, sharedopts->posturi, NULL, NULL, &postcur)); - - testutil_progress(opts, "get start"); - for (threadargs->njoins = 0; threadargs->done == 0; - threadargs->njoins++) { - testutil_check(session->begin_transaction(session, NULL)); - postcur->set_key(postcur, 54321); - testutil_check(postcur->search(postcur)); - while (postcur->next(postcur) == 0) { - testutil_check(postcur->get_key(postcur, &post)); - testutil_check(postcur->get_value(postcur, &post2, - &bal, &extra, &flag, &key)); - testutil_assert((flag > 0 && bal < 0) || - (flag == 0 && bal >= 0)); - - maincur->set_key(maincur, key); - testutil_check(maincur->search(maincur)); - testutil_check(maincur->get_value(maincur, &post2, - &bal2, &extra, &flag2, &key2)); - testutil_check(maincur->reset(maincur)); - testutil_assert((flag2 > 0 && bal2 < 0) || - (flag2 == 0 && bal2 >= 0)); - } - /* - * Reset the cursors, potentially allowing the insert - * threads to proceed. - */ - testutil_check(postcur->reset(postcur)); - if (threadargs->njoins % 100 == 0) - fprintf(stderr, "G"); - testutil_check(session->rollback_transaction(session, NULL)); - - __wt_seconds((WT_SESSION_IMPL *)session, &curtime); - if ((elapsed = curtime - prevtime) > MAX_GAP) { - testutil_progress(opts, "get time gap"); - fprintf(stderr, "\n" - "GAP: %" PRIu64 " secs after %d gets\n", - elapsed, threadargs->njoins); - threadargs->nfail++; - } - prevtime = curtime; - } - testutil_progress(opts, "get end"); - testutil_check(postcur->close(postcur)); - testutil_check(maincur->close(maincur)); - testutil_check(session->close(session, NULL)); - return (NULL); + SHARED_OPTS *sharedopts; + TEST_OPTS *opts; + THREAD_ARGS *threadargs; + WT_CURSOR *maincur, *postcur; + WT_SESSION *session; + uint64_t curtime, elapsed, prevtime; /* 1 second resolution enough */ + int bal, bal2, flag, flag2, key, key2, post, post2; + char *extra; + + threadargs = (THREAD_ARGS *)arg; + opts = threadargs->testopts; + sharedopts = threadargs->sharedopts; + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + __wt_seconds((WT_SESSION_IMPL *)session, &prevtime); + + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &maincur)); + testutil_check(session->open_cursor(session, sharedopts->posturi, NULL, NULL, &postcur)); + + testutil_progress(opts, "get start"); + for (threadargs->njoins = 0; threadargs->done == 0; threadargs->njoins++) { + testutil_check(session->begin_transaction(session, NULL)); + postcur->set_key(postcur, 54321); + testutil_check(postcur->search(postcur)); + while (postcur->next(postcur) == 0) { + testutil_check(postcur->get_key(postcur, &post)); + testutil_check(postcur->get_value(postcur, &post2, &bal, &extra, &flag, &key)); + testutil_assert((flag > 0 && bal < 0) || (flag == 0 && bal >= 0)); + + maincur->set_key(maincur, key); + testutil_check(maincur->search(maincur)); + testutil_check(maincur->get_value(maincur, &post2, &bal2, &extra, &flag2, &key2)); + testutil_check(maincur->reset(maincur)); + testutil_assert((flag2 > 0 && bal2 < 0) || (flag2 == 0 && bal2 >= 0)); + } + /* + * Reset the cursors, potentially allowing the insert threads to proceed. + */ + testutil_check(postcur->reset(postcur)); + if (threadargs->njoins % 100 == 0) + fprintf(stderr, "G"); + testutil_check(session->rollback_transaction(session, NULL)); + + __wt_seconds((WT_SESSION_IMPL *)session, &curtime); + if ((elapsed = curtime - prevtime) > MAX_GAP) { + testutil_progress(opts, "get time gap"); + fprintf(stderr, + "\n" + "GAP: %" PRIu64 " secs after %d gets\n", + elapsed, threadargs->njoins); + threadargs->nfail++; + } + prevtime = curtime; + } + testutil_progress(opts, "get end"); + testutil_check(postcur->close(postcur)); + testutil_check(maincur->close(maincur)); + testutil_check(session->close(session, NULL)); + return (NULL); } diff --git a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c index 54b56d597a4..ff59ee95267 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c @@ -69,19 +69,19 @@ */ /* - * This program does not run on Windows. The non-portable aspects at minimum - * are fork/exec the use of environment variables (used by fail_fs), and file - * name and build locations of dynamically loaded libraries. + * This program does not run on Windows. The non-portable aspects at minimum are fork/exec the use + * of environment variables (used by fail_fs), and file name and build locations of dynamically + * loaded libraries. */ -#define BIG_SIZE (1024 * 10) -#define BIG_CONTENTS "<Big String Contents>" -#define MAX_ARGS 20 -#define MAX_OP_RANGE 1000 -#define STDERR_FILE "stderr.txt" -#define STDOUT_FILE "stdout.txt" -#define TESTS_PER_CALIBRATION 2 -#define TESTS_WITH_RECALIBRATION 5 -#define VERBOSE_PRINT 10000 +#define BIG_SIZE (1024 * 10) +#define BIG_CONTENTS "<Big String Contents>" +#define MAX_ARGS 20 +#define MAX_OP_RANGE 1000 +#define STDERR_FILE "stderr.txt" +#define STDOUT_FILE "stdout.txt" +#define TESTS_PER_CALIBRATION 2 +#define TESTS_WITH_RECALIBRATION 5 +#define VERBOSE_PRINT 10000 static int check_results(TEST_OPTS *, uint64_t *); static void check_values(WT_CURSOR *, int, int, int, char *); @@ -90,400 +90,372 @@ static void cursor_count_items(WT_CURSOR *, uint64_t *); static void disable_failures(void); static void enable_failures(uint64_t, uint64_t); static void generate_key(uint64_t, int *); -static void generate_value(uint32_t, uint64_t, char *, int *, int *, int *, - char **); -static void run_check_subtest(TEST_OPTS *, const char *, uint64_t, bool, - uint64_t *); +static void generate_value(uint32_t, uint64_t, char *, int *, int *, int *, char **); +static void run_check_subtest(TEST_OPTS *, const char *, uint64_t, bool, uint64_t *); static int run_check_subtest_range(TEST_OPTS *, const char *, bool); static void run_check_subtest_range_retry(TEST_OPTS *, const char *, bool); static int run_process(TEST_OPTS *, const char *, char *[], int *); static void subtest_main(int, char *[], bool); static void subtest_populate(TEST_OPTS *, bool); -extern int __wt_optind; +extern int __wt_optind; /* * check_results -- - * Check all the tables and verify the results. + * Check all the tables and verify the results. */ static int check_results(TEST_OPTS *opts, uint64_t *foundp) { - WT_CURSOR *maincur, *maincur2, *v0cur, *v1cur, *v2cur; - WT_SESSION *session; - uint64_t count, idxcount, nrecords; - uint32_t rndint; - int key, key_got, ret, v0, v1, v2; - char *big, *bigref; - - testutil_check(create_big_string(&bigref)); - nrecords = opts->nrecords; - testutil_check(wiredtiger_open(opts->home, NULL, - "create,log=(enabled)", &opts->conn)); - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - - testutil_check(session->open_cursor(session, "table:subtest", NULL, - NULL, &maincur)); - testutil_check(session->open_cursor(session, "table:subtest2", NULL, - NULL, &maincur2)); - testutil_check(session->open_cursor(session, "index:subtest:v0", NULL, - NULL, &v0cur)); - testutil_check(session->open_cursor(session, "index:subtest:v1", NULL, - NULL, &v1cur)); - testutil_check(session->open_cursor(session, "index:subtest:v2", NULL, - NULL, &v2cur)); - - count = 0; - while ((ret = maincur->next(maincur)) == 0) { - testutil_check(maincur2->next(maincur2)); - testutil_check(maincur2->get_key(maincur2, &key_got)); - testutil_check(maincur2->get_value(maincur2, &rndint)); - - generate_key(count, &key); - generate_value(rndint, count, bigref, &v0, &v1, &v2, &big); - testutil_assert(key == key_got); - - /* Check the key/values in main table. */ - testutil_check(maincur->get_key(maincur, &key_got)); - testutil_assert(key == key_got); - check_values(maincur, v0, v1, v2, big); - - /* Check the values in the indices. */ - v0cur->set_key(v0cur, v0); - testutil_check(v0cur->search(v0cur)); - check_values(v0cur, v0, v1, v2, big); - v1cur->set_key(v1cur, v1); - testutil_check(v1cur->search(v1cur)); - check_values(v1cur, v0, v1, v2, big); - v2cur->set_key(v2cur, v2); - testutil_check(v2cur->search(v2cur)); - check_values(v2cur, v0, v1, v2, big); - - count++; - if (count % VERBOSE_PRINT == 0 && opts->verbose) - printf("checked %" PRIu64 "/%" PRIu64 "\n", count, - nrecords); - } - if (count % VERBOSE_PRINT != 0 && opts->verbose) - printf("checked %" PRIu64 "/%" PRIu64 "\n", count, nrecords); - - /* - * Always expect at least one entry, as populate does a - * checkpoint after the first insert. - */ - testutil_assert(count > 0); - testutil_assert(ret == WT_NOTFOUND); - testutil_assert(maincur2->next(maincur2) == WT_NOTFOUND); - cursor_count_items(v0cur, &idxcount); - testutil_assert(count == idxcount); - cursor_count_items(v1cur, &idxcount); - testutil_assert(count == idxcount); - cursor_count_items(v2cur, &idxcount); - testutil_assert(count == idxcount); - - testutil_check(opts->conn->close(opts->conn, NULL)); - opts->conn = NULL; - - free(bigref); - *foundp = count; - return (0); + WT_CURSOR *maincur, *maincur2, *v0cur, *v1cur, *v2cur; + WT_SESSION *session; + uint64_t count, idxcount, nrecords; + uint32_t rndint; + int key, key_got, ret, v0, v1, v2; + char *big, *bigref; + + testutil_check(create_big_string(&bigref)); + nrecords = opts->nrecords; + testutil_check(wiredtiger_open(opts->home, NULL, "create,log=(enabled)", &opts->conn)); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(session->open_cursor(session, "table:subtest", NULL, NULL, &maincur)); + testutil_check(session->open_cursor(session, "table:subtest2", NULL, NULL, &maincur2)); + testutil_check(session->open_cursor(session, "index:subtest:v0", NULL, NULL, &v0cur)); + testutil_check(session->open_cursor(session, "index:subtest:v1", NULL, NULL, &v1cur)); + testutil_check(session->open_cursor(session, "index:subtest:v2", NULL, NULL, &v2cur)); + + count = 0; + while ((ret = maincur->next(maincur)) == 0) { + testutil_check(maincur2->next(maincur2)); + testutil_check(maincur2->get_key(maincur2, &key_got)); + testutil_check(maincur2->get_value(maincur2, &rndint)); + + generate_key(count, &key); + generate_value(rndint, count, bigref, &v0, &v1, &v2, &big); + testutil_assert(key == key_got); + + /* Check the key/values in main table. */ + testutil_check(maincur->get_key(maincur, &key_got)); + testutil_assert(key == key_got); + check_values(maincur, v0, v1, v2, big); + + /* Check the values in the indices. */ + v0cur->set_key(v0cur, v0); + testutil_check(v0cur->search(v0cur)); + check_values(v0cur, v0, v1, v2, big); + v1cur->set_key(v1cur, v1); + testutil_check(v1cur->search(v1cur)); + check_values(v1cur, v0, v1, v2, big); + v2cur->set_key(v2cur, v2); + testutil_check(v2cur->search(v2cur)); + check_values(v2cur, v0, v1, v2, big); + + count++; + if (count % VERBOSE_PRINT == 0 && opts->verbose) + printf("checked %" PRIu64 "/%" PRIu64 "\n", count, nrecords); + } + if (count % VERBOSE_PRINT != 0 && opts->verbose) + printf("checked %" PRIu64 "/%" PRIu64 "\n", count, nrecords); + + /* + * Always expect at least one entry, as populate does a checkpoint after the first insert. + */ + testutil_assert(count > 0); + testutil_assert(ret == WT_NOTFOUND); + testutil_assert(maincur2->next(maincur2) == WT_NOTFOUND); + cursor_count_items(v0cur, &idxcount); + testutil_assert(count == idxcount); + cursor_count_items(v1cur, &idxcount); + testutil_assert(count == idxcount); + cursor_count_items(v2cur, &idxcount); + testutil_assert(count == idxcount); + + testutil_check(opts->conn->close(opts->conn, NULL)); + opts->conn = NULL; + + free(bigref); + *foundp = count; + return (0); } /* * check_values -- - * Check that the values in the cursor match the given values. + * Check that the values in the cursor match the given values. */ static void check_values(WT_CURSOR *cursor, int v0, int v1, int v2, char *big) { - int v0_got, v1_got, v2_got; - char *big_got; - - testutil_check(cursor->get_value(cursor, &v0_got, &v1_got, &v2_got, - &big_got)); - testutil_assert(v0 == v0_got); - testutil_assert(v1 == v1_got); - testutil_assert(v2 == v2_got); - testutil_assert(strcmp(big, big_got) == 0); + int v0_got, v1_got, v2_got; + char *big_got; + + testutil_check(cursor->get_value(cursor, &v0_got, &v1_got, &v2_got, &big_got)); + testutil_assert(v0 == v0_got); + testutil_assert(v1 == v1_got); + testutil_assert(v2 == v2_got); + testutil_assert(strcmp(big, big_got) == 0); } /* * create_big_string -- - * Create and fill the "reference" big array. + * Create and fill the "reference" big array. */ static int create_big_string(char **bigp) { - size_t i, mod; - char *big; - - if ((big = malloc(BIG_SIZE + 1)) == NULL) - return (ENOMEM); - mod = strlen(BIG_CONTENTS); - for (i = 0; i < BIG_SIZE; i++) { - big[i] = BIG_CONTENTS[i % mod]; - } - big[BIG_SIZE] = '\0'; - *bigp = big; - return (0); + size_t i, mod; + char *big; + + if ((big = malloc(BIG_SIZE + 1)) == NULL) + return (ENOMEM); + mod = strlen(BIG_CONTENTS); + for (i = 0; i < BIG_SIZE; i++) { + big[i] = BIG_CONTENTS[i % mod]; + } + big[BIG_SIZE] = '\0'; + *bigp = big; + return (0); } /* * cursor_count_items -- - * Count the number of items in the table by traversing - * through the cursor. + * Count the number of items in the table by traversing through the cursor. */ static void cursor_count_items(WT_CURSOR *cursor, uint64_t *countp) { - WT_DECL_RET; + WT_DECL_RET; - *countp = 0; + *countp = 0; - testutil_check(cursor->reset(cursor)); - while ((ret = cursor->next(cursor)) == 0) - (*countp)++; - testutil_assert(ret == WT_NOTFOUND); + testutil_check(cursor->reset(cursor)); + while ((ret = cursor->next(cursor)) == 0) + (*countp)++; + testutil_assert(ret == WT_NOTFOUND); } /* * disable_failures -- - * Disable failures in the fail file system. + * Disable failures in the fail file system. */ static void disable_failures(void) { - testutil_check(setenv("WT_FAIL_FS_ENABLE", "0", 1)); + testutil_check(setenv("WT_FAIL_FS_ENABLE", "0", 1)); } /* * enable_failures -- - * Enable failures in the fail file system. + * Enable failures in the fail file system. */ static void enable_failures(uint64_t allow_writes, uint64_t allow_reads) { - char value[100]; - - testutil_check(setenv("WT_FAIL_FS_ENABLE", "1", 1)); - testutil_check(__wt_snprintf( - value, sizeof(value), "%" PRIu64, allow_writes)); - testutil_check(setenv("WT_FAIL_FS_WRITE_ALLOW", value, 1)); - testutil_check(__wt_snprintf( - value, sizeof(value), "%" PRIu64, allow_reads)); - testutil_check(setenv("WT_FAIL_FS_READ_ALLOW", value, 1)); + char value[100]; + + testutil_check(setenv("WT_FAIL_FS_ENABLE", "1", 1)); + testutil_check(__wt_snprintf(value, sizeof(value), "%" PRIu64, allow_writes)); + testutil_check(setenv("WT_FAIL_FS_WRITE_ALLOW", value, 1)); + testutil_check(__wt_snprintf(value, sizeof(value), "%" PRIu64, allow_reads)); + testutil_check(setenv("WT_FAIL_FS_READ_ALLOW", value, 1)); } /* * generate_key -- - * Generate a key used by the "subtest" and "subtest2" tables. + * Generate a key used by the "subtest" and "subtest2" tables. */ static void generate_key(uint64_t i, int *keyp) { - *keyp = (int)i; + *keyp = (int)i; } /* * generate_value -- - * Generate values for the "subtest" table. + * Generate values for the "subtest" table. */ static void -generate_value(uint32_t rndint, uint64_t i, char *bigref, - int *v0p, int *v1p, int *v2p, char **bigp) +generate_value(uint32_t rndint, uint64_t i, char *bigref, int *v0p, int *v1p, int *v2p, char **bigp) { - *v0p = (int)(i * 7); - *v1p = (int)(i * 10007); - *v2p = (int)(i * 100000007); - *bigp = &bigref[rndint % BIG_SIZE]; + *v0p = (int)(i * 7); + *v1p = (int)(i * 10007); + *v2p = (int)(i * 100000007); + *bigp = &bigref[rndint % BIG_SIZE]; } /* * run_check_subtest -- - * Run the subtest with the given parameters and check the results. + * Run the subtest with the given parameters and check the results. */ static void -run_check_subtest(TEST_OPTS *opts, const char *debugger, uint64_t nops, - bool close_test, uint64_t *nresultsp) +run_check_subtest( + TEST_OPTS *opts, const char *debugger, uint64_t nops, bool close_test, uint64_t *nresultsp) { - int estatus, narg; - char rarg[20], sarg[20], *subtest_args[MAX_ARGS]; - - narg = 0; - if (debugger != NULL) { - subtest_args[narg++] = (char *)debugger; - subtest_args[narg++] = (char *)"--"; - } - - subtest_args[narg++] = (char *)opts->argv0; - /* "subtest" must appear before arguments */ - if (close_test) - subtest_args[narg++] = (char *)"subtest_close"; - else - subtest_args[narg++] = (char *)"subtest"; - subtest_args[narg++] = (char *)"-h"; - subtest_args[narg++] = opts->home; - subtest_args[narg++] = (char *)"-v"; /* subtest is always verbose */ - subtest_args[narg++] = (char *)"-p"; - subtest_args[narg++] = (char *)"-o"; - testutil_check(__wt_snprintf(sarg, sizeof(sarg), "%" PRIu64, nops)); - subtest_args[narg++] = sarg; /* number of operations */ - subtest_args[narg++] = (char *)"-n"; - testutil_check(__wt_snprintf( - rarg, sizeof(rarg), "%" PRIu64, opts->nrecords)); - subtest_args[narg++] = rarg; /* number of records */ - subtest_args[narg++] = NULL; - testutil_assert(narg <= MAX_ARGS); - if (opts->verbose) - printf("running a separate process with %" PRIu64 - " operations until fail...\n", nops); - testutil_clean_work_dir(opts->home); - testutil_check(run_process( - opts, debugger != NULL ? debugger : opts->argv0, - subtest_args, &estatus)); - if (opts->verbose) - printf("process exited %d\n", estatus); - - /* - * Verify results in parent process. - */ - testutil_check(check_results(opts, nresultsp)); + int estatus, narg; + char rarg[20], sarg[20], *subtest_args[MAX_ARGS]; + + narg = 0; + if (debugger != NULL) { + subtest_args[narg++] = (char *)debugger; + subtest_args[narg++] = (char *)"--"; + } + + subtest_args[narg++] = (char *)opts->argv0; + /* "subtest" must appear before arguments */ + if (close_test) + subtest_args[narg++] = (char *)"subtest_close"; + else + subtest_args[narg++] = (char *)"subtest"; + subtest_args[narg++] = (char *)"-h"; + subtest_args[narg++] = opts->home; + subtest_args[narg++] = (char *)"-v"; /* subtest is always verbose */ + subtest_args[narg++] = (char *)"-p"; + subtest_args[narg++] = (char *)"-o"; + testutil_check(__wt_snprintf(sarg, sizeof(sarg), "%" PRIu64, nops)); + subtest_args[narg++] = sarg; /* number of operations */ + subtest_args[narg++] = (char *)"-n"; + testutil_check(__wt_snprintf(rarg, sizeof(rarg), "%" PRIu64, opts->nrecords)); + subtest_args[narg++] = rarg; /* number of records */ + subtest_args[narg++] = NULL; + testutil_assert(narg <= MAX_ARGS); + if (opts->verbose) + printf("running a separate process with %" PRIu64 " operations until fail...\n", nops); + testutil_clean_work_dir(opts->home); + testutil_check( + run_process(opts, debugger != NULL ? debugger : opts->argv0, subtest_args, &estatus)); + if (opts->verbose) + printf("process exited %d\n", estatus); + + /* + * Verify results in parent process. + */ + testutil_check(check_results(opts, nresultsp)); } /* * run_check_subtest_range -- - * Run successive tests via binary search that determines the approximate - * crossover point between when data is recoverable or not. Once that is - * determined, run the subtest in a range near that crossover point. - * - * The theory is that running at the crossover point will tend to trigger - * "interesting" failures at the borderline when the checkpoint is about - * to, or has, succeeded. If any of those failures creates a WiredTiger - * home directory that cannot be recovered, the top level test will fail. - */ + * Run successive tests via binary search that determines the approximate crossover point + * between when data is recoverable or not. Once that is determined, run the subtest in a range + * near that crossover point. The theory is that running at the crossover point will tend to + * trigger "interesting" failures at the borderline when the checkpoint is about to, or has, + * succeeded. If any of those failures creates a WiredTiger home directory that cannot be + * recovered, the top level test will fail. + */ static int run_check_subtest_range(TEST_OPTS *opts, const char *debugger, bool close_test) { - uint64_t cutoff, high, low, mid, nops, nresults; - int i; - bool got_failure, got_success; - - if (opts->verbose) - printf("Determining best range of operations until failure, " - "with close_test %s.\n", - (close_test ? "enabled" : "disabled")); - - run_check_subtest(opts, debugger, 1, close_test, &cutoff); - low = 0; - high = MAX_OP_RANGE; - mid = (low + high) / 2; - while (low < mid - 5 || high > mid + 5) { - run_check_subtest(opts, debugger, mid, close_test, - &nresults); - if (nresults > cutoff) - high = mid; - else - low = mid; - mid = (low + high) / 2; - } - /* - * mid is the number of ops that is the crossover point. - * Run some tests near that point to try to trigger weird - * failures. If mid is too low or too high, it indicates - * there is a fundamental problem with the test. - */ - testutil_assert(mid > 1 && mid < MAX_OP_RANGE - 1); - if (opts->verbose) - printf("Retesting around %" PRIu64 " operations.\n", - mid); - - got_failure = false; - got_success = false; - for (i = 0; - i < TESTS_PER_CALIBRATION && (!got_failure || !got_success); i++) - for (nops = mid - 10; nops < mid + 10; nops++) { - run_check_subtest(opts, debugger, nops, - close_test, &nresults); - if (nresults > cutoff) - got_failure = true; - else - got_success = true; - } - - /* - * Check that it really ran with a crossover point. - * If not, perhaps we calibrated the range incorrectly. - * Tell caller to try again. - */ - if (!got_failure || !got_success) { - fprintf(stderr, "Warning: did not find a reliable test range.\n" - "midpoint=%" PRIu64 ", close_test=%d, got_failure=%d, " - "got_success=%d\n", mid, (int)close_test, (int)got_failure, - (int)got_success); - return (EAGAIN); - } - return (0); + uint64_t cutoff, high, low, mid, nops, nresults; + int i; + bool got_failure, got_success; + + if (opts->verbose) + printf( + "Determining best range of operations until failure, " + "with close_test %s.\n", + (close_test ? "enabled" : "disabled")); + + run_check_subtest(opts, debugger, 1, close_test, &cutoff); + low = 0; + high = MAX_OP_RANGE; + mid = (low + high) / 2; + while (low < mid - 5 || high > mid + 5) { + run_check_subtest(opts, debugger, mid, close_test, &nresults); + if (nresults > cutoff) + high = mid; + else + low = mid; + mid = (low + high) / 2; + } + /* + * mid is the number of ops that is the crossover point. Run some tests near that point to try + * to trigger weird failures. If mid is too low or too high, it indicates there is a fundamental + * problem with the test. + */ + testutil_assert(mid > 1 && mid < MAX_OP_RANGE - 1); + if (opts->verbose) + printf("Retesting around %" PRIu64 " operations.\n", mid); + + got_failure = false; + got_success = false; + for (i = 0; i < TESTS_PER_CALIBRATION && (!got_failure || !got_success); i++) + for (nops = mid - 10; nops < mid + 10; nops++) { + run_check_subtest(opts, debugger, nops, close_test, &nresults); + if (nresults > cutoff) + got_failure = true; + else + got_success = true; + } + + /* + * Check that it really ran with a crossover point. If not, perhaps we calibrated the range + * incorrectly. Tell caller to try again. + */ + if (!got_failure || !got_success) { + fprintf(stderr, + "Warning: did not find a reliable test range.\n" + "midpoint=%" PRIu64 + ", close_test=%d, got_failure=%d, " + "got_success=%d\n", + mid, (int)close_test, (int)got_failure, (int)got_success); + return (EAGAIN); + } + return (0); } /* * run_check_subtest_range_retry -- - * Repeatedly run the subtest range test, retrying some number of times - * as long as EBUSY is returned, a warning that the test did not - * adequately cover "both sides" of the test threshold. Such warning - * returns should be rare and are not hard failures, no WiredTiger bug - * is demonstrated. Rerunning the subtest range test will determine - * a new calibration for the range. + * Repeatedly run the subtest range test, retrying some number of times as long as EBUSY is + * returned, a warning that the test did not adequately cover "both sides" of the test + * threshold. Such warning returns should be rare and are not hard failures, no WiredTiger bug + * is demonstrated. Rerunning the subtest range test will determine a new calibration for the + * range. */ static void -run_check_subtest_range_retry(TEST_OPTS *opts, const char *debugger, - bool close_test) +run_check_subtest_range_retry(TEST_OPTS *opts, const char *debugger, bool close_test) { - WT_DECL_RET; - int tries; - - for (tries = 0; tries < TESTS_WITH_RECALIBRATION; tries++) { - if (tries != 0) { - fprintf(stderr, "Retrying after sleep...\n"); - sleep(5); - } - if ((ret = run_check_subtest_range( - opts, debugger, close_test)) == 0) - break; - testutil_assert(ret == EAGAIN); - } - if (tries == TESTS_WITH_RECALIBRATION) - /* - * If we couldn't successfully perform the test, - * we want to know about it. - */ - testutil_die(ret, "too many retries"); + WT_DECL_RET; + int tries; + + for (tries = 0; tries < TESTS_WITH_RECALIBRATION; tries++) { + if (tries != 0) { + fprintf(stderr, "Retrying after sleep...\n"); + sleep(5); + } + if ((ret = run_check_subtest_range(opts, debugger, close_test)) == 0) + break; + testutil_assert(ret == EAGAIN); + } + if (tries == TESTS_WITH_RECALIBRATION) + /* + * If we couldn't successfully perform the test, we want to know about it. + */ + testutil_die(ret, "too many retries"); } /* * run_process -- - * Run a program with arguments, wait until it completes. + * Run a program with arguments, wait until it completes. */ static int run_process(TEST_OPTS *opts, const char *prog, char *argv[], int *status) { - int pid; - char **arg; - - if (opts->verbose) { - printf("running: "); - for (arg = argv; *arg != NULL; arg++) - printf("%s ", *arg); - printf("\n"); - } - if ((pid = fork()) == 0) { - (void)execv(prog, argv); - testutil_die(errno, "%s", prog); - } else if (pid < 0) - return (errno); - - (void)waitpid(pid, status, 0); - return (0); + int pid; + char **arg; + + if (opts->verbose) { + printf("running: "); + for (arg = argv; *arg != NULL; arg++) + printf("%s ", *arg); + printf("\n"); + } + if ((pid = fork()) == 0) { + (void)execv(prog, argv); + testutil_die(errno, "%s", prog); + } else if (pid < 0) + return (errno); + + (void)waitpid(pid, status, 0); + return (0); } /* @@ -491,252 +463,232 @@ run_process(TEST_OPTS *opts, const char *prog, char *argv[], int *status) * Error event handler. */ static int -subtest_error_handler(WT_EVENT_HANDLER *handler, - WT_SESSION *session, int error, const char *message) +subtest_error_handler( + WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const char *message) { - (void)(handler); - (void)(session); - (void)(message); - - /* Exit on panic, there's no checking to be done. */ - if (error == WT_PANIC) - exit (1); - return (0); + (void)(handler); + (void)(session); + (void)(message); + + /* Exit on panic, there's no checking to be done. */ + if (error == WT_PANIC) + exit(1); + return (0); } static WT_EVENT_HANDLER event_handler = { - subtest_error_handler, - NULL, /* Message handler */ - NULL, /* Progress handler */ - NULL /* Close handler */ + subtest_error_handler, NULL, /* Message handler */ + NULL, /* Progress handler */ + NULL /* Close handler */ }; /* * subtest_main -- - * The main program for the subtest + * The main program for the subtest */ static void subtest_main(int argc, char *argv[], bool close_test) { - struct rlimit rlim; - TEST_OPTS *opts, _opts; - WT_SESSION *session; - char config[1024], filename[1024]; - const char *p; - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - memset(&rlim, 0, sizeof(rlim)); - - /* No core files during fault injection tests. */ - testutil_check(setrlimit(RLIMIT_CORE, &rlim)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - - /* Redirect stderr, stdout. */ - testutil_check(__wt_snprintf( - filename, sizeof(filename), "%s/%s", opts->home, STDERR_FILE)); - testutil_assert(freopen(filename, "a", stderr) != NULL); - testutil_check(__wt_snprintf( - filename, sizeof(filename), "%s/%s", opts->home, STDOUT_FILE)); - testutil_assert(freopen(filename, "a", stdout) != NULL); - - /* - * Use $top_builddir if it's available, otherwise assume we're building - * in build_posix and running in the test/csuite directory. - */ -#define WT_FAIL_FS_LIB "ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so" - if ((p = getenv("top_builddir")) == NULL) - p = "../../build_posix"; - testutil_check(__wt_snprintf(config, sizeof(config), - "create,cache_size=250M,log=(enabled)," - "transaction_sync=(enabled,method=none)," - "extensions=(%s/%s=" - "(early_load,config={environment=true,verbose=true}))", - p, WT_FAIL_FS_LIB)); - testutil_check( - wiredtiger_open(opts->home, &event_handler, config, &opts->conn)); - - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check(session->create(session, "table:subtest", - "key_format=i,value_format=iiiS," - "columns=(id,v0,v1,v2,big)")); - - testutil_check(session->create(session, "table:subtest2", - "key_format=i,value_format=i")); - - testutil_check(session->create(session, "index:subtest:v0", - "columns=(v0)")); - testutil_check(session->create(session, "index:subtest:v1", - "columns=(v1)")); - testutil_check(session->create(session, "index:subtest:v2", - "columns=(v2)")); - - testutil_check(session->close(session, NULL)); - - subtest_populate(opts, close_test); - - testutil_cleanup(opts); -} + struct rlimit rlim; + TEST_OPTS *opts, _opts; + WT_SESSION *session; + char config[1024], filename[1024]; + const char *p; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + memset(&rlim, 0, sizeof(rlim)); + + /* No core files during fault injection tests. */ + testutil_check(setrlimit(RLIMIT_CORE, &rlim)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + /* Redirect stderr, stdout. */ + testutil_check(__wt_snprintf(filename, sizeof(filename), "%s/%s", opts->home, STDERR_FILE)); + testutil_assert(freopen(filename, "a", stderr) != NULL); + testutil_check(__wt_snprintf(filename, sizeof(filename), "%s/%s", opts->home, STDOUT_FILE)); + testutil_assert(freopen(filename, "a", stdout) != NULL); /* - * This macro is used as a substitute for testutil_check, except that it is - * aware of when a failure may be expected due to the effects of the fail_fs. - * This macro is used only in subtest_populate(), it uses local variables. + * Use $top_builddir if it's available, otherwise assume we're building in build_posix and running + * in the test/csuite directory. */ -#define CHECK(expr, failmode) { \ - int _ret; \ - _ret = expr; \ - if (_ret != 0) { \ - if (!failmode || \ - (_ret != WT_RUN_RECOVERY && _ret != EIO)) { \ - fprintf(stderr, " BAD RETURN %d for \"%s\"\n", \ - _ret, #expr); \ - testutil_check(_ret); \ - } else \ - failed = true; \ - } \ +#define WT_FAIL_FS_LIB "ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so" + if ((p = getenv("top_builddir")) == NULL) + p = "../../build_posix"; + testutil_check(__wt_snprintf(config, sizeof(config), + "create,cache_size=250M,log=(enabled)," + "transaction_sync=(enabled,method=none)," + "extensions=(%s/%s=" + "(early_load,config={environment=true,verbose=true}))", + p, WT_FAIL_FS_LIB)); + testutil_check(wiredtiger_open(opts->home, &event_handler, config, &opts->conn)); + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(session->create(session, "table:subtest", + "key_format=i,value_format=iiiS," + "columns=(id,v0,v1,v2,big)")); + + testutil_check(session->create(session, "table:subtest2", "key_format=i,value_format=i")); + + testutil_check(session->create(session, "index:subtest:v0", "columns=(v0)")); + testutil_check(session->create(session, "index:subtest:v1", "columns=(v1)")); + testutil_check(session->create(session, "index:subtest:v2", "columns=(v2)")); + + testutil_check(session->close(session, NULL)); + + subtest_populate(opts, close_test); + + testutil_cleanup(opts); } /* + * This macro is used as a substitute for testutil_check, except that it is aware of when a failure + * may be expected due to the effects of the fail_fs. This macro is used only in subtest_populate(), + * it uses local variables. + */ +#define CHECK(expr, failmode) \ + { \ + int _ret; \ + _ret = expr; \ + if (_ret != 0) { \ + if (!failmode || (_ret != WT_RUN_RECOVERY && _ret != EIO)) { \ + fprintf(stderr, " BAD RETURN %d for \"%s\"\n", _ret, #expr); \ + testutil_check(_ret); \ + } else \ + failed = true; \ + } \ + } + +/* * subtest_populate -- - * Populate the tables. + * Populate the tables. */ static void subtest_populate(TEST_OPTS *opts, bool close_test) { - WT_CURSOR *maincur, *maincur2; - WT_RAND_STATE rnd; - WT_SESSION *session; - uint64_t i, nrecords; - uint32_t rndint; - int key, v0, v1, v2; - char *big, *bigref; - bool failed; - - failed = false; - __wt_random_init_seed(NULL, &rnd); - CHECK(create_big_string(&bigref), false); - nrecords = opts->nrecords; - - CHECK(opts->conn->open_session( - opts->conn, NULL, NULL, &session), false); - - CHECK(session->open_cursor(session, "table:subtest", NULL, - NULL, &maincur), false); - - CHECK(session->open_cursor(session, "table:subtest2", NULL, - NULL, &maincur2), false); - - for (i = 0; i < nrecords && !failed; i++) { - rndint = __wt_random(&rnd); - generate_key(i, &key); - generate_value(rndint, i, bigref, &v0, &v1, &v2, &big); - CHECK(session->begin_transaction(session, NULL), false); - maincur->set_key(maincur, key); - maincur->set_value(maincur, v0, v1, v2, big); - CHECK(maincur->insert(maincur), false); - - maincur2->set_key(maincur2, key); - maincur2->set_value(maincur2, rndint); - CHECK(maincur2->insert(maincur2), false); - CHECK(session->commit_transaction(session, NULL), false); - - if (i == 0) - /* - * Force an initial checkpoint, that helps to - * distinguish a clear failure from just not running - * long enough. - */ - CHECK(session->checkpoint(session, NULL), false); - - if ((i + 1) % VERBOSE_PRINT == 0 && opts->verbose) - printf(" %" PRIu64 "/%" PRIu64 "\n", - (i + 1), nrecords); - /* Attempt to isolate the failures to checkpointing. */ - if (i == (nrecords/100)) { - enable_failures(opts->nops, 1000000); - /* CHECK should expect failures. */ - CHECK(session->checkpoint(session, NULL), true); - disable_failures(); - if (failed && opts->verbose) - printf("checkpoint failed (expected).\n"); - } - } - - /* - * Closing handles after an extreme fail is likely to cause - * cascading failures (or crashes), so recommended practice is - * to immediately exit. We're interested in testing both with - * and without the recommended practice. - */ - if (failed) { - if (!close_test) { - fprintf(stderr, "exit early.\n"); - exit(0); - } else - fprintf(stderr, "closing after failure.\n"); - } - - free(bigref); - CHECK(maincur->close(maincur), false); - CHECK(maincur2->close(maincur2), false); - CHECK(session->close(session, NULL), false); + WT_CURSOR *maincur, *maincur2; + WT_RAND_STATE rnd; + WT_SESSION *session; + uint64_t i, nrecords; + uint32_t rndint; + int key, v0, v1, v2; + char *big, *bigref; + bool failed; + + failed = false; + __wt_random_init_seed(NULL, &rnd); + CHECK(create_big_string(&bigref), false); + nrecords = opts->nrecords; + + CHECK(opts->conn->open_session(opts->conn, NULL, NULL, &session), false); + + CHECK(session->open_cursor(session, "table:subtest", NULL, NULL, &maincur), false); + + CHECK(session->open_cursor(session, "table:subtest2", NULL, NULL, &maincur2), false); + + for (i = 0; i < nrecords && !failed; i++) { + rndint = __wt_random(&rnd); + generate_key(i, &key); + generate_value(rndint, i, bigref, &v0, &v1, &v2, &big); + CHECK(session->begin_transaction(session, NULL), false); + maincur->set_key(maincur, key); + maincur->set_value(maincur, v0, v1, v2, big); + CHECK(maincur->insert(maincur), false); + + maincur2->set_key(maincur2, key); + maincur2->set_value(maincur2, rndint); + CHECK(maincur2->insert(maincur2), false); + CHECK(session->commit_transaction(session, NULL), false); + + if (i == 0) + /* + * Force an initial checkpoint, that helps to distinguish a clear failure from just not + * running long enough. + */ + CHECK(session->checkpoint(session, NULL), false); + + if ((i + 1) % VERBOSE_PRINT == 0 && opts->verbose) + printf(" %" PRIu64 "/%" PRIu64 "\n", (i + 1), nrecords); + /* Attempt to isolate the failures to checkpointing. */ + if (i == (nrecords / 100)) { + enable_failures(opts->nops, 1000000); + /* CHECK should expect failures. */ + CHECK(session->checkpoint(session, NULL), true); + disable_failures(); + if (failed && opts->verbose) + printf("checkpoint failed (expected).\n"); + } + } + + /* + * Closing handles after an extreme fail is likely to cause cascading failures (or crashes), so + * recommended practice is to immediately exit. We're interested in testing both with and + * without the recommended practice. + */ + if (failed) { + if (!close_test) { + fprintf(stderr, "exit early.\n"); + exit(0); + } else + fprintf(stderr, "closing after failure.\n"); + } + + free(bigref); + CHECK(maincur->close(maincur), false); + CHECK(maincur2->close(maincur2), false); + CHECK(session->close(session, NULL), false); } /* * main -- - * The main program for the test. When invoked with "subtest" - * argument, run the subtest. Otherwise, run a separate process - * for each needed subtest, and check the results. + * The main program for the test. When invoked with "subtest" argument, run the subtest. + * Otherwise, run a separate process for each needed subtest, and check the results. */ int main(int argc, char *argv[]) { - TEST_OPTS *opts, _opts; - uint64_t nresults; - const char *debugger; - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - debugger = NULL; - - testutil_check(testutil_parse_opts(argc, argv, opts)); - argc -= __wt_optind; - argv += __wt_optind; - if (opts->nrecords == 0) - opts->nrecords = 50000; - - while (argc > 0) { - if (strcmp(argv[0], "subtest") == 0) { - subtest_main(argc, argv, false); - return (0); - } else if (strcmp(argv[0], "subtest_close") == 0) { - subtest_main(argc, argv, true); - return (0); - } else if (strcmp(argv[0], "gdb") == 0) - debugger = "/usr/bin/gdb"; - else - testutil_assert(false); - argc--; - argv++; - } - if (opts->verbose) { - printf("Number of operations until failure: %" PRIu64 - " (change with -o N)\n", opts->nops); - printf("Number of records: %" PRIu64 - " (change with -n N)\n", opts->nrecords); - } - if (opts->nops == 0) { - run_check_subtest_range_retry(opts, debugger, false); - run_check_subtest_range_retry(opts, debugger, true); - } else - run_check_subtest(opts, debugger, opts->nops, - opts->nrecords, &nresults); - - testutil_clean_work_dir(opts->home); - testutil_cleanup(opts); - - return (0); + TEST_OPTS *opts, _opts; + uint64_t nresults; + const char *debugger; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + debugger = NULL; + + testutil_check(testutil_parse_opts(argc, argv, opts)); + argc -= __wt_optind; + argv += __wt_optind; + if (opts->nrecords == 0) + opts->nrecords = 50000; + + while (argc > 0) { + if (strcmp(argv[0], "subtest") == 0) { + subtest_main(argc, argv, false); + return (0); + } else if (strcmp(argv[0], "subtest_close") == 0) { + subtest_main(argc, argv, true); + return (0); + } else if (strcmp(argv[0], "gdb") == 0) + debugger = "/usr/bin/gdb"; + else + testutil_assert(false); + argc--; + argv++; + } + if (opts->verbose) { + printf("Number of operations until failure: %" PRIu64 " (change with -o N)\n", opts->nops); + printf("Number of records: %" PRIu64 " (change with -n N)\n", opts->nrecords); + } + if (opts->nops == 0) { + run_check_subtest_range_retry(opts, debugger, false); + run_check_subtest_range_retry(opts, debugger, true); + } else + run_check_subtest(opts, debugger, opts->nops, opts->nrecords, &nresults); + + testutil_clean_work_dir(opts->home); + testutil_cleanup(opts); + + return (0); } diff --git a/src/third_party/wiredtiger/test/csuite/wt2999_join_extractor/main.c b/src/third_party/wiredtiger/test/csuite/wt2999_join_extractor/main.c index 796415adea9..3bf02ed3f3c 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2999_join_extractor/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2999_join_extractor/main.c @@ -40,129 +40,122 @@ * sets the key. */ static int -custom_extract1(WT_EXTRACTOR *extractor, WT_SESSION *session, - const WT_ITEM *key, const WT_ITEM *value, WT_CURSOR *result_cursor) +custom_extract1(WT_EXTRACTOR *extractor, WT_SESSION *session, const WT_ITEM *key, + const WT_ITEM *value, WT_CURSOR *result_cursor) { - WT_ITEM item; - int32_t v1; + WT_ITEM item; + int32_t v1; - (void)extractor; - (void)key; - testutil_check(wiredtiger_struct_unpack( - session, value->data, value->size, "u", &item)); + (void)extractor; + (void)key; + testutil_check(wiredtiger_struct_unpack(session, value->data, value->size, "u", &item)); - v1 = ((int*)item.data)[0]; - item.data = &v1; - item.size = sizeof(v1); + v1 = ((int *)item.data)[0]; + item.data = &v1; + item.size = sizeof(v1); - result_cursor->set_key(result_cursor, &item); - return (result_cursor->insert(result_cursor)); + result_cursor->set_key(result_cursor, &item); + return (result_cursor->insert(result_cursor)); } static int -custom_extract2(WT_EXTRACTOR *extractor, WT_SESSION *session, - const WT_ITEM *key, const WT_ITEM *value, WT_CURSOR *result_cursor) +custom_extract2(WT_EXTRACTOR *extractor, WT_SESSION *session, const WT_ITEM *key, + const WT_ITEM *value, WT_CURSOR *result_cursor) { - WT_ITEM item; - int32_t v2; + WT_ITEM item; + int32_t v2; - (void)extractor; - (void)key; - testutil_check(wiredtiger_struct_unpack( - session, value->data, value->size, "u", &item)); + (void)extractor; + (void)key; + testutil_check(wiredtiger_struct_unpack(session, value->data, value->size, "u", &item)); - v2 = ((int*)item.data)[1]; - item.data = &v2; - item.size = sizeof(v2); + v2 = ((int *)item.data)[1]; + item.data = &v2; + item.size = sizeof(v2); - result_cursor->set_key(result_cursor, &item); - return (result_cursor->insert(result_cursor)); + result_cursor->set_key(result_cursor, &item); + return (result_cursor->insert(result_cursor)); } -static WT_EXTRACTOR custom_extractor1 = { custom_extract1, NULL, NULL }; -static WT_EXTRACTOR custom_extractor2 = { custom_extract2, NULL, NULL }; +static WT_EXTRACTOR custom_extractor1 = {custom_extract1, NULL, NULL}; +static WT_EXTRACTOR custom_extractor2 = {custom_extract2, NULL, NULL}; int main(int argc, char *argv[]) { - TEST_OPTS *opts, _opts; - WT_CONNECTION *conn; - WT_CURSOR *cursor1, *cursor2, *jcursor; - WT_ITEM k, v; - WT_SESSION *session; - int32_t key, val[2]; - int i, ret; - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - - testutil_check(wiredtiger_open(opts->home, NULL, "create", &conn)); - opts->conn = conn; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - testutil_check(conn->add_extractor(conn, "custom_extractor1", - &custom_extractor1, NULL)); - testutil_check(conn->add_extractor(conn, "custom_extractor2", - &custom_extractor2, NULL)); - - testutil_check(session->create(session, - "table:main", "key_format=u,value_format=u,columns=(k,v)")); - testutil_check(session->create(session, - "index:main:index1", "key_format=u,extractor=custom_extractor1")); - testutil_check(session->create(session, - "index:main:index2", "key_format=u,extractor=custom_extractor2")); - - testutil_check(session->open_cursor(session, "table:main", NULL, NULL, - &cursor1)); - - v.data = val; - v.size = sizeof(val); - k.data = &key; - k.size = sizeof(key); - - key = 10; - val[0] = 20; - val[1] = 30; - for (i = 0; i < 100000; ++i) { - key += i; - val[0] += i; val[1] += i; - cursor1->set_key(cursor1, &k); - cursor1->set_value(cursor1, &v); - testutil_check(cursor1->insert(cursor1)); - } - - testutil_check(cursor1->close(cursor1)); - - testutil_check(session->open_cursor(session, "index:main:index1", NULL, - NULL, &cursor1)); - key = 20; - cursor1->set_key(cursor1, &k); - testutil_check(cursor1->search(cursor1)); - - testutil_check(session->open_cursor(session, "index:main:index2", NULL, - NULL, &cursor2)); - key = 30; - cursor2->set_key(cursor2, &k); - testutil_check(cursor2->search(cursor2)); - - testutil_check(session->open_cursor(session, "join:table:main", NULL, - NULL, &jcursor)); - testutil_check(session->join(session, jcursor, cursor1, "compare=gt")); - testutil_check(session->join(session, jcursor, cursor2, "compare=gt")); - - while ((ret = jcursor->next(jcursor)) == 0) //leak - ; - testutil_assert(ret == WT_NOTFOUND); - - testutil_check(jcursor->close(jcursor)); - testutil_check(cursor1->close(cursor1)); - testutil_check(cursor2->close(cursor2)); - - testutil_check(opts->conn->close(opts->conn, NULL)); - opts->conn = NULL; - testutil_cleanup(opts); - - return (EXIT_SUCCESS); + TEST_OPTS *opts, _opts; + WT_CONNECTION *conn; + WT_CURSOR *cursor1, *cursor2, *jcursor; + WT_ITEM k, v; + WT_SESSION *session; + int32_t key, val[2]; + int i, ret; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(wiredtiger_open(opts->home, NULL, "create", &conn)); + opts->conn = conn; + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + testutil_check(conn->add_extractor(conn, "custom_extractor1", &custom_extractor1, NULL)); + testutil_check(conn->add_extractor(conn, "custom_extractor2", &custom_extractor2, NULL)); + + testutil_check( + session->create(session, "table:main", "key_format=u,value_format=u,columns=(k,v)")); + testutil_check( + session->create(session, "index:main:index1", "key_format=u,extractor=custom_extractor1")); + testutil_check( + session->create(session, "index:main:index2", "key_format=u,extractor=custom_extractor2")); + + testutil_check(session->open_cursor(session, "table:main", NULL, NULL, &cursor1)); + + v.data = val; + v.size = sizeof(val); + k.data = &key; + k.size = sizeof(key); + + key = 10; + val[0] = 20; + val[1] = 30; + for (i = 0; i < 100000; ++i) { + key += i; + val[0] += i; + val[1] += i; + cursor1->set_key(cursor1, &k); + cursor1->set_value(cursor1, &v); + testutil_check(cursor1->insert(cursor1)); + } + + testutil_check(cursor1->close(cursor1)); + + testutil_check(session->open_cursor(session, "index:main:index1", NULL, NULL, &cursor1)); + key = 20; + cursor1->set_key(cursor1, &k); + testutil_check(cursor1->search(cursor1)); + + testutil_check(session->open_cursor(session, "index:main:index2", NULL, NULL, &cursor2)); + key = 30; + cursor2->set_key(cursor2, &k); + testutil_check(cursor2->search(cursor2)); + + testutil_check(session->open_cursor(session, "join:table:main", NULL, NULL, &jcursor)); + testutil_check(session->join(session, jcursor, cursor1, "compare=gt")); + testutil_check(session->join(session, jcursor, cursor2, "compare=gt")); + + while ((ret = jcursor->next(jcursor)) == 0) // leak + ; + testutil_assert(ret == WT_NOTFOUND); + + testutil_check(jcursor->close(jcursor)); + testutil_check(cursor1->close(cursor1)); + testutil_check(cursor2->close(cursor2)); + + testutil_check(opts->conn->close(opts->conn, NULL)); + opts->conn = NULL; + testutil_cleanup(opts); + + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c b/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c index 148d0062ddd..3d9dff6ac3e 100644 --- a/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c @@ -28,77 +28,69 @@ #include "test_util.h" /* - * JIRA ticket reference: WT-3120 - * Test case description: A simple file system extension built into - * a shared library. - * Failure mode: Loading the file system and closing the connection - * is enough to evoke the failure. This test does slightly more - * than that. + * JIRA ticket reference: WT-3120 Test case description: A simple file system extension built into a + * shared library. Failure mode: Loading the file system and closing the connection is enough to + * evoke the failure. This test does slightly more than that. */ int main(int argc, char *argv[]) { - TEST_OPTS *opts, _opts; - WT_CURSOR *cursor; - WT_SESSION *session; - char *kstr, *vstr, buf[1024]; - const char *p; + TEST_OPTS *opts, _opts; + WT_CURSOR *cursor; + WT_SESSION *session; + char *kstr, *vstr, buf[1024]; + const char *p; - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); - /* - * Use $top_builddir if it's available, otherwise assume we're building - * in build_posix and running in the test/csuite directory. - */ -#define WT_FAIL_FS_LIB "ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so" - if ((p = getenv("top_builddir")) == NULL) - p = "../../build_posix"; - testutil_check(__wt_snprintf(buf, sizeof(buf), - "create,extensions=(%s/%s=(early_load=true))", p, WT_FAIL_FS_LIB)); - testutil_check(wiredtiger_open(opts->home, NULL, buf, &opts->conn)); - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check(session->create(session, opts->uri, - "key_format=S,value_format=S")); +/* + * Use $top_builddir if it's available, otherwise assume we're building in build_posix and running + * in the test/csuite directory. + */ +#define WT_FAIL_FS_LIB "ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so" + if ((p = getenv("top_builddir")) == NULL) + p = "../../build_posix"; + testutil_check(__wt_snprintf( + buf, sizeof(buf), "create,extensions=(%s/%s=(early_load=true))", p, WT_FAIL_FS_LIB)); + testutil_check(wiredtiger_open(opts->home, NULL, buf, &opts->conn)); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(session->create(session, opts->uri, "key_format=S,value_format=S")); - testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, - &cursor)); - cursor->set_key(cursor, "a"); - cursor->set_value(cursor, "0"); - testutil_check(cursor->insert(cursor)); - cursor->set_key(cursor, "b"); - cursor->set_value(cursor, "1"); - testutil_check(cursor->insert(cursor)); - testutil_check(cursor->close(cursor)); - testutil_check(session->close(session, NULL)); + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &cursor)); + cursor->set_key(cursor, "a"); + cursor->set_value(cursor, "0"); + testutil_check(cursor->insert(cursor)); + cursor->set_key(cursor, "b"); + cursor->set_value(cursor, "1"); + testutil_check(cursor->insert(cursor)); + testutil_check(cursor->close(cursor)); + testutil_check(session->close(session, NULL)); - /* Force to disk and re-open. */ - testutil_check(opts->conn->close(opts->conn, NULL)); - testutil_check(wiredtiger_open(opts->home, NULL, NULL, &opts->conn)); + /* Force to disk and re-open. */ + testutil_check(opts->conn->close(opts->conn, NULL)); + testutil_check(wiredtiger_open(opts->home, NULL, NULL, &opts->conn)); - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, - &cursor)); - testutil_check(cursor->next(cursor)); - testutil_check(cursor->get_key(cursor, &kstr)); - testutil_check(cursor->get_value(cursor, &vstr)); - testutil_assert(strcmp(kstr, "a") == 0); - testutil_assert(strcmp(vstr, "0") == 0); - testutil_check(cursor->next(cursor)); - testutil_check(cursor->get_key(cursor, &kstr)); - testutil_check(cursor->get_value(cursor, &vstr)); - testutil_assert(strcmp(kstr, "b") == 0); - testutil_assert(strcmp(vstr, "1") == 0); - testutil_assert(cursor->next(cursor) == WT_NOTFOUND); - testutil_check(cursor->close(cursor)); - testutil_check(session->close(session, NULL)); - printf("Success\n"); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &cursor)); + testutil_check(cursor->next(cursor)); + testutil_check(cursor->get_key(cursor, &kstr)); + testutil_check(cursor->get_value(cursor, &vstr)); + testutil_assert(strcmp(kstr, "a") == 0); + testutil_assert(strcmp(vstr, "0") == 0); + testutil_check(cursor->next(cursor)); + testutil_check(cursor->get_key(cursor, &kstr)); + testutil_check(cursor->get_value(cursor, &vstr)); + testutil_assert(strcmp(kstr, "b") == 0); + testutil_assert(strcmp(vstr, "1") == 0); + testutil_assert(cursor->next(cursor) == WT_NOTFOUND); + testutil_check(cursor->close(cursor)); + testutil_check(session->close(session, NULL)); + printf("Success\n"); - testutil_cleanup(opts); - return (EXIT_SUCCESS); + testutil_cleanup(opts); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt3135_search_near_collator/main.c b/src/third_party/wiredtiger/test/csuite/wt3135_search_near_collator/main.c index 96a9f429d9f..44abd0af993 100644 --- a/src/third_party/wiredtiger/test/csuite/wt3135_search_near_collator/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt3135_search_near_collator/main.c @@ -28,214 +28,202 @@ #include "test_util.h" /* - * JIRA ticket reference: WT-3135 - * Test case description: Each set of data is ordered and contains - * five elements (0-4). We insert elements 1 and 3, and then do - * search_near and search for each element. For each set of data, we perform - * these tests first using a custom collator, and second using a custom collator - * and extractor. In each case there are index keys having variable length. - * Failure mode: In the reported test case, the custom compare routine is - * given a truncated key to compare, and the unpack functions return errors - * because the truncation appeared in the middle of a key. + * JIRA ticket reference: WT-3135 Test case description: Each set of data is ordered and contains + * five elements (0-4). We insert elements 1 and 3, and then do search_near and search for each + * element. For each set of data, we perform these tests first using a custom collator, and second + * using a custom collator and extractor. In each case there are index keys having variable length. + * Failure mode: In the reported test case, the custom compare routine is given a truncated key to + * compare, and the unpack functions return errors because the truncation appeared in the middle of + * a key. */ -#define TEST_ENTRY_COUNT 5 +#define TEST_ENTRY_COUNT 5 typedef const char *TEST_SET[TEST_ENTRY_COUNT]; -static TEST_SET test_sets[] = { - { "0", "01", "012", "0123", "01234" }, - { "A", "B", "C", "D", "E" }, - { "5", "54", "543", "5432", "54321" }, - { "54321", "5433", "544", "55", "6" } -}; -#define TEST_SET_COUNT (sizeof(test_sets) / sizeof(test_sets[0])) +static TEST_SET test_sets[] = {{"0", "01", "012", "0123", "01234"}, {"A", "B", "C", "D", "E"}, + {"5", "54", "543", "5432", "54321"}, {"54321", "5433", "544", "55", "6"}}; +#define TEST_SET_COUNT (sizeof(test_sets) / sizeof(test_sets[0])) static bool item_str_equal(WT_ITEM *item, const char *str) { - return (item->size == strlen(str) + 1 && strncmp((char *)item->data, - str, item->size) == 0); + return (item->size == strlen(str) + 1 && strncmp((char *)item->data, str, item->size) == 0); } static int compare_int(int64_t a, int64_t b) { - return (a < b ? -1 : (a > b ? 1 : 0)); + return (a < b ? -1 : (a > b ? 1 : 0)); } static int index_compare_primary(WT_PACK_STREAM *s1, WT_PACK_STREAM *s2, int *cmp) { - int64_t pkey1, pkey2; - int rc1, rc2; - - rc1 = wiredtiger_unpack_int(s1, &pkey1); - rc2 = wiredtiger_unpack_int(s2, &pkey2); - - if (rc1 == 0 && rc2 == 0) - *cmp = compare_int(pkey1, pkey2); - else if (rc1 != 0 && rc2 != 0) - *cmp = 0; - else if (rc1 != 0) - *cmp = -1; - else - *cmp = 1; - return (0); + int64_t pkey1, pkey2; + int rc1, rc2; + + rc1 = wiredtiger_unpack_int(s1, &pkey1); + rc2 = wiredtiger_unpack_int(s2, &pkey2); + + if (rc1 == 0 && rc2 == 0) + *cmp = compare_int(pkey1, pkey2); + else if (rc1 != 0 && rc2 != 0) + *cmp = 0; + else if (rc1 != 0) + *cmp = -1; + else + *cmp = 1; + return (0); } static int -index_compare_S(WT_COLLATOR *collator, WT_SESSION *session, - const WT_ITEM *key1, const WT_ITEM *key2, int *cmp) +index_compare_S( + WT_COLLATOR *collator, WT_SESSION *session, const WT_ITEM *key1, const WT_ITEM *key2, int *cmp) { - WT_PACK_STREAM *s1, *s2; - const char *skey1, *skey2; + WT_PACK_STREAM *s1, *s2; + const char *skey1, *skey2; - (void)collator; + (void)collator; - testutil_check(wiredtiger_unpack_start(session, "Si", key1->data, - key1->size, &s1)); - testutil_check(wiredtiger_unpack_start(session, "Si", key2->data, - key2->size, &s2)); + testutil_check(wiredtiger_unpack_start(session, "Si", key1->data, key1->size, &s1)); + testutil_check(wiredtiger_unpack_start(session, "Si", key2->data, key2->size, &s2)); - testutil_check(wiredtiger_unpack_str(s1, &skey1)); - testutil_check(wiredtiger_unpack_str(s2, &skey2)); + testutil_check(wiredtiger_unpack_str(s1, &skey1)); + testutil_check(wiredtiger_unpack_str(s2, &skey2)); - if ((*cmp = strcmp(skey1, skey2)) == 0) - testutil_check(index_compare_primary(s1, s2, cmp)); + if ((*cmp = strcmp(skey1, skey2)) == 0) + testutil_check(index_compare_primary(s1, s2, cmp)); - testutil_check(wiredtiger_pack_close(s1, NULL)); - testutil_check(wiredtiger_pack_close(s2, NULL)); + testutil_check(wiredtiger_pack_close(s1, NULL)); + testutil_check(wiredtiger_pack_close(s2, NULL)); - return (0); + return (0); } static int -index_compare_u(WT_COLLATOR *collator, WT_SESSION *session, - const WT_ITEM *key1, const WT_ITEM *key2, int *cmp) +index_compare_u( + WT_COLLATOR *collator, WT_SESSION *session, const WT_ITEM *key1, const WT_ITEM *key2, int *cmp) { - WT_ITEM skey1, skey2; - WT_PACK_STREAM *s1, *s2; + WT_ITEM skey1, skey2; + WT_PACK_STREAM *s1, *s2; - (void)collator; + (void)collator; - testutil_check(wiredtiger_unpack_start(session, "ui", key1->data, - key1->size, &s1)); - testutil_check(wiredtiger_unpack_start(session, "ui", key2->data, - key2->size, &s2)); + testutil_check(wiredtiger_unpack_start(session, "ui", key1->data, key1->size, &s1)); + testutil_check(wiredtiger_unpack_start(session, "ui", key2->data, key2->size, &s2)); - testutil_check(wiredtiger_unpack_item(s1, &skey1)); - testutil_check(wiredtiger_unpack_item(s2, &skey2)); + testutil_check(wiredtiger_unpack_item(s1, &skey1)); + testutil_check(wiredtiger_unpack_item(s2, &skey2)); - if ((*cmp = strcmp(skey1.data, skey2.data)) == 0) - testutil_check(index_compare_primary(s1, s2, cmp)); + if ((*cmp = strcmp(skey1.data, skey2.data)) == 0) + testutil_check(index_compare_primary(s1, s2, cmp)); - testutil_check(wiredtiger_pack_close(s1, NULL)); - testutil_check(wiredtiger_pack_close(s2, NULL)); + testutil_check(wiredtiger_pack_close(s1, NULL)); + testutil_check(wiredtiger_pack_close(s2, NULL)); - return (0); + return (0); } static int -index_extractor_u(WT_EXTRACTOR *extractor, WT_SESSION *session, - const WT_ITEM *key, const WT_ITEM *value, WT_CURSOR *result_cursor) +index_extractor_u(WT_EXTRACTOR *extractor, WT_SESSION *session, const WT_ITEM *key, + const WT_ITEM *value, WT_CURSOR *result_cursor) { - (void)extractor; - (void)session; - (void)key; + (void)extractor; + (void)session; + (void)key; - result_cursor->set_key(result_cursor, value); - return result_cursor->insert(result_cursor); + result_cursor->set_key(result_cursor, value); + return result_cursor->insert(result_cursor); } -static WT_COLLATOR collator_S = { index_compare_S, NULL, NULL }; -static WT_COLLATOR collator_u = { index_compare_u, NULL, NULL }; -static WT_EXTRACTOR extractor_u = { index_extractor_u, NULL, NULL }; +static WT_COLLATOR collator_S = {index_compare_S, NULL, NULL}; +static WT_COLLATOR collator_u = {index_compare_u, NULL, NULL}; +static WT_EXTRACTOR extractor_u = {index_extractor_u, NULL, NULL}; /* - * Check search() and search_near() using the test string indicated - * by test_index. + * Check search() and search_near() using the test string indicated by test_index. */ static void search_using_str(WT_CURSOR *cursor, TEST_SET test_set, int test_index) { - int exact, ret; - const char *result; - const char *str_01, *str_0123, *test_str; - - testutil_assert(test_index >= 0 && test_index <= 4); - str_01 = test_set[1]; - str_0123 = test_set[3]; - test_str = test_set[test_index]; - - cursor->set_key(cursor, test_str); - testutil_check(cursor->search_near(cursor, &exact)); - testutil_check(cursor->get_key(cursor, &result)); - - if (test_index == 0) - testutil_assert(strcmp(result, str_01) == 0 && exact > 0); - else if (test_index == 1) - testutil_assert(strcmp(result, str_01) == 0 && exact == 0); - else if (test_index == 2) - testutil_assert((strcmp(result, str_0123) == 0 && exact > 0) || - (strcmp(result, str_01) == 0 && exact < 0)); - else if (test_index == 3) - testutil_assert(strcmp(result, str_0123) == 0 && exact == 0); - else if (test_index == 4) - testutil_assert(strcmp(result, str_0123) == 0 && exact < 0); - - cursor->set_key(cursor, test_str); - ret = cursor->search(cursor); - - if (test_index == 0 || test_index == 2 || test_index == 4) - testutil_assert(ret == WT_NOTFOUND); - else if (test_index == 1 || test_index == 3) - testutil_assert(ret == 0); + int exact, ret; + const char *result; + const char *str_01, *str_0123, *test_str; + + testutil_assert(test_index >= 0 && test_index <= 4); + str_01 = test_set[1]; + str_0123 = test_set[3]; + test_str = test_set[test_index]; + + cursor->set_key(cursor, test_str); + testutil_check(cursor->search_near(cursor, &exact)); + testutil_check(cursor->get_key(cursor, &result)); + + if (test_index == 0) + testutil_assert(strcmp(result, str_01) == 0 && exact > 0); + else if (test_index == 1) + testutil_assert(strcmp(result, str_01) == 0 && exact == 0); + else if (test_index == 2) + testutil_assert((strcmp(result, str_0123) == 0 && exact > 0) || + (strcmp(result, str_01) == 0 && exact < 0)); + else if (test_index == 3) + testutil_assert(strcmp(result, str_0123) == 0 && exact == 0); + else if (test_index == 4) + testutil_assert(strcmp(result, str_0123) == 0 && exact < 0); + + cursor->set_key(cursor, test_str); + ret = cursor->search(cursor); + + if (test_index == 0 || test_index == 2 || test_index == 4) + testutil_assert(ret == WT_NOTFOUND); + else if (test_index == 1 || test_index == 3) + testutil_assert(ret == 0); } /* - * Check search() and search_near() using the test string indicated - * by test_index against a table containing a variable sized item. + * Check search() and search_near() using the test string indicated by test_index against a table + * containing a variable sized item. */ static void search_using_item(WT_CURSOR *cursor, TEST_SET test_set, int test_index) { - WT_ITEM item; - size_t testlen; - int exact, ret; - const char *str_01, *str_0123, *test_str; - - testutil_assert(test_index >= 0 && test_index <= 4); - str_01 = test_set[1]; - str_0123 = test_set[3]; - test_str = test_set[test_index]; - - testlen = strlen(test_str) + 1; - item.data = test_str; - item.size = testlen; - cursor->set_key(cursor, &item); - testutil_check(cursor->search_near(cursor, &exact)); - testutil_check(cursor->get_key(cursor, &item)); - - if (test_index == 0) - testutil_assert(item_str_equal(&item, str_01) && exact > 0); - else if (test_index == 1) - testutil_assert(item_str_equal(&item, str_01) && exact == 0); - else if (test_index == 2) - testutil_assert((item_str_equal(&item, str_0123) && exact > 0) - || (item_str_equal(&item, str_01) && exact < 0)); - else if (test_index == 3) - testutil_assert(item_str_equal(&item, str_0123) && exact == 0); - else if (test_index == 4) - testutil_assert(item_str_equal(&item, str_0123) && exact < 0); - - item.data = test_str; - item.size = testlen; - cursor->set_key(cursor, &item); - ret = cursor->search(cursor); - - if (test_index == 0 || test_index == 2 || test_index == 4) - testutil_assert(ret == WT_NOTFOUND); - else if (test_index == 1 || test_index == 3) - testutil_assert(ret == 0); + WT_ITEM item; + size_t testlen; + int exact, ret; + const char *str_01, *str_0123, *test_str; + + testutil_assert(test_index >= 0 && test_index <= 4); + str_01 = test_set[1]; + str_0123 = test_set[3]; + test_str = test_set[test_index]; + + testlen = strlen(test_str) + 1; + item.data = test_str; + item.size = testlen; + cursor->set_key(cursor, &item); + testutil_check(cursor->search_near(cursor, &exact)); + testutil_check(cursor->get_key(cursor, &item)); + + if (test_index == 0) + testutil_assert(item_str_equal(&item, str_01) && exact > 0); + else if (test_index == 1) + testutil_assert(item_str_equal(&item, str_01) && exact == 0); + else if (test_index == 2) + testutil_assert((item_str_equal(&item, str_0123) && exact > 0) || + (item_str_equal(&item, str_01) && exact < 0)); + else if (test_index == 3) + testutil_assert(item_str_equal(&item, str_0123) && exact == 0); + else if (test_index == 4) + testutil_assert(item_str_equal(&item, str_0123) && exact < 0); + + item.data = test_str; + item.size = testlen; + cursor->set_key(cursor, &item); + ret = cursor->search(cursor); + + if (test_index == 0 || test_index == 2 || test_index == 4) + testutil_assert(ret == WT_NOTFOUND); + else if (test_index == 1 || test_index == 3) + testutil_assert(ret == 0); } /* @@ -244,117 +232,104 @@ search_using_item(WT_CURSOR *cursor, TEST_SET test_set, int test_index) static void test_one_set(WT_SESSION *session, TEST_SET set) { - WT_CURSOR *cursor; - WT_ITEM item; - int32_t i; - - /* - * Part 1: Using a custom collator, insert some elements - * and verify results from search_near. - */ - - testutil_check(session->create(session, - "table:main", "key_format=i,value_format=S,columns=(k,v)")); - testutil_check(session->create(session, - "index:main:def_collator", "columns=(v)")); - testutil_check(session->create(session, - "index:main:custom_collator", - "columns=(v),collator=collator_S")); - - /* Insert only elements #1 and #3. */ - testutil_check(session->open_cursor(session, - "table:main", NULL, NULL, &cursor)); - cursor->set_key(cursor, 0); - cursor->set_value(cursor, set[1]); - testutil_check(cursor->insert(cursor)); - cursor->set_key(cursor, 1); - cursor->set_value(cursor, set[3]); - testutil_check(cursor->insert(cursor)); - testutil_check(cursor->close(cursor)); - - /* Check all elements in def_collator index. */ - testutil_check(session->open_cursor(session, - "index:main:def_collator", NULL, NULL, &cursor)); - for (i = 0; i < (int32_t)TEST_ENTRY_COUNT; i++) - search_using_str(cursor, set, i); - testutil_check(cursor->close(cursor)); - - /* Check all elements in custom_collator index */ - testutil_check(session->open_cursor(session, - "index:main:custom_collator", NULL, NULL, &cursor)); - for (i = 0; i < (int32_t)TEST_ENTRY_COUNT; i++) - search_using_str(cursor, set, i); - testutil_check(cursor->close(cursor)); - - /* - * Part 2: perform the same checks using a custom collator and - * extractor. - */ - testutil_check(session->create(session, - "table:main2", "key_format=i,value_format=u,columns=(k,v)")); - - testutil_check(session->create(session, "index:main2:idx_w_coll", - "key_format=u,collator=collator_u,extractor=extractor_u")); - - testutil_check(session->open_cursor(session, - "table:main2", NULL, NULL, &cursor)); - - memset(&item, 0, sizeof(item)); - item.size = strlen(set[1]) + 1; - item.data = set[1]; - cursor->set_key(cursor, 1); - cursor->set_value(cursor, &item); - testutil_check(cursor->insert(cursor)); - - item.size = strlen(set[3]) + 1; - item.data = set[3]; - cursor->set_key(cursor, 3); - cursor->set_value(cursor, &item); - testutil_check(cursor->insert(cursor)); - - testutil_check(cursor->close(cursor)); - - testutil_check(session->open_cursor(session, - "index:main2:idx_w_coll", NULL, NULL, &cursor)); - for (i = 0; i < (int32_t)TEST_ENTRY_COUNT; i++) - search_using_item(cursor, set, i); - testutil_check(cursor->close(cursor)); - - testutil_check(session->drop(session, "table:main", NULL)); - testutil_check(session->drop(session, "table:main2", NULL)); + WT_CURSOR *cursor; + WT_ITEM item; + int32_t i; + + /* + * Part 1: Using a custom collator, insert some elements and verify results from search_near. + */ + + testutil_check( + session->create(session, "table:main", "key_format=i,value_format=S,columns=(k,v)")); + testutil_check(session->create(session, "index:main:def_collator", "columns=(v)")); + testutil_check( + session->create(session, "index:main:custom_collator", "columns=(v),collator=collator_S")); + + /* Insert only elements #1 and #3. */ + testutil_check(session->open_cursor(session, "table:main", NULL, NULL, &cursor)); + cursor->set_key(cursor, 0); + cursor->set_value(cursor, set[1]); + testutil_check(cursor->insert(cursor)); + cursor->set_key(cursor, 1); + cursor->set_value(cursor, set[3]); + testutil_check(cursor->insert(cursor)); + testutil_check(cursor->close(cursor)); + + /* Check all elements in def_collator index. */ + testutil_check(session->open_cursor(session, "index:main:def_collator", NULL, NULL, &cursor)); + for (i = 0; i < (int32_t)TEST_ENTRY_COUNT; i++) + search_using_str(cursor, set, i); + testutil_check(cursor->close(cursor)); + + /* Check all elements in custom_collator index */ + testutil_check( + session->open_cursor(session, "index:main:custom_collator", NULL, NULL, &cursor)); + for (i = 0; i < (int32_t)TEST_ENTRY_COUNT; i++) + search_using_str(cursor, set, i); + testutil_check(cursor->close(cursor)); + + /* + * Part 2: perform the same checks using a custom collator and extractor. + */ + testutil_check( + session->create(session, "table:main2", "key_format=i,value_format=u,columns=(k,v)")); + + testutil_check(session->create( + session, "index:main2:idx_w_coll", "key_format=u,collator=collator_u,extractor=extractor_u")); + + testutil_check(session->open_cursor(session, "table:main2", NULL, NULL, &cursor)); + + memset(&item, 0, sizeof(item)); + item.size = strlen(set[1]) + 1; + item.data = set[1]; + cursor->set_key(cursor, 1); + cursor->set_value(cursor, &item); + testutil_check(cursor->insert(cursor)); + + item.size = strlen(set[3]) + 1; + item.data = set[3]; + cursor->set_key(cursor, 3); + cursor->set_value(cursor, &item); + testutil_check(cursor->insert(cursor)); + + testutil_check(cursor->close(cursor)); + + testutil_check(session->open_cursor(session, "index:main2:idx_w_coll", NULL, NULL, &cursor)); + for (i = 0; i < (int32_t)TEST_ENTRY_COUNT; i++) + search_using_item(cursor, set, i); + testutil_check(cursor->close(cursor)); + + testutil_check(session->drop(session, "table:main", NULL)); + testutil_check(session->drop(session, "table:main2", NULL)); } int main(int argc, char *argv[]) { - TEST_OPTS *opts, _opts; - WT_SESSION *session; - size_t i; - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - - testutil_check(wiredtiger_open(opts->home, NULL, "create", - &opts->conn)); - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - - /* Add any collators and extractors used by tests */ - testutil_check(opts->conn->add_collator(opts->conn, "collator_S", - &collator_S, NULL)); - testutil_check(opts->conn->add_collator(opts->conn, "collator_u", - &collator_u, NULL)); - testutil_check(opts->conn->add_extractor(opts->conn, "extractor_u", - &extractor_u, NULL)); - - for (i = 0; i < TEST_SET_COUNT; i++) { - printf("test set %" WT_SIZET_FMT "\n", i); - test_one_set(session, test_sets[i]); - } - - testutil_check(session->close(session, NULL)); - testutil_cleanup(opts); - return (EXIT_SUCCESS); + TEST_OPTS *opts, _opts; + WT_SESSION *session; + size_t i; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(wiredtiger_open(opts->home, NULL, "create", &opts->conn)); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + /* Add any collators and extractors used by tests */ + testutil_check(opts->conn->add_collator(opts->conn, "collator_S", &collator_S, NULL)); + testutil_check(opts->conn->add_collator(opts->conn, "collator_u", &collator_u, NULL)); + testutil_check(opts->conn->add_extractor(opts->conn, "extractor_u", &extractor_u, NULL)); + + for (i = 0; i < TEST_SET_COUNT; i++) { + printf("test set %" WT_SIZET_FMT "\n", i); + test_one_set(session, test_sets[i]); + } + + testutil_check(session->close(session, NULL)); + testutil_cleanup(opts); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt3184_dup_index_collator/main.c b/src/third_party/wiredtiger/test/csuite/wt3184_dup_index_collator/main.c index de25db68ceb..151d7687f8a 100644 --- a/src/third_party/wiredtiger/test/csuite/wt3184_dup_index_collator/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt3184_dup_index_collator/main.c @@ -28,141 +28,132 @@ #include "test_util.h" /* - * JIRA ticket reference: WT-3184 - * Test case description: Each set of data is ordered and contains - * five elements (0-4). We insert elements 1 and 3, and then do - * search_near and search for each element. For each set of data, we perform - * these tests first using a custom collator, and second using a custom collator - * and extractor. In each case there are index keys having variable length. - * Failure mode: In the reported test case, the custom compare routine is - * given a truncated key to compare, and the unpack functions return errors - * because the truncation appeared in the middle of a key. + * JIRA ticket reference: WT-3184 Test case description: Each set of data is ordered and contains + * five elements (0-4). We insert elements 1 and 3, and then do search_near and search for each + * element. For each set of data, we perform these tests first using a custom collator, and second + * using a custom collator and extractor. In each case there are index keys having variable length. + * Failure mode: In the reported test case, the custom compare routine is given a truncated key to + * compare, and the unpack functions return errors because the truncation appeared in the middle of + * a key. */ static int compare_int(int32_t a, int32_t b) { - return (a < b ? -1 : (a > b ? 1 : 0)); + return (a < b ? -1 : (a > b ? 1 : 0)); } static int32_t item_to_int(WT_ITEM *item) { - testutil_assert(item->size == sizeof(int32_t)); - return (*(int32_t *)item->data); + testutil_assert(item->size == sizeof(int32_t)); + return (*(int32_t *)item->data); } static int compare_int_items(WT_ITEM *itema, WT_ITEM *itemb) { - testutil_assert(itema->size == sizeof(int32_t)); - testutil_assert(itemb->size == sizeof(int32_t)); - return (compare_int(item_to_int(itema), item_to_int(itemb))); + testutil_assert(itema->size == sizeof(int32_t)); + testutil_assert(itemb->size == sizeof(int32_t)); + return (compare_int(item_to_int(itema), item_to_int(itemb))); } static void print_int_item(const char *str, const WT_ITEM *item) { - if (item->size > 0) { - testutil_assert(item->size == sizeof(int32_t)); - printf("%s%" PRId32, str, *(int32_t *)item->data); - } else - printf("%s<empty>", str); + if (item->size > 0) { + testutil_assert(item->size == sizeof(int32_t)); + printf("%s%" PRId32, str, *(int32_t *)item->data); + } else + printf("%s<empty>", str); } static int -index_compare(WT_COLLATOR *collator, WT_SESSION *session, - const WT_ITEM *key1, const WT_ITEM *key2, int *cmp) +index_compare( + WT_COLLATOR *collator, WT_SESSION *session, const WT_ITEM *key1, const WT_ITEM *key2, int *cmp) { - WT_ITEM ikey1, ikey2, pkey1, pkey2; - - (void)collator; - testutil_check(wiredtiger_struct_unpack(session, - key1->data, key1->size, "uu", &ikey1, &pkey1)); - testutil_check(wiredtiger_struct_unpack(session, - key2->data, key2->size, "uu", &ikey2, &pkey2)); - - print_int_item("index_compare: index key1 = ", &ikey1); - print_int_item(", primary key1 = ", &pkey1); - print_int_item(", index key2 = ", &ikey2); - print_int_item(", primary key2 = ", &pkey2); - printf("\n"); - - if ((*cmp = compare_int_items(&ikey1, &ikey2)) != 0) - return (0); - - if (pkey1.size != 0 && pkey2.size != 0) - *cmp = compare_int_items(&pkey1, &pkey2); - else if (pkey1.size != 0) - *cmp = 1; - else if (pkey2.size != 0) - *cmp = -1; - else - *cmp = 0; - - return (0); + WT_ITEM ikey1, ikey2, pkey1, pkey2; + + (void)collator; + testutil_check(wiredtiger_struct_unpack(session, key1->data, key1->size, "uu", &ikey1, &pkey1)); + testutil_check(wiredtiger_struct_unpack(session, key2->data, key2->size, "uu", &ikey2, &pkey2)); + + print_int_item("index_compare: index key1 = ", &ikey1); + print_int_item(", primary key1 = ", &pkey1); + print_int_item(", index key2 = ", &ikey2); + print_int_item(", primary key2 = ", &pkey2); + printf("\n"); + + if ((*cmp = compare_int_items(&ikey1, &ikey2)) != 0) + return (0); + + if (pkey1.size != 0 && pkey2.size != 0) + *cmp = compare_int_items(&pkey1, &pkey2); + else if (pkey1.size != 0) + *cmp = 1; + else if (pkey2.size != 0) + *cmp = -1; + else + *cmp = 0; + + return (0); } -static WT_COLLATOR index_coll = { index_compare, NULL, NULL }; +static WT_COLLATOR index_coll = {index_compare, NULL, NULL}; int main(int argc, char *argv[]) { - TEST_OPTS *opts, _opts; - WT_CURSOR *cursor, *cursor1; - WT_ITEM got, k, v; - WT_SESSION *session; - int32_t ki, vi; - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - - testutil_check(wiredtiger_open(opts->home, NULL, "create", - &opts->conn)); - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - - testutil_check(opts->conn->add_collator(opts->conn, "index_coll", - &index_coll, NULL)); - - testutil_check(session->create(session, - "table:main", "key_format=u,value_format=u,columns=(k,v)")); - testutil_check(session->create(session, - "index:main:index", "columns=(v),collator=index_coll")); - - printf("adding new record\n"); - testutil_check(session->open_cursor(session, "table:main", NULL, NULL, - &cursor)); - - ki = 13; - vi = 17; - - k.data = &ki; k.size = sizeof(ki); - v.data = &vi; v.size = sizeof(vi); - - cursor->set_key(cursor, &k); - cursor->set_value(cursor, &v); - testutil_check(cursor->insert(cursor)); - testutil_check(cursor->close(cursor)); - - printf("positioning index cursor\n"); - - testutil_check(session->open_cursor(session, "index:main:index", NULL, - NULL, &cursor)); - cursor->set_key(cursor, &v); - testutil_check(cursor->search(cursor)); - - printf("duplicating cursor\n"); - testutil_check(session->open_cursor(session, NULL, cursor, NULL, - &cursor1)); - testutil_check(cursor->get_value(cursor, &got)); - testutil_assert(item_to_int(&got) == 17); - testutil_check(cursor1->get_value(cursor1, &got)); - testutil_assert(item_to_int(&got) == 17); - - testutil_check(session->close(session, NULL)); - testutil_cleanup(opts); - return (EXIT_SUCCESS); + TEST_OPTS *opts, _opts; + WT_CURSOR *cursor, *cursor1; + WT_ITEM got, k, v; + WT_SESSION *session; + int32_t ki, vi; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(wiredtiger_open(opts->home, NULL, "create", &opts->conn)); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(opts->conn->add_collator(opts->conn, "index_coll", &index_coll, NULL)); + + testutil_check( + session->create(session, "table:main", "key_format=u,value_format=u,columns=(k,v)")); + testutil_check(session->create(session, "index:main:index", "columns=(v),collator=index_coll")); + + printf("adding new record\n"); + testutil_check(session->open_cursor(session, "table:main", NULL, NULL, &cursor)); + + ki = 13; + vi = 17; + + k.data = &ki; + k.size = sizeof(ki); + v.data = &vi; + v.size = sizeof(vi); + + cursor->set_key(cursor, &k); + cursor->set_value(cursor, &v); + testutil_check(cursor->insert(cursor)); + testutil_check(cursor->close(cursor)); + + printf("positioning index cursor\n"); + + testutil_check(session->open_cursor(session, "index:main:index", NULL, NULL, &cursor)); + cursor->set_key(cursor, &v); + testutil_check(cursor->search(cursor)); + + printf("duplicating cursor\n"); + testutil_check(session->open_cursor(session, NULL, cursor, NULL, &cursor1)); + testutil_check(cursor->get_value(cursor, &got)); + testutil_assert(item_to_int(&got) == 17); + testutil_check(cursor1->get_value(cursor1, &got)); + testutil_assert(item_to_int(&got) == 17); + + testutil_check(session->close(session, NULL)); + testutil_cleanup(opts); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt3338_partial_update/main.c b/src/third_party/wiredtiger/test/csuite/wt3338_partial_update/main.c index 5a413c0df3b..5689a996de9 100644 --- a/src/third_party/wiredtiger/test/csuite/wt3338_partial_update/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt3338_partial_update/main.c @@ -28,215 +28,199 @@ #include "test_util.h" /* - * JIRA ticket reference: WT-3338 - * Test case description: Smoke-test the partial update construction. + * JIRA ticket reference: WT-3338 Test case description: Smoke-test the partial update construction. */ -#define DEBUG 0 +#define DEBUG 0 -#define DATASIZE 1024 -#define MAX_MODIFY_ENTRIES 37 /* Maximum modify vectors */ +#define DATASIZE 1024 +#define MAX_MODIFY_ENTRIES 37 /* Maximum modify vectors */ -static WT_MODIFY entries[MAX_MODIFY_ENTRIES]; /* Entries vector */ -static int nentries; /* Entries count */ +static WT_MODIFY entries[MAX_MODIFY_ENTRIES]; /* Entries vector */ +static int nentries; /* Entries count */ /* - * The replacement bytes array is 2x the maximum replacement string so we can - * offset into it by the maximum replacement string and still take a maximum - * replacement string without going past the end of the buffer. + * The replacement bytes array is 2x the maximum replacement string so we can offset into it by the + * maximum replacement string and still take a maximum replacement string without going past the end + * of the buffer. */ -#define MAX_REPL_BYTES 17 -static char modify_repl[MAX_REPL_BYTES * 2]; /* Replacement bytes */ +#define MAX_REPL_BYTES 17 +static char modify_repl[MAX_REPL_BYTES * 2]; /* Replacement bytes */ -static WT_RAND_STATE rnd; /* RNG state */ +static WT_RAND_STATE rnd; /* RNG state */ /* * show -- - * Dump out a buffer. + * Dump out a buffer. */ static void show(WT_ITEM *buf, const char *tag) { - size_t i; - const uint8_t *a; + size_t i; + const uint8_t *a; - fprintf(stderr, "%s: %" WT_SIZET_FMT " bytes\n\t", tag, buf->size); - for (a = buf->data, i = 0; i < buf->size; ++i, ++a) - fprintf(stderr, " %c", isprint(*a) ? *a : '.'); - fprintf(stderr, "\n"); + fprintf(stderr, "%s: %" WT_SIZET_FMT " bytes\n\t", tag, buf->size); + for (a = buf->data, i = 0; i < buf->size; ++i, ++a) + fprintf(stderr, " %c", isprint(*a) ? *a : '.'); + fprintf(stderr, "\n"); } /* * modify_repl_init -- - * Initialize the replacement information. + * Initialize the replacement information. */ static void modify_repl_init(void) { - size_t i; + size_t i; - for (i = 0; i < sizeof(modify_repl); ++i) - modify_repl[i] = 'Z' - (i % 26); + for (i = 0; i < sizeof(modify_repl); ++i) + modify_repl[i] = 'Z' - (i % 26); } /* * modify_build -- - * Generate a set of modify vectors. + * Generate a set of modify vectors. */ static void modify_build(void) { - int i; - - /* Mess up the entries. */ - memset(entries, 0xff, sizeof(entries)); - - /* - * Randomly select a number of byte changes, offsets and lengths. - * Allow a value of 0, the API should accept it. - */ - nentries = (int)(__wt_random(&rnd) % (MAX_MODIFY_ENTRIES + 1)); - for (i = 0; i < nentries; ++i) { - entries[i].data.data = - modify_repl + __wt_random(&rnd) % MAX_REPL_BYTES; - entries[i].data.size = - (size_t)(__wt_random(&rnd) % MAX_REPL_BYTES); - entries[i].offset = (size_t)(__wt_random(&rnd) % DATASIZE); - entries[i].size = (size_t)(__wt_random(&rnd) % MAX_REPL_BYTES); - } + int i; + + /* Mess up the entries. */ + memset(entries, 0xff, sizeof(entries)); + + /* + * Randomly select a number of byte changes, offsets and lengths. Allow a value of 0, the API + * should accept it. + */ + nentries = (int)(__wt_random(&rnd) % (MAX_MODIFY_ENTRIES + 1)); + for (i = 0; i < nentries; ++i) { + entries[i].data.data = modify_repl + __wt_random(&rnd) % MAX_REPL_BYTES; + entries[i].data.size = (size_t)(__wt_random(&rnd) % MAX_REPL_BYTES); + entries[i].offset = (size_t)(__wt_random(&rnd) % DATASIZE); + entries[i].size = (size_t)(__wt_random(&rnd) % MAX_REPL_BYTES); + } #if DEBUG - for (i = 0; i < nentries; ++i) - printf( - "%d: {%.*s} %" WT_SIZET_FMT " bytes replacing %" - WT_SIZET_FMT " bytes @ %" WT_SIZET_FMT "\n", - i, (int)entries[i].data.size, (char *)entries[i].data.data, - entries[i].data.size, entries[i].size, entries[i].offset); + for (i = 0; i < nentries; ++i) + printf("%d: {%.*s} %" WT_SIZET_FMT " bytes replacing %" WT_SIZET_FMT + " bytes @ %" WT_SIZET_FMT "\n", + i, (int)entries[i].data.size, (char *)entries[i].data.data, entries[i].data.size, + entries[i].size, entries[i].offset); #endif } /* * slow_apply_api -- - * Apply a set of modification changes using a different algorithm. + * Apply a set of modification changes using a different algorithm. */ static void slow_apply_api(WT_ITEM *orig) { - static WT_ITEM _tb; - WT_ITEM *ta, *tb, *tmp, _tmp; - size_t len, size; - int i; - - ta = orig; - tb = &_tb; - - /* Mess up anything not initialized in the buffers. */ - memset((uint8_t *)ta->mem + ta->size, 0xff, ta->memsize - ta->size); - memset((uint8_t *)tb->mem, 0xff, tb->memsize); - - /* - * Process the entries to figure out how large a buffer we need. This is - * a bit pessimistic because we're ignoring replacement bytes, but it's - * a simpler calculation. - */ - for (size = ta->size, i = 0; i < nentries; ++i) { - if (entries[i].offset >= size) - size = entries[i].offset; - size += entries[i].data.size; - } - - testutil_check(__wt_buf_grow(NULL, ta, size)); - testutil_check(__wt_buf_grow(NULL, tb, size)); + static WT_ITEM _tb; + WT_ITEM *ta, *tb, *tmp, _tmp; + size_t len, size; + int i; + + ta = orig; + tb = &_tb; + + /* Mess up anything not initialized in the buffers. */ + memset((uint8_t *)ta->mem + ta->size, 0xff, ta->memsize - ta->size); + memset((uint8_t *)tb->mem, 0xff, tb->memsize); + + /* + * Process the entries to figure out how large a buffer we need. This is a bit pessimistic + * because we're ignoring replacement bytes, but it's a simpler calculation. + */ + for (size = ta->size, i = 0; i < nentries; ++i) { + if (entries[i].offset >= size) + size = entries[i].offset; + size += entries[i].data.size; + } + + testutil_check(__wt_buf_grow(NULL, ta, size)); + testutil_check(__wt_buf_grow(NULL, tb, size)); #if DEBUG - show(ta, "slow-apply start"); + show(ta, "slow-apply start"); #endif - /* - * From the starting buffer, create a new buffer b based on changes - * in the entries array. We're doing a brute force solution here to - * test the faster solution implemented in the library. - */ - for (i = 0; i < nentries; ++i) { - /* Take leading bytes from the original, plus any gap bytes. */ - if (entries[i].offset >= ta->size) { - memcpy(tb->mem, ta->mem, ta->size); - if (entries[i].offset > ta->size) - memset((uint8_t *)tb->mem + ta->size, - '\0', entries[i].offset - ta->size); - } else - if (entries[i].offset > 0) - memcpy(tb->mem, ta->mem, entries[i].offset); - tb->size = entries[i].offset; - - /* Take replacement bytes. */ - if (entries[i].data.size > 0) { - memcpy((uint8_t *)tb->mem + tb->size, - entries[i].data.data, entries[i].data.size); - tb->size += entries[i].data.size; - } - - /* Take trailing bytes from the original. */ - len = entries[i].offset + entries[i].size; - if (ta->size > len) { - memcpy((uint8_t *)tb->mem + tb->size, - (uint8_t *)ta->mem + len, ta->size - len); - tb->size += ta->size - len; - } - testutil_assert(tb->size <= size); - - /* Swap the buffers and do it again. */ - tmp = ta; - ta = tb; - tb = tmp; - } - ta->data = ta->mem; - tb->data = tb->mem; - - /* - * The final results may not be in the original buffer, in which case - * we swap them back around. - */ - if (ta != orig) { - _tmp = *ta; - *ta = *tb; - *tb = _tmp; - } + /* + * From the starting buffer, create a new buffer b based on changes in the entries array. We're + * doing a brute force solution here to test the faster solution implemented in the library. + */ + for (i = 0; i < nentries; ++i) { + /* Take leading bytes from the original, plus any gap bytes. */ + if (entries[i].offset >= ta->size) { + memcpy(tb->mem, ta->mem, ta->size); + if (entries[i].offset > ta->size) + memset((uint8_t *)tb->mem + ta->size, '\0', entries[i].offset - ta->size); + } else if (entries[i].offset > 0) + memcpy(tb->mem, ta->mem, entries[i].offset); + tb->size = entries[i].offset; + + /* Take replacement bytes. */ + if (entries[i].data.size > 0) { + memcpy((uint8_t *)tb->mem + tb->size, entries[i].data.data, entries[i].data.size); + tb->size += entries[i].data.size; + } + + /* Take trailing bytes from the original. */ + len = entries[i].offset + entries[i].size; + if (ta->size > len) { + memcpy((uint8_t *)tb->mem + tb->size, (uint8_t *)ta->mem + len, ta->size - len); + tb->size += ta->size - len; + } + testutil_assert(tb->size <= size); + + /* Swap the buffers and do it again. */ + tmp = ta; + ta = tb; + tb = tmp; + } + ta->data = ta->mem; + tb->data = tb->mem; + + /* + * The final results may not be in the original buffer, in which case we swap them back around. + */ + if (ta != orig) { + _tmp = *ta; + *ta = *tb; + *tb = _tmp; + } #if DEBUG - show(ta, "slow-apply finish"); + show(ta, "slow-apply finish"); #endif } /* * compare -- - * Compare two results. + * Compare two results. */ static void compare(WT_ITEM *orig, WT_ITEM *local, WT_ITEM *library) { - size_t i, max; - const uint8_t *p, *t; - - max = WT_MIN(local->size, library->size); - if (local->size != library->size || - memcmp(local->data, library->data, local->size) != 0) { - for (i = 0, - p = local->data, t = library->data; i < max; ++i, ++p, ++t) - if (*p != *t) - break; - fprintf(stderr, "results differ: "); - if (max == 0) - fprintf(stderr, - "identical up to %" WT_SIZET_FMT " bytes\n", max); - else - fprintf(stderr, - "first mismatch at offset %" WT_SIZET_FMT "\n", i); - show(orig, "original"); - show(local, "local results"); - show(library, "library results"); - } - testutil_assert( - local->size == library->size && memcmp( - local->data, library->data, local->size) == 0); + size_t i, max; + const uint8_t *p, *t; + + max = WT_MIN(local->size, library->size); + if (local->size != library->size || memcmp(local->data, library->data, local->size) != 0) { + for (i = 0, p = local->data, t = library->data; i < max; ++i, ++p, ++t) + if (*p != *t) + break; + fprintf(stderr, "results differ: "); + if (max == 0) + fprintf(stderr, "identical up to %" WT_SIZET_FMT " bytes\n", max); + else + fprintf(stderr, "first mismatch at offset %" WT_SIZET_FMT "\n", i); + show(orig, "original"); + show(local, "local results"); + show(library, "library results"); + } + testutil_assert( + local->size == library->size && memcmp(local->data, library->data, local->size) == 0); } /* @@ -259,114 +243,101 @@ compare(WT_ITEM *orig, WT_ITEM *local, WT_ITEM *library) static void modify_run(TEST_OPTS *opts) { - WT_CURSOR *cursor, _cursor; - WT_DECL_RET; - WT_ITEM *localA, _localA, *localB, _localB; - WT_SESSION_IMPL *session; - size_t len; - int i, j; - u_char *p; - bool verbose; - - session = (WT_SESSION_IMPL *)opts->session; - verbose = opts->verbose; - - /* Initialize the RNG. */ - __wt_random_init_seed(session, &rnd); - - /* Set up replacement information. */ - modify_repl_init(); - - /* We need three WT_ITEMs, one of them part of a fake cursor. */ - localA = &_localA; - memset(&_localA, 0, sizeof(_localA)); - localB = &_localB; - memset(&_localB, 0, sizeof(_localB)); - cursor = &_cursor; - memset(&_cursor, 0, sizeof(_cursor)); - cursor->session = (WT_SESSION *)session; - cursor->value_format = "u"; - -#define NRUNS 10000 - for (i = 0; i < NRUNS; ++i) { - /* Create an initial value. */ - len = (size_t)(__wt_random(&rnd) % MAX_REPL_BYTES); - testutil_check(__wt_buf_set(session, localA, modify_repl, len)); - - for (j = 0; j < 1000; ++j) { - /* Make lower case so modifications are easy to see. */ - for (p = localA->mem; - WT_PTRDIFF(p, localA->mem) < localA->size; p++) - *p = __wt_tolower(*p); - - /* Copy the current value into the second item. */ - testutil_check(__wt_buf_set( - session, localB, localA->data, localA->size)); - - /* - * Create a random set of modify vectors, run the - * underlying library modification function, then - * compare the result against our implementation - * of modify. - */ - modify_build(); - testutil_check(__wt_buf_set(session, - &cursor->value, localA->data, localA->size)); - testutil_check(__wt_modify_apply_api( - cursor, entries, nentries)); - slow_apply_api(localA); - compare(localB, localA, &cursor->value); - - /* - * Call the WiredTiger function to build a modification - * vector for the change, and repeat the test using the - * WiredTiger modification vector, then compare results - * against our implementation of modify. - */ - nentries = WT_ELEMENTS(entries); - ret = wiredtiger_calc_modify(opts->session, - localB, localA, - WT_MAX(localB->size, localA->size) + 100, - entries, &nentries); - if (ret == WT_NOTFOUND) - continue; - testutil_check(ret); - testutil_check(__wt_buf_set(session, - &cursor->value, localB->data, localB->size)); - testutil_check(__wt_modify_apply_api( - cursor, entries, nentries)); - compare(localB, localA, &cursor->value); - } - if (verbose) { - printf("%d (%d%%)\r", i, (i * 100) / NRUNS); - fflush(stdout); - } - } - if (verbose) - printf("%d (100%%)\n", i); - - __wt_buf_free(session, localA); - __wt_buf_free(session, localB); - __wt_buf_free(session, &cursor->value); + WT_CURSOR *cursor, _cursor; + WT_DECL_RET; + WT_ITEM *localA, _localA, *localB, _localB; + WT_SESSION_IMPL *session; + size_t len; + int i, j; + u_char *p; + bool verbose; + + session = (WT_SESSION_IMPL *)opts->session; + verbose = opts->verbose; + + /* Initialize the RNG. */ + __wt_random_init_seed(session, &rnd); + + /* Set up replacement information. */ + modify_repl_init(); + + /* We need three WT_ITEMs, one of them part of a fake cursor. */ + localA = &_localA; + memset(&_localA, 0, sizeof(_localA)); + localB = &_localB; + memset(&_localB, 0, sizeof(_localB)); + cursor = &_cursor; + memset(&_cursor, 0, sizeof(_cursor)); + cursor->session = (WT_SESSION *)session; + cursor->value_format = "u"; + +#define NRUNS 10000 + for (i = 0; i < NRUNS; ++i) { + /* Create an initial value. */ + len = (size_t)(__wt_random(&rnd) % MAX_REPL_BYTES); + testutil_check(__wt_buf_set(session, localA, modify_repl, len)); + + for (j = 0; j < 1000; ++j) { + /* Make lower case so modifications are easy to see. */ + for (p = localA->mem; WT_PTRDIFF(p, localA->mem) < localA->size; p++) + *p = __wt_tolower(*p); + + /* Copy the current value into the second item. */ + testutil_check(__wt_buf_set(session, localB, localA->data, localA->size)); + + /* + * Create a random set of modify vectors, run the underlying library modification + * function, then compare the result against our implementation of modify. + */ + modify_build(); + testutil_check(__wt_buf_set(session, &cursor->value, localA->data, localA->size)); + testutil_check(__wt_modify_apply_api(cursor, entries, nentries)); + slow_apply_api(localA); + compare(localB, localA, &cursor->value); + + /* + * Call the WiredTiger function to build a modification vector for the change, and + * repeat the test using the WiredTiger modification vector, then compare results + * against our implementation of modify. + */ + nentries = WT_ELEMENTS(entries); + ret = wiredtiger_calc_modify(opts->session, localB, localA, + WT_MAX(localB->size, localA->size) + 100, entries, &nentries); + if (ret == WT_NOTFOUND) + continue; + testutil_check(ret); + testutil_check(__wt_buf_set(session, &cursor->value, localB->data, localB->size)); + testutil_check(__wt_modify_apply_api(cursor, entries, nentries)); + compare(localB, localA, &cursor->value); + } + if (verbose) { + printf("%d (%d%%)\r", i, (i * 100) / NRUNS); + fflush(stdout); + } + } + if (verbose) + printf("%d (100%%)\n", i); + + __wt_buf_free(session, localA); + __wt_buf_free(session, localB); + __wt_buf_free(session, &cursor->value); } int main(int argc, char *argv[]) { - TEST_OPTS *opts, _opts; - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - testutil_check( - wiredtiger_open(opts->home, NULL, "create", &opts->conn)); - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &opts->session)); - - /* Run the test. */ - modify_run(opts); - - testutil_cleanup(opts); - return (EXIT_SUCCESS); + TEST_OPTS *opts, _opts; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + testutil_check(wiredtiger_open(opts->home, NULL, "create", &opts->conn)); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &opts->session)); + + /* Run the test. */ + modify_run(opts); + + testutil_cleanup(opts); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c b/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c index 9dcd065a0c9..97b2a1a03a2 100644 --- a/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c @@ -46,84 +46,72 @@ static WT_THREAD_RET do_ops(void *); static WT_THREAD_RET monitor(void *); /* - * Time delay to introduce into checkpoints in seconds. Should be at-least - * double the maximum time that any one of the operations should take. Currently - * this is set to 10 seconds and we expect no single operation to take longer - * than 5 seconds. + * Time delay to introduce into checkpoints in seconds. Should be at-least double the maximum time + * that any one of the operations should take. Currently this is set to 10 seconds and we expect no + * single operation to take longer than 5 seconds. */ -#define MAX_EXECUTION_TIME 10 -#define N_THREADS 10 +#define MAX_EXECUTION_TIME 10 +#define N_THREADS 10 /* - * Number of seconds to execute for. Initially set to 15 minutes, as we need to - * run long enough to be certain we have captured any blockages. In initial - * testing 5 minutes was enough to reproduce the issue, so we run for 3x that - * here to ensure we reproduce before declaring success. + * Number of seconds to execute for. Initially set to 15 minutes, as we need to run long enough to + * be certain we have captured any blockages. In initial testing 5 minutes was enough to reproduce + * the issue, so we run for 3x that here to ensure we reproduce before declaring success. */ -#define RUNTIME 900.0 +#define RUNTIME 900.0 -static WT_EVENT_HANDLER event_handler = { - handle_op_error, - handle_op_message, - NULL, - NULL -}; +static WT_EVENT_HANDLER event_handler = {handle_op_error, handle_op_message, NULL, NULL}; int main(int argc, char *argv[]) { - TEST_OPTS *opts, _opts; - TEST_PER_THREAD_OPTS thread_args[N_THREADS]; - pthread_t ckpt_thread, mon_thread, threads[N_THREADS]; - int i; - - /* - * This test should not run unless long tests flag is set. The test - * runs for 15 minutes. - */ - if (!testutil_is_flag_set("TESTUTIL_ENABLE_TIMING_TESTS")) - return (EXIT_SUCCESS); - - opts = &_opts; - opts->unique_id = 0; - memset(opts, 0, sizeof(*opts)); - - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - - testutil_check(wiredtiger_open(opts->home, &event_handler, - "create,cache_size=1G,timing_stress_for_test=[checkpoint_slow]", - &opts->conn)); - - testutil_check(pthread_create( - &ckpt_thread, NULL, do_checkpoints, opts)); - - for (i = 0; i < N_THREADS; ++i) { - thread_args[i].testopts = opts; - thread_args[i].thread_counter = 0; - thread_args[i].threadnum = i; - testutil_check(pthread_create( - &threads[i], NULL, do_ops, &thread_args[i])); - } - - /* - * Pass the whole array of thread arguments to the monitoring thread. - * This thread will need to monitor each threads counter to track if it - * is stuck. - */ - testutil_check(pthread_create(&mon_thread, NULL, monitor, thread_args)); - - for (i = 0; i < N_THREADS; ++i) - testutil_check(pthread_join(threads[i], NULL)); - - testutil_check(pthread_join(mon_thread, NULL)); - - testutil_check(pthread_join(ckpt_thread, NULL)); - - printf("Success\n"); - - testutil_cleanup(opts); - return (EXIT_SUCCESS); + TEST_OPTS *opts, _opts; + TEST_PER_THREAD_OPTS thread_args[N_THREADS]; + pthread_t ckpt_thread, mon_thread, threads[N_THREADS]; + int i; + + /* + * This test should not run unless long tests flag is set. The test runs for 15 minutes. + */ + if (!testutil_is_flag_set("TESTUTIL_ENABLE_TIMING_TESTS")) + return (EXIT_SUCCESS); + + opts = &_opts; + opts->unique_id = 0; + memset(opts, 0, sizeof(*opts)); + + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(wiredtiger_open(opts->home, &event_handler, + "create,cache_size=1G,timing_stress_for_test=[checkpoint_slow]", &opts->conn)); + + testutil_check(pthread_create(&ckpt_thread, NULL, do_checkpoints, opts)); + + for (i = 0; i < N_THREADS; ++i) { + thread_args[i].testopts = opts; + thread_args[i].thread_counter = 0; + thread_args[i].threadnum = i; + testutil_check(pthread_create(&threads[i], NULL, do_ops, &thread_args[i])); + } + + /* + * Pass the whole array of thread arguments to the monitoring thread. This thread will need to + * monitor each threads counter to track if it is stuck. + */ + testutil_check(pthread_create(&mon_thread, NULL, monitor, thread_args)); + + for (i = 0; i < N_THREADS; ++i) + testutil_check(pthread_join(threads[i], NULL)); + + testutil_check(pthread_join(mon_thread, NULL)); + + testutil_check(pthread_join(ckpt_thread, NULL)); + + printf("Success\n"); + + testutil_cleanup(opts); + return (EXIT_SUCCESS); } /* @@ -132,88 +120,86 @@ main(int argc, char *argv[]) static WT_THREAD_RET do_checkpoints(void *_opts) { - TEST_OPTS *opts; - WT_DECL_RET; - WT_SESSION *session; - time_t now, start; - - opts = (TEST_OPTS *)_opts; - (void)time(&start); - (void)time(&now); - - while (difftime(now, start) < RUNTIME) { - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - - if ((ret = session->checkpoint(session, "force")) != 0) - if (ret != EBUSY && ret != ENOENT) - testutil_die(ret, "session.checkpoint"); - - testutil_check(session->close(session, NULL)); - - /* - * A short sleep to let operations process and avoid back to - * back checkpoints locking up resources. - */ - sleep(1); - (void)time(&now); - } - - return (WT_THREAD_RET_VALUE); + TEST_OPTS *opts; + WT_DECL_RET; + WT_SESSION *session; + time_t now, start; + + opts = (TEST_OPTS *)_opts; + (void)time(&start); + (void)time(&now); + + while (difftime(now, start) < RUNTIME) { + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + if ((ret = session->checkpoint(session, "force")) != 0) + if (ret != EBUSY && ret != ENOENT) + testutil_die(ret, "session.checkpoint"); + + testutil_check(session->close(session, NULL)); + + /* + * A short sleep to let operations process and avoid back to back checkpoints locking up + * resources. + */ + sleep(1); + (void)time(&now); + } + + return (WT_THREAD_RET_VALUE); } /* - * Function to monitor running operations and abort to dump core in the event - * that we catch an operation running long. + * Function to monitor running operations and abort to dump core in the event that we catch an + * operation running long. */ static WT_THREAD_RET monitor(void *args) { - TEST_PER_THREAD_OPTS *thread_args; - time_t now, start; - int ctr, i, last_ops[N_THREADS]; - - thread_args = (TEST_PER_THREAD_OPTS *)args; - - (void)time(&start); - (void)time(&now); - - memset(last_ops, 0, sizeof(int) + N_THREADS); - - while (difftime(now, start) < RUNTIME) { - /* - * Checkpoints will run for slightly over MAX_EXECUTION_TIME. - * MAX_EXECUTION_TIME should always be long enough that we can - * complete any single operation in 1/2 that time. - */ - sleep(MAX_EXECUTION_TIME / 2); - - for (i = 0; i < N_THREADS; i++) { - ctr = thread_args[i].thread_counter; - - /* Ignore any threads which may not have started yet. */ - if (ctr == 0) - continue; - - /* - * We track how many operations each thread has done. If - * we have slept and the counter remains the same for a - * thread it is stuck and should drop a core so the - * cause of the hang can be investigated. - */ - if (ctr != last_ops[i]) - last_ops[i] = ctr; - else { - printf("Thread %d had a task running" - " for more than %d seconds\n", - i, MAX_EXECUTION_TIME / 2); - abort(); - } - } - (void)time(&now); - } - - return (WT_THREAD_RET_VALUE); + TEST_PER_THREAD_OPTS *thread_args; + time_t now, start; + int ctr, i, last_ops[N_THREADS]; + + thread_args = (TEST_PER_THREAD_OPTS *)args; + + (void)time(&start); + (void)time(&now); + + memset(last_ops, 0, sizeof(int) + N_THREADS); + + while (difftime(now, start) < RUNTIME) { + /* + * Checkpoints will run for slightly over MAX_EXECUTION_TIME. MAX_EXECUTION_TIME should + * always be long enough that we can complete any single operation in 1/2 that time. + */ + sleep(MAX_EXECUTION_TIME / 2); + + for (i = 0; i < N_THREADS; i++) { + ctr = thread_args[i].thread_counter; + + /* Ignore any threads which may not have started yet. */ + if (ctr == 0) + continue; + + /* + * We track how many operations each thread has done. If we have slept and the counter + * remains the same for a thread it is stuck and should drop a core so the cause of the + * hang can be investigated. + */ + if (ctr != last_ops[i]) + last_ops[i] = ctr; + else { + printf( + "Thread %d had a task running" + " for more than %d seconds\n", + i, MAX_EXECUTION_TIME / 2); + abort(); + } + } + (void)time(&now); + } + + return (WT_THREAD_RET_VALUE); } /* @@ -222,36 +208,36 @@ monitor(void *args) static WT_THREAD_RET do_ops(void *args) { - WT_RAND_STATE rnd; - time_t now, start; - - __wt_random_init_seed(NULL, &rnd); - (void)time(&start); - (void)time(&now); - - while (difftime(now, start) < RUNTIME) { - switch (__wt_random(&rnd) % 6) { - case 0: - op_bulk(args); - break; - case 1: - op_create(args); - break; - case 2: - op_cursor(args); - break; - case 3: - op_drop(args); - break; - case 4: - op_bulk_unique(args); - break; - case 5: - op_create_unique(args); - break; - } - (void)time(&now); - } - - return (WT_THREAD_RET_VALUE); + WT_RAND_STATE rnd; + time_t now, start; + + __wt_random_init_seed(NULL, &rnd); + (void)time(&start); + (void)time(&now); + + while (difftime(now, start) < RUNTIME) { + switch (__wt_random(&rnd) % 6) { + case 0: + op_bulk(args); + break; + case 1: + op_create(args); + break; + case 2: + op_cursor(args); + break; + case 3: + op_drop(args); + break; + case 4: + op_bulk_unique(args); + break; + case 5: + op_create_unique(args); + break; + } + (void)time(&now); + } + + return (WT_THREAD_RET_VALUE); } diff --git a/src/third_party/wiredtiger/test/csuite/wt3874_pad_byte_collator/main.c b/src/third_party/wiredtiger/test/csuite/wt3874_pad_byte_collator/main.c index f086fa415de..e1880f2a431 100644 --- a/src/third_party/wiredtiger/test/csuite/wt3874_pad_byte_collator/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt3874_pad_byte_collator/main.c @@ -28,81 +28,76 @@ #include "test_util.h" /* - * JIRA ticket reference: WT-3874 - * Test case description: Set up a collator that only uses the first - * byte of a record for comparison; all other bytes are considered padding. - * With that collator for a table, insert an item, then remove that - * item (with different padding). - * Failure mode: An assertion is fired when we get back the key as stored - * in the record, if we compare it to the given key without taking into - * account the collator. + * JIRA ticket reference: WT-3874 Test case description: Set up a collator that only uses the first + * byte of a record for comparison; all other bytes are considered padding. With that collator for a + * table, insert an item, then remove that item (with different padding). Failure mode: An assertion + * is fired when we get back the key as stored in the record, if we compare it to the given key + * without taking into account the collator. */ -#define KEY_SIZE 20 +#define KEY_SIZE 20 static int -my_compare(WT_COLLATOR *collator, WT_SESSION *session, - const WT_ITEM *v1, const WT_ITEM *v2, int *cmp) +my_compare( + WT_COLLATOR *collator, WT_SESSION *session, const WT_ITEM *v1, const WT_ITEM *v2, int *cmp) { - (void)collator; - (void)session; + (void)collator; + (void)session; - if (v1->size < 1 || v2->size < 1) - return (EINVAL); - *cmp = strncmp((const char *)v1->data, (const char *)v2->data, 1); - return (0); + if (v1->size < 1 || v2->size < 1) + return (EINVAL); + *cmp = strncmp((const char *)v1->data, (const char *)v2->data, 1); + return (0); } -static WT_COLLATOR my_coll = { my_compare, NULL, NULL }; +static WT_COLLATOR my_coll = {my_compare, NULL, NULL}; int main(int argc, char *argv[]) { - TEST_OPTS *opts, _opts; - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_ITEM key; - WT_SESSION *session; - char buf[KEY_SIZE]; + TEST_OPTS *opts, _opts; + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_ITEM key; + WT_SESSION *session; + char buf[KEY_SIZE]; - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - srand(123); + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + srand(123); - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); - testutil_check(wiredtiger_open(opts->home, NULL, "create,log=(enabled)", - &opts->conn)); - conn = opts->conn; - testutil_check(conn->add_collator(conn, "my_coll", &my_coll, NULL)); - testutil_check(conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(wiredtiger_open(opts->home, NULL, "create,log=(enabled)", &opts->conn)); + conn = opts->conn; + testutil_check(conn->add_collator(conn, "my_coll", &my_coll, NULL)); + testutil_check(conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check(session->create(session, "table:main", - "key_format=u,value_format=u,collator=my_coll")); + testutil_check( + session->create(session, "table:main", "key_format=u,value_format=u,collator=my_coll")); - testutil_check(session->open_cursor( - session, "table:main", NULL, NULL, &cursor)); + testutil_check(session->open_cursor(session, "table:main", NULL, NULL, &cursor)); - memset(buf, 'X', sizeof(buf)); - buf[0] = 'a'; + memset(buf, 'X', sizeof(buf)); + buf[0] = 'a'; - key.data = buf; - key.size = sizeof(buf); - cursor->set_key(cursor, &key); - cursor->set_value(cursor, &key); - testutil_check(cursor->insert(cursor)); + key.data = buf; + key.size = sizeof(buf); + cursor->set_key(cursor, &key); + cursor->set_value(cursor, &key); + testutil_check(cursor->insert(cursor)); - testutil_check(session->checkpoint(session, NULL)); + testutil_check(session->checkpoint(session, NULL)); - /* Use a different padding. */ - memset(buf, 'Y', sizeof(buf)); - buf[0] = 'a'; + /* Use a different padding. */ + memset(buf, 'Y', sizeof(buf)); + buf[0] = 'a'; - cursor->set_key(cursor, &key); - testutil_check(cursor->remove(cursor)); + cursor->set_key(cursor, &key); + testutil_check(cursor->remove(cursor)); - testutil_check(session->close(session, NULL)); - testutil_cleanup(opts); - return (EXIT_SUCCESS); + testutil_check(session->close(session, NULL)); + testutil_cleanup(opts); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c b/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c index 3e7d52de0a5..c48b73d51c9 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c @@ -29,139 +29,126 @@ #include <signal.h> -static const char * const uri = "table:large"; +static const char *const uri = "table:large"; -#define DATASIZE (1024 * 1024) -#define MODIFY_COUNT (1024) -#define NUM_DOCS 2 +#define DATASIZE (1024 * 1024) +#define MODIFY_COUNT (1024) +#define NUM_DOCS 2 static void on_alarm(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void on_alarm(int signo) { - (void)signo; /* Unused parameter */ - fprintf(stderr, "cursor->modify timed out \n"); - abort(); + (void)signo; /* Unused parameter */ + fprintf(stderr, "cursor->modify timed out \n"); + abort(); - /* NOTREACHED */ + /* NOTREACHED */ } static int ignore_errors = 0; static int -handle_error(WT_EVENT_HANDLER *handler, - WT_SESSION *session, int error, const char *message) +handle_error(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const char *message) { - (void)(handler); - - /* Skip the error messages we're expecting to see. */ - if (ignore_errors > 0 && - (strstr(message, "requires key be set") != NULL || - strstr(message, "requires value be set") != NULL)) { - --ignore_errors; - return (0); - } - - (void)fprintf(stderr, "%s: %s\n", - message, session->strerror(session, error)); - return (0); + (void)(handler); + + /* Skip the error messages we're expecting to see. */ + if (ignore_errors > 0 && (strstr(message, "requires key be set") != NULL || + strstr(message, "requires value be set") != NULL)) { + --ignore_errors; + return (0); + } + + (void)fprintf(stderr, "%s: %s\n", message, session->strerror(session, error)); + return (0); } -static WT_EVENT_HANDLER event_handler = { - handle_error, - NULL, - NULL, - NULL -}; +static WT_EVENT_HANDLER event_handler = {handle_error, NULL, NULL, NULL}; int main(int argc, char *argv[]) { - TEST_OPTS *opts, _opts; - WT_CURSOR *c; - WT_ITEM value; - WT_MODIFY modify_entry; - WT_SESSION *session, *session2; - uint64_t i, j, offset; - char *large_doc; - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); - - testutil_check(wiredtiger_open(opts->home, &event_handler, - "create," - "cache_size=1G," - "statistics_log=(json,wait=1)", &opts->conn)); - - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check(session->create(session, uri, - "key_format=Q,value_format=u," - "leaf_item_max=64M,leaf_page_max=32k,memory_page_max=1M")); - - testutil_check(session->open_cursor(session, uri, NULL, NULL, &c)); - - /* Value is initialized with 'v' and has not significance to it. */ - large_doc = dmalloc(DATASIZE); - memset(large_doc, 'v', DATASIZE); - value.data = large_doc; - value.size = DATASIZE; - - /* Insert records. */ - for (i = 0; i < NUM_DOCS; i++) { - c->set_key(c, i); - c->set_value(c, &value); - testutil_check(c->insert(c)); - } - - testutil_check(c->close(c)); - if (opts->verbose) - printf("%d documents inserted\n", NUM_DOCS); - - /* Setup Transaction to pin the cache */ - testutil_check( - session->begin_transaction(session, "isolation=snapshot")); - - /* Set an alarm so we can debug hangs. */ - (void)signal(SIGALRM, on_alarm); - - /* Start another session to perform small updates. */ - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session2)); - testutil_check(session2->open_cursor(session2, uri, NULL, NULL, &c)); - - j = offset = 0; - while (++j < MODIFY_COUNT) { - for (i = 0; i < NUM_DOCS; i++) { - /* Position the cursor. */ - testutil_check(session2->begin_transaction( - session2, "isolation=snapshot")); - c->set_key(c, i); - modify_entry.data.data = - "abcdefghijklmnopqrstuvwxyz"; - modify_entry.data.size = strlen(modify_entry.data.data); - modify_entry.offset = offset; - modify_entry.size = modify_entry.data.size; - (void)alarm(1); - testutil_check(c->modify(c, &modify_entry, 1)); - (void)alarm(0); - testutil_check( - session2->commit_transaction(session2, NULL)); - } - /* - * Modify operations are done similar to append sequence. - * This has no bearing on the test outcome. - */ - offset += modify_entry.data.size; - offset = offset < DATASIZE ? offset : 0; - if (opts->verbose) - printf("modify count %" PRIu64"\n", j * NUM_DOCS); - } - - free(large_doc); - testutil_cleanup(opts); - - return (EXIT_SUCCESS); + TEST_OPTS *opts, _opts; + WT_CURSOR *c; + WT_ITEM value; + WT_MODIFY modify_entry; + WT_SESSION *session, *session2; + uint64_t i, j, offset; + char *large_doc; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(wiredtiger_open(opts->home, &event_handler, + "create," + "cache_size=1G," + "statistics_log=(json,wait=1)", + &opts->conn)); + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(session->create(session, uri, + "key_format=Q,value_format=u," + "leaf_item_max=64M,leaf_page_max=32k,memory_page_max=1M")); + + testutil_check(session->open_cursor(session, uri, NULL, NULL, &c)); + + /* Value is initialized with 'v' and has not significance to it. */ + large_doc = dmalloc(DATASIZE); + memset(large_doc, 'v', DATASIZE); + value.data = large_doc; + value.size = DATASIZE; + + /* Insert records. */ + for (i = 0; i < NUM_DOCS; i++) { + c->set_key(c, i); + c->set_value(c, &value); + testutil_check(c->insert(c)); + } + + testutil_check(c->close(c)); + if (opts->verbose) + printf("%d documents inserted\n", NUM_DOCS); + + /* Setup Transaction to pin the cache */ + testutil_check(session->begin_transaction(session, "isolation=snapshot")); + + /* Set an alarm so we can debug hangs. */ + (void)signal(SIGALRM, on_alarm); + + /* Start another session to perform small updates. */ + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session2)); + testutil_check(session2->open_cursor(session2, uri, NULL, NULL, &c)); + + j = offset = 0; + while (++j < MODIFY_COUNT) { + for (i = 0; i < NUM_DOCS; i++) { + /* Position the cursor. */ + testutil_check(session2->begin_transaction(session2, "isolation=snapshot")); + c->set_key(c, i); + modify_entry.data.data = "abcdefghijklmnopqrstuvwxyz"; + modify_entry.data.size = strlen(modify_entry.data.data); + modify_entry.offset = offset; + modify_entry.size = modify_entry.data.size; + (void)alarm(1); + testutil_check(c->modify(c, &modify_entry, 1)); + (void)alarm(0); + testutil_check(session2->commit_transaction(session2, NULL)); + } + /* + * Modify operations are done similar to append sequence. This has no bearing on the test + * outcome. + */ + offset += modify_entry.data.size; + offset = offset < DATASIZE ? offset : 0; + if (opts->verbose) + printf("modify count %" PRIu64 "\n", j * NUM_DOCS); + } + + free(large_doc); + testutil_cleanup(opts); + + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt4117_checksum/main.c b/src/third_party/wiredtiger/test/csuite/wt4117_checksum/main.c index 9a6eb13f92e..2a591544039 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4117_checksum/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4117_checksum/main.c @@ -28,67 +28,66 @@ #include "test_util.h" /* - * JIRA ticket reference: WT-4117 - * Test case description: Smoke-test the CRC32C external API. + * JIRA ticket reference: WT-4117 Test case description: Smoke-test the CRC32C external API. */ static inline void check(uint32_t crc32c, uint32_t expected, size_t len, const char *msg) { - testutil_checkfmt(crc32c == expected ? 0 : 1, - "%s checksum mismatch of %" WT_SIZET_FMT " bytes: %#08x != %#08x\n", - msg, len, crc32c, expected); + testutil_checkfmt(crc32c == expected ? 0 : 1, + "%s checksum mismatch of %" WT_SIZET_FMT " bytes: %#08x != %#08x\n", msg, len, crc32c, + expected); } static void run(void) { - size_t len; - uint32_t crc32c, (*func)(const void *, size_t); - uint8_t *data; + size_t len; + uint32_t crc32c, (*func)(const void *, size_t); + uint8_t *data; - /* Allocate aligned memory for the data. */ - data = dcalloc(100, sizeof(uint8_t)); + /* Allocate aligned memory for the data. */ + data = dcalloc(100, sizeof(uint8_t)); - /* Get a pointer to the CRC32C function. */ - func = wiredtiger_crc32c_func(); + /* Get a pointer to the CRC32C function. */ + func = wiredtiger_crc32c_func(); - /* - * Some simple known checksums. - */ - len = 1; - crc32c = func(data, len); - check(crc32c, (uint32_t)0x527d5351, len, "nul x1"); + /* + * Some simple known checksums. + */ + len = 1; + crc32c = func(data, len); + check(crc32c, (uint32_t)0x527d5351, len, "nul x1"); - len = 2; - crc32c = func(data, len); - check(crc32c, (uint32_t)0xf16177d2, len, "nul x2"); + len = 2; + crc32c = func(data, len); + check(crc32c, (uint32_t)0xf16177d2, len, "nul x2"); - len = 3; - crc32c = func(data, len); - check(crc32c, (uint32_t)0x6064a37a, len, "nul x3"); + len = 3; + crc32c = func(data, len); + check(crc32c, (uint32_t)0x6064a37a, len, "nul x3"); - len = 4; - crc32c = func(data, len); - check(crc32c, (uint32_t)0x48674bc7, len, "nul x4"); + len = 4; + crc32c = func(data, len); + check(crc32c, (uint32_t)0x48674bc7, len, "nul x4"); - len = strlen("123456789"); - memcpy(data, "123456789", len); - crc32c = func(data, len); - check(crc32c, (uint32_t)0xe3069283, len, "known string #1"); + len = strlen("123456789"); + memcpy(data, "123456789", len); + crc32c = func(data, len); + check(crc32c, (uint32_t)0xe3069283, len, "known string #1"); - len = strlen("The quick brown fox jumps over the lazy dog"); - memcpy(data, "The quick brown fox jumps over the lazy dog", len); - crc32c = func(data, len); - check(crc32c, (uint32_t)0x22620404, len, "known string #2"); + len = strlen("The quick brown fox jumps over the lazy dog"); + memcpy(data, "The quick brown fox jumps over the lazy dog", len); + crc32c = func(data, len); + check(crc32c, (uint32_t)0x22620404, len, "known string #2"); - free(data); + free(data); } int main(void) { - run(); + run(); - return (EXIT_SUCCESS); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c b/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c index 32b9f8f42a8..97dee1822ae 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c @@ -30,19 +30,19 @@ #include <sys/wait.h> #include <signal.h> -#define CORRUPT "file:zzz-corrupt.SS" -#define KEY "key" -#define VALUE "value,value,value" +#define CORRUPT "file:zzz-corrupt.SS" +#define KEY "key" +#define VALUE "value,value,value" -#define SAVE "SAVE" +#define SAVE "SAVE" /* - * NOTE: This assumes the default page size of 4096. If that changes these - * sizes need to change along with it. + * NOTE: This assumes the default page size of 4096. If that changes these sizes need to change + * along with it. */ -#define APP_MD_SIZE 4096 -#define APP_BUF_SIZE (3 * 1024) -#define APP_STR "Long app metadata intended to force a page per entry. " +#define APP_MD_SIZE 4096 +#define APP_BUF_SIZE (3 * 1024) +#define APP_STR "Long app metadata intended to force a page per entry. " static uint64_t data_val; static const char *home; @@ -51,527 +51,481 @@ static bool test_out_of_sync = false; static WT_SESSION *wt_session; static int -handle_message(WT_EVENT_HANDLER *handler, - WT_SESSION *session, int error, const char *message) +handle_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const char *message) { - (void)(handler); - - (void)fprintf(stderr, "%s: %s\n", - message, session->strerror(session, error)); - if (test_abort) { - fprintf(stderr, "Got unexpected error. Aborting\n"); - abort(); - } - return (0); + (void)(handler); + + (void)fprintf(stderr, "%s: %s\n", message, session->strerror(session, error)); + if (test_abort) { + fprintf(stderr, "Got unexpected error. Aborting\n"); + abort(); + } + return (0); } -static WT_EVENT_HANDLER event_handler = { - handle_message, - NULL, - NULL, - NULL -}; +static WT_EVENT_HANDLER event_handler = {handle_message, NULL, NULL, NULL}; typedef struct table_info { - const char *name; - const char *kvformat; - bool verified; + const char *name; + const char *kvformat; + bool verified; } TABLE_INFO; /* * byte_str -- - * A byte-string version to find a sub-string. The metadata we read - * contains a lot of zeroes so we cannot use string-based functions. + * A byte-string version to find a sub-string. The metadata we read contains a lot of zeroes so + * we cannot use string-based functions. */ static uint8_t * byte_str(uint8_t *buf, size_t bufsize, const char *str) { - size_t buflen, slen; - uint8_t *end, *p, *s; - int c; - - p = buf; - end = buf + bufsize; - s = NULL; - c = (int)str[0]; - buflen = bufsize; - slen = strlen(str); - /* - * Find the first character and then compare. - */ - while ((s = memchr(p, c, buflen)) != NULL) { - /* - * If we don't have enough buffer left to compare we do not - * have a match. - */ - buflen = (size_t)(end - s); - if (buflen < slen) - return (NULL); - if (memcmp(s, str, slen) == 0) - return (s); - /* - * This one didn't match, increment in the buffer and find the - * next one. - */ - ++s; - --buflen; - p = s; - } - return (NULL); + size_t buflen, slen; + uint8_t *end, *p, *s; + int c; + + p = buf; + end = buf + bufsize; + s = NULL; + c = (int)str[0]; + buflen = bufsize; + slen = strlen(str); + /* + * Find the first character and then compare. + */ + while ((s = memchr(p, c, buflen)) != NULL) { + /* + * If we don't have enough buffer left to compare we do not have a match. + */ + buflen = (size_t)(end - s); + if (buflen < slen) + return (NULL); + if (memcmp(s, str, slen) == 0) + return (s); + /* + * This one didn't match, increment in the buffer and find the next one. + */ + ++s; + --buflen; + p = s; + } + return (NULL); } /* * cursor_insert -- - * Insert some data into a table. + * Insert some data into a table. */ static void cursor_insert(const char *uri, uint64_t i) { - WT_CURSOR *cursor; - WT_ITEM vu; - char keybuf[100], valuebuf[100]; - bool recno; - - memset(&vu, 0, sizeof(vu)); - - /* Open a cursor. */ - testutil_check(wt_session->open_cursor( - wt_session, uri, NULL, NULL, &cursor)); - /* Operations change based on the key/value formats. */ - recno = strcmp(cursor->key_format, "r") == 0; - if (recno) - cursor->set_key(cursor, i); - else { - testutil_check(__wt_snprintf(keybuf, sizeof(keybuf), - "%s-%" PRIu64, KEY, i)); - cursor->set_key(cursor, keybuf); - } - strcpy(valuebuf, VALUE); - cursor->set_value(cursor, valuebuf); - testutil_check(cursor->insert(cursor)); - testutil_check(cursor->close(cursor)); + WT_CURSOR *cursor; + WT_ITEM vu; + char keybuf[100], valuebuf[100]; + bool recno; + + memset(&vu, 0, sizeof(vu)); + + /* Open a cursor. */ + testutil_check(wt_session->open_cursor(wt_session, uri, NULL, NULL, &cursor)); + /* Operations change based on the key/value formats. */ + recno = strcmp(cursor->key_format, "r") == 0; + if (recno) + cursor->set_key(cursor, i); + else { + testutil_check(__wt_snprintf(keybuf, sizeof(keybuf), "%s-%" PRIu64, KEY, i)); + cursor->set_key(cursor, keybuf); + } + strcpy(valuebuf, VALUE); + cursor->set_value(cursor, valuebuf); + testutil_check(cursor->insert(cursor)); + testutil_check(cursor->close(cursor)); } /* * create_data -- - * Create a table and insert a piece of data. + * Create a table and insert a piece of data. */ static void create_data(TABLE_INFO *t) { - size_t len; - uint64_t i; - char buf[APP_BUF_SIZE], cfg[APP_MD_SIZE]; - - memset(buf, 0, sizeof(buf)); - memset(cfg, 0, sizeof(cfg)); - - /* - * Create an app-specific metadata string that fills most of page - * so that each table in the metadata has its own page. - */ - len = strlen(APP_STR); - for (i = 0; i + len < APP_BUF_SIZE; i += len) - testutil_check(__wt_snprintf( - &buf[i], APP_BUF_SIZE - i, "%s", APP_STR)); - testutil_check(__wt_snprintf(cfg, sizeof(cfg), - "%s,app_metadata=\"%s\"", t->kvformat, buf)); - testutil_check(wt_session->create(wt_session, t->name, cfg)); - data_val = 1; - cursor_insert(t->name, data_val); + size_t len; + uint64_t i; + char buf[APP_BUF_SIZE], cfg[APP_MD_SIZE]; + + memset(buf, 0, sizeof(buf)); + memset(cfg, 0, sizeof(cfg)); + + /* + * Create an app-specific metadata string that fills most of page so that each table in the + * metadata has its own page. + */ + len = strlen(APP_STR); + for (i = 0; i + len < APP_BUF_SIZE; i += len) + testutil_check(__wt_snprintf(&buf[i], APP_BUF_SIZE - i, "%s", APP_STR)); + testutil_check(__wt_snprintf(cfg, sizeof(cfg), "%s,app_metadata=\"%s\"", t->kvformat, buf)); + testutil_check(wt_session->create(wt_session, t->name, cfg)); + data_val = 1; + cursor_insert(t->name, data_val); } /* * corrupt_metadata -- - * Corrupt the file by scribbling on the provided URI string. + * Corrupt the file by scribbling on the provided URI string. */ static void corrupt_file(const char *file_name, const char *uri) { - struct stat sb; - FILE *fp; - size_t meta_size; - long off; - uint8_t *buf, *corrupt; - char path[256]; - bool corrupted; - - /* - * Open the file, read its contents. Find the string "corrupt" and - * modify one byte at that offset. That will cause a checksum error - * when WiredTiger next reads it. - */ - testutil_check(__wt_snprintf( - path, sizeof(path), "%s/%s", home, file_name)); - if ((fp = fopen(path, "r+")) == NULL) - testutil_die(errno, "fopen: %s", path); - testutil_check(fstat(fileno(fp), &sb)); - meta_size = (size_t)sb.st_size; - buf = dcalloc(meta_size, 1); - if (fread(buf, 1, meta_size, fp) != meta_size) - testutil_die(errno, "fread: %" WT_SIZET_FMT, meta_size); - corrupted = false; - /* - * Corrupt all occurrences of the string in the file. - */ - while ((corrupt = byte_str(buf, meta_size, uri)) != NULL) { - corrupted = true; - testutil_assert(*(char *)corrupt != 'X'); - *(char *)corrupt = 'X'; - off = (long)(corrupt - buf); - if (fseek(fp, off, SEEK_SET) != 0) - testutil_die(errno, "fseek: %ld", off); - if (fwrite("X", 1, 1, fp) != 1) - testutil_die(errno, "fwrite"); - } - if (!corrupted) - testutil_die(errno, "corrupt string did not occur"); - if (fclose(fp) != 0) - testutil_die(errno, "fclose"); - free(buf); + struct stat sb; + FILE *fp; + size_t meta_size; + long off; + uint8_t *buf, *corrupt; + char path[256]; + bool corrupted; + + /* + * Open the file, read its contents. Find the string "corrupt" and modify one byte at that + * offset. That will cause a checksum error when WiredTiger next reads it. + */ + testutil_check(__wt_snprintf(path, sizeof(path), "%s/%s", home, file_name)); + if ((fp = fopen(path, "r+")) == NULL) + testutil_die(errno, "fopen: %s", path); + testutil_check(fstat(fileno(fp), &sb)); + meta_size = (size_t)sb.st_size; + buf = dcalloc(meta_size, 1); + if (fread(buf, 1, meta_size, fp) != meta_size) + testutil_die(errno, "fread: %" WT_SIZET_FMT, meta_size); + corrupted = false; + /* + * Corrupt all occurrences of the string in the file. + */ + while ((corrupt = byte_str(buf, meta_size, uri)) != NULL) { + corrupted = true; + testutil_assert(*(char *)corrupt != 'X'); + *(char *)corrupt = 'X'; + off = (long)(corrupt - buf); + if (fseek(fp, off, SEEK_SET) != 0) + testutil_die(errno, "fseek: %ld", off); + if (fwrite("X", 1, 1, fp) != 1) + testutil_die(errno, "fwrite"); + } + if (!corrupted) + testutil_die(errno, "corrupt string did not occur"); + if (fclose(fp) != 0) + testutil_die(errno, "fclose"); + free(buf); } /* * file_exists -- - * Return if the file exists. + * Return if the file exists. */ static int file_exists(const char *path) { - struct stat sb; + struct stat sb; - return (stat(path, &sb) == 0); + return (stat(path, &sb) == 0); } /* * reset_verified -- - * Reset the verified field in the table array. + * Reset the verified field in the table array. */ static void reset_verified(TABLE_INFO *tables) { - TABLE_INFO *t; + TABLE_INFO *t; - for (t = tables; t->name != NULL; t++) - t->verified = false; + for (t = tables; t->name != NULL; t++) + t->verified = false; } /* * verify_metadata -- - * Verify all the tables expected are in the metadata. We expect all but - * the "corrupt" table name. + * Verify all the tables expected are in the metadata. We expect all but the "corrupt" table + * name. */ static void verify_metadata(WT_CONNECTION *conn, TABLE_INFO *tables) { - TABLE_INFO *t; - WT_CURSOR *cursor; - WT_DECL_RET; - const char *kv; - - /* - * Open a metadata cursor. - */ - testutil_check(conn->open_session(conn, NULL, NULL, &wt_session)); - testutil_check(wt_session->open_cursor( - wt_session, "metadata:", NULL, NULL, &cursor)); - reset_verified(tables); - - /* - * We have to walk the cursor and walk the tables to match up that - * the expected tables are in the metadata. It is not efficient, but - * the list of tables is small. Walk the cursor once and the array - * of tables each time. - */ - while ((ret = cursor->next(cursor)) == 0) { - testutil_check(cursor->get_key(cursor, &kv)); - for (t = tables; t->name != NULL; t++) { - if (strcmp(t->name, kv) == 0) { - testutil_assert(t->verified == false); - t->verified = true; - break; - } - } - } - testutil_assert(ret == WT_NOTFOUND); - testutil_check(cursor->close(cursor)); - /* - * Any tables that were salvaged, make sure we can read the data. - * The corrupt table should never be salvaged. - */ - for (t = tables; t->name != NULL; t++) { - if (strcmp(t->name, CORRUPT) == 0 && !test_out_of_sync) - testutil_assert(t->verified == false); - else if (t->verified != true) - printf("%s not seen in metadata\n", t->name); - else { - testutil_check(wt_session->open_cursor( - wt_session, t->name, NULL, NULL, &cursor)); - while ((ret = cursor->next(cursor)) == 0) { - testutil_check(cursor->get_value(cursor, &kv)); - testutil_assert(strcmp(kv, VALUE) == 0); - } - testutil_assert(ret == WT_NOTFOUND); - testutil_check(cursor->close(cursor)); - printf("%s metadata salvaged and data verified\n", - t->name); - } - } + TABLE_INFO *t; + WT_CURSOR *cursor; + WT_DECL_RET; + const char *kv; + + /* + * Open a metadata cursor. + */ + testutil_check(conn->open_session(conn, NULL, NULL, &wt_session)); + testutil_check(wt_session->open_cursor(wt_session, "metadata:", NULL, NULL, &cursor)); + reset_verified(tables); + + /* + * We have to walk the cursor and walk the tables to match up that the expected tables are in + * the metadata. It is not efficient, but the list of tables is small. Walk the cursor once and + * the array of tables each time. + */ + while ((ret = cursor->next(cursor)) == 0) { + testutil_check(cursor->get_key(cursor, &kv)); + for (t = tables; t->name != NULL; t++) { + if (strcmp(t->name, kv) == 0) { + testutil_assert(t->verified == false); + t->verified = true; + break; + } + } + } + testutil_assert(ret == WT_NOTFOUND); + testutil_check(cursor->close(cursor)); + /* + * Any tables that were salvaged, make sure we can read the data. The corrupt table should never + * be salvaged. + */ + for (t = tables; t->name != NULL; t++) { + if (strcmp(t->name, CORRUPT) == 0 && !test_out_of_sync) + testutil_assert(t->verified == false); + else if (t->verified != true) + printf("%s not seen in metadata\n", t->name); + else { + testutil_check(wt_session->open_cursor(wt_session, t->name, NULL, NULL, &cursor)); + while ((ret = cursor->next(cursor)) == 0) { + testutil_check(cursor->get_value(cursor, &kv)); + testutil_assert(strcmp(kv, VALUE) == 0); + } + testutil_assert(ret == WT_NOTFOUND); + testutil_check(cursor->close(cursor)); + printf("%s metadata salvaged and data verified\n", t->name); + } + } } /* * copy_database -- - * Copy the database to the specified suffix. In addition, make a copy - * of the metadata and turtle files in that new directory. + * Copy the database to the specified suffix. In addition, make a copy of the metadata and + * turtle files in that new directory. */ static void copy_database(const char *sfx) { - WT_DECL_RET; - char buf[1024]; - - testutil_check(__wt_snprintf(buf, sizeof(buf), - "rm -rf ./%s.%s; mkdir ./%s.%s; " - "cp -p %s/* ./%s.%s", - home, sfx, home, sfx, home, home, sfx)); - printf("copy: %s\n", buf); - if ((ret = system(buf)) < 0) - testutil_die(ret, "system: %s", buf); - - /* - * Now, in the copied directory make a save copy of the - * metadata and turtle files to move around and restore - * as needed during testing. - */ - testutil_check(__wt_snprintf(buf, sizeof(buf), - "cp -p %s.%s/%s %s.%s/%s.%s", - home, sfx, WT_METADATA_TURTLE, - home, sfx, WT_METADATA_TURTLE, SAVE)); - if ((ret = system(buf)) < 0) - testutil_die(ret, "system: %s", buf); - testutil_check(__wt_snprintf(buf, sizeof(buf), - "cp -p %s.%s/%s %s.%s/%s.%s", - home, sfx, WT_METAFILE, - home, sfx, WT_METAFILE, SAVE)); - if ((ret = system(buf)) < 0) - testutil_die(ret, "system: %s", buf); + WT_DECL_RET; + char buf[1024]; + + testutil_check(__wt_snprintf(buf, sizeof(buf), + "rm -rf ./%s.%s; mkdir ./%s.%s; " + "cp -p %s/* ./%s.%s", + home, sfx, home, sfx, home, home, sfx)); + printf("copy: %s\n", buf); + if ((ret = system(buf)) < 0) + testutil_die(ret, "system: %s", buf); + + /* + * Now, in the copied directory make a save copy of the metadata and turtle files to move around + * and restore as needed during testing. + */ + testutil_check(__wt_snprintf(buf, sizeof(buf), "cp -p %s.%s/%s %s.%s/%s.%s", home, sfx, + WT_METADATA_TURTLE, home, sfx, WT_METADATA_TURTLE, SAVE)); + if ((ret = system(buf)) < 0) + testutil_die(ret, "system: %s", buf); + testutil_check(__wt_snprintf(buf, sizeof(buf), "cp -p %s.%s/%s %s.%s/%s.%s", home, sfx, + WT_METAFILE, home, sfx, WT_METAFILE, SAVE)); + if ((ret = system(buf)) < 0) + testutil_die(ret, "system: %s", buf); } /* * wt_open_corrupt -- - * Call wiredtiger_open and expect a corruption error. + * Call wiredtiger_open and expect a corruption error. */ -static void wt_open_corrupt(const char *) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void wt_open_corrupt(const char *) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void wt_open_corrupt(const char *sfx) { - WT_CONNECTION *conn; - WT_DECL_RET; - char buf[1024]; - - if (sfx != NULL) - testutil_check(__wt_snprintf(buf, sizeof(buf), - "%s.%s", home, sfx)); - else - testutil_check(__wt_snprintf(buf, sizeof(buf), "%s", home)); - ret = wiredtiger_open(buf, &event_handler, NULL, &conn); - /* - * Not all out of sync combinations lead to corruption. We keep - * the previous checkpoint in the file so some combinations of - * future or old turtle files and metadata files will succeed. - */ - if (ret != WT_TRY_SALVAGE && ret != 0) - fprintf(stderr, - "OPEN_CORRUPT: wiredtiger_open returned %d\n", ret); - testutil_assert(ret == WT_TRY_SALVAGE || ret == 0); - exit (EXIT_SUCCESS); + WT_CONNECTION *conn; + WT_DECL_RET; + char buf[1024]; + + if (sfx != NULL) + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s.%s", home, sfx)); + else + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s", home)); + ret = wiredtiger_open(buf, &event_handler, NULL, &conn); + /* + * Not all out of sync combinations lead to corruption. We keep the previous checkpoint in the + * file so some combinations of future or old turtle files and metadata files will succeed. + */ + if (ret != WT_TRY_SALVAGE && ret != 0) + fprintf(stderr, "OPEN_CORRUPT: wiredtiger_open returned %d\n", ret); + testutil_assert(ret == WT_TRY_SALVAGE || ret == 0); + exit(EXIT_SUCCESS); } static int open_with_error(const char *sfx) { - pid_t pid; - int status; - - /* - * Call wiredtiger_open. We expect to see a corruption panic so we - * run this in a forked process. In diagnostic mode, the panic will - * cause an abort and core dump. So we want to catch that and - * continue running with salvage. - */ - printf("=== open corrupt in child ===\n"); - if ((pid = fork()) < 0) - testutil_die(errno, "fork"); - if (pid == 0) { /* child */ - wt_open_corrupt(sfx); - return (EXIT_SUCCESS); - } - /* parent */ - if (waitpid(pid, &status, 0) == -1) - testutil_die(errno, "waitpid"); - return (EXIT_SUCCESS); + pid_t pid; + int status; + + /* + * Call wiredtiger_open. We expect to see a corruption panic so we run this in a forked process. + * In diagnostic mode, the panic will cause an abort and core dump. So we want to catch that and + * continue running with salvage. + */ + printf("=== open corrupt in child ===\n"); + if ((pid = fork()) < 0) + testutil_die(errno, "fork"); + if (pid == 0) { /* child */ + wt_open_corrupt(sfx); + return (EXIT_SUCCESS); + } + /* parent */ + if (waitpid(pid, &status, 0) == -1) + testutil_die(errno, "waitpid"); + return (EXIT_SUCCESS); } static void open_with_salvage(const char *sfx, TABLE_INFO *table_data) { - WT_CONNECTION *conn; - char buf[1024]; - - printf("=== wt_open with salvage ===\n"); - /* - * Then call wiredtiger_open with the salvage configuration setting. - * That should succeed. We should be able to then verify the contents - * of the metadata file. - */ - test_abort = true; - if (sfx != NULL) - testutil_check(__wt_snprintf(buf, sizeof(buf), - "%s.%s", home, sfx)); - else - testutil_check(__wt_snprintf(buf, sizeof(buf), "%s", home)); - testutil_check(wiredtiger_open(buf, - &event_handler, "salvage=true", &conn)); - testutil_assert(conn != NULL); - if (sfx != NULL) - testutil_check(__wt_snprintf(buf, sizeof(buf), - "%s.%s/%s", home, sfx, WT_METAFILE_SLVG)); - else - testutil_check(__wt_snprintf(buf, sizeof(buf), - "%s/%s", home, WT_METAFILE_SLVG)); - testutil_assert(file_exists(buf)); - - /* - * Confirm we salvaged the metadata file by looking for the saved - * copy of the original metadata. - */ - printf("verify with salvaged connection\n"); - verify_metadata(conn, &table_data[0]); - testutil_check(conn->close(conn, NULL)); + WT_CONNECTION *conn; + char buf[1024]; + + printf("=== wt_open with salvage ===\n"); + /* + * Then call wiredtiger_open with the salvage configuration setting. That should succeed. We + * should be able to then verify the contents of the metadata file. + */ + test_abort = true; + if (sfx != NULL) + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s.%s", home, sfx)); + else + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s", home)); + testutil_check(wiredtiger_open(buf, &event_handler, "salvage=true", &conn)); + testutil_assert(conn != NULL); + if (sfx != NULL) + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s.%s/%s", home, sfx, WT_METAFILE_SLVG)); + else + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s/%s", home, WT_METAFILE_SLVG)); + testutil_assert(file_exists(buf)); + + /* + * Confirm we salvaged the metadata file by looking for the saved copy of the original metadata. + */ + printf("verify with salvaged connection\n"); + verify_metadata(conn, &table_data[0]); + testutil_check(conn->close(conn, NULL)); } static void open_normal(const char *sfx, TABLE_INFO *table_data) { - WT_CONNECTION *conn; - char buf[1024]; - - printf("=== wt_open normal ===\n"); - if (sfx != NULL) - testutil_check(__wt_snprintf(buf, sizeof(buf), - "%s.%s", home, sfx)); - else - testutil_check(__wt_snprintf(buf, sizeof(buf), "%s", home)); - testutil_check(wiredtiger_open(buf, &event_handler, NULL, &conn)); - verify_metadata(conn, &table_data[0]); - testutil_check(conn->close(conn, NULL)); + WT_CONNECTION *conn; + char buf[1024]; + + printf("=== wt_open normal ===\n"); + if (sfx != NULL) + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s.%s", home, sfx)); + else + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s", home)); + testutil_check(wiredtiger_open(buf, &event_handler, NULL, &conn)); + verify_metadata(conn, &table_data[0]); + testutil_check(conn->close(conn, NULL)); } static void run_all_verification(const char *sfx, TABLE_INFO *t) { - testutil_check(open_with_error(sfx)); - open_with_salvage(sfx, t); - open_normal(sfx, t); + testutil_check(open_with_error(sfx)); + open_with_salvage(sfx, t); + open_normal(sfx, t); } int main(int argc, char *argv[]) { - /* - * Add a bunch of tables so that some of the metadata ends up on - * other pages and a good number of tables are available after - * salvage completes. - */ - TABLE_INFO table_data[] = { - { "file:aaa-file.SS", "key_format=S,value_format=S", false }, - { "file:bbb-file.rS", "key_format=r,value_format=S", false }, - { "lsm:ccc-lsm.SS", "key_format=S,value_format=S", false }, - { "table:ddd-table.SS", "key_format=S,value_format=S", false }, - { "table:eee-table.rS", "key_format=r,value_format=S", false }, - { "file:fff-file.SS", "key_format=S,value_format=S", false }, - { "file:ggg-file.rS", "key_format=r,value_format=S", false }, - { "lsm:hhh-lsm.SS", "key_format=S,value_format=S", false }, - { "table:iii-table.SS", "key_format=S,value_format=S", false }, - { "table:jjj-table.rS", "key_format=r,value_format=S", false }, - { CORRUPT, "key_format=S,value_format=S", false }, - { NULL, NULL, false } - }; - TABLE_INFO *t; - TEST_OPTS *opts, _opts; - WT_DECL_RET; - char buf[1024]; - - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - /* - * Set a global. We use this everywhere. - */ - home = opts->home; - testutil_make_work_dir(home); - - testutil_check( - wiredtiger_open(home, &event_handler, "create", &opts->conn)); - - testutil_check(opts->conn->open_session( - opts->conn, NULL, NULL, &wt_session)); - /* - * Create a bunch of different tables. - */ - for (t = table_data; t->name != NULL; t++) - create_data(t); - - testutil_check(opts->conn->close(opts->conn, NULL)); - opts->conn = NULL; - - /* - * Make copy of original directory. - */ - copy_database(SAVE); - /* - * Damage/corrupt WiredTiger.wt. - */ - printf("corrupt metadata\n"); - corrupt_file(WT_METAFILE, CORRUPT); - testutil_check(__wt_snprintf(buf, sizeof(buf), - "cp -p %s/WiredTiger.wt ./%s.SAVE/WiredTiger.wt.CORRUPT", - home, home)); - printf("copy: %s\n", buf); - if ((ret = system(buf)) < 0) - testutil_die(ret, "system: %s", buf); - run_all_verification(NULL, &table_data[0]); - - /* - * Damage/corrupt WiredTiger.turtle. - */ - printf("corrupt turtle\n"); - corrupt_file(WT_METADATA_TURTLE, WT_METAFILE_URI); - testutil_check(__wt_snprintf(buf, sizeof(buf), - "cp -p %s/WiredTiger.turtle ./%s.SAVE/WiredTiger.turtle.CORRUPT", - home, home)); - printf("copy: %s\n", buf); - if ((ret = system(buf)) < 0) - testutil_die(ret, "system: %s", buf); - run_all_verification(NULL, &table_data[0]); - - /* - * We need to set up the string before we clean up - * the structure. Then after the clean up we will - * run this command. - */ - testutil_check(__wt_snprintf(buf, sizeof(buf), - "rm -rf core* %s*", home)); - testutil_cleanup(opts); - - /* - * We've created a lot of extra directories and possibly some core - * files from child process aborts. Manually clean them up. - */ - printf("cleanup and remove: %s\n", buf); - if ((ret = system(buf)) < 0) - testutil_die(ret, "system: %s", buf); - - return (EXIT_SUCCESS); + /* + * Add a bunch of tables so that some of the metadata ends up on other pages and a good number + * of tables are available after salvage completes. + */ + TABLE_INFO table_data[] = {{"file:aaa-file.SS", "key_format=S,value_format=S", false}, + {"file:bbb-file.rS", "key_format=r,value_format=S", false}, + {"lsm:ccc-lsm.SS", "key_format=S,value_format=S", false}, + {"table:ddd-table.SS", "key_format=S,value_format=S", false}, + {"table:eee-table.rS", "key_format=r,value_format=S", false}, + {"file:fff-file.SS", "key_format=S,value_format=S", false}, + {"file:ggg-file.rS", "key_format=r,value_format=S", false}, + {"lsm:hhh-lsm.SS", "key_format=S,value_format=S", false}, + {"table:iii-table.SS", "key_format=S,value_format=S", false}, + {"table:jjj-table.rS", "key_format=r,value_format=S", false}, + {CORRUPT, "key_format=S,value_format=S", false}, {NULL, NULL, false}}; + TABLE_INFO *t; + TEST_OPTS *opts, _opts; + WT_DECL_RET; + char buf[1024]; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + /* + * Set a global. We use this everywhere. + */ + home = opts->home; + testutil_make_work_dir(home); + + testutil_check(wiredtiger_open(home, &event_handler, "create", &opts->conn)); + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &wt_session)); + /* + * Create a bunch of different tables. + */ + for (t = table_data; t->name != NULL; t++) + create_data(t); + + testutil_check(opts->conn->close(opts->conn, NULL)); + opts->conn = NULL; + + /* + * Make copy of original directory. + */ + copy_database(SAVE); + /* + * Damage/corrupt WiredTiger.wt. + */ + printf("corrupt metadata\n"); + corrupt_file(WT_METAFILE, CORRUPT); + testutil_check(__wt_snprintf( + buf, sizeof(buf), "cp -p %s/WiredTiger.wt ./%s.SAVE/WiredTiger.wt.CORRUPT", home, home)); + printf("copy: %s\n", buf); + if ((ret = system(buf)) < 0) + testutil_die(ret, "system: %s", buf); + run_all_verification(NULL, &table_data[0]); + + /* + * Damage/corrupt WiredTiger.turtle. + */ + printf("corrupt turtle\n"); + corrupt_file(WT_METADATA_TURTLE, WT_METAFILE_URI); + testutil_check(__wt_snprintf(buf, sizeof(buf), + "cp -p %s/WiredTiger.turtle ./%s.SAVE/WiredTiger.turtle.CORRUPT", home, home)); + printf("copy: %s\n", buf); + if ((ret = system(buf)) < 0) + testutil_die(ret, "system: %s", buf); + run_all_verification(NULL, &table_data[0]); + + /* + * We need to set up the string before we clean up the structure. Then after the clean up we + * will run this command. + */ + testutil_check(__wt_snprintf(buf, sizeof(buf), "rm -rf core* %s*", home)); + testutil_cleanup(opts); + + /* + * We've created a lot of extra directories and possibly some core files from child process + * aborts. Manually clean them up. + */ + printf("cleanup and remove: %s\n", buf); + if ((ret = system(buf)) < 0) + testutil_die(ret, "system: %s", buf); + + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c b/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c index 40b4c543500..4e182453703 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4333_handle_locks/main.c @@ -29,8 +29,8 @@ #include <signal.h> -#define MAXKEY 10000 -#define PERIOD 60 +#define MAXKEY 10000 +#define PERIOD 60 static WT_CONNECTION *conn; static uint64_t worker, worker_busy, verify, verify_busy; @@ -42,353 +42,323 @@ static char *uri_list[750]; static void uri_init(void) { - WT_CURSOR *cursor; - WT_SESSION *session; - u_int i, key; - char buf[128]; - - for (i = 0; i < uris; ++i) - if (uri_list[i] == NULL) { - testutil_check( - __wt_snprintf(buf, sizeof(buf), "table:%u", i)); - uri_list[i] = dstrdup(buf); - } - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - /* Initialize the file contents. */ - for (i = 0; i < uris; ++i) { - testutil_check(__wt_snprintf(buf, sizeof(buf), - "key_format=S,value_format=S," - "allocation_size=4K,leaf_page_max=32KB,")); - testutil_check(session->create(session, uri_list[i], buf)); - testutil_check(session->open_cursor( - session, uri_list[i], NULL, NULL, &cursor)); - for (key = 1; key < MAXKEY; ++key) { - testutil_check(__wt_snprintf( - buf, sizeof(buf), "key:%020u", key)); - cursor->set_key(cursor, buf); - cursor->set_value(cursor, buf); - testutil_check(cursor->insert(cursor)); - } - testutil_check(cursor->close(cursor)); - } - - /* Create a checkpoint we can use for readonly handles. */ - testutil_check(session->checkpoint(session, NULL)); - - testutil_check(session->close(session, NULL)); + WT_CURSOR *cursor; + WT_SESSION *session; + u_int i, key; + char buf[128]; + + for (i = 0; i < uris; ++i) + if (uri_list[i] == NULL) { + testutil_check(__wt_snprintf(buf, sizeof(buf), "table:%u", i)); + uri_list[i] = dstrdup(buf); + } + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + /* Initialize the file contents. */ + for (i = 0; i < uris; ++i) { + testutil_check(__wt_snprintf(buf, sizeof(buf), + "key_format=S,value_format=S," + "allocation_size=4K,leaf_page_max=32KB,")); + testutil_check(session->create(session, uri_list[i], buf)); + testutil_check(session->open_cursor(session, uri_list[i], NULL, NULL, &cursor)); + for (key = 1; key < MAXKEY; ++key) { + testutil_check(__wt_snprintf(buf, sizeof(buf), "key:%020u", key)); + cursor->set_key(cursor, buf); + cursor->set_value(cursor, buf); + testutil_check(cursor->insert(cursor)); + } + testutil_check(cursor->close(cursor)); + } + + /* Create a checkpoint we can use for readonly handles. */ + testutil_check(session->checkpoint(session, NULL)); + + testutil_check(session->close(session, NULL)); } static void uri_teardown(void) { - u_int i; + u_int i; - for (i = 0; i < WT_ELEMENTS(uri_list); ++i) - free(uri_list[i]); + for (i = 0; i < WT_ELEMENTS(uri_list); ++i) + free(uri_list[i]); } static void op(WT_SESSION *session, WT_RAND_STATE *rnd, WT_CURSOR **cpp) { - WT_CURSOR *cursor; - WT_DECL_RET; - u_int i, key; - char buf[128]; - bool readonly; - - /* Close any open cursor in the slot we're about to reuse. */ - if (*cpp != NULL) { - testutil_check((*cpp)->close(*cpp)); - *cpp = NULL; - } - - cursor = NULL; - readonly = __wt_random(rnd) % 2 == 0; - - /* Loop to open an object handle. */ - for (i = __wt_random(rnd) % uris; !done; __wt_yield()) { - /* Use a checkpoint handle for 50% of reads. */ - ret = session->open_cursor(session, uri_list[i], NULL, - readonly && (i % 2 == 0) ? - "checkpoint=WiredTigerCheckpoint" : NULL, &cursor); - if (ret != EBUSY) { - testutil_check(ret); - break; - } - (void)__wt_atomic_add64(&worker_busy, 1); - } - if (cursor == NULL) - return; - - /* Operate on some number of key/value pairs. */ - for (key = 1; - !done && key < MAXKEY; key += __wt_random(rnd) % 37, __wt_yield()) { - testutil_check( - __wt_snprintf(buf, sizeof(buf), "key:%020u", key)); - cursor->set_key(cursor, buf); - if (readonly) - testutil_check(cursor->search(cursor)); - else { - cursor->set_value(cursor, buf); - testutil_check(cursor->insert(cursor)); - } - } - - /* Close the cursor half the time, otherwise cache it. */ - if (__wt_random(rnd) % 2 == 0) - testutil_check(cursor->close(cursor)); - else { - testutil_check(cursor->reset(cursor)); - *cpp = cursor; - } - - (void)__wt_atomic_add64(&worker, 1); + WT_CURSOR *cursor; + WT_DECL_RET; + u_int i, key; + char buf[128]; + bool readonly; + + /* Close any open cursor in the slot we're about to reuse. */ + if (*cpp != NULL) { + testutil_check((*cpp)->close(*cpp)); + *cpp = NULL; + } + + cursor = NULL; + readonly = __wt_random(rnd) % 2 == 0; + + /* Loop to open an object handle. */ + for (i = __wt_random(rnd) % uris; !done; __wt_yield()) { + /* Use a checkpoint handle for 50% of reads. */ + ret = session->open_cursor(session, uri_list[i], NULL, + readonly && (i % 2 == 0) ? "checkpoint=WiredTigerCheckpoint" : NULL, &cursor); + if (ret != EBUSY) { + testutil_check(ret); + break; + } + (void)__wt_atomic_add64(&worker_busy, 1); + } + if (cursor == NULL) + return; + + /* Operate on some number of key/value pairs. */ + for (key = 1; !done && key < MAXKEY; key += __wt_random(rnd) % 37, __wt_yield()) { + testutil_check(__wt_snprintf(buf, sizeof(buf), "key:%020u", key)); + cursor->set_key(cursor, buf); + if (readonly) + testutil_check(cursor->search(cursor)); + else { + cursor->set_value(cursor, buf); + testutil_check(cursor->insert(cursor)); + } + } + + /* Close the cursor half the time, otherwise cache it. */ + if (__wt_random(rnd) % 2 == 0) + testutil_check(cursor->close(cursor)); + else { + testutil_check(cursor->reset(cursor)); + *cpp = cursor; + } + + (void)__wt_atomic_add64(&worker, 1); } static void * wthread(void *arg) { - WT_CURSOR *cursor_list[10]; - WT_RAND_STATE rnd; - WT_SESSION *session; - u_int next; + WT_CURSOR *cursor_list[10]; + WT_RAND_STATE rnd; + WT_SESSION *session; + u_int next; - (void)arg; + (void)arg; - memset(cursor_list, 0, sizeof(cursor_list)); + memset(cursor_list, 0, sizeof(cursor_list)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); - for (next = 0; !done;) { - if (++next == WT_ELEMENTS(cursor_list)) - next = 0; - op(session, &rnd, &cursor_list[next]); - } + for (next = 0; !done;) { + if (++next == WT_ELEMENTS(cursor_list)) + next = 0; + op(session, &rnd, &cursor_list[next]); + } - return (NULL); + return (NULL); } static void * vthread(void *arg) { - WT_CURSOR *cursor_list[10]; - WT_DECL_RET; - WT_RAND_STATE rnd; - WT_SESSION *session; - u_int i, next; - - (void)arg; - - memset(cursor_list, 0, sizeof(cursor_list)); - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); - - for (next = 0; !done;) { - if (++next == WT_ELEMENTS(cursor_list)) - next = 0; - op(session, &rnd, &cursor_list[next]); - - while (!done) { - i = __wt_random(&rnd) % uris; - ret = session->verify(session, uri_list[i], NULL); - if (ret == EBUSY) { - (void)__wt_atomic_add64(&verify_busy, 1); - continue; - } - - testutil_check(ret); - (void)__wt_atomic_add64(&verify, 1); - break; - } - } - - return (NULL); + WT_CURSOR *cursor_list[10]; + WT_DECL_RET; + WT_RAND_STATE rnd; + WT_SESSION *session; + u_int i, next; + + (void)arg; + + memset(cursor_list, 0, sizeof(cursor_list)); + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); + + for (next = 0; !done;) { + if (++next == WT_ELEMENTS(cursor_list)) + next = 0; + op(session, &rnd, &cursor_list[next]); + + while (!done) { + i = __wt_random(&rnd) % uris; + ret = session->verify(session, uri_list[i], NULL); + if (ret == EBUSY) { + (void)__wt_atomic_add64(&verify_busy, 1); + continue; + } + + testutil_check(ret); + (void)__wt_atomic_add64(&verify, 1); + break; + } + } + + return (NULL); } static void on_alarm(int signo) { - (void)signo; /* Unused parameter */ + (void)signo; /* Unused parameter */ - done = true; + done = true; } static void sweep_stats(void) { - static const int list[] = { - WT_STAT_CONN_CURSOR_SWEEP_BUCKETS, - WT_STAT_CONN_CURSOR_SWEEP_CLOSED, - WT_STAT_CONN_CURSOR_SWEEP_EXAMINED, - WT_STAT_CONN_CURSOR_SWEEP, - WT_STAT_CONN_DH_SWEEP_REF, - WT_STAT_CONN_DH_SWEEP_CLOSE, - WT_STAT_CONN_DH_SWEEP_REMOVE, - WT_STAT_CONN_DH_SWEEP_TOD, - WT_STAT_CONN_DH_SWEEPS, - WT_STAT_CONN_DH_SESSION_SWEEPS, - -1 - }; - WT_SESSION *session; - WT_CURSOR *cursor; - uint64_t value; - int i; - const char *desc, *pvalue; - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->open_cursor( - session, "statistics:", NULL, NULL, &cursor)); - for (i = 0;; ++i) { - if (list[i] == -1) - break; - cursor->set_key(cursor, list[i]); - testutil_check(cursor->search(cursor)); - testutil_check( - cursor->get_value(cursor, &desc, &pvalue, &value)); - printf("\t" "%s=%s\n", desc, pvalue); - } + static const int list[] = {WT_STAT_CONN_CURSOR_SWEEP_BUCKETS, WT_STAT_CONN_CURSOR_SWEEP_CLOSED, + WT_STAT_CONN_CURSOR_SWEEP_EXAMINED, WT_STAT_CONN_CURSOR_SWEEP, WT_STAT_CONN_DH_SWEEP_REF, + WT_STAT_CONN_DH_SWEEP_CLOSE, WT_STAT_CONN_DH_SWEEP_REMOVE, WT_STAT_CONN_DH_SWEEP_TOD, + WT_STAT_CONN_DH_SWEEPS, WT_STAT_CONN_DH_SESSION_SWEEPS, -1}; + WT_SESSION *session; + WT_CURSOR *cursor; + uint64_t value; + int i; + const char *desc, *pvalue; + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->open_cursor(session, "statistics:", NULL, NULL, &cursor)); + for (i = 0;; ++i) { + if (list[i] == -1) + break; + cursor->set_key(cursor, list[i]); + testutil_check(cursor->search(cursor)); + testutil_check(cursor->get_value(cursor, &desc, &pvalue, &value)); + printf( + "\t" + "%s=%s\n", + desc, pvalue); + } } static void runone(bool config_cache) { - pthread_t idlist[1000]; - u_int i, j; - char buf[256], home[256]; - - done = false; - - testutil_work_dir_from_path( - home, sizeof(home), "WT_TEST.wt4333_handle_locks"); - testutil_make_work_dir(home); - - testutil_check(__wt_snprintf(buf, sizeof(buf), - "create" - ", cache_cursors=%s" - ", cache_size=5GB" - ", checkpoint_sync=true" - ", eviction=(threads_max=5)" - ", file_manager=(" - "close_handle_minimum=1,close_idle_time=1,close_scan_interval=1)" - ", mmap=true" - ", session_max=%u" - ", statistics=(all)", - config_cache ? "true" : "false", - workers + 100)); - testutil_check(wiredtiger_open(home, NULL, buf, &conn)); - - printf("%s: %d seconds, cache_cursors=%s, %u workers, %u files\n", - progname, PERIOD, config_cache ? "true" : "false", workers, uris); - - uri_init(); - - /* 75% readers, 25% writers. */ - for (i = 0; i < workers; ++i) - testutil_check(pthread_create(&idlist[i], NULL, wthread, NULL)); - testutil_check(pthread_create(&idlist[i], NULL, vthread, NULL)); - ++i; - - (void)alarm(PERIOD); - - for (j = 0; j < i; ++j) - testutil_check(pthread_join(idlist[j], NULL)); - - printf( - "\t" "worker %" PRIu64 - ", worker_busy %" PRIu64 - ", verify %" PRIu64 - ", verify_busy %" PRIu64 - "\n", - worker, worker_busy, verify, verify_busy); - - if (verbose) - sweep_stats(); - - testutil_check(conn->close(conn, NULL)); + pthread_t idlist[1000]; + u_int i, j; + char buf[256], home[256]; + + done = false; + + testutil_work_dir_from_path(home, sizeof(home), "WT_TEST.wt4333_handle_locks"); + testutil_make_work_dir(home); + + testutil_check(__wt_snprintf(buf, sizeof(buf), + "create" + ", cache_cursors=%s" + ", cache_size=5GB" + ", checkpoint_sync=true" + ", eviction=(threads_max=5)" + ", file_manager=(" + "close_handle_minimum=1,close_idle_time=1,close_scan_interval=1)" + ", mmap=true" + ", session_max=%u" + ", statistics=(all)", + config_cache ? "true" : "false", workers + 100)); + testutil_check(wiredtiger_open(home, NULL, buf, &conn)); + + printf("%s: %d seconds, cache_cursors=%s, %u workers, %u files\n", progname, PERIOD, + config_cache ? "true" : "false", workers, uris); + + uri_init(); + + /* 75% readers, 25% writers. */ + for (i = 0; i < workers; ++i) + testutil_check(pthread_create(&idlist[i], NULL, wthread, NULL)); + testutil_check(pthread_create(&idlist[i], NULL, vthread, NULL)); + ++i; + + (void)alarm(PERIOD); + + for (j = 0; j < i; ++j) + testutil_check(pthread_join(idlist[j], NULL)); + + printf( + "\t" + "worker %" PRIu64 ", worker_busy %" PRIu64 ", verify %" PRIu64 ", verify_busy %" PRIu64 "\n", + worker, worker_busy, verify, verify_busy); + + if (verbose) + sweep_stats(); + + testutil_check(conn->close(conn, NULL)); } static int run(int argc, char *argv[]) { - static const struct { - u_int workers; - u_int uris; - bool cache_cursors; - } runs[] = { - { 1, 1, false}, - { 1, 1, true}, - { 8, 1, false}, - { 8, 1, true}, - { 16, 1, false}, - { 16, 1, true}, - { 16, WT_ELEMENTS(uri_list), false}, - { 16, WT_ELEMENTS(uri_list), true}, - {200, 100, false}, - {200, 100, true}, - {200, WT_ELEMENTS(uri_list), false}, - {200, WT_ELEMENTS(uri_list), true}, - {300, 100, false}, - {300, 100, true}, - {600, WT_ELEMENTS(uri_list), false}, - {600, WT_ELEMENTS(uri_list), true}, - }; - WT_RAND_STATE rnd; - u_int i, n; - int ch; - - (void)testutil_set_progname(argv); - __wt_random_init_seed(NULL, &rnd); - - while ((ch = __wt_getopt(argv[0], argc, argv, "v")) != EOF) { - switch (ch) { - case 'v': - verbose = true; - break; - default: - fprintf(stderr, "usage: %s [-v]\n", argv[0]); - return (EXIT_FAILURE); - } - } - - (void)signal(SIGALRM, on_alarm); - - /* Each test in the table runs for a minute, run 5 tests at random. */ - for (i = 0; i < 5; ++i) { - n = __wt_random(&rnd) % WT_ELEMENTS(runs); - workers = runs[n].workers; - uris = runs[n].uris; - runone(runs[n].cache_cursors); - } - - uri_teardown(); - - return (EXIT_SUCCESS); + static const struct { + u_int workers; + u_int uris; + bool cache_cursors; + } runs[] = { + {1, 1, false}, {1, 1, true}, {8, 1, false}, {8, 1, true}, {16, 1, false}, {16, 1, true}, + {16, WT_ELEMENTS(uri_list), false}, {16, WT_ELEMENTS(uri_list), true}, {200, 100, false}, + {200, 100, true}, {200, WT_ELEMENTS(uri_list), false}, {200, WT_ELEMENTS(uri_list), true}, + {300, 100, false}, {300, 100, true}, {600, WT_ELEMENTS(uri_list), false}, + {600, WT_ELEMENTS(uri_list), true}, + }; + WT_RAND_STATE rnd; + u_int i, n; + int ch; + + (void)testutil_set_progname(argv); + __wt_random_init_seed(NULL, &rnd); + + while ((ch = __wt_getopt(argv[0], argc, argv, "v")) != EOF) { + switch (ch) { + case 'v': + verbose = true; + break; + default: + fprintf(stderr, "usage: %s [-v]\n", argv[0]); + return (EXIT_FAILURE); + } + } + + (void)signal(SIGALRM, on_alarm); + + /* Each test in the table runs for a minute, run 5 tests at random. */ + for (i = 0; i < 5; ++i) { + n = __wt_random(&rnd) % WT_ELEMENTS(runs); + workers = runs[n].workers; + uris = runs[n].uris; + runone(runs[n].cache_cursors); + } + + uri_teardown(); + + return (EXIT_SUCCESS); } int main(int argc, char *argv[]) { - bool skip; + bool skip; - skip = false; + skip = false; - /* - * Bypass this test for valgrind. It has a fairly low thread limit. - */ - if (testutil_is_flag_set("TESTUTIL_BYPASS_VALGRIND")) - skip = true; + /* + * Bypass this test for valgrind. It has a fairly low thread limit. + */ + if (testutil_is_flag_set("TESTUTIL_BYPASS_VALGRIND")) + skip = true; - /* - * Bypass this test for OS X. We periodically see it hang without error, - * leaving a zombie process that never exits (WT-4613, BUILD-7616). - */ +/* + * Bypass this test for OS X. We periodically see it hang without error, leaving a zombie process + * that never exits (WT-4613, BUILD-7616). + */ #if defined(__APPLE__) - skip = true; + skip = true; #endif - return (skip ? EXIT_SUCCESS : run(argc, argv)); + return (skip ? EXIT_SUCCESS : run(argc, argv)); } diff --git a/src/third_party/wiredtiger/test/csuite/wt4699_json/main.c b/src/third_party/wiredtiger/test/csuite/wt4699_json/main.c index 636798c0ffd..e6b71156974 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4699_json/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4699_json/main.c @@ -28,70 +28,62 @@ #include "test_util.h" /* - * JIRA ticket reference: WT-4699 - * Test case description: Use a JSON dump cursor on a projection, - * and overwrite the projection string. - * Failure mode: On the first retrieval of a JSON key/value, a configure - * parse error is returned. + * JIRA ticket reference: WT-4699 Test case description: Use a JSON dump cursor on a projection, and + * overwrite the projection string. Failure mode: On the first retrieval of a JSON key/value, a + * configure parse error is returned. */ int main(int argc, char *argv[]) { - TEST_OPTS *opts, _opts; - WT_CURSOR *c; - WT_SESSION *session; - char *jsonkey, *jsonvalue; - char projection[1000]; + TEST_OPTS *opts, _opts; + WT_CURSOR *c; + WT_SESSION *session; + char *jsonkey, *jsonvalue; + char projection[1000]; - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); - testutil_check(wiredtiger_open(opts->home, NULL, - "create", &opts->conn)); - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(wiredtiger_open(opts->home, NULL, "create", &opts->conn)); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); - /* Create a single record in a table with two fields in its value. */ - testutil_check(session->create(session, opts->uri, - "key_format=i,value_format=ii,columns=(k,v0,v1)")); - testutil_check( - session->open_cursor(session, opts->uri, NULL, NULL, &c)); - c->set_key(c, 1); - c->set_value(c, 1, 1); - testutil_check(c->insert(c)); - testutil_check(c->close(c)); + /* Create a single record in a table with two fields in its value. */ + testutil_check( + session->create(session, opts->uri, "key_format=i,value_format=ii,columns=(k,v0,v1)")); + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &c)); + c->set_key(c, 1); + c->set_value(c, 1, 1); + testutil_check(c->insert(c)); + testutil_check(c->close(c)); - /* - * Open a dump JSON cursor on a projection of the table. - * The fields will be listed in a different order. - */ - strcpy(projection, opts->uri); - strcat(projection, "(v1,v0,k)"); - testutil_check( - session->open_cursor(session, projection, NULL, "dump=json", &c)); - testutil_check(c->next(c)); - /* Overwrite the projection, with not enough columns */ - strcpy(projection, opts->uri); - strcat(projection, "(aaa,bbb)"); - testutil_check(c->get_key(c, &jsonkey)); + /* + * Open a dump JSON cursor on a projection of the table. The fields will be listed in a + * different order. + */ + strcpy(projection, opts->uri); + strcat(projection, "(v1,v0,k)"); + testutil_check(session->open_cursor(session, projection, NULL, "dump=json", &c)); + testutil_check(c->next(c)); + /* Overwrite the projection, with not enough columns */ + strcpy(projection, opts->uri); + strcat(projection, "(aaa,bbb)"); + testutil_check(c->get_key(c, &jsonkey)); - /* - * Here's where we would get the parse error. - * When a JSON dump is performed on a projection, we need to format - * all the field names and values in the order listed. - * The implementation uses the projection string from the - * open_cursor call to determine the field names. - */ - testutil_check(c->get_value(c, &jsonvalue)); - testutil_assert(strstr(jsonvalue, "aaa") == NULL); - printf("KEY: %s\n", jsonkey); - printf("VALUE: %s\n", jsonvalue); - testutil_assert(c->next(c) == WT_NOTFOUND); - testutil_check(c->close(c)); - testutil_check(session->close(session, NULL)); - testutil_cleanup(opts); - return (EXIT_SUCCESS); + /* + * Here's where we would get the parse error. When a JSON dump is performed on a projection, we + * need to format all the field names and values in the order listed. The implementation uses + * the projection string from the open_cursor call to determine the field names. + */ + testutil_check(c->get_value(c, &jsonvalue)); + testutil_assert(strstr(jsonvalue, "aaa") == NULL); + printf("KEY: %s\n", jsonkey); + printf("VALUE: %s\n", jsonvalue); + testutil_assert(c->next(c) == WT_NOTFOUND); + testutil_check(c->close(c)); + testutil_check(session->close(session, NULL)); + testutil_cleanup(opts); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/csuite/wt4803_cache_overflow_abort/main.c b/src/third_party/wiredtiger/test/csuite/wt4803_cache_overflow_abort/main.c index 7d9b0baf132..264dbbb5679 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4803_cache_overflow_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4803_cache_overflow_abort/main.c @@ -31,209 +31,185 @@ #include <sys/wait.h> /* - * JIRA ticket reference: WT-4803 - * Test case description: This test is checking the functionality of the - * lookaside file_max configuration. When the size of the lookaside file exceeds - * this value, we expect to panic. - * Failure mode: If we receive a panic in the test cases we weren't expecting to + * JIRA ticket reference: WT-4803 Test case description: This test is checking the functionality of + * the lookaside file_max configuration. When the size of the lookaside file exceeds this value, we + * expect to panic. Failure mode: If we receive a panic in the test cases we weren't expecting to * and vice versa. */ -#define NUM_KEYS 2000 +#define NUM_KEYS 2000 /* - * This is a global flag that should be set before running test_las_workload. - * It lets the child process know whether it should be expecting a panic or not - * so that it can adjust its exit code as needed. + * This is a global flag that should be set before running test_las_workload. It lets the child + * process know whether it should be expecting a panic or not so that it can adjust its exit code as + * needed. */ static bool expect_panic; static int -handle_message(WT_EVENT_HANDLER *handler, - WT_SESSION *session, int error, const char *message) +handle_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const char *message) { - WT_UNUSED(handler); - WT_UNUSED(session); - - (void)fprintf( - stderr, "%s: %s\n", message, session->strerror(session, error)); - - if (error == WT_PANIC && - strstr(message, "exceeds maximum size") != NULL) { - fprintf(stderr, "Got cache overflow error (expect_panic=%s)\n", - expect_panic ? "true" : "false"); - - /* - * If we're expecting a panic, exit with zero to indicate to the - * parent that this test was successful. - * - * If not, don't intercept. We'll naturally exit with non-zero - * if we're terminating due to panic. - */ - if (expect_panic) - exit(EXIT_SUCCESS); - } - - return (0); + WT_UNUSED(handler); + WT_UNUSED(session); + + (void)fprintf(stderr, "%s: %s\n", message, session->strerror(session, error)); + + if (error == WT_PANIC && strstr(message, "exceeds maximum size") != NULL) { + fprintf( + stderr, "Got cache overflow error (expect_panic=%s)\n", expect_panic ? "true" : "false"); + + /* + * If we're expecting a panic, exit with zero to indicate to the + * parent that this test was successful. + * + * If not, don't intercept. We'll naturally exit with non-zero + * if we're terminating due to panic. + */ + if (expect_panic) + exit(EXIT_SUCCESS); + } + + return (0); } -static WT_EVENT_HANDLER event_handler = { - handle_message, - NULL, - NULL, - NULL -}; +static WT_EVENT_HANDLER event_handler = {handle_message, NULL, NULL, NULL}; static void las_workload(TEST_OPTS *opts, const char *las_file_max) { - WT_CURSOR *cursor; - WT_SESSION *other_session, *session; - int i; - char buf[WT_MEGABYTE], open_config[128]; - - testutil_check(__wt_snprintf(open_config, sizeof(open_config), - "create,cache_size=50MB,cache_overflow=(file_max=%s)", - las_file_max)); - - testutil_check(wiredtiger_open( - opts->home, &event_handler, open_config, &opts->conn)); - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check( - session->create(session, opts->uri, "key_format=i,value_format=S")); - testutil_check( - session->open_cursor(session, opts->uri, NULL, NULL, &cursor)); - - memset(buf, 0xA, WT_MEGABYTE); - buf[WT_MEGABYTE - 1] = '\0'; - - /* Populate the table. */ - for (i = 0; i < NUM_KEYS; ++i) { - cursor->set_key(cursor, i); - cursor->set_value(cursor, buf); - testutil_check(cursor->insert(cursor)); - } - - /* - * Open a snapshot isolation transaction in another session. This forces - * the cache to retain all previous values. Then update all keys with a - * new value in the original session while keeping that snapshot - * transaction open. With the large value buffer, small cache and lots - * of keys, this will force a lot of lookaside usage. - * - * When the file_max setting is small, the maximum size should easily be - * reached and we should panic. When the maximum size is large or not - * set, then we should succeed. - */ - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &other_session)); - testutil_check(other_session->begin_transaction( - other_session, "isolation=snapshot")); - - memset(buf, 0xB, WT_MEGABYTE); - buf[WT_MEGABYTE - 1] = '\0'; - - for (i = 0; i < NUM_KEYS; ++i) { - cursor->set_key(cursor, i); - cursor->set_value(cursor, buf); - testutil_check(cursor->update(cursor)); - } - - /* - * Cleanup. - * We do not get here when the file_max size is small because we will - * have already hit the maximum and exited. This code only executes on - * the successful path. - */ - testutil_check( - other_session->rollback_transaction(other_session, NULL)); - testutil_check(other_session->close(other_session, NULL)); - - testutil_check(cursor->close(cursor)); - testutil_check(session->close(session, NULL)); + WT_CURSOR *cursor; + WT_SESSION *other_session, *session; + int i; + char buf[WT_MEGABYTE], open_config[128]; + + testutil_check(__wt_snprintf(open_config, sizeof(open_config), + "create,cache_size=50MB,cache_overflow=(file_max=%s)", las_file_max)); + + testutil_check(wiredtiger_open(opts->home, &event_handler, open_config, &opts->conn)); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(session->create(session, opts->uri, "key_format=i,value_format=S")); + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &cursor)); + + memset(buf, 0xA, WT_MEGABYTE); + buf[WT_MEGABYTE - 1] = '\0'; + + /* Populate the table. */ + for (i = 0; i < NUM_KEYS; ++i) { + cursor->set_key(cursor, i); + cursor->set_value(cursor, buf); + testutil_check(cursor->insert(cursor)); + } + + /* + * Open a snapshot isolation transaction in another session. This forces + * the cache to retain all previous values. Then update all keys with a + * new value in the original session while keeping that snapshot + * transaction open. With the large value buffer, small cache and lots + * of keys, this will force a lot of lookaside usage. + * + * When the file_max setting is small, the maximum size should easily be + * reached and we should panic. When the maximum size is large or not + * set, then we should succeed. + */ + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &other_session)); + testutil_check(other_session->begin_transaction(other_session, "isolation=snapshot")); + + memset(buf, 0xB, WT_MEGABYTE); + buf[WT_MEGABYTE - 1] = '\0'; + + for (i = 0; i < NUM_KEYS; ++i) { + cursor->set_key(cursor, i); + cursor->set_value(cursor, buf); + testutil_check(cursor->update(cursor)); + } + + /* + * Cleanup. We do not get here when the file_max size is small because we will have already hit + * the maximum and exited. This code only executes on the successful path. + */ + testutil_check(other_session->rollback_transaction(other_session, NULL)); + testutil_check(other_session->close(other_session, NULL)); + + testutil_check(cursor->close(cursor)); + testutil_check(session->close(session, NULL)); } static int test_las_workload(TEST_OPTS *opts, const char *las_file_max) { - pid_t pid; - int status; - - /* - * We're going to run this workload for different configurations of - * file_max. So clean out the work directory each time. - */ - testutil_make_work_dir(opts->home); - - /* - * Since it's possible that the workload will panic and abort, we will - * fork the process and execute the workload in the child process. - * - * This way, we can safely check the exit code of the child process and - * confirm that it is what we expected. - */ - pid = fork(); - if (pid < 0) - /* Failed fork. */ - testutil_die(errno, "fork"); - else if (pid == 0) { - /* Child process from here. */ - las_workload(opts, las_file_max); - - /* - * If we're expecting a panic during the workload, we shouldn't - * get to this point. Exit with non-zero to indicate to parent - * that we should fail this test. - */ - fprintf(stderr, - "Successfully completed workload (expect_panic=%s)\n", - expect_panic ? "true" : "false"); - - if (expect_panic) - exit(EXIT_FAILURE); - else - exit(EXIT_SUCCESS); - } - - /* Parent process from here. */ - if (waitpid(pid, &status, 0) == -1) - testutil_die(errno, "waitpid"); - - return (status); + pid_t pid; + int status; + + /* + * We're going to run this workload for different configurations of file_max. So clean out the + * work directory each time. + */ + testutil_make_work_dir(opts->home); + + /* + * Since it's possible that the workload will panic and abort, we will + * fork the process and execute the workload in the child process. + * + * This way, we can safely check the exit code of the child process and + * confirm that it is what we expected. + */ + pid = fork(); + if (pid < 0) + /* Failed fork. */ + testutil_die(errno, "fork"); + else if (pid == 0) { + /* Child process from here. */ + las_workload(opts, las_file_max); + + /* + * If we're expecting a panic during the workload, we shouldn't get to this point. Exit with + * non-zero to indicate to parent that we should fail this test. + */ + fprintf(stderr, "Successfully completed workload (expect_panic=%s)\n", + expect_panic ? "true" : "false"); + + if (expect_panic) + exit(EXIT_FAILURE); + else + exit(EXIT_SUCCESS); + } + + /* Parent process from here. */ + if (waitpid(pid, &status, 0) == -1) + testutil_die(errno, "waitpid"); + + return (status); } int main(int argc, char **argv) { - TEST_OPTS opts; - - memset(&opts, 0x0, sizeof(opts)); - testutil_check(testutil_parse_opts(argc, argv, &opts)); - - /* - * The lookaside is unbounded. - * We don't expect any failure since we can use as much as needed. - */ - expect_panic = false; - testutil_check(test_las_workload(&opts, "0")); - - /* - * The lookaside is limited to 5GB. - * This is more than enough for this workload so we don't expect any - * failure. - */ - expect_panic = false; - testutil_check(test_las_workload(&opts, "5GB")); - - /* - * The lookaside is limited to 100MB. - * This is insufficient for this workload so we're expecting a failure. - */ - expect_panic = true; - testutil_check(test_las_workload(&opts, "100MB")); - - testutil_cleanup(&opts); - - return (0); + TEST_OPTS opts; + + memset(&opts, 0x0, sizeof(opts)); + testutil_check(testutil_parse_opts(argc, argv, &opts)); + + /* + * The lookaside is unbounded. We don't expect any failure since we can use as much as needed. + */ + expect_panic = false; + testutil_check(test_las_workload(&opts, "0")); + + /* + * The lookaside is limited to 5GB. This is more than enough for this workload so we don't + * expect any failure. + */ + expect_panic = false; + testutil_check(test_las_workload(&opts, "5GB")); + + /* + * The lookaside is limited to 100MB. This is insufficient for this workload so we're expecting + * a failure. + */ + expect_panic = true; + testutil_check(test_las_workload(&opts, "100MB")); + + testutil_cleanup(&opts); + + return (0); } diff --git a/src/third_party/wiredtiger/test/csuite/wt4891_meta_ckptlist_get_alloc/main.c b/src/third_party/wiredtiger/test/csuite/wt4891_meta_ckptlist_get_alloc/main.c index 4be4d5308ce..81745e6aaab 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4891_meta_ckptlist_get_alloc/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4891_meta_ckptlist_get_alloc/main.c @@ -26,66 +26,58 @@ * OTHER DEALINGS IN THE SOFTWARE. */ #include "test_util.h" -#define CHECKPOINT_COUNT 10 +#define CHECKPOINT_COUNT 10 /* - * JIRA ticket reference: WT-4891 - * Test case description: Test wt_meta_ckptlist_get by creating a number of - * checkpoints and then running __wt_verify. - * Failure mode: If the bug still exists then this test will cause an - * error in address sanitized builds. + * JIRA ticket reference: WT-4891 Test case description: Test wt_meta_ckptlist_get by creating a + * number of checkpoints and then running __wt_verify. Failure mode: If the bug still exists then + * this test will cause an error in address sanitized builds. */ int main(int argc, char *argv[]) { - TEST_OPTS *opts, _opts; - WT_CURSOR *cursor, *cursor_ckpt; - WT_SESSION *session; - int i; + TEST_OPTS *opts, _opts; + WT_CURSOR *cursor, *cursor_ckpt; + WT_SESSION *session; + int i; - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); - testutil_make_work_dir(opts->home); + testutil_make_work_dir(opts->home); - testutil_check(wiredtiger_open( - opts->home, NULL, "create", &opts->conn)); + testutil_check(wiredtiger_open(opts->home, NULL, "create", &opts->conn)); - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check( - session->create(session, opts->uri, "key_format=S,value_format=i")); + testutil_check(session->create(session, opts->uri, "key_format=S,value_format=i")); - testutil_check( - session->open_cursor(session, opts->uri, NULL, NULL, &cursor)); + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &cursor)); - /* - * Create checkpoints and keep them active by around by opening a - * checkpoint cursor for each one. - */ - for (i = 0; i < CHECKPOINT_COUNT; ++i) { - testutil_check( - session->begin_transaction(session, "isolation=snapshot")); - cursor->set_key(cursor, "key1"); - cursor->set_value(cursor, i); - testutil_check(cursor->update(cursor)); - testutil_check(session->commit_transaction(session, NULL)); - testutil_check(session->checkpoint(session, NULL)); - testutil_check(session->open_cursor(session, opts->uri, NULL, - "checkpoint=WiredTigerCheckpoint", &cursor_ckpt)); - } + /* + * Create checkpoints and keep them active by around by opening a checkpoint cursor for each + * one. + */ + for (i = 0; i < CHECKPOINT_COUNT; ++i) { + testutil_check(session->begin_transaction(session, "isolation=snapshot")); + cursor->set_key(cursor, "key1"); + cursor->set_value(cursor, i); + testutil_check(cursor->update(cursor)); + testutil_check(session->commit_transaction(session, NULL)); + testutil_check(session->checkpoint(session, NULL)); + testutil_check(session->open_cursor( + session, opts->uri, NULL, "checkpoint=WiredTigerCheckpoint", &cursor_ckpt)); + } - testutil_check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); - testutil_check(session->verify(session, opts->uri, NULL)); + testutil_check(session->verify(session, opts->uri, NULL)); - testutil_cleanup(opts); + testutil_cleanup(opts); - return (0); + return (0); } diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order.c index e0c05c85c9f..714108a0ed9 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order.c +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order.c @@ -28,15 +28,14 @@ #include "cursor_order.h" -static char home[512]; /* Program working dir */ -static FILE *logfp; /* Log file */ +static char home[512]; /* Program working dir */ +static FILE *logfp; /* Log file */ -static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); -static int handle_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); -static void onint(int) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); +static int handle_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); +static void onint(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void shutdown(void); -static int usage(void); +static int usage(void); static void wt_connect(SHARED_CONFIG *, char *); static void wt_shutdown(SHARED_CONFIG *); @@ -46,245 +45,231 @@ extern char *__wt_optarg; int main(int argc, char *argv[]) { - SHARED_CONFIG _cfg, *cfg; - int ch, cnt, runs; - char *config_open, *working_dir; - - (void)testutil_set_progname(argv); - - cfg = &_cfg; - config_open = NULL; - working_dir = NULL; - runs = 1; - - /* - * Explicitly initialize the shared configuration object before - * parsing command line options. - */ - cfg->append_inserters = 1; - cfg->conn = NULL; - cfg->ftype = ROW; - cfg->max_nops = 1000000; - cfg->multiple_files = false; - cfg->nkeys = 1000; - cfg->reverse_scanners = 5; - cfg->reverse_scan_ops = 10; - cfg->thread_finish = false; - cfg->vary_nops = false; - - while ((ch = __wt_getopt( - progname, argc, argv, "C:Fk:h:l:n:R:r:t:vw:W:")) != EOF) - switch (ch) { - case 'C': /* wiredtiger_open config */ - config_open = __wt_optarg; - break; - case 'F': /* multiple files */ - cfg->multiple_files = true; - break; - case 'h': - working_dir = __wt_optarg; - break; - case 'k': /* rows */ - cfg->nkeys = (uint64_t)atol(__wt_optarg); - break; - case 'l': /* log */ - if ((logfp = fopen(__wt_optarg, "w")) == NULL) { - fprintf(stderr, - "%s: %s\n", __wt_optarg, strerror(errno)); - return (EXIT_FAILURE); - } - break; - case 'n': /* operations */ - cfg->max_nops = (uint64_t)atol(__wt_optarg); - break; - case 'R': - cfg->reverse_scanners = (uint64_t)atol(__wt_optarg); - break; - case 'r': /* runs */ - runs = atoi(__wt_optarg); - break; - case 't': - switch (__wt_optarg[0]) { - case 'f': - cfg->ftype = FIX; - break; - case 'r': - cfg->ftype = ROW; - break; - case 'v': - cfg->ftype = VAR; - break; - default: - return (usage()); - } - break; - case 'v': /* vary operation count */ - cfg->vary_nops = true; - break; - case 'w': - cfg->reverse_scan_ops = (uint64_t)atol(__wt_optarg); - break; - case 'W': - cfg->append_inserters = (uint64_t)atol(__wt_optarg); - break; - default: - return (usage()); - } - - argc -= __wt_optind; - if (argc != 0) - return (usage()); - - testutil_work_dir_from_path(home, 512, working_dir); - - if (cfg->vary_nops && !cfg->multiple_files) { - fprintf(stderr, - "Variable op counts only supported with multiple tables\n"); - return (usage()); - } - - /* Clean up on signal. */ - (void)signal(SIGINT, onint); - - printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); - for (cnt = 1; runs == 0 || cnt <= runs; ++cnt) { - printf( - " %d: %" PRIu64 - " reverse scanners, %" PRIu64 " writers\n", - cnt, cfg->reverse_scanners, cfg->append_inserters); - - shutdown(); /* Clean up previous runs */ - - wt_connect(cfg, config_open); /* WiredTiger connection */ - - ops_start(cfg); - - wt_shutdown(cfg); /* WiredTiger shut down */ - } - return (0); + SHARED_CONFIG _cfg, *cfg; + int ch, cnt, runs; + char *config_open, *working_dir; + + (void)testutil_set_progname(argv); + + cfg = &_cfg; + config_open = NULL; + working_dir = NULL; + runs = 1; + + /* + * Explicitly initialize the shared configuration object before parsing command line options. + */ + cfg->append_inserters = 1; + cfg->conn = NULL; + cfg->ftype = ROW; + cfg->max_nops = 1000000; + cfg->multiple_files = false; + cfg->nkeys = 1000; + cfg->reverse_scanners = 5; + cfg->reverse_scan_ops = 10; + cfg->thread_finish = false; + cfg->vary_nops = false; + + while ((ch = __wt_getopt(progname, argc, argv, "C:Fk:h:l:n:R:r:t:vw:W:")) != EOF) + switch (ch) { + case 'C': /* wiredtiger_open config */ + config_open = __wt_optarg; + break; + case 'F': /* multiple files */ + cfg->multiple_files = true; + break; + case 'h': + working_dir = __wt_optarg; + break; + case 'k': /* rows */ + cfg->nkeys = (uint64_t)atol(__wt_optarg); + break; + case 'l': /* log */ + if ((logfp = fopen(__wt_optarg, "w")) == NULL) { + fprintf(stderr, "%s: %s\n", __wt_optarg, strerror(errno)); + return (EXIT_FAILURE); + } + break; + case 'n': /* operations */ + cfg->max_nops = (uint64_t)atol(__wt_optarg); + break; + case 'R': + cfg->reverse_scanners = (uint64_t)atol(__wt_optarg); + break; + case 'r': /* runs */ + runs = atoi(__wt_optarg); + break; + case 't': + switch (__wt_optarg[0]) { + case 'f': + cfg->ftype = FIX; + break; + case 'r': + cfg->ftype = ROW; + break; + case 'v': + cfg->ftype = VAR; + break; + default: + return (usage()); + } + break; + case 'v': /* vary operation count */ + cfg->vary_nops = true; + break; + case 'w': + cfg->reverse_scan_ops = (uint64_t)atol(__wt_optarg); + break; + case 'W': + cfg->append_inserters = (uint64_t)atol(__wt_optarg); + break; + default: + return (usage()); + } + + argc -= __wt_optind; + if (argc != 0) + return (usage()); + + testutil_work_dir_from_path(home, 512, working_dir); + + if (cfg->vary_nops && !cfg->multiple_files) { + fprintf(stderr, "Variable op counts only supported with multiple tables\n"); + return (usage()); + } + + /* Clean up on signal. */ + (void)signal(SIGINT, onint); + + printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); + for (cnt = 1; runs == 0 || cnt <= runs; ++cnt) { + printf(" %d: %" PRIu64 " reverse scanners, %" PRIu64 " writers\n", cnt, + cfg->reverse_scanners, cfg->append_inserters); + + shutdown(); /* Clean up previous runs */ + + wt_connect(cfg, config_open); /* WiredTiger connection */ + + ops_start(cfg); + + wt_shutdown(cfg); /* WiredTiger shut down */ + } + return (0); } /* * wt_connect -- - * Configure the WiredTiger connection. + * Configure the WiredTiger connection. */ static void wt_connect(SHARED_CONFIG *cfg, char *config_open) { - static WT_EVENT_HANDLER event_handler = { - handle_error, - handle_message, - NULL, - NULL /* Close handler. */ - }; - char config[512]; - - testutil_clean_work_dir(home); - testutil_make_work_dir(home); - - testutil_check(__wt_snprintf(config, sizeof(config), - "create,statistics=(all),error_prefix=\"%s\",%s%s", - progname, - config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open)); - - testutil_check(wiredtiger_open( - home, &event_handler, config, &cfg->conn)); + static WT_EVENT_HANDLER event_handler = { + handle_error, handle_message, NULL, NULL /* Close handler. */ + }; + char config[512]; + + testutil_clean_work_dir(home); + testutil_make_work_dir(home); + + testutil_check( + __wt_snprintf(config, sizeof(config), "create,statistics=(all),error_prefix=\"%s\",%s%s", + progname, config_open == NULL ? "" : ",", config_open == NULL ? "" : config_open)); + + testutil_check(wiredtiger_open(home, &event_handler, config, &cfg->conn)); } /* * wt_shutdown -- - * Flush the file to disk and shut down the WiredTiger connection. + * Flush the file to disk and shut down the WiredTiger connection. */ static void wt_shutdown(SHARED_CONFIG *cfg) { - WT_CONNECTION *conn; - WT_SESSION *session; + WT_CONNECTION *conn; + WT_SESSION *session; - conn = cfg->conn; + conn = cfg->conn; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->checkpoint(session, NULL)); + testutil_check(session->checkpoint(session, NULL)); - testutil_check(conn->close(conn, NULL)); + testutil_check(conn->close(conn, NULL)); } /* * shutdown -- - * Clean up from previous runs. + * Clean up from previous runs. */ static void shutdown(void) { - testutil_clean_work_dir(home); + testutil_clean_work_dir(home); } static int -handle_error(WT_EVENT_HANDLER *handler, - WT_SESSION *session, int error, const char *errmsg) +handle_error(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const char *errmsg) { - (void)(handler); - (void)(session); - (void)(error); + (void)(handler); + (void)(session); + (void)(error); - return (fprintf(stderr, "%s\n", errmsg) < 0 ? -1 : 0); + return (fprintf(stderr, "%s\n", errmsg) < 0 ? -1 : 0); } static int -handle_message(WT_EVENT_HANDLER *handler, - WT_SESSION *session, const char *message) +handle_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message) { - (void)(handler); - (void)(session); + (void)(handler); + (void)(session); - if (logfp != NULL) - return (fprintf(logfp, "%s\n", message) < 0 ? -1 : 0); + if (logfp != NULL) + return (fprintf(logfp, "%s\n", message) < 0 ? -1 : 0); - return (printf("%s\n", message) < 0 ? -1 : 0); + return (printf("%s\n", message) < 0 ? -1 : 0); } /* * onint -- - * Interrupt signal handler. + * Interrupt signal handler. */ static void onint(int signo) { - (void)(signo); + (void)(signo); - shutdown(); + shutdown(); - fprintf(stderr, "\n"); - exit(EXIT_FAILURE); + fprintf(stderr, "\n"); + exit(EXIT_FAILURE); } /* * usage -- - * Display usage statement and exit failure. + * Display usage statement and exit failure. */ static int usage(void) { - fprintf(stderr, - "usage: %s " - "[-FLv] [-C wiredtiger-config] [-k keys] [-l log]\n\t" - "[-n ops] [-R reverse_scanners] [-r runs] [-t f|r|v] " - "[-W append_inserters]\n", - progname); - fprintf(stderr, "%s", - "\t-C specify wiredtiger_open configuration arguments\n" - "\t-F create a file per thread\n" - "\t-k set number of keys to load\n" - "\t-L log print per operation\n" - "\t-l specify a log file\n" - "\t-n set number of operations each thread does\n" - "\t-R set number of reverse scanner threads\n" - "\t-r set number of runs (0 for continuous)\n" - "\t-t set a file type (fix | row | var)\n" - "\t-v do a different number of operations on different tables\n" - "\t-w set number of items to walk in a reverse scan\n" - "\t-W set number of threads doing append inserts\n"); - return (EXIT_FAILURE); + fprintf(stderr, + "usage: %s " + "[-FLv] [-C wiredtiger-config] [-k keys] [-l log]\n\t" + "[-n ops] [-R reverse_scanners] [-r runs] [-t f|r|v] " + "[-W append_inserters]\n", + progname); + fprintf(stderr, "%s", + "\t-C specify wiredtiger_open configuration arguments\n" + "\t-F create a file per thread\n" + "\t-k set number of keys to load\n" + "\t-L log print per operation\n" + "\t-l specify a log file\n" + "\t-n set number of operations each thread does\n" + "\t-R set number of reverse scanner threads\n" + "\t-r set number of runs (0 for continuous)\n" + "\t-t set a file type (fix | row | var)\n" + "\t-v do a different number of operations on different tables\n" + "\t-w set number of items to walk in a reverse scan\n" + "\t-W set number of threads doing append inserts\n"); + return (EXIT_FAILURE); } diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order.h b/src/third_party/wiredtiger/test/cursor_order/cursor_order.h index 7c95f7b6e71..400ece2d2a2 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order.h +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order.h @@ -30,22 +30,22 @@ #include "test_util.h" -#define FNAME "file:cursor_order.%03d" /* File name */ +#define FNAME "file:cursor_order.%03d" /* File name */ -typedef enum { FIX, ROW, VAR } __ftype; /* File type */ +typedef enum { FIX, ROW, VAR } __ftype; /* File type */ typedef struct { - uint64_t append_inserters; /* Number of append threads */ - WT_CONNECTION *conn; /* WiredTiger connection */ - __ftype ftype; - uint64_t key_range; /* Current key range */ - uint64_t max_nops; /* Operations per thread */ - bool multiple_files; /* File per thread */ - uint64_t nkeys; /* Keys to load */ - uint64_t reverse_scanners; /* Number of scan threads */ - uint64_t reverse_scan_ops; /* Keys to visit per scan */ - bool thread_finish; /* Signal to finish run. */ - bool vary_nops; /* Operations per thread */ + uint64_t append_inserters; /* Number of append threads */ + WT_CONNECTION *conn; /* WiredTiger connection */ + __ftype ftype; + uint64_t key_range; /* Current key range */ + uint64_t max_nops; /* Operations per thread */ + bool multiple_files; /* File per thread */ + uint64_t nkeys; /* Keys to load */ + uint64_t reverse_scanners; /* Number of scan threads */ + uint64_t reverse_scan_ops; /* Keys to visit per scan */ + bool thread_finish; /* Signal to finish run. */ + bool vary_nops; /* Operations per thread */ } SHARED_CONFIG; diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c index 94bcb40f667..898bc75486c 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c @@ -31,91 +31,88 @@ static void file_create(SHARED_CONFIG *cfg, const char *name) { - WT_CONNECTION *conn; - WT_SESSION *session; - int ret; - char config[128]; + WT_CONNECTION *conn; + WT_SESSION *session; + int ret; + char config[128]; - conn = cfg->conn; + conn = cfg->conn; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(__wt_snprintf(config, sizeof(config), - "key_format=%s," - "internal_page_max=%d," - "split_deepen_min_child=200," - "leaf_page_max=%d," - "%s", - cfg->ftype == ROW ? "S" : "r", 16 * 1024, 128 * 1024, - cfg->ftype == FIX ? ",value_format=3t" : "")); + testutil_check(__wt_snprintf(config, sizeof(config), + "key_format=%s," + "internal_page_max=%d," + "split_deepen_min_child=200," + "leaf_page_max=%d," + "%s", + cfg->ftype == ROW ? "S" : "r", 16 * 1024, 128 * 1024, + cfg->ftype == FIX ? ",value_format=3t" : "")); - if ((ret = session->create(session, name, config)) != 0) - if (ret != EEXIST) - testutil_die(ret, "session.create"); + if ((ret = session->create(session, name, config)) != 0) + if (ret != EEXIST) + testutil_die(ret, "session.create"); - testutil_check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); } void load(SHARED_CONFIG *cfg, const char *name) { - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_ITEM *value, _value; - WT_SESSION *session; - size_t len; - uint64_t keyno; - char keybuf[64], valuebuf[64]; - - conn = cfg->conn; - - file_create(cfg, name); - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - testutil_check( - session->open_cursor(session, name, NULL, "bulk", &cursor)); - - value = &_value; - for (keyno = 1; keyno <= cfg->nkeys; ++keyno) { - if (cfg->ftype == ROW) { - testutil_check(__wt_snprintf( - keybuf, sizeof(keybuf), "%016" PRIu64, keyno)); - cursor->set_key(cursor, keybuf); - } else - cursor->set_key(cursor, (uint32_t)keyno); - value->data = valuebuf; - if (cfg->ftype == FIX) - cursor->set_value(cursor, 0x01); - else { - testutil_check(__wt_snprintf_len_set( - valuebuf, sizeof(valuebuf), - &len, "%37" PRIu64, keyno)); - value->size = (uint32_t)len; - cursor->set_value(cursor, value); - } - testutil_check(cursor->insert(cursor)); - } - - /* Setup the starting key range for the workload phase. */ - cfg->key_range = cfg->nkeys; - testutil_check(cursor->close(cursor)); - testutil_check(session->checkpoint(session, NULL)); - - testutil_check(session->close(session, NULL)); + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_ITEM *value, _value; + WT_SESSION *session; + size_t len; + uint64_t keyno; + char keybuf[64], valuebuf[64]; + + conn = cfg->conn; + + file_create(cfg, name); + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + testutil_check(session->open_cursor(session, name, NULL, "bulk", &cursor)); + + value = &_value; + for (keyno = 1; keyno <= cfg->nkeys; ++keyno) { + if (cfg->ftype == ROW) { + testutil_check(__wt_snprintf(keybuf, sizeof(keybuf), "%016" PRIu64, keyno)); + cursor->set_key(cursor, keybuf); + } else + cursor->set_key(cursor, (uint32_t)keyno); + value->data = valuebuf; + if (cfg->ftype == FIX) + cursor->set_value(cursor, 0x01); + else { + testutil_check( + __wt_snprintf_len_set(valuebuf, sizeof(valuebuf), &len, "%37" PRIu64, keyno)); + value->size = (uint32_t)len; + cursor->set_value(cursor, value); + } + testutil_check(cursor->insert(cursor)); + } + + /* Setup the starting key range for the workload phase. */ + cfg->key_range = cfg->nkeys; + testutil_check(cursor->close(cursor)); + testutil_check(session->checkpoint(session, NULL)); + + testutil_check(session->close(session, NULL)); } void verify(SHARED_CONFIG *cfg, const char *name) { - WT_CONNECTION *conn; - WT_SESSION *session; + WT_CONNECTION *conn; + WT_SESSION *session; - conn = cfg->conn; + conn = cfg->conn; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->verify(session, name, NULL)); + testutil_check(session->verify(session, name, NULL)); - testutil_check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); } diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c index d5001dd38d6..7186ef52ed1 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c @@ -29,18 +29,18 @@ #include "cursor_order.h" static WT_THREAD_RET append_insert(void *); -static void print_stats(SHARED_CONFIG *); +static void print_stats(SHARED_CONFIG *); static WT_THREAD_RET reverse_scan(void *); typedef struct { - char *name; /* object name */ - uint64_t nops; /* Thread op count */ + char *name; /* object name */ + uint64_t nops; /* Thread op count */ - WT_RAND_STATE rnd; /* RNG */ + WT_RAND_STATE rnd; /* RNG */ - int append_insert; /* cursor.insert */ - int reverse_scans; /* cursor.prev sequences */ - SHARED_CONFIG *cfg; + int append_insert; /* cursor.insert */ + int reverse_scans; /* cursor.prev sequences */ + SHARED_CONFIG *cfg; } INFO; static INFO *run_info; @@ -48,301 +48,272 @@ static INFO *run_info; void ops_start(SHARED_CONFIG *cfg) { - struct timeval start, stop; - wt_thread_t *tids; - double seconds; - uint64_t i, name_index, offset, total_nops; - - tids = NULL; /* Keep GCC 4.1 happy. */ - total_nops = 0; - - /* Create per-thread structures. */ - run_info = dcalloc((size_t) - (cfg->reverse_scanners + cfg->append_inserters), sizeof(*run_info)); - tids = dcalloc((size_t) - (cfg->reverse_scanners + cfg->append_inserters), sizeof(*tids)); - - /* Create the files and load the initial records. */ - for (i = 0; i < cfg->append_inserters; ++i) { - run_info[i].cfg = cfg; - if (i == 0 || cfg->multiple_files) { - run_info[i].name = dmalloc(64); - testutil_check(__wt_snprintf( - run_info[i].name, 64, FNAME, (int)i)); - - /* Vary by orders of magnitude */ - if (cfg->vary_nops) - run_info[i].nops = - WT_MAX(1000, cfg->max_nops >> i); - load(cfg, run_info[i].name); - } else - run_info[i].name = run_info[0].name; - - /* Setup op count if not varying ops. */ - if (run_info[i].nops == 0) - run_info[i].nops = cfg->max_nops; - total_nops += run_info[i].nops; - } - - /* Setup the reverse scanner configurations */ - for (i = 0; i < cfg->reverse_scanners; ++i) { - offset = i + cfg->append_inserters; - run_info[offset].cfg = cfg; - if (cfg->multiple_files) { - run_info[offset].name = dmalloc(64); - /* Have reverse scans read from tables with writes. */ - name_index = i % cfg->append_inserters; - testutil_check(__wt_snprintf( - run_info[offset].name, 64, FNAME, (int)name_index)); - - /* Vary by orders of magnitude */ - if (cfg->vary_nops) - run_info[offset].nops = - WT_MAX(1000, cfg->max_nops >> name_index); - } else - run_info[offset].name = run_info[0].name; - - /* Setup op count if not varying ops. */ - if (run_info[offset].nops == 0) - run_info[offset].nops = cfg->max_nops; - total_nops += run_info[offset].nops; - } - - (void)gettimeofday(&start, NULL); - - /* Create threads. */ - for (i = 0; i < cfg->reverse_scanners; ++i) - testutil_check(__wt_thread_create(NULL, - &tids[i], reverse_scan, (void *)(uintptr_t)i)); - for (; i < cfg->reverse_scanners + cfg->append_inserters; ++i) - testutil_check(__wt_thread_create(NULL, - &tids[i], append_insert, (void *)(uintptr_t)i)); - - /* Wait for the threads. */ - for (i = 0; i < cfg->reverse_scanners + cfg->append_inserters; ++i) - testutil_check(__wt_thread_join(NULL, &tids[i])); - - (void)gettimeofday(&stop, NULL); - seconds = (stop.tv_sec - start.tv_sec) + - (stop.tv_usec - start.tv_usec) * 1e-6; - fprintf(stderr, "timer: %.2lf seconds (%d ops/second)\n", - seconds, (int) - (((double)(cfg->reverse_scanners + cfg->append_inserters) * - total_nops) / seconds)); - - /* Verify the files. */ - for (i = 0; i < cfg->reverse_scanners + cfg->append_inserters; ++i) { - verify(cfg, run_info[i].name); - if (!cfg->multiple_files) - break; - } - - /* Output run statistics. */ - print_stats(cfg); - - /* Free allocated memory. */ - for (i = 0; i < cfg->reverse_scanners + cfg->append_inserters; ++i) { - free(run_info[i].name); - if (!cfg->multiple_files) - break; - } - - free(run_info); - free(tids); + struct timeval start, stop; + wt_thread_t *tids; + double seconds; + uint64_t i, name_index, offset, total_nops; + + tids = NULL; /* Keep GCC 4.1 happy. */ + total_nops = 0; + + /* Create per-thread structures. */ + run_info = dcalloc((size_t)(cfg->reverse_scanners + cfg->append_inserters), sizeof(*run_info)); + tids = dcalloc((size_t)(cfg->reverse_scanners + cfg->append_inserters), sizeof(*tids)); + + /* Create the files and load the initial records. */ + for (i = 0; i < cfg->append_inserters; ++i) { + run_info[i].cfg = cfg; + if (i == 0 || cfg->multiple_files) { + run_info[i].name = dmalloc(64); + testutil_check(__wt_snprintf(run_info[i].name, 64, FNAME, (int)i)); + + /* Vary by orders of magnitude */ + if (cfg->vary_nops) + run_info[i].nops = WT_MAX(1000, cfg->max_nops >> i); + load(cfg, run_info[i].name); + } else + run_info[i].name = run_info[0].name; + + /* Setup op count if not varying ops. */ + if (run_info[i].nops == 0) + run_info[i].nops = cfg->max_nops; + total_nops += run_info[i].nops; + } + + /* Setup the reverse scanner configurations */ + for (i = 0; i < cfg->reverse_scanners; ++i) { + offset = i + cfg->append_inserters; + run_info[offset].cfg = cfg; + if (cfg->multiple_files) { + run_info[offset].name = dmalloc(64); + /* Have reverse scans read from tables with writes. */ + name_index = i % cfg->append_inserters; + testutil_check(__wt_snprintf(run_info[offset].name, 64, FNAME, (int)name_index)); + + /* Vary by orders of magnitude */ + if (cfg->vary_nops) + run_info[offset].nops = WT_MAX(1000, cfg->max_nops >> name_index); + } else + run_info[offset].name = run_info[0].name; + + /* Setup op count if not varying ops. */ + if (run_info[offset].nops == 0) + run_info[offset].nops = cfg->max_nops; + total_nops += run_info[offset].nops; + } + + (void)gettimeofday(&start, NULL); + + /* Create threads. */ + for (i = 0; i < cfg->reverse_scanners; ++i) + testutil_check(__wt_thread_create(NULL, &tids[i], reverse_scan, (void *)(uintptr_t)i)); + for (; i < cfg->reverse_scanners + cfg->append_inserters; ++i) + testutil_check(__wt_thread_create(NULL, &tids[i], append_insert, (void *)(uintptr_t)i)); + + /* Wait for the threads. */ + for (i = 0; i < cfg->reverse_scanners + cfg->append_inserters; ++i) + testutil_check(__wt_thread_join(NULL, &tids[i])); + + (void)gettimeofday(&stop, NULL); + seconds = (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec) * 1e-6; + fprintf(stderr, "timer: %.2lf seconds (%d ops/second)\n", seconds, + (int)(((double)(cfg->reverse_scanners + cfg->append_inserters) * total_nops) / seconds)); + + /* Verify the files. */ + for (i = 0; i < cfg->reverse_scanners + cfg->append_inserters; ++i) { + verify(cfg, run_info[i].name); + if (!cfg->multiple_files) + break; + } + + /* Output run statistics. */ + print_stats(cfg); + + /* Free allocated memory. */ + for (i = 0; i < cfg->reverse_scanners + cfg->append_inserters; ++i) { + free(run_info[i].name); + if (!cfg->multiple_files) + break; + } + + free(run_info); + free(tids); } /* * reverse_scan_op -- - * Walk a cursor back from the end of the file. + * Walk a cursor back from the end of the file. */ static inline void -reverse_scan_op( - SHARED_CONFIG *cfg, WT_SESSION *session, WT_CURSOR *cursor, INFO *s) +reverse_scan_op(SHARED_CONFIG *cfg, WT_SESSION *session, WT_CURSOR *cursor, INFO *s) { - uint64_t i, initial_key_range, prev_key, this_key; - int ret; - char *strkey; - - WT_UNUSED(session); - WT_UNUSED(s); - - /* Make GCC 4.1 happy */ - prev_key = this_key = 0; - - /* Reset the cursor */ - testutil_check(cursor->reset(cursor)); - - /* Save the key range. */ - initial_key_range = cfg->key_range - cfg->append_inserters; - - for (i = 0; i < cfg->reverse_scan_ops; i++) { - if ((ret = cursor->prev(cursor)) != 0) { - if (ret == WT_NOTFOUND) - break; - testutil_die(ret, "cursor.prev"); - } - - if (cfg->ftype == ROW) { - testutil_check(cursor->get_key(cursor, &strkey)); - this_key = (uint64_t)atol(strkey); - } else - testutil_check(cursor->get_key( - cursor, (uint64_t *)&this_key)); - - if (i == 0 && this_key < initial_key_range) - testutil_die(ret, - "cursor scan start range wrong first prev %" PRIu64 - " initial range: %" PRIu64, - this_key, initial_key_range); - if (i != 0 && this_key >= prev_key) - testutil_die(ret, - "cursor scan out of order this: %" PRIu64 - " prev: %" PRIu64, - this_key, prev_key); - prev_key = this_key; - } + uint64_t i, initial_key_range, prev_key, this_key; + int ret; + char *strkey; + + WT_UNUSED(session); + WT_UNUSED(s); + + /* Make GCC 4.1 happy */ + prev_key = this_key = 0; + + /* Reset the cursor */ + testutil_check(cursor->reset(cursor)); + + /* Save the key range. */ + initial_key_range = cfg->key_range - cfg->append_inserters; + + for (i = 0; i < cfg->reverse_scan_ops; i++) { + if ((ret = cursor->prev(cursor)) != 0) { + if (ret == WT_NOTFOUND) + break; + testutil_die(ret, "cursor.prev"); + } + + if (cfg->ftype == ROW) { + testutil_check(cursor->get_key(cursor, &strkey)); + this_key = (uint64_t)atol(strkey); + } else + testutil_check(cursor->get_key(cursor, (uint64_t *)&this_key)); + + if (i == 0 && this_key < initial_key_range) + testutil_die(ret, + "cursor scan start range wrong first prev %" PRIu64 " initial range: %" PRIu64, + this_key, initial_key_range); + if (i != 0 && this_key >= prev_key) + testutil_die( + ret, "cursor scan out of order this: %" PRIu64 " prev: %" PRIu64, this_key, prev_key); + prev_key = this_key; + } } /* * reverse_scan -- - * Reader thread start function. + * Reader thread start function. */ static WT_THREAD_RET reverse_scan(void *arg) { - INFO *s; - SHARED_CONFIG *cfg; - WT_CURSOR *cursor; - WT_SESSION *session; - uintmax_t id; - uint64_t i; - char tid[128]; - - id = (uintmax_t)arg; - s = &run_info[id]; - cfg = s->cfg; - testutil_check(__wt_thread_str(tid, sizeof(tid))); - __wt_random_init(&s->rnd); - - printf(" reverse scan thread %2" PRIuMAX - " starting: tid: %s, file: %s\n", - id, tid, s->name); - - __wt_yield(); /* Get all the threads created. */ - - testutil_check(cfg->conn->open_session( - cfg->conn, NULL, "isolation=snapshot", &session)); - testutil_check(session->open_cursor( - session, s->name, NULL, NULL, &cursor)); - for (i = 0; i < s->nops && !cfg->thread_finish; - ++i, ++s->reverse_scans, __wt_yield()) - reverse_scan_op(cfg, session, cursor, s); - testutil_check(session->close(session, NULL)); - - printf(" reverse scan thread %2" PRIuMAX - " stopping: tid: %s, file: %s\n", - id, tid, s->name); - - /* Notify all other threads to finish once the first thread is done */ - cfg->thread_finish = true; - - return (WT_THREAD_RET_VALUE); + INFO *s; + SHARED_CONFIG *cfg; + WT_CURSOR *cursor; + WT_SESSION *session; + uintmax_t id; + uint64_t i; + char tid[128]; + + id = (uintmax_t)arg; + s = &run_info[id]; + cfg = s->cfg; + testutil_check(__wt_thread_str(tid, sizeof(tid))); + __wt_random_init(&s->rnd); + + printf(" reverse scan thread %2" PRIuMAX " starting: tid: %s, file: %s\n", id, tid, s->name); + + __wt_yield(); /* Get all the threads created. */ + + testutil_check(cfg->conn->open_session(cfg->conn, NULL, "isolation=snapshot", &session)); + testutil_check(session->open_cursor(session, s->name, NULL, NULL, &cursor)); + for (i = 0; i < s->nops && !cfg->thread_finish; ++i, ++s->reverse_scans, __wt_yield()) + reverse_scan_op(cfg, session, cursor, s); + testutil_check(session->close(session, NULL)); + + printf(" reverse scan thread %2" PRIuMAX " stopping: tid: %s, file: %s\n", id, tid, s->name); + + /* Notify all other threads to finish once the first thread is done */ + cfg->thread_finish = true; + + return (WT_THREAD_RET_VALUE); } /* * append_insert_op -- - * Write operation. + * Write operation. */ static inline void -append_insert_op( - SHARED_CONFIG *cfg, WT_SESSION *session, WT_CURSOR *cursor, INFO *s) +append_insert_op(SHARED_CONFIG *cfg, WT_SESSION *session, WT_CURSOR *cursor, INFO *s) { - WT_ITEM *value, _value; - size_t len; - uint64_t keyno; - char keybuf[64], valuebuf[64]; - - WT_UNUSED(session); - - value = &_value; - - keyno = __wt_atomic_add64(&cfg->key_range, 1); - if (cfg->ftype == ROW) { - testutil_check(__wt_snprintf( - keybuf, sizeof(keybuf), "%016" PRIu64, keyno)); - cursor->set_key(cursor, keybuf); - } else - cursor->set_key(cursor, (uint32_t)keyno); - - ++s->append_insert; - value->data = valuebuf; - if (cfg->ftype == FIX) - cursor->set_value(cursor, 0x10); - else { - testutil_check(__wt_snprintf_len_set( - valuebuf, sizeof(valuebuf), &len, "XXX %37" PRIu64, keyno)); - value->size = (uint32_t)len; - cursor->set_value(cursor, value); - } - testutil_check(cursor->insert(cursor)); + WT_ITEM *value, _value; + size_t len; + uint64_t keyno; + char keybuf[64], valuebuf[64]; + + WT_UNUSED(session); + + value = &_value; + + keyno = __wt_atomic_add64(&cfg->key_range, 1); + if (cfg->ftype == ROW) { + testutil_check(__wt_snprintf(keybuf, sizeof(keybuf), "%016" PRIu64, keyno)); + cursor->set_key(cursor, keybuf); + } else + cursor->set_key(cursor, (uint32_t)keyno); + + ++s->append_insert; + value->data = valuebuf; + if (cfg->ftype == FIX) + cursor->set_value(cursor, 0x10); + else { + testutil_check( + __wt_snprintf_len_set(valuebuf, sizeof(valuebuf), &len, "XXX %37" PRIu64, keyno)); + value->size = (uint32_t)len; + cursor->set_value(cursor, value); + } + testutil_check(cursor->insert(cursor)); } /* * append_insert -- - * Writer thread start function. + * Writer thread start function. */ static WT_THREAD_RET append_insert(void *arg) { - INFO *s; - SHARED_CONFIG *cfg; - WT_CURSOR *cursor; - WT_SESSION *session; - uintmax_t id; - uint64_t i; - char tid[128]; - - id = (uintmax_t)arg; - s = &run_info[id]; - cfg = s->cfg; - testutil_check(__wt_thread_str(tid, sizeof(tid))); - __wt_random_init(&s->rnd); - - printf("write thread %2" PRIuMAX " starting: tid: %s, file: %s\n", - id, tid, s->name); - - __wt_yield(); /* Get all the threads created. */ - - testutil_check(cfg->conn->open_session( - cfg->conn, NULL, "isolation=snapshot", &session)); - testutil_check(session->open_cursor( - session, s->name, NULL, NULL, &cursor)); - for (i = 0; i < s->nops && !cfg->thread_finish; ++i, __wt_yield()) - append_insert_op(cfg, session, cursor, s); - testutil_check(session->close(session, NULL)); - - printf("write thread %2" PRIuMAX " stopping: tid: %s, file: %s\n", - id, tid, s->name); - - /* Notify all other threads to finish once the first thread is done */ - cfg->thread_finish = true; - - return (WT_THREAD_RET_VALUE); + INFO *s; + SHARED_CONFIG *cfg; + WT_CURSOR *cursor; + WT_SESSION *session; + uintmax_t id; + uint64_t i; + char tid[128]; + + id = (uintmax_t)arg; + s = &run_info[id]; + cfg = s->cfg; + testutil_check(__wt_thread_str(tid, sizeof(tid))); + __wt_random_init(&s->rnd); + + printf("write thread %2" PRIuMAX " starting: tid: %s, file: %s\n", id, tid, s->name); + + __wt_yield(); /* Get all the threads created. */ + + testutil_check(cfg->conn->open_session(cfg->conn, NULL, "isolation=snapshot", &session)); + testutil_check(session->open_cursor(session, s->name, NULL, NULL, &cursor)); + for (i = 0; i < s->nops && !cfg->thread_finish; ++i, __wt_yield()) + append_insert_op(cfg, session, cursor, s); + testutil_check(session->close(session, NULL)); + + printf("write thread %2" PRIuMAX " stopping: tid: %s, file: %s\n", id, tid, s->name); + + /* Notify all other threads to finish once the first thread is done */ + cfg->thread_finish = true; + + return (WT_THREAD_RET_VALUE); } /* * print_stats -- - * Display reverse scan/writer thread stats. + * Display reverse scan/writer thread stats. */ static void print_stats(SHARED_CONFIG *cfg) { - INFO *s; - uint64_t id, total_threads; - - total_threads = cfg->reverse_scanners + cfg->append_inserters; - s = run_info; - for (id = 0; id < total_threads; ++id, ++s) - printf("%3d: reverse scans %6d, append inserts %6d\n", - (int)id, (int)s->reverse_scans, (int)s->append_insert); + INFO *s; + uint64_t id, total_threads; + + total_threads = cfg->reverse_scanners + cfg->append_inserters; + s = run_info; + for (id = 0; id < total_threads; ++id, ++s) + printf("%3d: reverse scans %6d, append inserts %6d\n", (int)id, (int)s->reverse_scans, + (int)s->append_insert); } diff --git a/src/third_party/wiredtiger/test/fops/file.c b/src/third_party/wiredtiger/test/fops/file.c index c0a6fd699b7..96faf5fb9b6 100644 --- a/src/third_party/wiredtiger/test/fops/file.c +++ b/src/third_party/wiredtiger/test/fops/file.c @@ -33,263 +33,244 @@ static u_int uid = 1; void obj_bulk(void) { - WT_CURSOR *c; - WT_SESSION *session; - int ret; - bool create; - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - if (use_txn) - testutil_check(session->begin_transaction(session, NULL)); - create = false; - if ((ret = session->create(session, uri, config)) != 0) - if (ret != EEXIST && ret != EBUSY) - testutil_die(ret, "session.create"); - - if (ret == 0) { - create = true; - __wt_yield(); - if ((ret = session->open_cursor( - session, uri, NULL, "bulk", &c)) == 0) { - testutil_check(c->close(c)); - } else if (ret != ENOENT && ret != EBUSY && ret != EINVAL) - testutil_die(ret, "session.open_cursor bulk"); - } - - if (use_txn) { - /* If create fails, rollback else will commit.*/ - if (!create) - ret = session->rollback_transaction(session, NULL); - else - ret = session->commit_transaction(session, NULL); - - if (ret == EINVAL) - testutil_die(ret, "session.commit bulk"); - } - testutil_check(session->close(session, NULL)); + WT_CURSOR *c; + WT_SESSION *session; + int ret; + bool create; + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + create = false; + if ((ret = session->create(session, uri, config)) != 0) + if (ret != EEXIST && ret != EBUSY) + testutil_die(ret, "session.create"); + + if (ret == 0) { + create = true; + __wt_yield(); + if ((ret = session->open_cursor(session, uri, NULL, "bulk", &c)) == 0) { + testutil_check(c->close(c)); + } else if (ret != ENOENT && ret != EBUSY && ret != EINVAL) + testutil_die(ret, "session.open_cursor bulk"); + } + + if (use_txn) { + /* If create fails, rollback else will commit.*/ + if (!create) + ret = session->rollback_transaction(session, NULL); + else + ret = session->commit_transaction(session, NULL); + + if (ret == EINVAL) + testutil_die(ret, "session.commit bulk"); + } + testutil_check(session->close(session, NULL)); } void obj_bulk_unique(int force) { - WT_CURSOR *c; - WT_SESSION *session; - int ret; - char new_uri[64]; - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - /* Generate a unique object name. */ - testutil_check(pthread_rwlock_wrlock(&single)); - testutil_check(__wt_snprintf( - new_uri, sizeof(new_uri), "%s.%u", uri, ++uid)); - testutil_check(pthread_rwlock_unlock(&single)); - - if (use_txn) - testutil_check(session->begin_transaction(session, NULL)); - testutil_check(session->create(session, new_uri, config)); - - __wt_yield(); - /* - * Opening a bulk cursor may have raced with a forced checkpoint - * which created a checkpoint of the empty file, and triggers an EINVAL - */ - if ((ret = session->open_cursor( - session, new_uri, NULL, "bulk", &c)) == 0) - testutil_check(c->close(c)); - else if (ret != EINVAL) - testutil_die(ret, - "session.open_cursor bulk unique: %s, new_uri"); - - while ((ret = session->drop( - session, new_uri, force ? "force" : NULL)) != 0) - if (ret != EBUSY) - testutil_die(ret, "session.drop: %s", new_uri); - - if (use_txn && - (ret = session->commit_transaction(session, NULL)) != 0 && - ret != EINVAL) - testutil_die(ret, "session.commit bulk unique"); - testutil_check(session->close(session, NULL)); + WT_CURSOR *c; + WT_SESSION *session; + int ret; + char new_uri[64]; + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + /* Generate a unique object name. */ + testutil_check(pthread_rwlock_wrlock(&single)); + testutil_check(__wt_snprintf(new_uri, sizeof(new_uri), "%s.%u", uri, ++uid)); + testutil_check(pthread_rwlock_unlock(&single)); + + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + testutil_check(session->create(session, new_uri, config)); + + __wt_yield(); + /* + * Opening a bulk cursor may have raced with a forced checkpoint which created a checkpoint of + * the empty file, and triggers an EINVAL + */ + if ((ret = session->open_cursor(session, new_uri, NULL, "bulk", &c)) == 0) + testutil_check(c->close(c)); + else if (ret != EINVAL) + testutil_die(ret, "session.open_cursor bulk unique: %s, new_uri"); + + while ((ret = session->drop(session, new_uri, force ? "force" : NULL)) != 0) + if (ret != EBUSY) + testutil_die(ret, "session.drop: %s", new_uri); + + if (use_txn && (ret = session->commit_transaction(session, NULL)) != 0 && ret != EINVAL) + testutil_die(ret, "session.commit bulk unique"); + testutil_check(session->close(session, NULL)); } void obj_cursor(void) { - WT_CURSOR *cursor; - WT_SESSION *session; - int ret; - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - if (use_txn) - testutil_check(session->begin_transaction(session, NULL)); - if ((ret = - session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0) { - if (ret != ENOENT && ret != EBUSY) - testutil_die(ret, "session.open_cursor"); - } else - testutil_check(cursor->close(cursor)); - - if (use_txn && - (ret = session->commit_transaction(session, NULL)) != 0 && - ret != EINVAL) - testutil_die(ret, "session.commit cursor"); - testutil_check(session->close(session, NULL)); + WT_CURSOR *cursor; + WT_SESSION *session; + int ret; + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + if ((ret = session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0) { + if (ret != ENOENT && ret != EBUSY) + testutil_die(ret, "session.open_cursor"); + } else + testutil_check(cursor->close(cursor)); + + if (use_txn && (ret = session->commit_transaction(session, NULL)) != 0 && ret != EINVAL) + testutil_die(ret, "session.commit cursor"); + testutil_check(session->close(session, NULL)); } void obj_create(void) { - WT_SESSION *session; - int ret; - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - if (use_txn) - testutil_check(session->begin_transaction(session, NULL)); - if ((ret = session->create(session, uri, config)) != 0) - if (ret != EEXIST && ret != EBUSY) - testutil_die(ret, "session.create"); - - if (use_txn && - (ret = session->commit_transaction(session, NULL)) != 0 && - ret != EINVAL) - testutil_die(ret, "session.commit create"); - testutil_check(session->close(session, NULL)); + WT_SESSION *session; + int ret; + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + if ((ret = session->create(session, uri, config)) != 0) + if (ret != EEXIST && ret != EBUSY) + testutil_die(ret, "session.create"); + + if (use_txn && (ret = session->commit_transaction(session, NULL)) != 0 && ret != EINVAL) + testutil_die(ret, "session.commit create"); + testutil_check(session->close(session, NULL)); } void obj_create_unique(int force) { - WT_SESSION *session; - int ret; - char new_uri[64]; - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - /* Generate a unique object name. */ - testutil_check(pthread_rwlock_wrlock(&single)); - testutil_check(__wt_snprintf( - new_uri, sizeof(new_uri), "%s.%u", uri, ++uid)); - testutil_check(pthread_rwlock_unlock(&single)); - - if (use_txn) - testutil_check(session->begin_transaction(session, NULL)); - testutil_check(session->create(session, new_uri, config)); - if (use_txn && - (ret = session->commit_transaction(session, NULL)) != 0 && - ret != EINVAL) - testutil_die(ret, "session.commit create unique"); - - __wt_yield(); - if (use_txn) - testutil_check(session->begin_transaction(session, NULL)); - while ((ret = session->drop( - session, new_uri, force ? "force" : NULL)) != 0) - if (ret != EBUSY) - testutil_die(ret, "session.drop: %s", new_uri); - if (use_txn && - (ret = session->commit_transaction(session, NULL)) != 0 && - ret != EINVAL) - testutil_die(ret, "session.commit create unique"); - - testutil_check(session->close(session, NULL)); + WT_SESSION *session; + int ret; + char new_uri[64]; + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + /* Generate a unique object name. */ + testutil_check(pthread_rwlock_wrlock(&single)); + testutil_check(__wt_snprintf(new_uri, sizeof(new_uri), "%s.%u", uri, ++uid)); + testutil_check(pthread_rwlock_unlock(&single)); + + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + testutil_check(session->create(session, new_uri, config)); + if (use_txn && (ret = session->commit_transaction(session, NULL)) != 0 && ret != EINVAL) + testutil_die(ret, "session.commit create unique"); + + __wt_yield(); + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + while ((ret = session->drop(session, new_uri, force ? "force" : NULL)) != 0) + if (ret != EBUSY) + testutil_die(ret, "session.drop: %s", new_uri); + if (use_txn && (ret = session->commit_transaction(session, NULL)) != 0 && ret != EINVAL) + testutil_die(ret, "session.commit create unique"); + + testutil_check(session->close(session, NULL)); } void obj_drop(int force) { - WT_SESSION *session; - int ret; - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - if (use_txn) - testutil_check(session->begin_transaction(session, NULL)); - if ((ret = session->drop(session, uri, force ? "force" : NULL)) != 0) - if (ret != ENOENT && ret != EBUSY) - testutil_die(ret, "session.drop"); - - if (use_txn) { - /* - * As the operations are being performed concurrently, - * return value can be ENOENT or EBUSY will set - * error to transaction opened by session. In these - * cases the transaction has to be aborted. - */ - if (ret != ENOENT && ret != EBUSY) - ret = session->commit_transaction(session, NULL); - else - ret = session->rollback_transaction(session, NULL); - if (ret == EINVAL) - testutil_die(ret, "session.commit drop"); - } - testutil_check(session->close(session, NULL)); + WT_SESSION *session; + int ret; + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + if (use_txn) + testutil_check(session->begin_transaction(session, NULL)); + if ((ret = session->drop(session, uri, force ? "force" : NULL)) != 0) + if (ret != ENOENT && ret != EBUSY) + testutil_die(ret, "session.drop"); + + if (use_txn) { + /* + * As the operations are being performed concurrently, return value can be ENOENT or EBUSY + * will set error to transaction opened by session. In these cases the transaction has to be + * aborted. + */ + if (ret != ENOENT && ret != EBUSY) + ret = session->commit_transaction(session, NULL); + else + ret = session->rollback_transaction(session, NULL); + if (ret == EINVAL) + testutil_die(ret, "session.commit drop"); + } + testutil_check(session->close(session, NULL)); } void obj_checkpoint(void) { - WT_SESSION *session; - int ret; + WT_SESSION *session; + int ret; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); - /* - * Force the checkpoint so it has to be taken. Forced checkpoints can - * race with other metadata operations and return EBUSY - we'd expect - * applications using forced checkpoints to retry on EBUSY. - */ - if ((ret = session->checkpoint(session, "force")) != 0) - if (ret != EBUSY && ret != ENOENT) - testutil_die(ret, "session.checkpoint"); + /* + * Force the checkpoint so it has to be taken. Forced checkpoints can race with other metadata + * operations and return EBUSY - we'd expect applications using forced checkpoints to retry on + * EBUSY. + */ + if ((ret = session->checkpoint(session, "force")) != 0) + if (ret != EBUSY && ret != ENOENT) + testutil_die(ret, "session.checkpoint"); - testutil_check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); } void obj_rebalance(void) { - WT_SESSION *session; - int ret; + WT_SESSION *session; + int ret; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); - if ((ret = session->rebalance(session, uri, NULL)) != 0) - if (ret != ENOENT && ret != EBUSY) - testutil_die(ret, "session.rebalance"); + if ((ret = session->rebalance(session, uri, NULL)) != 0) + if (ret != ENOENT && ret != EBUSY) + testutil_die(ret, "session.rebalance"); - testutil_check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); } void obj_upgrade(void) { - WT_SESSION *session; - int ret; + WT_SESSION *session; + int ret; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); - if ((ret = session->upgrade(session, uri, NULL)) != 0) - if (ret != ENOENT && ret != EBUSY) - testutil_die(ret, "session.upgrade"); + if ((ret = session->upgrade(session, uri, NULL)) != 0) + if (ret != ENOENT && ret != EBUSY) + testutil_die(ret, "session.upgrade"); - testutil_check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); } void obj_verify(void) { - WT_SESSION *session; - int ret; + WT_SESSION *session; + int ret; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); - if ((ret = session->verify(session, uri, NULL)) != 0) - if (ret != ENOENT && ret != EBUSY) - testutil_die(ret, "session.verify"); + if ((ret = session->verify(session, uri, NULL)) != 0) + if (ret != ENOENT && ret != EBUSY) + testutil_die(ret, "session.verify"); - testutil_check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); } diff --git a/src/third_party/wiredtiger/test/fops/fops.c b/src/third_party/wiredtiger/test/fops/fops.c index 507a8b8838b..e30702ab876 100644 --- a/src/third_party/wiredtiger/test/fops/fops.c +++ b/src/third_party/wiredtiger/test/fops/fops.c @@ -29,19 +29,19 @@ #include "thread.h" static WT_THREAD_RET fop(void *); -static void print_stats(u_int); +static void print_stats(u_int); typedef struct { - int bulk; /* bulk load */ - int bulk_unique; /* bulk load of new file */ - int ckpt; /* session.checkpoint */ - int create; /* session.create */ - int create_unique; /* session.create of new file */ - int cursor; /* session.open_cursor */ - int drop; /* session.drop */ - int rebalance; /* session.rebalance */ - int upgrade; /* session.upgrade */ - int verify; /* session.verify */ + int bulk; /* bulk load */ + int bulk_unique; /* bulk load of new file */ + int ckpt; /* session.checkpoint */ + int create; /* session.create */ + int create_unique; /* session.create of new file */ + int cursor; /* session.open_cursor */ + int drop; /* session.drop */ + int rebalance; /* session.rebalance */ + int upgrade; /* session.upgrade */ + int verify; /* session.verify */ } STATS; static STATS *run_stats; @@ -49,122 +49,120 @@ static STATS *run_stats; void fop_start(u_int nthreads) { - struct timeval start, stop; - wt_thread_t *tids; - double seconds; - u_int i; + struct timeval start, stop; + wt_thread_t *tids; + double seconds; + u_int i; - tids = NULL; /* Silence GCC 4.1 warning. */ + tids = NULL; /* Silence GCC 4.1 warning. */ - /* Create statistics and thread structures. */ - run_stats = dcalloc((size_t)(nthreads), sizeof(*run_stats)); - tids = dcalloc((size_t)(nthreads), sizeof(*tids)); + /* Create statistics and thread structures. */ + run_stats = dcalloc((size_t)(nthreads), sizeof(*run_stats)); + tids = dcalloc((size_t)(nthreads), sizeof(*tids)); - (void)gettimeofday(&start, NULL); + (void)gettimeofday(&start, NULL); - /* Create threads. */ - for (i = 0; i < nthreads; ++i) - testutil_check(__wt_thread_create( - NULL, &tids[i], fop, (void *)(uintptr_t)i)); + /* Create threads. */ + for (i = 0; i < nthreads; ++i) + testutil_check(__wt_thread_create(NULL, &tids[i], fop, (void *)(uintptr_t)i)); - /* Wait for the threads. */ - for (i = 0; i < nthreads; ++i) - testutil_check(__wt_thread_join(NULL, &tids[i])); + /* Wait for the threads. */ + for (i = 0; i < nthreads; ++i) + testutil_check(__wt_thread_join(NULL, &tids[i])); - (void)gettimeofday(&stop, NULL); - seconds = (stop.tv_sec - start.tv_sec) + - (stop.tv_usec - start.tv_usec) * 1e-6; + (void)gettimeofday(&stop, NULL); + seconds = (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec) * 1e-6; - print_stats(nthreads); - printf("timer: %.2lf seconds (%d ops/second)\n", - seconds, (int)((nthreads * nops) / seconds)); + print_stats(nthreads); + printf("timer: %.2lf seconds (%d ops/second)\n", seconds, (int)((nthreads * nops) / seconds)); - free(run_stats); - free(tids); + free(run_stats); + free(tids); } /* * fop -- - * File operation function. + * File operation function. */ static WT_THREAD_RET fop(void *arg) { - STATS *s; - uintptr_t id; - WT_RAND_STATE rnd; - u_int i; - - id = (uintptr_t)arg; - __wt_yield(); /* Get all the threads created. */ - - s = &run_stats[id]; - __wt_random_init(&rnd); - - for (i = 0; i < nops; ++i, __wt_yield()) - switch (__wt_random(&rnd) % 10) { - case 0: - ++s->bulk; - obj_bulk(); - break; - case 1: - ++s->create; - obj_create(); - break; - case 2: - ++s->cursor; - obj_cursor(); - break; - case 3: - ++s->drop; - obj_drop(__wt_random(&rnd) & 1); - break; - case 4: - ++s->ckpt; - obj_checkpoint(); - break; - case 5: - ++s->upgrade; - obj_upgrade(); - break; - case 6: - ++s->rebalance; - obj_rebalance(); - break; - case 7: - ++s->verify; - obj_verify(); - break; - case 8: - ++s->bulk_unique; - obj_bulk_unique(__wt_random(&rnd) & 1); - break; - case 9: - ++s->create_unique; - obj_create_unique(__wt_random(&rnd) & 1); - break; - } - - return (WT_THREAD_RET_VALUE); + STATS *s; + uintptr_t id; + WT_RAND_STATE rnd; + u_int i; + + id = (uintptr_t)arg; + __wt_yield(); /* Get all the threads created. */ + + s = &run_stats[id]; + __wt_random_init(&rnd); + + for (i = 0; i < nops; ++i, __wt_yield()) + switch (__wt_random(&rnd) % 10) { + case 0: + ++s->bulk; + obj_bulk(); + break; + case 1: + ++s->create; + obj_create(); + break; + case 2: + ++s->cursor; + obj_cursor(); + break; + case 3: + ++s->drop; + obj_drop(__wt_random(&rnd) & 1); + break; + case 4: + ++s->ckpt; + obj_checkpoint(); + break; + case 5: + ++s->upgrade; + obj_upgrade(); + break; + case 6: + ++s->rebalance; + obj_rebalance(); + break; + case 7: + ++s->verify; + obj_verify(); + break; + case 8: + ++s->bulk_unique; + obj_bulk_unique(__wt_random(&rnd) & 1); + break; + case 9: + ++s->create_unique; + obj_create_unique(__wt_random(&rnd) & 1); + break; + } + + return (WT_THREAD_RET_VALUE); } /* * print_stats -- - * Display file operation thread stats. + * Display file operation thread stats. */ static void print_stats(u_int nthreads) { - STATS *s; - u_int id; - - s = run_stats; - for (id = 0; id < nthreads; ++id, ++s) - printf( - "%2u:" - "\t" "bulk %3d, checkpoint %3d, create %3d, cursor %3d,\n" - "\t" "drop %3d, rebalance %3d, upgrade %3d, verify %3d\n", - id, s->bulk + s->bulk_unique, s->ckpt, - s->create + s->create_unique, s->cursor, - s->drop, s->rebalance, s->upgrade, s->verify); + STATS *s; + u_int id; + + s = run_stats; + for (id = 0; id < nthreads; ++id, ++s) + printf( + "%2u:" + "\t" + "bulk %3d, checkpoint %3d, create %3d, cursor %3d,\n" + "\t" + "drop %3d, rebalance %3d, upgrade %3d, verify %3d\n", + id, s->bulk + s->bulk_unique, s->ckpt, s->create + s->create_unique, s->cursor, s->drop, + s->rebalance, s->upgrade, s->verify); } diff --git a/src/third_party/wiredtiger/test/fops/t.c b/src/third_party/wiredtiger/test/fops/t.c index ed95c4e29be..6da4a7e748b 100644 --- a/src/third_party/wiredtiger/test/fops/t.c +++ b/src/third_party/wiredtiger/test/fops/t.c @@ -28,23 +28,22 @@ #include "thread.h" -bool use_txn; /* Operations with user txn */ -WT_CONNECTION *conn; /* WiredTiger connection */ -pthread_rwlock_t single; /* Single thread */ -u_int nops; /* Operations */ -const char *uri; /* Object */ -const char *config; /* Object config */ +bool use_txn; /* Operations with user txn */ +WT_CONNECTION *conn; /* WiredTiger connection */ +pthread_rwlock_t single; /* Single thread */ +u_int nops; /* Operations */ +const char *uri; /* Object */ +const char *config; /* Object config */ -static FILE *logfp; /* Log file */ +static FILE *logfp; /* Log file */ static char home[512]; -static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); -static int handle_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); -static void onint(int) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); +static int handle_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); +static void onint(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void shutdown(void); -static int usage(void); +static int usage(void); static void wt_startup(char *); static void wt_shutdown(void); @@ -54,212 +53,197 @@ extern char *__wt_optarg; int main(int argc, char *argv[]) { - static struct config { - const char *uri; - const char *desc; - const char *config; - } *cp, configs[] = { - { "file:wt", NULL, NULL }, - { "table:wt", NULL, NULL }, + static struct config { + const char *uri; + const char *desc; + const char *config; + } * cp, configs[] = {{"file:wt", NULL, NULL}, {"table:wt", NULL, NULL}, /* Configure for a modest cache size. */ -#define LSM_CONFIG "lsm=(chunk_size=1m,merge_max=2),leaf_page_max=4k" - { "lsm:wt", NULL, LSM_CONFIG }, - { "table:wt", " [lsm]", "type=lsm," LSM_CONFIG }, - { NULL, NULL, NULL } - }; - u_int nthreads; - int ch, cnt, runs; - char *config_open, *working_dir; - - (void)testutil_set_progname(argv); - - testutil_check(pthread_rwlock_init(&single, NULL)); - - nops = 1000; - nthreads = 10; - runs = 1; - use_txn = false; - config_open = working_dir = NULL; - while ((ch = __wt_getopt(progname, argc, argv, "C:h:l:n:r:t:x")) != EOF) - switch (ch) { - case 'C': /* wiredtiger_open config */ - config_open = __wt_optarg; - break; - case 'h': - working_dir = __wt_optarg; - break; - case 'l': /* log */ - if ((logfp = fopen(__wt_optarg, "w")) == NULL) { - fprintf(stderr, - "%s: %s\n", __wt_optarg, strerror(errno)); - return (EXIT_FAILURE); - } - break; - case 'n': /* operations */ - nops = (u_int)atoi(__wt_optarg); - break; - case 'r': /* runs */ - runs = atoi(__wt_optarg); - break; - case 't': - nthreads = (u_int)atoi(__wt_optarg); - break; - case 'x': - use_txn = true; - break; - default: - return (usage()); - } - - argc -= __wt_optind; - if (argc != 0) - return (usage()); - - testutil_work_dir_from_path(home, 512, working_dir); - - /* Clean up on signal. */ - (void)signal(SIGINT, onint); - - printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); - for (cnt = 1; runs == 0 || cnt <= runs; ++cnt) { - shutdown(); /* Clean up previous runs */ - - for (cp = configs; cp->uri != NULL; ++cp) { - uri = cp->uri; - config = cp->config; - printf("%5d: %u threads on %s%s\n", cnt, nthreads, uri, - cp->desc == NULL ? "" : cp->desc); - - wt_startup(config_open); - - fop_start(nthreads); - - wt_shutdown(); - printf("\n"); - } - } - return (0); +#define LSM_CONFIG "lsm=(chunk_size=1m,merge_max=2),leaf_page_max=4k" + {"lsm:wt", NULL, LSM_CONFIG}, {"table:wt", " [lsm]", "type=lsm," LSM_CONFIG}, + {NULL, NULL, NULL}}; + u_int nthreads; + int ch, cnt, runs; + char *config_open, *working_dir; + + (void)testutil_set_progname(argv); + + testutil_check(pthread_rwlock_init(&single, NULL)); + + nops = 1000; + nthreads = 10; + runs = 1; + use_txn = false; + config_open = working_dir = NULL; + while ((ch = __wt_getopt(progname, argc, argv, "C:h:l:n:r:t:x")) != EOF) + switch (ch) { + case 'C': /* wiredtiger_open config */ + config_open = __wt_optarg; + break; + case 'h': + working_dir = __wt_optarg; + break; + case 'l': /* log */ + if ((logfp = fopen(__wt_optarg, "w")) == NULL) { + fprintf(stderr, "%s: %s\n", __wt_optarg, strerror(errno)); + return (EXIT_FAILURE); + } + break; + case 'n': /* operations */ + nops = (u_int)atoi(__wt_optarg); + break; + case 'r': /* runs */ + runs = atoi(__wt_optarg); + break; + case 't': + nthreads = (u_int)atoi(__wt_optarg); + break; + case 'x': + use_txn = true; + break; + default: + return (usage()); + } + + argc -= __wt_optind; + if (argc != 0) + return (usage()); + + testutil_work_dir_from_path(home, 512, working_dir); + + /* Clean up on signal. */ + (void)signal(SIGINT, onint); + + printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); + for (cnt = 1; runs == 0 || cnt <= runs; ++cnt) { + shutdown(); /* Clean up previous runs */ + + for (cp = configs; cp->uri != NULL; ++cp) { + uri = cp->uri; + config = cp->config; + printf( + "%5d: %u threads on %s%s\n", cnt, nthreads, uri, cp->desc == NULL ? "" : cp->desc); + + wt_startup(config_open); + + fop_start(nthreads); + + wt_shutdown(); + printf("\n"); + } + } + return (0); } /* * wt_startup -- - * Configure the WiredTiger connection. + * Configure the WiredTiger connection. */ static void wt_startup(char *config_open) { - static WT_EVENT_HANDLER event_handler = { - handle_error, - handle_message, - NULL, - NULL /* Close handler. */ - }; - char config_buf[128]; - - testutil_make_work_dir(home); - - testutil_check(__wt_snprintf(config_buf, sizeof(config_buf), - "create,error_prefix=\"%s\",cache_size=5MB%s%s," - "operation_tracking=(enabled=false)", - progname, - config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open)); - testutil_check( - wiredtiger_open(home, &event_handler, config_buf, &conn)); + static WT_EVENT_HANDLER event_handler = { + handle_error, handle_message, NULL, NULL /* Close handler. */ + }; + char config_buf[128]; + + testutil_make_work_dir(home); + + testutil_check(__wt_snprintf(config_buf, sizeof(config_buf), + "create,error_prefix=\"%s\",cache_size=5MB%s%s," + "operation_tracking=(enabled=false)", + progname, config_open == NULL ? "" : ",", config_open == NULL ? "" : config_open)); + testutil_check(wiredtiger_open(home, &event_handler, config_buf, &conn)); } /* * wt_shutdown -- - * Flush the file to disk and shut down the WiredTiger connection. + * Flush the file to disk and shut down the WiredTiger connection. */ static void wt_shutdown(void) { - testutil_check(conn->close(conn, NULL)); + testutil_check(conn->close(conn, NULL)); } /* * shutdown -- - * Clean up from previous runs. + * Clean up from previous runs. */ static void shutdown(void) { - testutil_clean_work_dir(home); + testutil_clean_work_dir(home); } static int -handle_error(WT_EVENT_HANDLER *handler, - WT_SESSION *session, int error, const char *errmsg) +handle_error(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const char *errmsg) { - (void)(handler); - (void)(session); - (void)(error); + (void)(handler); + (void)(session); + (void)(error); - /* Ignore complaints about missing files. */ - if (error == ENOENT) - return (0); + /* Ignore complaints about missing files. */ + if (error == ENOENT) + return (0); - /* Ignore complaints about failure to open bulk cursors. */ - if (strstr( - errmsg, "bulk-load is only supported on newly created") != NULL) - return (0); + /* Ignore complaints about failure to open bulk cursors. */ + if (strstr(errmsg, "bulk-load is only supported on newly created") != NULL) + return (0); - return (fprintf(stderr, "%s\n", errmsg) < 0 ? -1 : 0); + return (fprintf(stderr, "%s\n", errmsg) < 0 ? -1 : 0); } static int -handle_message(WT_EVENT_HANDLER *handler, - WT_SESSION *session, const char *message) +handle_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message) { - (void)(handler); - (void)(session); + (void)(handler); + (void)(session); - /* Ignore messages about failing to create forced checkpoints. */ - if (strstr( - message, "forced or named checkpoint") != NULL) - return (0); + /* Ignore messages about failing to create forced checkpoints. */ + if (strstr(message, "forced or named checkpoint") != NULL) + return (0); - if (logfp != NULL) - return (fprintf(logfp, "%s\n", message) < 0 ? -1 : 0); + if (logfp != NULL) + return (fprintf(logfp, "%s\n", message) < 0 ? -1 : 0); - return (printf("%s\n", message) < 0 ? -1 : 0); + return (printf("%s\n", message) < 0 ? -1 : 0); } /* * onint -- - * Interrupt signal handler. + * Interrupt signal handler. */ static void onint(int signo) { - (void)(signo); + (void)(signo); - shutdown(); + shutdown(); - fprintf(stderr, "\n"); - exit(EXIT_FAILURE); + fprintf(stderr, "\n"); + exit(EXIT_FAILURE); } /* * usage -- - * Display usage statement and exit failure. + * Display usage statement and exit failure. */ static int usage(void) { - fprintf(stderr, - "usage: %s " - "[-C wiredtiger-config] [-l log] [-n ops] [-r runs] [-t threads] " - "[-x] \n", - progname); - fprintf(stderr, "%s", - "\t-C specify wiredtiger_open configuration arguments\n" - "\t-h home (default 'WT_TEST')\n" - "\t-l specify a log file\n" - "\t-n set number of operations each thread does\n" - "\t-r set number of runs\n" - "\t-t set number of threads\n" - "\t-x operations within user transaction \n"); - return (EXIT_FAILURE); + fprintf(stderr, + "usage: %s " + "[-C wiredtiger-config] [-l log] [-n ops] [-r runs] [-t threads] " + "[-x] \n", + progname); + fprintf(stderr, "%s", + "\t-C specify wiredtiger_open configuration arguments\n" + "\t-h home (default 'WT_TEST')\n" + "\t-l specify a log file\n" + "\t-n set number of operations each thread does\n" + "\t-r set number of runs\n" + "\t-t set number of threads\n" + "\t-x operations within user transaction \n"); + return (EXIT_FAILURE); } diff --git a/src/third_party/wiredtiger/test/fops/thread.h b/src/third_party/wiredtiger/test/fops/thread.h index d63e882f1f0..feaa795fdc8 100644 --- a/src/third_party/wiredtiger/test/fops/thread.h +++ b/src/third_party/wiredtiger/test/fops/thread.h @@ -30,15 +30,15 @@ #include <signal.h> -extern bool use_txn; /* Operations with user txn */ -extern WT_CONNECTION *conn; /* WiredTiger connection */ +extern bool use_txn; /* Operations with user txn */ +extern WT_CONNECTION *conn; /* WiredTiger connection */ -extern u_int nops; /* Operations per thread */ +extern u_int nops; /* Operations per thread */ -extern const char *uri; /* Object */ -extern const char *config; /* Object config */ +extern const char *uri; /* Object */ +extern const char *config; /* Object config */ -extern pthread_rwlock_t single; /* Single-thread */ +extern pthread_rwlock_t single; /* Single-thread */ void fop_start(u_int); void obj_bulk(void); diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c index ba858a28d5b..9d2bb241efc 100644 --- a/src/third_party/wiredtiger/test/format/backup.c +++ b/src/third_party/wiredtiger/test/format/backup.c @@ -30,165 +30,151 @@ /* * check_copy -- - * Confirm the backup worked. + * Confirm the backup worked. */ static void check_copy(void) { - WT_CONNECTION *conn; - WT_DECL_RET; - WT_SESSION *session; - - wts_open(g.home_backup, false, &conn); - - testutil_checkfmt( - conn->open_session(conn, NULL, NULL, &session), - "%s", g.home_backup); - - /* - * Verify can return EBUSY if the handle isn't available. Don't yield - * and retry, in the case of LSM, the handle may not be available for - * a long time. - */ - ret = session->verify(session, g.uri, NULL); - testutil_assertfmt(ret == 0 || ret == EBUSY, - "WT_SESSION.verify: %s: %s", g.home_backup, g.uri); - - testutil_checkfmt(conn->close(conn, NULL), "%s", g.home_backup); + WT_CONNECTION *conn; + WT_DECL_RET; + WT_SESSION *session; + + wts_open(g.home_backup, false, &conn); + + testutil_checkfmt(conn->open_session(conn, NULL, NULL, &session), "%s", g.home_backup); + + /* + * Verify can return EBUSY if the handle isn't available. Don't yield and retry, in the case of + * LSM, the handle may not be available for a long time. + */ + ret = session->verify(session, g.uri, NULL); + testutil_assertfmt(ret == 0 || ret == EBUSY, "WT_SESSION.verify: %s: %s", g.home_backup, g.uri); + + testutil_checkfmt(conn->close(conn, NULL), "%s", g.home_backup); } /* * copy_file -- - * Copy a single file into the backup directories. + * Copy a single file into the backup directories. */ static void copy_file(WT_SESSION *session, const char *name) { - size_t len; - char *first, *second; - - len = strlen("BACKUP") + strlen(name) + 10; - first = dmalloc(len); - testutil_check(__wt_snprintf(first, len, "BACKUP/%s", name)); - testutil_check(__wt_copy_and_sync(session, name, first)); - - /* - * Save another copy of the original file to make debugging recovery - * errors easier. - */ - len = strlen("BACKUP_COPY") + strlen(name) + 10; - second = dmalloc(len); - testutil_check(__wt_snprintf(second, len, "BACKUP_COPY/%s", name)); - testutil_check(__wt_copy_and_sync(session, first, second)); - - free(first); - free(second); + size_t len; + char *first, *second; + + len = strlen("BACKUP") + strlen(name) + 10; + first = dmalloc(len); + testutil_check(__wt_snprintf(first, len, "BACKUP/%s", name)); + testutil_check(__wt_copy_and_sync(session, name, first)); + + /* + * Save another copy of the original file to make debugging recovery errors easier. + */ + len = strlen("BACKUP_COPY") + strlen(name) + 10; + second = dmalloc(len); + testutil_check(__wt_snprintf(second, len, "BACKUP_COPY/%s", name)); + testutil_check(__wt_copy_and_sync(session, first, second)); + + free(first); + free(second); } /* * backup -- - * Periodically do a backup and verify it. + * Periodically do a backup and verify it. */ WT_THREAD_RET backup(void *arg) { - WT_CONNECTION *conn; - WT_CURSOR *backup_cursor; - WT_DECL_RET; - WT_SESSION *session; - u_int incremental, period; - const char *config, *key; - bool full; - - (void)(arg); - - conn = g.wts_conn; - - /* Open a session. */ - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - /* - * Perform a full backup at somewhere under 10 seconds (that way there's - * at least one), then at larger intervals, optionally do incremental - * backups between full backups. - */ - incremental = 0; - for (period = mmrand(NULL, 1, 10);; period = mmrand(NULL, 20, 45)) { - /* Sleep for short periods so we don't make the run wait. */ - while (period > 0 && !g.workers_finished) { - --period; - __wt_sleep(1, 0); - } - - /* - * We can't drop named checkpoints while there's a backup in - * progress, serialize backups with named checkpoints. Wait - * for the checkpoint to complete, otherwise backups might be - * starved out. - */ - testutil_check(pthread_rwlock_wrlock(&g.backup_lock)); - if (g.workers_finished) { - testutil_check(pthread_rwlock_unlock(&g.backup_lock)); - break; - } - - if (incremental) { - config = "target=(\"log:\")"; - full = false; - } else { - /* Re-create the backup directory. */ - testutil_checkfmt( - system(g.home_backup_init), - "%s", "backup directory creation failed"); - - config = NULL; - full = true; - } - - /* - * open_cursor can return EBUSY if concurrent with a metadata - * operation, retry in that case. - */ - while ((ret = session->open_cursor( - session, "backup:", NULL, config, &backup_cursor)) == EBUSY) - __wt_yield(); - if (ret != 0) - testutil_die(ret, "session.open_cursor: backup"); - - while ((ret = backup_cursor->next(backup_cursor)) == 0) { - testutil_check( - backup_cursor->get_key(backup_cursor, &key)); - copy_file(session, key); - } - if (ret != WT_NOTFOUND) - testutil_die(ret, "backup-cursor"); - - /* After an incremental backup, truncate the log files. */ - if (incremental) - testutil_check(session->truncate( - session, "log:", backup_cursor, NULL, NULL)); - - testutil_check(backup_cursor->close(backup_cursor)); - testutil_check(pthread_rwlock_unlock(&g.backup_lock)); - - /* - * If automatic log archival isn't configured, optionally do - * incremental backups after each full backup. If we're not - * doing any more incrementals, verify the backup (we can't - * verify intermediate states, once we perform recovery on the - * backup database, we can't do any more incremental backups). - */ - if (full) - incremental = - g.c_logging_archive ? 1 : mmrand(NULL, 1, 5); - if (--incremental == 0) - check_copy(); - } - - if (incremental != 0) - check_copy(); - - testutil_check(session->close(session, NULL)); - - return (WT_THREAD_RET_VALUE); + WT_CONNECTION *conn; + WT_CURSOR *backup_cursor; + WT_DECL_RET; + WT_SESSION *session; + u_int incremental, period; + const char *config, *key; + bool full; + + (void)(arg); + + conn = g.wts_conn; + + /* Open a session. */ + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + /* + * Perform a full backup at somewhere under 10 seconds (that way there's at least one), then at + * larger intervals, optionally do incremental backups between full backups. + */ + incremental = 0; + for (period = mmrand(NULL, 1, 10);; period = mmrand(NULL, 20, 45)) { + /* Sleep for short periods so we don't make the run wait. */ + while (period > 0 && !g.workers_finished) { + --period; + __wt_sleep(1, 0); + } + + /* + * We can't drop named checkpoints while there's a backup in progress, serialize backups + * with named checkpoints. Wait for the checkpoint to complete, otherwise backups might be + * starved out. + */ + testutil_check(pthread_rwlock_wrlock(&g.backup_lock)); + if (g.workers_finished) { + testutil_check(pthread_rwlock_unlock(&g.backup_lock)); + break; + } + + if (incremental) { + config = "target=(\"log:\")"; + full = false; + } else { + /* Re-create the backup directory. */ + testutil_checkfmt(system(g.home_backup_init), "%s", "backup directory creation failed"); + + config = NULL; + full = true; + } + + /* + * open_cursor can return EBUSY if concurrent with a metadata operation, retry in that case. + */ + while ( + (ret = session->open_cursor(session, "backup:", NULL, config, &backup_cursor)) == EBUSY) + __wt_yield(); + if (ret != 0) + testutil_die(ret, "session.open_cursor: backup"); + + while ((ret = backup_cursor->next(backup_cursor)) == 0) { + testutil_check(backup_cursor->get_key(backup_cursor, &key)); + copy_file(session, key); + } + if (ret != WT_NOTFOUND) + testutil_die(ret, "backup-cursor"); + + /* After an incremental backup, truncate the log files. */ + if (incremental) + testutil_check(session->truncate(session, "log:", backup_cursor, NULL, NULL)); + + testutil_check(backup_cursor->close(backup_cursor)); + testutil_check(pthread_rwlock_unlock(&g.backup_lock)); + + /* + * If automatic log archival isn't configured, optionally do incremental backups after each + * full backup. If we're not doing any more incrementals, verify the backup (we can't verify + * intermediate states, once we perform recovery on the backup database, we can't do any + * more incremental backups). + */ + if (full) + incremental = g.c_logging_archive ? 1 : mmrand(NULL, 1, 5); + if (--incremental == 0) + check_copy(); + } + + if (incremental != 0) + check_copy(); + + testutil_check(session->close(session, NULL)); + + return (WT_THREAD_RET_VALUE); } diff --git a/src/third_party/wiredtiger/test/format/bulk.c b/src/third_party/wiredtiger/test/format/bulk.c index 550d5f74d38..303b0e4dbca 100644 --- a/src/third_party/wiredtiger/test/format/bulk.c +++ b/src/third_party/wiredtiger/test/format/bulk.c @@ -30,172 +30,165 @@ /* * bulk_begin_transaction -- - * Begin a bulk-load transaction. + * Begin a bulk-load transaction. */ static void bulk_begin_transaction(WT_SESSION *session) { - uint64_t ts; - char buf[64]; - - wiredtiger_begin_transaction(session, "isolation=snapshot"); - ts = __wt_atomic_addv64(&g.timestamp, 1); - testutil_check(__wt_snprintf( - buf, sizeof(buf), "read_timestamp=%" PRIx64, ts)); - testutil_check(session->timestamp_transaction(session, buf)); + uint64_t ts; + char buf[64]; + + wiredtiger_begin_transaction(session, "isolation=snapshot"); + ts = __wt_atomic_addv64(&g.timestamp, 1); + testutil_check(__wt_snprintf(buf, sizeof(buf), "read_timestamp=%" PRIx64, ts)); + testutil_check(session->timestamp_transaction(session, buf)); } /* * bulk_commit_transaction -- - * Commit a bulk-load transaction. + * Commit a bulk-load transaction. */ static void bulk_commit_transaction(WT_SESSION *session) { - uint64_t ts; - char buf[64]; + uint64_t ts; + char buf[64]; - ts = __wt_atomic_addv64(&g.timestamp, 1); - testutil_check(__wt_snprintf( - buf, sizeof(buf), "commit_timestamp=%" PRIx64, ts)); - testutil_check(session->commit_transaction(session, buf)); + ts = __wt_atomic_addv64(&g.timestamp, 1); + testutil_check(__wt_snprintf(buf, sizeof(buf), "commit_timestamp=%" PRIx64, ts)); + testutil_check(session->commit_transaction(session, buf)); } /* * bulk_rollback_transaction -- - * Rollback a bulk-load transaction. + * Rollback a bulk-load transaction. */ static void bulk_rollback_transaction(WT_SESSION *session) { - testutil_check(session->rollback_transaction(session, NULL)); + testutil_check(session->rollback_transaction(session, NULL)); } void wts_load(void) { - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_DECL_RET; - WT_ITEM key, value; - WT_SESSION *session; - bool is_bulk; - - conn = g.wts_conn; - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - logop(session, "%s", "=============== bulk load start"); - - /* - * No bulk load with custom collators, the order of insertion will not - * match the collation order. - */ - is_bulk = true; - if (g.c_reverse) - is_bulk = false; - - /* - * open_cursor can return EBUSY if concurrent with a metadata - * operation, retry in that case. - */ - while ((ret = session->open_cursor(session, g.uri, NULL, - is_bulk ? "bulk,append" : NULL, &cursor)) == EBUSY) - __wt_yield(); - testutil_check(ret); - - /* Set up the key/value buffers. */ - key_gen_init(&key); - val_gen_init(&value); - - if (g.c_txn_timestamps) - bulk_begin_transaction(session); - - for (;;) { - if (++g.key_cnt > g.c_rows) { - g.key_cnt = g.rows = g.c_rows; - break; - } - - /* Report on progress every 100 inserts. */ - if (g.key_cnt % 10000 == 0) { - track("bulk load", g.key_cnt, NULL); - - if (g.c_txn_timestamps) { - bulk_commit_transaction(session); - bulk_begin_transaction(session); - } - } - - key_gen(&key, g.key_cnt); - val_gen(NULL, &value, g.key_cnt); - - switch (g.type) { - case FIX: - if (!is_bulk) - cursor->set_key(cursor, g.key_cnt); - cursor->set_value(cursor, *(uint8_t *)value.data); - logop(session, "%-10s %" PRIu64 " {0x%02" PRIx8 "}", - "bulk", g.key_cnt, ((uint8_t *)value.data)[0]); - break; - case VAR: - if (!is_bulk) - cursor->set_key(cursor, g.key_cnt); - cursor->set_value(cursor, &value); - logop(session, "%-10s %" PRIu64 " {%.*s}", "bulk", - g.key_cnt, (int)value.size, (char *)value.data); - break; - case ROW: - cursor->set_key(cursor, &key); - cursor->set_value(cursor, &value); - logop(session, - "%-10s %" PRIu64 " {%.*s}, {%.*s}", "bulk", - g.key_cnt, - (int)key.size, (char *)key.data, - (int)value.size, (char *)value.data); - break; - } - - /* - * We don't want to size the cache to ensure the initial data - * set can load in the in-memory case, guaranteeing the load - * succeeds probably means future updates are also guaranteed - * to succeed, which isn't what we want. If we run out of space - * in the initial load, reset the row counter and continue. - * - * Decrease inserts, they can't be successful if we're at the - * cache limit, and increase the delete percentage to get some - * extra space once the run starts. - */ - if ((ret = cursor->insert(cursor)) != 0) { - testutil_assert( - ret == WT_CACHE_FULL || ret == WT_ROLLBACK); - - if (g.c_txn_timestamps) { - bulk_rollback_transaction(session); - bulk_begin_transaction(session); - } - - g.rows = --g.key_cnt; - g.c_rows = (uint32_t)g.key_cnt; - - if (g.c_insert_pct > 5) - g.c_insert_pct = 5; - if (g.c_delete_pct < 20) - g.c_delete_pct += 20; - break; - } - } - - if (g.c_txn_timestamps) - bulk_commit_transaction(session); - - testutil_check(cursor->close(cursor)); - - logop(session, "%s", "=============== bulk load stop"); - - testutil_check(session->close(session, NULL)); - - key_gen_teardown(&key); - val_gen_teardown(&value); + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_DECL_RET; + WT_ITEM key, value; + WT_SESSION *session; + bool is_bulk; + + conn = g.wts_conn; + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + logop(session, "%s", "=============== bulk load start"); + + /* + * No bulk load with custom collators, the order of insertion will not match the collation + * order. + */ + is_bulk = true; + if (g.c_reverse) + is_bulk = false; + + /* + * open_cursor can return EBUSY if concurrent with a metadata operation, retry in that case. + */ + while ((ret = session->open_cursor( + session, g.uri, NULL, is_bulk ? "bulk,append" : NULL, &cursor)) == EBUSY) + __wt_yield(); + testutil_check(ret); + + /* Set up the key/value buffers. */ + key_gen_init(&key); + val_gen_init(&value); + + if (g.c_txn_timestamps) + bulk_begin_transaction(session); + + for (;;) { + if (++g.key_cnt > g.c_rows) { + g.key_cnt = g.rows = g.c_rows; + break; + } + + /* Report on progress every 100 inserts. */ + if (g.key_cnt % 10000 == 0) { + track("bulk load", g.key_cnt, NULL); + + if (g.c_txn_timestamps) { + bulk_commit_transaction(session); + bulk_begin_transaction(session); + } + } + + key_gen(&key, g.key_cnt); + val_gen(NULL, &value, g.key_cnt); + + switch (g.type) { + case FIX: + if (!is_bulk) + cursor->set_key(cursor, g.key_cnt); + cursor->set_value(cursor, *(uint8_t *)value.data); + logop(session, "%-10s %" PRIu64 " {0x%02" PRIx8 "}", "bulk", g.key_cnt, + ((uint8_t *)value.data)[0]); + break; + case VAR: + if (!is_bulk) + cursor->set_key(cursor, g.key_cnt); + cursor->set_value(cursor, &value); + logop(session, "%-10s %" PRIu64 " {%.*s}", "bulk", g.key_cnt, (int)value.size, + (char *)value.data); + break; + case ROW: + cursor->set_key(cursor, &key); + cursor->set_value(cursor, &value); + logop(session, "%-10s %" PRIu64 " {%.*s}, {%.*s}", "bulk", g.key_cnt, (int)key.size, + (char *)key.data, (int)value.size, (char *)value.data); + break; + } + + /* + * We don't want to size the cache to ensure the initial data + * set can load in the in-memory case, guaranteeing the load + * succeeds probably means future updates are also guaranteed + * to succeed, which isn't what we want. If we run out of space + * in the initial load, reset the row counter and continue. + * + * Decrease inserts, they can't be successful if we're at the + * cache limit, and increase the delete percentage to get some + * extra space once the run starts. + */ + if ((ret = cursor->insert(cursor)) != 0) { + testutil_assert(ret == WT_CACHE_FULL || ret == WT_ROLLBACK); + + if (g.c_txn_timestamps) { + bulk_rollback_transaction(session); + bulk_begin_transaction(session); + } + + g.rows = --g.key_cnt; + g.c_rows = (uint32_t)g.key_cnt; + + if (g.c_insert_pct > 5) + g.c_insert_pct = 5; + if (g.c_delete_pct < 20) + g.c_delete_pct += 20; + break; + } + } + + if (g.c_txn_timestamps) + bulk_commit_transaction(session); + + testutil_check(cursor->close(cursor)); + + logop(session, "%s", "=============== bulk load stop"); + + testutil_check(session->close(session, NULL)); + + key_gen_teardown(&key); + val_gen_teardown(&value); } diff --git a/src/third_party/wiredtiger/test/format/compact.c b/src/third_party/wiredtiger/test/format/compact.c index 01b43351cd3..e0492b7d5d6 100644 --- a/src/third_party/wiredtiger/test/format/compact.c +++ b/src/third_party/wiredtiger/test/format/compact.c @@ -30,50 +30,49 @@ /* * compaction -- - * Periodically do a compaction operation. + * Periodically do a compaction operation. */ WT_THREAD_RET compact(void *arg) { - WT_CONNECTION *conn; - WT_DECL_RET; - WT_SESSION *session; - u_int period; + WT_CONNECTION *conn; + WT_DECL_RET; + WT_SESSION *session; + u_int period; - (void)(arg); + (void)(arg); - /* Open a session. */ - conn = g.wts_conn; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); + /* Open a session. */ + conn = g.wts_conn; + testutil_check(conn->open_session(conn, NULL, NULL, &session)); - /* - * Perform compaction at somewhere under 15 seconds (so we get at - * least one done), and then at 23 second intervals. - */ - for (period = mmrand(NULL, 1, 15);; period = 23) { - /* Sleep for short periods so we don't make the run wait. */ - while (period > 0 && !g.workers_finished) { - --period; - __wt_sleep(1, 0); - } - if (g.workers_finished) - break; + /* + * Perform compaction at somewhere under 15 seconds (so we get at least one done), and then at + * 23 second intervals. + */ + for (period = mmrand(NULL, 1, 15);; period = 23) { + /* Sleep for short periods so we don't make the run wait. */ + while (period > 0 && !g.workers_finished) { + --period; + __wt_sleep(1, 0); + } + if (g.workers_finished) + break; - /* - * Compact can return EBUSY if concurrent with alter or if there - * is eviction pressure, or we collide with checkpoints. - * - * Compact returns ETIMEDOUT if the compaction doesn't finish in - * in some number of seconds. We don't configure a timeout and - * occasionally exceed the default of 1200 seconds. - */ - ret = session->compact(session, g.uri, NULL); - if (ret != 0 && - ret != EBUSY && ret != ETIMEDOUT && ret != WT_ROLLBACK) - testutil_die(ret, "session.compact"); - } + /* + * Compact can return EBUSY if concurrent with alter or if there + * is eviction pressure, or we collide with checkpoints. + * + * Compact returns ETIMEDOUT if the compaction doesn't finish in + * in some number of seconds. We don't configure a timeout and + * occasionally exceed the default of 1200 seconds. + */ + ret = session->compact(session, g.uri, NULL); + if (ret != 0 && ret != EBUSY && ret != ETIMEDOUT && ret != WT_ROLLBACK) + testutil_die(ret, "session.compact"); + } - testutil_check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); - return (WT_THREAD_RET_VALUE); + return (WT_THREAD_RET_VALUE); } diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index dd655ac3b8d..712bd27fffb 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -29,864 +29,852 @@ #include "format.h" #include "config.h" -static void config_cache(void); -static void config_checkpoint(void); -static void config_checksum(void); -static void config_compression(const char *); -static void config_encryption(void); +static void config_cache(void); +static void config_checkpoint(void); +static void config_checksum(void); +static void config_compression(const char *); +static void config_encryption(void); static const char *config_file_type(u_int); -static bool config_fix(void); -static void config_in_memory(void); -static void config_in_memory_reset(void); -static int config_is_perm(const char *); -static void config_lrt(void); -static void config_lsm_reset(void); -static void config_map_checkpoint(const char *, u_int *); -static void config_map_checksum(const char *, u_int *); -static void config_map_compression(const char *, u_int *); -static void config_map_encryption(const char *, u_int *); -static void config_map_file_type(const char *, u_int *); -static void config_map_isolation(const char *, u_int *); -static void config_pct(void); -static void config_reset(void); -static void config_transaction(void); +static bool config_fix(void); +static void config_in_memory(void); +static void config_in_memory_reset(void); +static int config_is_perm(const char *); +static void config_lrt(void); +static void config_lsm_reset(void); +static void config_map_checkpoint(const char *, u_int *); +static void config_map_checksum(const char *, u_int *); +static void config_map_compression(const char *, u_int *); +static void config_map_encryption(const char *, u_int *); +static void config_map_file_type(const char *, u_int *); +static void config_map_isolation(const char *, u_int *); +static void config_pct(void); +static void config_reset(void); +static void config_transaction(void); /* * config_setup -- - * Initialize configuration for a run. + * Initialize configuration for a run. */ void config_setup(void) { - CONFIG *cp; - char buf[128]; - - /* Clear any temporary values. */ - config_reset(); - - /* Periodically run in-memory. */ - config_in_memory(); - - /* - * Choose a file format and a data source: they're interrelated (LSM is - * only compatible with row-store) and other items depend on them. - */ - if (!config_is_perm("file_type")) { - if (config_is_perm("data_source") && DATASOURCE("lsm")) - config_single("file_type=row", false); - else - switch (mmrand(NULL, 1, 10)) { - case 1: case 2: case 3: /* 30% */ - config_single("file_type=var", false); - break; - case 4: /* 10% */ - if (config_fix()) { - config_single("file_type=fix", false); - break; - } - /* FALLTHROUGH */ /* 60% */ - case 5: case 6: case 7: case 8: case 9: case 10: - config_single("file_type=row", false); - break; - } - } - config_map_file_type(g.c_file_type, &g.type); - - if (!config_is_perm("data_source")) { - config_single("data_source=table", false); - switch (mmrand(NULL, 1, 5)) { - case 1: /* 20% */ - config_single("data_source=file", false); - break; - case 2: /* 20% */ - /* - * LSM requires a row-store and backing disk. - * - * Configuring truncation or timestamps results in LSM - * cache problems, don't configure LSM if those set. - * - * XXX - * Remove the timestamp test when WT-4162 resolved. - */ - if (g.type != ROW || g.c_in_memory) - break; - if (config_is_perm( - "transaction_timestamps") && g.c_txn_timestamps) - break; - if (config_is_perm("truncate") && g.c_truncate) - break; - config_single("data_source=lsm", false); - break; - case 3: case 4: case 5: /* 60% */ - break; - } - } - - /* - * If data_source and file_type were both "permanent", we may still - * have a mismatch. - */ - if (DATASOURCE("lsm") && g.type != ROW) { - fprintf(stderr, - "%s: lsm data_source is only compatible with row file_type\n", - progname); - exit(EXIT_FAILURE); - } - - /* - * Build the top-level object name: we're overloading data_source in - * our configuration, LSM objects are "tables", but files are tested - * as well. - */ - g.uri = dmalloc(256); - strcpy(g.uri, DATASOURCE("file") ? "file:" : "table:"); - strcat(g.uri, WT_NAME); - - /* Fill in random values for the rest of the run. */ - for (cp = c; cp->name != NULL; ++cp) { - if (F_ISSET(cp, C_IGNORE | C_PERM | C_TEMP)) - continue; - - /* - * Boolean flags are 0 or 1, where the variable's "min" value - * is the percent chance the flag is "on" (so "on" if random - * rolled <= N, otherwise "off"). - */ - if (F_ISSET(cp, C_BOOL)) - testutil_check(__wt_snprintf(buf, sizeof(buf), - "%s=%s", - cp->name, - mmrand(NULL, 1, 100) <= cp->min ? "on" : "off")); - else - testutil_check(__wt_snprintf(buf, sizeof(buf), - "%s=%" PRIu32, - cp->name, mmrand(NULL, cp->min, cp->maxrand))); - config_single(buf, false); - } - - /* Only row-store tables support collation order. */ - if (g.type != ROW) - config_single("reverse=off", false); - - /* First, transaction configuration, it configures other features. */ - config_transaction(); - - /* Simple selection. */ - config_checkpoint(); - config_checksum(); - config_compression("compression"); - config_compression("logging_compression"); - config_encryption(); - config_lrt(); - - /* Configuration based on the configuration already chosen. */ - config_pct(); - config_cache(); - - /* Give in-memory and LSM configurations a final review. */ - if (g.c_in_memory != 0) - config_in_memory_reset(); - if (DATASOURCE("lsm")) - config_lsm_reset(); - - /* - * Key/value minimum/maximum are related, correct unless specified by - * the configuration. - */ - if (!config_is_perm("key_min") && g.c_key_min > g.c_key_max) - g.c_key_min = g.c_key_max; - if (!config_is_perm("key_max") && g.c_key_max < g.c_key_min) - g.c_key_max = g.c_key_min; - if (g.c_key_min > g.c_key_max) - testutil_die(EINVAL, "key_min may not be larger than key_max"); - - if (!config_is_perm("value_min") && g.c_value_min > g.c_value_max) - g.c_value_min = g.c_value_max; - if (!config_is_perm("value_max") && g.c_value_max < g.c_value_min) - g.c_value_max = g.c_value_min; - if (g.c_value_min > g.c_value_max) - testutil_die(EINVAL, - "value_min may not be larger than value_max"); - - /* - * Run-length is configured by a number of operations and a timer. - * - * If the operation count and the timer are both configured, do nothing. - * If only the timer is configured, clear the operations count. - * If only the operation count is configured, limit the run to 6 hours. - * If neither is configured, leave the operations count alone and limit - * the run to 30 minutes. - * - * In other words, if we rolled the dice on everything, do a short run. - * If we chose a number of operations but the rest of the configuration - * means operations take a long time to complete (for example, a small - * cache and many worker threads), don't let it run forever. - */ - if (config_is_perm("timer")) { - if (!config_is_perm("ops")) - config_single("ops=0", false); - } else { - if (!config_is_perm("ops")) - config_single("timer=30", false); - else - config_single("timer=360", false); - } - - /* Reset the key count. */ - g.key_cnt = 0; + CONFIG *cp; + char buf[128]; + + /* Clear any temporary values. */ + config_reset(); + + /* Periodically run in-memory. */ + config_in_memory(); + + /* + * Choose a file format and a data source: they're interrelated (LSM is only compatible with + * row-store) and other items depend on them. + */ + if (!config_is_perm("file_type")) { + if (config_is_perm("data_source") && DATASOURCE("lsm")) + config_single("file_type=row", false); + else + switch (mmrand(NULL, 1, 10)) { + case 1: + case 2: + case 3: /* 30% */ + config_single("file_type=var", false); + break; + case 4: /* 10% */ + if (config_fix()) { + config_single("file_type=fix", false); + break; + } + /* FALLTHROUGH */ /* 60% */ + case 5: + case 6: + case 7: + case 8: + case 9: + case 10: + config_single("file_type=row", false); + break; + } + } + config_map_file_type(g.c_file_type, &g.type); + + if (!config_is_perm("data_source")) { + config_single("data_source=table", false); + switch (mmrand(NULL, 1, 5)) { + case 1: /* 20% */ + config_single("data_source=file", false); + break; + case 2: /* 20% */ + /* + * LSM requires a row-store and backing disk. + * + * Configuring truncation or timestamps results in LSM + * cache problems, don't configure LSM if those set. + * + * XXX + * Remove the timestamp test when WT-4162 resolved. + */ + if (g.type != ROW || g.c_in_memory) + break; + if (config_is_perm("transaction_timestamps") && g.c_txn_timestamps) + break; + if (config_is_perm("truncate") && g.c_truncate) + break; + config_single("data_source=lsm", false); + break; + case 3: + case 4: + case 5: /* 60% */ + break; + } + } + + /* + * If data_source and file_type were both "permanent", we may still have a mismatch. + */ + if (DATASOURCE("lsm") && g.type != ROW) { + fprintf(stderr, "%s: lsm data_source is only compatible with row file_type\n", progname); + exit(EXIT_FAILURE); + } + + /* + * Build the top-level object name: we're overloading data_source in our configuration, LSM + * objects are "tables", but files are tested as well. + */ + g.uri = dmalloc(256); + strcpy(g.uri, DATASOURCE("file") ? "file:" : "table:"); + strcat(g.uri, WT_NAME); + + /* Fill in random values for the rest of the run. */ + for (cp = c; cp->name != NULL; ++cp) { + if (F_ISSET(cp, C_IGNORE | C_PERM | C_TEMP)) + continue; + + /* + * Boolean flags are 0 or 1, where the variable's "min" value is the percent chance the flag + * is "on" (so "on" if random rolled <= N, otherwise "off"). + */ + if (F_ISSET(cp, C_BOOL)) + testutil_check(__wt_snprintf( + buf, sizeof(buf), "%s=%s", cp->name, mmrand(NULL, 1, 100) <= cp->min ? "on" : "off")); + else + testutil_check(__wt_snprintf( + buf, sizeof(buf), "%s=%" PRIu32, cp->name, mmrand(NULL, cp->min, cp->maxrand))); + config_single(buf, false); + } + + /* Only row-store tables support collation order. */ + if (g.type != ROW) + config_single("reverse=off", false); + + /* First, transaction configuration, it configures other features. */ + config_transaction(); + + /* Simple selection. */ + config_checkpoint(); + config_checksum(); + config_compression("compression"); + config_compression("logging_compression"); + config_encryption(); + config_lrt(); + + /* Configuration based on the configuration already chosen. */ + config_pct(); + config_cache(); + + /* Give in-memory and LSM configurations a final review. */ + if (g.c_in_memory != 0) + config_in_memory_reset(); + if (DATASOURCE("lsm")) + config_lsm_reset(); + + /* + * Key/value minimum/maximum are related, correct unless specified by the configuration. + */ + if (!config_is_perm("key_min") && g.c_key_min > g.c_key_max) + g.c_key_min = g.c_key_max; + if (!config_is_perm("key_max") && g.c_key_max < g.c_key_min) + g.c_key_max = g.c_key_min; + if (g.c_key_min > g.c_key_max) + testutil_die(EINVAL, "key_min may not be larger than key_max"); + + if (!config_is_perm("value_min") && g.c_value_min > g.c_value_max) + g.c_value_min = g.c_value_max; + if (!config_is_perm("value_max") && g.c_value_max < g.c_value_min) + g.c_value_max = g.c_value_min; + if (g.c_value_min > g.c_value_max) + testutil_die(EINVAL, "value_min may not be larger than value_max"); + + /* + * Run-length is configured by a number of operations and a timer. + * + * If the operation count and the timer are both configured, do nothing. + * If only the timer is configured, clear the operations count. + * If only the operation count is configured, limit the run to 6 hours. + * If neither is configured, leave the operations count alone and limit + * the run to 30 minutes. + * + * In other words, if we rolled the dice on everything, do a short run. + * If we chose a number of operations but the rest of the configuration + * means operations take a long time to complete (for example, a small + * cache and many worker threads), don't let it run forever. + */ + if (config_is_perm("timer")) { + if (!config_is_perm("ops")) + config_single("ops=0", false); + } else { + if (!config_is_perm("ops")) + config_single("timer=30", false); + else + config_single("timer=360", false); + } + + /* Reset the key count. */ + g.key_cnt = 0; } /* * config_cache -- - * Cache configuration. + * Cache configuration. */ static void config_cache(void) { - uint32_t required; - - /* Page sizes are powers-of-two for bad historic reasons. */ - g.intl_page_max = 1U << g.c_intl_page_max; - g.leaf_page_max = 1U << g.c_leaf_page_max; - - /* Check if a minimum cache size has been specified. */ - if (config_is_perm("cache")) { - if (config_is_perm("cache_minimum") && - g.c_cache_minimum != 0 && g.c_cache < g.c_cache_minimum) - testutil_die(EINVAL, - "minimum cache set larger than cache " - "(%" PRIu32 " > %" PRIu32 ")", - g.c_cache_minimum, g.c_cache); - return; - } - - g.c_cache = WT_MAX(g.c_cache, g.c_cache_minimum); - - /* - * Maximum internal/leaf page size sanity. - * - * Ensure we can service at least one operation per-thread concurrently - * without filling the cache with pinned pages, that is, every thread - * consuming an internal page and a leaf page (or a pair of leaf pages - * for cursor movements). - * - * Maximum memory pages are in units of MB. - * - * This code is what dramatically increases the cache size when there - * are lots of threads, it grows the cache to several megabytes per - * thread. - */ - g.c_cache = WT_MAX(g.c_cache, - 2 * g.c_threads * g.c_memory_page_max); - - /* - * Ensure cache size sanity for LSM runs. An LSM tree open requires 3 - * chunks plus a page for each participant in up to three concurrent - * merges. Integrate a thread count into that calculation by requiring - * 3 chunks/pages per configured thread. That might be overkill, but - * LSM runs are more sensitive to small caches than other runs, and a - * generous cache avoids stalls we're not interested in chasing. - */ - if (DATASOURCE("lsm")) { - required = WT_LSM_TREE_MINIMUM_SIZE( - g.c_chunk_size * WT_MEGABYTE, - g.c_threads * g.c_merge_max, g.c_threads * g.leaf_page_max); - required = (required + (WT_MEGABYTE - 1)) / WT_MEGABYTE; - if (g.c_cache < required) - g.c_cache = required; - } + uint32_t required; + + /* Page sizes are powers-of-two for bad historic reasons. */ + g.intl_page_max = 1U << g.c_intl_page_max; + g.leaf_page_max = 1U << g.c_leaf_page_max; + + /* Check if a minimum cache size has been specified. */ + if (config_is_perm("cache")) { + if (config_is_perm("cache_minimum") && g.c_cache_minimum != 0 && + g.c_cache < g.c_cache_minimum) + testutil_die(EINVAL, + "minimum cache set larger than cache " + "(%" PRIu32 " > %" PRIu32 ")", + g.c_cache_minimum, g.c_cache); + return; + } + + g.c_cache = WT_MAX(g.c_cache, g.c_cache_minimum); + + /* + * Maximum internal/leaf page size sanity. + * + * Ensure we can service at least one operation per-thread concurrently + * without filling the cache with pinned pages, that is, every thread + * consuming an internal page and a leaf page (or a pair of leaf pages + * for cursor movements). + * + * Maximum memory pages are in units of MB. + * + * This code is what dramatically increases the cache size when there + * are lots of threads, it grows the cache to several megabytes per + * thread. + */ + g.c_cache = WT_MAX(g.c_cache, 2 * g.c_threads * g.c_memory_page_max); + + /* + * Ensure cache size sanity for LSM runs. An LSM tree open requires 3 + * chunks plus a page for each participant in up to three concurrent + * merges. Integrate a thread count into that calculation by requiring + * 3 chunks/pages per configured thread. That might be overkill, but + * LSM runs are more sensitive to small caches than other runs, and a + * generous cache avoids stalls we're not interested in chasing. + */ + if (DATASOURCE("lsm")) { + required = WT_LSM_TREE_MINIMUM_SIZE( + g.c_chunk_size * WT_MEGABYTE, g.c_threads * g.c_merge_max, g.c_threads * g.leaf_page_max); + required = (required + (WT_MEGABYTE - 1)) / WT_MEGABYTE; + if (g.c_cache < required) + g.c_cache = required; + } } /* * config_checkpoint -- - * Checkpoint configuration. + * Checkpoint configuration. */ static void config_checkpoint(void) { - /* Choose a checkpoint mode if nothing was specified. */ - if (!config_is_perm("checkpoints")) - switch (mmrand(NULL, 1, 20)) { - case 1: case 2: case 3: case 4: /* 20% */ - config_single("checkpoints=wiredtiger", false); - break; - case 5: /* 5 % */ - config_single("checkpoints=off", false); - break; - default: /* 75% */ - config_single("checkpoints=on", false); - break; - } + /* Choose a checkpoint mode if nothing was specified. */ + if (!config_is_perm("checkpoints")) + switch (mmrand(NULL, 1, 20)) { + case 1: + case 2: + case 3: + case 4: /* 20% */ + config_single("checkpoints=wiredtiger", false); + break; + case 5: /* 5 % */ + config_single("checkpoints=off", false); + break; + default: /* 75% */ + config_single("checkpoints=on", false); + break; + } } /* * config_checksum -- - * Checksum configuration. + * Checksum configuration. */ static void config_checksum(void) { - /* Choose a checksum mode if nothing was specified. */ - if (!config_is_perm("checksum")) - switch (mmrand(NULL, 1, 10)) { - case 1: /* 10% */ - config_single("checksum=on", false); - break; - case 2: /* 10% */ - config_single("checksum=off", false); - break; - default: /* 80% */ - config_single("checksum=uncompressed", false); - break; - } + /* Choose a checksum mode if nothing was specified. */ + if (!config_is_perm("checksum")) + switch (mmrand(NULL, 1, 10)) { + case 1: /* 10% */ + config_single("checksum=on", false); + break; + case 2: /* 10% */ + config_single("checksum=off", false); + break; + default: /* 80% */ + config_single("checksum=uncompressed", false); + break; + } } /* * config_compression -- - * Compression configuration. + * Compression configuration. */ static void config_compression(const char *conf_name) { - char confbuf[128]; - const char *cstr; - - /* Return if already specified. */ - if (config_is_perm(conf_name)) - return; - - /* - * Don't configure a compression engine for logging if logging isn't - * configured (it won't break, but it's confusing). - */ - cstr = "none"; - if (strcmp(conf_name, "logging_compression") == 0 && g.c_logging == 0) { - testutil_check(__wt_snprintf( - confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr)); - config_single(confbuf, false); - return; - } - - /* - * Select a compression type from the list of built-in engines. - * - * Listed percentages are only correct if all of the possible engines - * are compiled in. - */ - switch (mmrand(NULL, 1, 20)) { + char confbuf[128]; + const char *cstr; + + /* Return if already specified. */ + if (config_is_perm(conf_name)) + return; + + /* + * Don't configure a compression engine for logging if logging isn't configured (it won't break, + * but it's confusing). + */ + cstr = "none"; + if (strcmp(conf_name, "logging_compression") == 0 && g.c_logging == 0) { + testutil_check(__wt_snprintf(confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr)); + config_single(confbuf, false); + return; + } + + /* + * Select a compression type from the list of built-in engines. + * + * Listed percentages are only correct if all of the possible engines + * are compiled in. + */ + switch (mmrand(NULL, 1, 20)) { #ifdef HAVE_BUILTIN_EXTENSION_LZ4 - case 1: case 2: case 3: /* 15% lz4 */ - cstr = "lz4"; - break; + case 1: + case 2: + case 3: /* 15% lz4 */ + cstr = "lz4"; + break; #endif #ifdef HAVE_BUILTIN_EXTENSION_SNAPPY - case 4: case 5: case 6: case 7: /* 30% snappy */ - case 8: case 9: - cstr = "snappy"; - break; + case 4: + case 5: + case 6: + case 7: /* 30% snappy */ + case 8: + case 9: + cstr = "snappy"; + break; #endif #ifdef HAVE_BUILTIN_EXTENSION_ZLIB - case 10: case 11: case 12: case 13: /* 20% zlib */ - cstr = "zlib"; - break; + case 10: + case 11: + case 12: + case 13: /* 20% zlib */ + cstr = "zlib"; + break; #endif #ifdef HAVE_BUILTIN_EXTENSION_ZSTD - case 14: case 15: case 16: case 17: /* 20% zstd */ - cstr = "zstd"; - break; + case 14: + case 15: + case 16: + case 17: /* 20% zstd */ + cstr = "zstd"; + break; #endif - case 18: case 19: case 20: /* 15% no compression */ - default: - break; - } - - testutil_check(__wt_snprintf( - confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr)); - config_single(confbuf, false); + case 18: + case 19: + case 20: /* 15% no compression */ + default: + break; + } + + testutil_check(__wt_snprintf(confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr)); + config_single(confbuf, false); } /* * config_encryption -- - * Encryption configuration. + * Encryption configuration. */ static void config_encryption(void) { - const char *cstr; - - /* - * Encryption: choose something if encryption wasn't specified. - */ - if (!config_is_perm("encryption")) { - cstr = "encryption=none"; - switch (mmrand(NULL, 1, 10)) { - case 1: case 2: case 3: case 4: case 5: /* 70% no encryption */ - case 6: case 7: - break; - case 8: case 9: case 10: /* 30% rotn */ - cstr = "encryption=rotn-7"; - break; - } - - config_single(cstr, false); - } + const char *cstr; + + /* + * Encryption: choose something if encryption wasn't specified. + */ + if (!config_is_perm("encryption")) { + cstr = "encryption=none"; + switch (mmrand(NULL, 1, 10)) { + case 1: + case 2: + case 3: + case 4: + case 5: /* 70% no encryption */ + case 6: + case 7: + break; + case 8: + case 9: + case 10: /* 30% rotn */ + cstr = "encryption=rotn-7"; + break; + } + + config_single(cstr, false); + } } /* * config_fix -- - * Fixed-length column-store configuration. + * Fixed-length column-store configuration. */ static bool config_fix(void) { - /* - * Fixed-length column stores don't support the lookaside table (so, no - * long running transactions), or modify operations. - */ - if (config_is_perm("long_running_txn")) - return (false); - if (config_is_perm("modify_pct")) - return (false); - return (true); + /* + * Fixed-length column stores don't support the lookaside table (so, no long running + * transactions), or modify operations. + */ + if (config_is_perm("long_running_txn")) + return (false); + if (config_is_perm("modify_pct")) + return (false); + return (true); } /* * config_in_memory -- - * Periodically set up an in-memory configuration. + * Periodically set up an in-memory configuration. */ static void config_in_memory(void) { - /* - * Configure in-memory before configuring anything else, in-memory has - * many related requirements. Don't configure in-memory if there's any - * incompatible configurations, so we don't have to configure in-memory - * every time we configure something like LSM, that's too painful. - */ - if (config_is_perm("backups")) - return; - if (config_is_perm("checkpoints")) - return; - if (config_is_perm("compression")) - return; - if (config_is_perm("data_source") && DATASOURCE("lsm")) - return; - if (config_is_perm("logging")) - return; - if (config_is_perm("rebalance")) - return; - if (config_is_perm("salvage")) - return; - if (config_is_perm("verify")) - return; - - if (!config_is_perm("in_memory") && mmrand(NULL, 1, 20) == 1) - g.c_in_memory = 1; + /* + * Configure in-memory before configuring anything else, in-memory has many related + * requirements. Don't configure in-memory if there's any incompatible configurations, so we + * don't have to configure in-memory every time we configure something like LSM, that's too + * painful. + */ + if (config_is_perm("backups")) + return; + if (config_is_perm("checkpoints")) + return; + if (config_is_perm("compression")) + return; + if (config_is_perm("data_source") && DATASOURCE("lsm")) + return; + if (config_is_perm("logging")) + return; + if (config_is_perm("rebalance")) + return; + if (config_is_perm("salvage")) + return; + if (config_is_perm("verify")) + return; + + if (!config_is_perm("in_memory") && mmrand(NULL, 1, 20) == 1) + g.c_in_memory = 1; } /* * config_in_memory_reset -- - * In-memory configuration review. + * In-memory configuration review. */ static void config_in_memory_reset(void) { - uint32_t cache; - - /* Turn off a lot of stuff. */ - if (!config_is_perm("alter")) - config_single("alter=off", false); - if (!config_is_perm("backups")) - config_single("backups=off", false); - if (!config_is_perm("checkpoints")) - config_single("checkpoints=off", false); - if (!config_is_perm("compression")) - config_single("compression=none", false); - if (!config_is_perm("logging")) - config_single("logging=off", false); - if (!config_is_perm("rebalance")) - config_single("rebalance=off", false); - if (!config_is_perm("salvage")) - config_single("salvage=off", false); - if (!config_is_perm("verify")) - config_single("verify=off", false); - - /* - * Keep keys/values small, overflow items aren't an issue for in-memory - * configurations and it keeps us from overflowing the cache. - */ - if (!config_is_perm("key_max")) - config_single("key_max=32", false); - if (!config_is_perm("value_max")) - config_single("value_max=80", false); - - /* - * Size the cache relative to the initial data set, use 2x the base - * size as a minimum. - */ - if (!config_is_perm("cache")) { - cache = g.c_value_max; - if (g.type == ROW) - cache += g.c_key_max; - cache *= g.c_rows; - cache *= 2; - cache /= WT_MEGABYTE; - if (g.c_cache < cache) - g.c_cache = cache; - } + uint32_t cache; + + /* Turn off a lot of stuff. */ + if (!config_is_perm("alter")) + config_single("alter=off", false); + if (!config_is_perm("backups")) + config_single("backups=off", false); + if (!config_is_perm("checkpoints")) + config_single("checkpoints=off", false); + if (!config_is_perm("compression")) + config_single("compression=none", false); + if (!config_is_perm("logging")) + config_single("logging=off", false); + if (!config_is_perm("rebalance")) + config_single("rebalance=off", false); + if (!config_is_perm("salvage")) + config_single("salvage=off", false); + if (!config_is_perm("verify")) + config_single("verify=off", false); + + /* + * Keep keys/values small, overflow items aren't an issue for in-memory configurations and it + * keeps us from overflowing the cache. + */ + if (!config_is_perm("key_max")) + config_single("key_max=32", false); + if (!config_is_perm("value_max")) + config_single("value_max=80", false); + + /* + * Size the cache relative to the initial data set, use 2x the base size as a minimum. + */ + if (!config_is_perm("cache")) { + cache = g.c_value_max; + if (g.type == ROW) + cache += g.c_key_max; + cache *= g.c_rows; + cache *= 2; + cache /= WT_MEGABYTE; + if (g.c_cache < cache) + g.c_cache = cache; + } } /* * config_lsm_reset -- - * LSM configuration review. + * LSM configuration review. */ static void config_lsm_reset(void) { - /* - * Turn off truncate for LSM runs (some configurations with truncate - * always result in a timeout). - */ - if (!config_is_perm("truncate")) - config_single("truncate=off", false); - - /* - * LSM doesn't currently play nicely with timestamps, don't choose the - * pair unless forced to. If we turn off timestamps, make sure we turn - * off prepare as well, it requires timestamps. Remove this code with - * WT-4162. - */ - if (!config_is_perm("prepare") && - !config_is_perm("transaction_timestamps")) { - config_single("prepare=off", false); - config_single("transaction_timestamps=off", false); - } + /* + * Turn off truncate for LSM runs (some configurations with truncate always result in a + * timeout). + */ + if (!config_is_perm("truncate")) + config_single("truncate=off", false); + + /* + * LSM doesn't currently play nicely with timestamps, don't choose the pair unless forced to. If + * we turn off timestamps, make sure we turn off prepare as well, it requires timestamps. Remove + * this code with WT-4162. + */ + if (!config_is_perm("prepare") && !config_is_perm("transaction_timestamps")) { + config_single("prepare=off", false); + config_single("transaction_timestamps=off", false); + } } /* * config_lrt -- - * Long-running transaction configuration. + * Long-running transaction configuration. */ static void config_lrt(void) { - /* - * WiredTiger doesn't support a lookaside file for fixed-length column - * stores. - */ - if (g.type == FIX && g.c_long_running_txn) { - if (config_is_perm("long_running_txn")) - testutil_die(EINVAL, - "long_running_txn not supported with fixed-length " - "column store"); - config_single("long_running_txn=off", false); - } + /* + * WiredTiger doesn't support a lookaside file for fixed-length column stores. + */ + if (g.type == FIX && g.c_long_running_txn) { + if (config_is_perm("long_running_txn")) + testutil_die(EINVAL, + "long_running_txn not supported with fixed-length " + "column store"); + config_single("long_running_txn=off", false); + } } /* * config_pct -- - * Configure operation percentages. + * Configure operation percentages. */ static void config_pct(void) { - static struct { - const char *name; /* Operation */ - uint32_t *vp; /* Value store */ - u_int order; /* Order of assignment */ - } list[] = { - { "delete_pct", &g.c_delete_pct, 0 }, - { "insert_pct", &g.c_insert_pct, 0 }, -#define CONFIG_MODIFY_ENTRY 2 - { "modify_pct", &g.c_modify_pct, 0 }, - { "read_pct", &g.c_read_pct, 0 }, - { "write_pct", &g.c_write_pct, 0 }, - }; - u_int i, max_order, max_slot, n, pct; - - /* - * Walk the list of operations, checking for an illegal configuration - * and creating a random order in the list. - */ - pct = 0; - for (i = 0; i < WT_ELEMENTS(list); ++i) - if (config_is_perm(list[i].name)) - pct += *list[i].vp; - else - list[i].order = mmrand(NULL, 1, 1000); - if (pct > 100) - testutil_die(EINVAL, - "operation percentages do not total to 100%%"); - - /* Cursor modify isn't possible for fixed-length column store. */ - if (g.type == FIX) { - if (config_is_perm("modify_pct") && g.c_modify_pct != 0) - testutil_die(EINVAL, - "WT_CURSOR.modify not supported by fixed-length " - "column store"); - list[CONFIG_MODIFY_ENTRY].order = 0; - *list[CONFIG_MODIFY_ENTRY].vp = 0; - } - - /* - * Cursor modify isn't possible for anything besides snapshot isolation - * transactions. If both forced, it's an error. The run-time operations - * code converts modify operations into updates if we're in some other - * transaction type, but if we're never going to be able to do a modify, - * turn it off in the CONFIG output to avoid misleading debuggers. - */ - if (g.c_isolation_flag == ISOLATION_READ_COMMITTED || - g.c_isolation_flag == ISOLATION_READ_UNCOMMITTED) { - if (config_is_perm("isolation") && - config_is_perm("modify_pct") && g.c_modify_pct != 0) - testutil_die(EINVAL, - "WT_CURSOR.modify only supported with " - "snapshot isolation transactions"); - - list[CONFIG_MODIFY_ENTRY].order = 0; - *list[CONFIG_MODIFY_ENTRY].vp = 0; - } - - /* - * Walk the list, allocating random numbers of operations in a random - * order. - * - * If the "order" field is non-zero, we need to create a value for this - * operation. Find the largest order field in the array; if one non-zero - * order field is found, it's the last entry and gets the remainder of - * the operations. - */ - for (pct = 100 - pct;;) { - for (i = n = - max_order = max_slot = 0; i < WT_ELEMENTS(list); ++i) { - if (list[i].order != 0) - ++n; - if (list[i].order > max_order) { - max_order = list[i].order; - max_slot = i; - } - } - if (n == 0) - break; - if (n == 1) { - *list[max_slot].vp = pct; - break; - } - *list[max_slot].vp = mmrand(NULL, 0, pct); - list[max_slot].order = 0; - pct -= *list[max_slot].vp; - } - - testutil_assert(g.c_delete_pct + g.c_insert_pct + - g.c_modify_pct + g.c_read_pct + g.c_write_pct == 100); + static struct { + const char *name; /* Operation */ + uint32_t *vp; /* Value store */ + u_int order; /* Order of assignment */ + } list[] = { + {"delete_pct", &g.c_delete_pct, 0}, {"insert_pct", &g.c_insert_pct, 0}, +#define CONFIG_MODIFY_ENTRY 2 + {"modify_pct", &g.c_modify_pct, 0}, {"read_pct", &g.c_read_pct, 0}, + {"write_pct", &g.c_write_pct, 0}, + }; + u_int i, max_order, max_slot, n, pct; + + /* + * Walk the list of operations, checking for an illegal configuration and creating a random + * order in the list. + */ + pct = 0; + for (i = 0; i < WT_ELEMENTS(list); ++i) + if (config_is_perm(list[i].name)) + pct += *list[i].vp; + else + list[i].order = mmrand(NULL, 1, 1000); + if (pct > 100) + testutil_die(EINVAL, "operation percentages do not total to 100%%"); + + /* Cursor modify isn't possible for fixed-length column store. */ + if (g.type == FIX) { + if (config_is_perm("modify_pct") && g.c_modify_pct != 0) + testutil_die(EINVAL, + "WT_CURSOR.modify not supported by fixed-length " + "column store"); + list[CONFIG_MODIFY_ENTRY].order = 0; + *list[CONFIG_MODIFY_ENTRY].vp = 0; + } + + /* + * Cursor modify isn't possible for anything besides snapshot isolation transactions. If both + * forced, it's an error. The run-time operations code converts modify operations into updates + * if we're in some other transaction type, but if we're never going to be able to do a modify, + * turn it off in the CONFIG output to avoid misleading debuggers. + */ + if (g.c_isolation_flag == ISOLATION_READ_COMMITTED || + g.c_isolation_flag == ISOLATION_READ_UNCOMMITTED) { + if (config_is_perm("isolation") && config_is_perm("modify_pct") && g.c_modify_pct != 0) + testutil_die(EINVAL, + "WT_CURSOR.modify only supported with " + "snapshot isolation transactions"); + + list[CONFIG_MODIFY_ENTRY].order = 0; + *list[CONFIG_MODIFY_ENTRY].vp = 0; + } + + /* + * Walk the list, allocating random numbers of operations in a random + * order. + * + * If the "order" field is non-zero, we need to create a value for this + * operation. Find the largest order field in the array; if one non-zero + * order field is found, it's the last entry and gets the remainder of + * the operations. + */ + for (pct = 100 - pct;;) { + for (i = n = max_order = max_slot = 0; i < WT_ELEMENTS(list); ++i) { + if (list[i].order != 0) + ++n; + if (list[i].order > max_order) { + max_order = list[i].order; + max_slot = i; + } + } + if (n == 0) + break; + if (n == 1) { + *list[max_slot].vp = pct; + break; + } + *list[max_slot].vp = mmrand(NULL, 0, pct); + list[max_slot].order = 0; + pct -= *list[max_slot].vp; + } + + testutil_assert( + g.c_delete_pct + g.c_insert_pct + g.c_modify_pct + g.c_read_pct + g.c_write_pct == 100); } /* * config_transaction -- - * Transaction configuration. + * Transaction configuration. */ static void config_transaction(void) { - bool prepare_requires_ts; - - /* - * We can't prepare a transaction if logging is configured or timestamps - * aren't configured. Further, for repeatable reads to work in timestamp - * testing, all updates must be within a snapshot-isolation transaction. - * Check for incompatible configurations, then let prepare and timestamp - * drive the remaining configuration. - */ - prepare_requires_ts = false; - if (g.c_prepare) { - if (config_is_perm("prepare")) { - if (g.c_logging && config_is_perm("logging")) - testutil_die(EINVAL, - "prepare is incompatible with logging"); - if (!g.c_txn_timestamps && - config_is_perm("transaction_timestamps")) - testutil_die(EINVAL, - "prepare requires transaction timestamps"); - } else - if ((g.c_logging && config_is_perm("logging")) || - (!g.c_txn_timestamps && - config_is_perm("transaction_timestamps"))) - config_single("prepare=off", false); - if (g.c_prepare) { - prepare_requires_ts = true; - if (g.c_logging) - config_single("logging=off", false); - if (!g.c_txn_timestamps) - config_single( - "transaction_timestamps=on", false); - } - } - - if (g.c_txn_timestamps) { - if (prepare_requires_ts || - config_is_perm("transaction_timestamps")) { - if (g.c_isolation_flag != ISOLATION_SNAPSHOT && - config_is_perm("isolation")) - testutil_die(EINVAL, - "transaction_timestamps or prepare require " - "isolation=snapshot"); - if (g.c_txn_freq != 100 && - config_is_perm("transaction-frequency")) - testutil_die(EINVAL, - "transaction_timestamps or prepare require " - "transaction-frequency=100"); - } else - if ((g.c_isolation_flag != ISOLATION_SNAPSHOT && - config_is_perm("isolation")) || - (g.c_txn_freq != 100 && - config_is_perm("transaction-frequency"))) - config_single( - "transaction_timestamps=off", false); - } - if (g.c_txn_timestamps) { - if (g.c_isolation_flag != ISOLATION_SNAPSHOT) - config_single("isolation=snapshot", false); - if (g.c_txn_freq != 100) - config_single("transaction-frequency=100", false); - } else - if (!config_is_perm("isolation")) - switch (mmrand(NULL, 1, 4)) { - case 1: - config_single("isolation=random", false); - break; - case 2: - config_single( - "isolation=read-uncommitted", false); - break; - case 3: - config_single( - "isolation=read-committed", false); - break; - case 4: - default: - config_single("isolation=snapshot", false); - break; - } + bool prepare_requires_ts; + + /* + * We can't prepare a transaction if logging is configured or timestamps aren't configured. + * Further, for repeatable reads to work in timestamp testing, all updates must be within a + * snapshot-isolation transaction. Check for incompatible configurations, then let prepare and + * timestamp drive the remaining configuration. + */ + prepare_requires_ts = false; + if (g.c_prepare) { + if (config_is_perm("prepare")) { + if (g.c_logging && config_is_perm("logging")) + testutil_die(EINVAL, "prepare is incompatible with logging"); + if (!g.c_txn_timestamps && config_is_perm("transaction_timestamps")) + testutil_die(EINVAL, "prepare requires transaction timestamps"); + } else if ((g.c_logging && config_is_perm("logging")) || + (!g.c_txn_timestamps && config_is_perm("transaction_timestamps"))) + config_single("prepare=off", false); + if (g.c_prepare) { + prepare_requires_ts = true; + if (g.c_logging) + config_single("logging=off", false); + if (!g.c_txn_timestamps) + config_single("transaction_timestamps=on", false); + } + } + + if (g.c_txn_timestamps) { + if (prepare_requires_ts || config_is_perm("transaction_timestamps")) { + if (g.c_isolation_flag != ISOLATION_SNAPSHOT && config_is_perm("isolation")) + testutil_die(EINVAL, + "transaction_timestamps or prepare require " + "isolation=snapshot"); + if (g.c_txn_freq != 100 && config_is_perm("transaction-frequency")) + testutil_die(EINVAL, + "transaction_timestamps or prepare require " + "transaction-frequency=100"); + } else if ((g.c_isolation_flag != ISOLATION_SNAPSHOT && config_is_perm("isolation")) || + (g.c_txn_freq != 100 && config_is_perm("transaction-frequency"))) + config_single("transaction_timestamps=off", false); + } + if (g.c_txn_timestamps) { + if (g.c_isolation_flag != ISOLATION_SNAPSHOT) + config_single("isolation=snapshot", false); + if (g.c_txn_freq != 100) + config_single("transaction-frequency=100", false); + } else if (!config_is_perm("isolation")) + switch (mmrand(NULL, 1, 4)) { + case 1: + config_single("isolation=random", false); + break; + case 2: + config_single("isolation=read-uncommitted", false); + break; + case 3: + config_single("isolation=read-committed", false); + break; + case 4: + default: + config_single("isolation=snapshot", false); + break; + } } /* * config_error -- - * Display configuration information on error. + * Display configuration information on error. */ void config_error(void) { - CONFIG *cp; - - /* Display configuration names. */ - fprintf(stderr, "\n"); - fprintf(stderr, "Configuration names:\n"); - for (cp = c; cp->name != NULL; ++cp) - if (strlen(cp->name) > 17) - fprintf(stderr, - "%s\n%17s: %s\n", cp->name, " ", cp->desc); - else - fprintf(stderr, "%17s: %s\n", cp->name, cp->desc); + CONFIG *cp; + + /* Display configuration names. */ + fprintf(stderr, "\n"); + fprintf(stderr, "Configuration names:\n"); + for (cp = c; cp->name != NULL; ++cp) + if (strlen(cp->name) > 17) + fprintf(stderr, "%s\n%17s: %s\n", cp->name, " ", cp->desc); + else + fprintf(stderr, "%17s: %s\n", cp->name, cp->desc); } /* * config_print -- - * Print configuration information. + * Print configuration information. */ void config_print(bool error_display) { - CONFIG *cp; - FILE *fp; - - if (error_display) - fp = stdout; - else - if ((fp = fopen(g.home_config, "w")) == NULL) - testutil_die(errno, "fopen: %s", g.home_config); - - fprintf(fp, "############################################\n"); - fprintf(fp, "# RUN PARAMETERS\n"); - fprintf(fp, "############################################\n"); - - /* Display configuration values. */ - for (cp = c; cp->name != NULL; ++cp) - if (F_ISSET(cp, C_STRING)) - fprintf(fp, "%s=%s\n", cp->name, - *cp->vstr == NULL ? "" : *cp->vstr); - else - fprintf(fp, "%s=%" PRIu32 "\n", cp->name, *cp->v); - - fprintf(fp, "############################################\n"); - - /* Flush so we're up-to-date on error. */ - (void)fflush(fp); - - if (fp != stdout) - fclose_and_clear(&fp); + CONFIG *cp; + FILE *fp; + + if (error_display) + fp = stdout; + else if ((fp = fopen(g.home_config, "w")) == NULL) + testutil_die(errno, "fopen: %s", g.home_config); + + fprintf(fp, "############################################\n"); + fprintf(fp, "# RUN PARAMETERS\n"); + fprintf(fp, "############################################\n"); + + /* Display configuration values. */ + for (cp = c; cp->name != NULL; ++cp) + if (F_ISSET(cp, C_STRING)) + fprintf(fp, "%s=%s\n", cp->name, *cp->vstr == NULL ? "" : *cp->vstr); + else + fprintf(fp, "%s=%" PRIu32 "\n", cp->name, *cp->v); + + fprintf(fp, "############################################\n"); + + /* Flush so we're up-to-date on error. */ + (void)fflush(fp); + + if (fp != stdout) + fclose_and_clear(&fp); } /* * config_file -- - * Read configuration values from a file. + * Read configuration values from a file. */ void config_file(const char *name) { - FILE *fp; - char buf[256], *p; - - if ((fp = fopen(name, "r")) == NULL) - testutil_die(errno, "fopen: %s", name); - while (fgets(buf, sizeof(buf), fp) != NULL) { - for (p = buf; *p != '\0' && *p != '\n'; ++p) - ; - *p = '\0'; - if (buf[0] == '\0' || buf[0] == '#') - continue; - config_single(buf, true); - } - fclose_and_clear(&fp); + FILE *fp; + char buf[256], *p; + + if ((fp = fopen(name, "r")) == NULL) + testutil_die(errno, "fopen: %s", name); + while (fgets(buf, sizeof(buf), fp) != NULL) { + for (p = buf; *p != '\0' && *p != '\n'; ++p) + ; + *p = '\0'; + if (buf[0] == '\0' || buf[0] == '#') + continue; + config_single(buf, true); + } + fclose_and_clear(&fp); } /* * config_clear -- - * Clear all configuration values. + * Clear all configuration values. */ void config_clear(void) { - CONFIG *cp; - - /* Clear all allocated configuration data. */ - for (cp = c; cp->name != NULL; ++cp) - if (cp->vstr != NULL) { - free((void *)*cp->vstr); - *cp->vstr = NULL; - } - free(g.uri); - g.uri = NULL; + CONFIG *cp; + + /* Clear all allocated configuration data. */ + for (cp = c; cp->name != NULL; ++cp) + if (cp->vstr != NULL) { + free((void *)*cp->vstr); + *cp->vstr = NULL; + } + free(g.uri); + g.uri = NULL; } /* * config_reset -- - * Clear per-run configuration values. + * Clear per-run configuration values. */ static void config_reset(void) { - CONFIG *cp; - - /* Clear temporary allocated configuration data. */ - for (cp = c; cp->name != NULL; ++cp) { - F_CLR(cp, C_TEMP); - if (!F_ISSET(cp, C_PERM) && cp->vstr != NULL) { - free((void *)*cp->vstr); - *cp->vstr = NULL; - } - } - free(g.uri); - g.uri = NULL; + CONFIG *cp; + + /* Clear temporary allocated configuration data. */ + for (cp = c; cp->name != NULL; ++cp) { + F_CLR(cp, C_TEMP); + if (!F_ISSET(cp, C_PERM) && cp->vstr != NULL) { + free((void *)*cp->vstr); + *cp->vstr = NULL; + } + } + free(g.uri); + g.uri = NULL; } /* @@ -896,247 +884,226 @@ config_reset(void) static CONFIG * config_find(const char *s, size_t len, bool fatal) { - CONFIG *cp; - - for (cp = c; cp->name != NULL; ++cp) - if (strncmp(s, cp->name, len) == 0 && cp->name[len] == '\0') - return (cp); - - /* - * Optionally ignore unknown keywords, it makes it easier to run old - * CONFIG files. - */ - if (fatal) { - fprintf(stderr, - "%s: %s: unknown required configuration keyword\n", - progname, s); - exit(EXIT_FAILURE); - } - fprintf(stderr, - "%s: %s: WARNING, ignoring unknown configuration keyword\n", - progname, s); - return (NULL); + CONFIG *cp; + + for (cp = c; cp->name != NULL; ++cp) + if (strncmp(s, cp->name, len) == 0 && cp->name[len] == '\0') + return (cp); + + /* + * Optionally ignore unknown keywords, it makes it easier to run old CONFIG files. + */ + if (fatal) { + fprintf(stderr, "%s: %s: unknown required configuration keyword\n", progname, s); + exit(EXIT_FAILURE); + } + fprintf(stderr, "%s: %s: WARNING, ignoring unknown configuration keyword\n", progname, s); + return (NULL); } /* * config_single -- - * Set a single configuration structure value. + * Set a single configuration structure value. */ void config_single(const char *s, bool perm) { - CONFIG *cp; - long vlong; - uint32_t v; - char *p; - const char *ep; - - if ((ep = strchr(s, '=')) == NULL) { - fprintf(stderr, - "%s: %s: illegal configuration value\n", progname, s); - exit(EXIT_FAILURE); - } - - if ((cp = config_find(s, (size_t)(ep - s), false)) == NULL) - return; - - F_SET(cp, perm ? C_PERM : C_TEMP); - ++ep; - - if (F_ISSET(cp, C_STRING)) { - /* - * Free the previous setting if a configuration has been - * passed in twice. - */ - if (*cp->vstr != NULL) { - free(*cp->vstr); - *cp->vstr = NULL; - } - - if (strncmp(s, "checkpoints", strlen("checkpoints")) == 0) { - config_map_checkpoint(ep, &g.c_checkpoint_flag); - *cp->vstr = dstrdup(ep); - } else if (strncmp(s, "checksum", strlen("checksum")) == 0) { - config_map_checksum(ep, &g.c_checksum_flag); - *cp->vstr = dstrdup(ep); - } else if (strncmp(s, - "compression", strlen("compression")) == 0) { - config_map_compression(ep, &g.c_compression_flag); - *cp->vstr = dstrdup(ep); - } else if (strncmp(s, - "data_source", strlen("data_source")) == 0 && - strncmp("file", ep, strlen("file")) != 0 && - strncmp("lsm", ep, strlen("lsm")) != 0 && - strncmp("table", ep, strlen("table")) != 0) { - fprintf(stderr, - "Invalid data source option: %s\n", ep); - exit(EXIT_FAILURE); - } else if (strncmp(s, - "encryption", strlen("encryption")) == 0) { - config_map_encryption(ep, &g.c_encryption_flag); - *cp->vstr = dstrdup(ep); - } else if (strncmp(s, "file_type", strlen("file_type")) == 0) { - config_map_file_type(ep, &g.type); - *cp->vstr = dstrdup(config_file_type(g.type)); - } else if (strncmp(s, "isolation", strlen("isolation")) == 0) { - config_map_isolation(ep, &g.c_isolation_flag); - *cp->vstr = dstrdup(ep); - } else if (strncmp(s, "logging_compression", - strlen("logging_compression")) == 0) { - config_map_compression(ep, - &g.c_logging_compression_flag); - *cp->vstr = dstrdup(ep); - } else - *cp->vstr = dstrdup(ep); - - return; - } - - vlong = -1; - if (F_ISSET(cp, C_BOOL)) { - if (strncmp(ep, "off", strlen("off")) == 0) - vlong = 0; - else if (strncmp(ep, "on", strlen("on")) == 0) - vlong = 1; - } - if (vlong == -1) { - vlong = strtol(ep, &p, 10); - if (*p != '\0') { - fprintf(stderr, "%s: %s: illegal numeric value\n", - progname, s); - exit(EXIT_FAILURE); - } - } - v = (uint32_t)vlong; - if (F_ISSET(cp, C_BOOL)) { - if (v != 0 && v != 1) { - fprintf(stderr, "%s: %s: value of boolean not 0 or 1\n", - progname, s); - exit(EXIT_FAILURE); - } - } else if (v < cp->min || v > cp->maxset) { - fprintf(stderr, "%s: %s: value outside min/max values of %" - PRIu32 "-%" PRIu32 "\n", - progname, s, cp->min, cp->maxset); - exit(EXIT_FAILURE); - } - - *cp->v = v; + CONFIG *cp; + long vlong; + uint32_t v; + char *p; + const char *ep; + + if ((ep = strchr(s, '=')) == NULL) { + fprintf(stderr, "%s: %s: illegal configuration value\n", progname, s); + exit(EXIT_FAILURE); + } + + if ((cp = config_find(s, (size_t)(ep - s), false)) == NULL) + return; + + F_SET(cp, perm ? C_PERM : C_TEMP); + ++ep; + + if (F_ISSET(cp, C_STRING)) { + /* + * Free the previous setting if a configuration has been passed in twice. + */ + if (*cp->vstr != NULL) { + free(*cp->vstr); + *cp->vstr = NULL; + } + + if (strncmp(s, "checkpoints", strlen("checkpoints")) == 0) { + config_map_checkpoint(ep, &g.c_checkpoint_flag); + *cp->vstr = dstrdup(ep); + } else if (strncmp(s, "checksum", strlen("checksum")) == 0) { + config_map_checksum(ep, &g.c_checksum_flag); + *cp->vstr = dstrdup(ep); + } else if (strncmp(s, "compression", strlen("compression")) == 0) { + config_map_compression(ep, &g.c_compression_flag); + *cp->vstr = dstrdup(ep); + } else if (strncmp(s, "data_source", strlen("data_source")) == 0 && + strncmp("file", ep, strlen("file")) != 0 && strncmp("lsm", ep, strlen("lsm")) != 0 && + strncmp("table", ep, strlen("table")) != 0) { + fprintf(stderr, "Invalid data source option: %s\n", ep); + exit(EXIT_FAILURE); + } else if (strncmp(s, "encryption", strlen("encryption")) == 0) { + config_map_encryption(ep, &g.c_encryption_flag); + *cp->vstr = dstrdup(ep); + } else if (strncmp(s, "file_type", strlen("file_type")) == 0) { + config_map_file_type(ep, &g.type); + *cp->vstr = dstrdup(config_file_type(g.type)); + } else if (strncmp(s, "isolation", strlen("isolation")) == 0) { + config_map_isolation(ep, &g.c_isolation_flag); + *cp->vstr = dstrdup(ep); + } else if (strncmp(s, "logging_compression", strlen("logging_compression")) == 0) { + config_map_compression(ep, &g.c_logging_compression_flag); + *cp->vstr = dstrdup(ep); + } else + *cp->vstr = dstrdup(ep); + + return; + } + + vlong = -1; + if (F_ISSET(cp, C_BOOL)) { + if (strncmp(ep, "off", strlen("off")) == 0) + vlong = 0; + else if (strncmp(ep, "on", strlen("on")) == 0) + vlong = 1; + } + if (vlong == -1) { + vlong = strtol(ep, &p, 10); + if (*p != '\0') { + fprintf(stderr, "%s: %s: illegal numeric value\n", progname, s); + exit(EXIT_FAILURE); + } + } + v = (uint32_t)vlong; + if (F_ISSET(cp, C_BOOL)) { + if (v != 0 && v != 1) { + fprintf(stderr, "%s: %s: value of boolean not 0 or 1\n", progname, s); + exit(EXIT_FAILURE); + } + } else if (v < cp->min || v > cp->maxset) { + fprintf(stderr, "%s: %s: value outside min/max values of %" PRIu32 "-%" PRIu32 "\n", + progname, s, cp->min, cp->maxset); + exit(EXIT_FAILURE); + } + + *cp->v = v; } /* * config_map_file_type -- - * Map a file type configuration to a flag. + * Map a file type configuration to a flag. */ static void config_map_file_type(const char *s, u_int *vp) { - if (strcmp(s, "fix") == 0 || - strcmp(s, "fixed-length column-store") == 0) - *vp = FIX; - else if (strcmp(s, "var") == 0 || - strcmp(s, "variable-length column-store") == 0) - *vp = VAR; - else if (strcmp(s, "row") == 0 || - strcmp(s, "row-store") == 0) - *vp = ROW; - else - testutil_die(EINVAL, "illegal file type configuration: %s", s); + if (strcmp(s, "fix") == 0 || strcmp(s, "fixed-length column-store") == 0) + *vp = FIX; + else if (strcmp(s, "var") == 0 || strcmp(s, "variable-length column-store") == 0) + *vp = VAR; + else if (strcmp(s, "row") == 0 || strcmp(s, "row-store") == 0) + *vp = ROW; + else + testutil_die(EINVAL, "illegal file type configuration: %s", s); } /* * config_map_checkpoint -- - * Map a checkpoint configuration to a flag. + * Map a checkpoint configuration to a flag. */ static void config_map_checkpoint(const char *s, u_int *vp) { - /* Checkpoint configuration used to be 1/0, let it continue to work. */ - if (strcmp(s, "on") == 0 || strcmp(s, "1") == 0) - *vp = CHECKPOINT_ON; - else if (strcmp(s, "off") == 0 || strcmp(s, "0") == 0) - *vp = CHECKPOINT_OFF; - else if (strcmp(s, "wiredtiger") == 0) - *vp = CHECKPOINT_WIREDTIGER; - else - testutil_die(EINVAL, "illegal checkpoint configuration: %s", s); + /* Checkpoint configuration used to be 1/0, let it continue to work. */ + if (strcmp(s, "on") == 0 || strcmp(s, "1") == 0) + *vp = CHECKPOINT_ON; + else if (strcmp(s, "off") == 0 || strcmp(s, "0") == 0) + *vp = CHECKPOINT_OFF; + else if (strcmp(s, "wiredtiger") == 0) + *vp = CHECKPOINT_WIREDTIGER; + else + testutil_die(EINVAL, "illegal checkpoint configuration: %s", s); } /* * config_map_checksum -- - * Map a checksum configuration to a flag. + * Map a checksum configuration to a flag. */ static void config_map_checksum(const char *s, u_int *vp) { - if (strcmp(s, "on") == 0) - *vp = CHECKSUM_ON; - else if (strcmp(s, "off") == 0) - *vp = CHECKSUM_ON; - else if (strcmp(s, "uncompressed") == 0) - *vp = CHECKSUM_UNCOMPRESSED; - else - testutil_die(EINVAL, "illegal checksum configuration: %s", s); + if (strcmp(s, "on") == 0) + *vp = CHECKSUM_ON; + else if (strcmp(s, "off") == 0) + *vp = CHECKSUM_ON; + else if (strcmp(s, "uncompressed") == 0) + *vp = CHECKSUM_UNCOMPRESSED; + else + testutil_die(EINVAL, "illegal checksum configuration: %s", s); } /* * config_map_compression -- - * Map a compression configuration to a flag. + * Map a compression configuration to a flag. */ static void config_map_compression(const char *s, u_int *vp) { - if (strcmp(s, "none") == 0) - *vp = COMPRESS_NONE; - else if (strcmp(s, "lz4") == 0) - *vp = COMPRESS_LZ4; - else if (strcmp(s, "lz4-noraw") == 0) /* CONFIG compatibility */ - *vp = COMPRESS_LZ4; - else if (strcmp(s, "snappy") == 0) - *vp = COMPRESS_SNAPPY; - else if (strcmp(s, "zlib") == 0) - *vp = COMPRESS_ZLIB; - else if (strcmp(s, "zlib-noraw") == 0) /* CONFIG compatibility */ - *vp = COMPRESS_ZLIB; - else if (strcmp(s, "zstd") == 0) - *vp = COMPRESS_ZSTD; - else - testutil_die(EINVAL, - "illegal compression configuration: %s", s); + if (strcmp(s, "none") == 0) + *vp = COMPRESS_NONE; + else if (strcmp(s, "lz4") == 0) + *vp = COMPRESS_LZ4; + else if (strcmp(s, "lz4-noraw") == 0) /* CONFIG compatibility */ + *vp = COMPRESS_LZ4; + else if (strcmp(s, "snappy") == 0) + *vp = COMPRESS_SNAPPY; + else if (strcmp(s, "zlib") == 0) + *vp = COMPRESS_ZLIB; + else if (strcmp(s, "zlib-noraw") == 0) /* CONFIG compatibility */ + *vp = COMPRESS_ZLIB; + else if (strcmp(s, "zstd") == 0) + *vp = COMPRESS_ZSTD; + else + testutil_die(EINVAL, "illegal compression configuration: %s", s); } /* * config_map_encryption -- - * Map a encryption configuration to a flag. + * Map a encryption configuration to a flag. */ static void config_map_encryption(const char *s, u_int *vp) { - if (strcmp(s, "none") == 0) - *vp = ENCRYPT_NONE; - else if (strcmp(s, "rotn-7") == 0) - *vp = ENCRYPT_ROTN_7; - else - testutil_die(EINVAL, "illegal encryption configuration: %s", s); + if (strcmp(s, "none") == 0) + *vp = ENCRYPT_NONE; + else if (strcmp(s, "rotn-7") == 0) + *vp = ENCRYPT_ROTN_7; + else + testutil_die(EINVAL, "illegal encryption configuration: %s", s); } /* * config_map_isolation -- - * Map an isolation configuration to a flag. + * Map an isolation configuration to a flag. */ static void config_map_isolation(const char *s, u_int *vp) { - if (strcmp(s, "random") == 0) - *vp = ISOLATION_RANDOM; - else if (strcmp(s, "read-uncommitted") == 0) - *vp = ISOLATION_READ_UNCOMMITTED; - else if (strcmp(s, "read-committed") == 0) - *vp = ISOLATION_READ_COMMITTED; - else if (strcmp(s, "snapshot") == 0) - *vp = ISOLATION_SNAPSHOT; - else - testutil_die(EINVAL, "illegal isolation configuration: %s", s); + if (strcmp(s, "random") == 0) + *vp = ISOLATION_RANDOM; + else if (strcmp(s, "read-uncommitted") == 0) + *vp = ISOLATION_READ_UNCOMMITTED; + else if (strcmp(s, "read-committed") == 0) + *vp = ISOLATION_READ_COMMITTED; + else if (strcmp(s, "snapshot") == 0) + *vp = ISOLATION_SNAPSHOT; + else + testutil_die(EINVAL, "illegal isolation configuration: %s", s); } /* @@ -1146,28 +1113,28 @@ config_map_isolation(const char *s, u_int *vp) static int config_is_perm(const char *s) { - CONFIG *cp; + CONFIG *cp; - cp = config_find(s, strlen(s), true); - return (F_ISSET(cp, C_PERM) ? 1 : 0); + cp = config_find(s, strlen(s), true); + return (F_ISSET(cp, C_PERM) ? 1 : 0); } /* * config_file_type -- - * Return the file type as a string. + * Return the file type as a string. */ static const char * config_file_type(u_int type) { - switch (type) { - case FIX: - return ("fixed-length column-store"); - case VAR: - return ("variable-length column-store"); - case ROW: - return ("row-store"); - default: - break; - } - return ("error: unknown file type"); + switch (type) { + case FIX: + return ("fixed-length column-store"); + case VAR: + return ("variable-length column-store"); + case ROW: + return ("row-store"); + default: + break; + } + return ("error: unknown file type"); } diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index c1aafcd214e..58decce75af 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -27,396 +27,285 @@ */ /* - * Configuration for the wts program is an array of string-based parameters. - * This is the structure used to declare them. + * Configuration for the wts program is an array of string-based parameters. This is the structure + * used to declare them. */ typedef struct { - const char *name; /* Configuration item */ - const char *desc; /* Configuration description */ + const char *name; /* Configuration item */ + const char *desc; /* Configuration description */ - /* Value is a boolean, yes if roll of 1-to-100 is <= CONFIG->min. */ -#define C_BOOL 0x01u +/* Value is a boolean, yes if roll of 1-to-100 is <= CONFIG->min. */ +#define C_BOOL 0x01u - /* Not a simple randomization, handle outside the main loop. */ -#define C_IGNORE 0x02u +/* Not a simple randomization, handle outside the main loop. */ +#define C_IGNORE 0x02u - /* Value was set from command-line or file, ignore for all runs. */ -#define C_PERM 0x04u +/* Value was set from command-line or file, ignore for all runs. */ +#define C_PERM 0x04u - /* Value isn't random for this run, ignore just for this run. */ -#define C_TEMP 0x08u +/* Value isn't random for this run, ignore just for this run. */ +#define C_TEMP 0x08u - /* Value is a string. */ -#define C_STRING 0x20u - u_int flags; +/* Value is a string. */ +#define C_STRING 0x20u + u_int flags; - uint32_t min; /* Minimum value */ - uint32_t maxrand; /* Maximum value randomly chosen */ - uint32_t maxset; /* Maximum value explicitly set */ - uint32_t *v; /* Value for this run */ - char **vstr; /* Value for string options */ + uint32_t min; /* Minimum value */ + uint32_t maxrand; /* Maximum value randomly chosen */ + uint32_t maxset; /* Maximum value explicitly set */ + uint32_t *v; /* Value for this run */ + char **vstr; /* Value for string options */ } CONFIG; -#define COMPRESSION_LIST \ - "(none | lz4 | snappy | zlib | zstd)" +#define COMPRESSION_LIST "(none | lz4 | snappy | zlib | zstd)" -static CONFIG c[] = { - { "abort", - "if timed run should drop core", /* 0% */ - C_BOOL, 0, 0, 0, &g.c_abort, NULL }, +static CONFIG c[] = {{"abort", "if timed run should drop core", /* 0% */ + C_BOOL, 0, 0, 0, &g.c_abort, NULL}, - { "alter", - "if altering the table is enabled", /* 10% */ - C_BOOL, 10, 0, 0, &g.c_alter, NULL }, + {"alter", "if altering the table is enabled", /* 10% */ + C_BOOL, 10, 0, 0, &g.c_alter, NULL}, - { "assert_commit_timestamp", - "if assert commit_timestamp", /* 5% */ - C_BOOL, 5, 0, 0, &g.c_assert_commit_timestamp, NULL }, + {"assert_commit_timestamp", "if assert commit_timestamp", /* 5% */ + C_BOOL, 5, 0, 0, &g.c_assert_commit_timestamp, NULL}, - { "assert_read_timestamp", - "if assert read_timestamp", /* 5% */ - C_BOOL, 5, 0, 0, &g.c_assert_read_timestamp, NULL }, + {"assert_read_timestamp", "if assert read_timestamp", /* 5% */ + C_BOOL, 5, 0, 0, &g.c_assert_read_timestamp, NULL}, - { "auto_throttle", - "if LSM inserts are throttled", /* 90% */ - C_BOOL, 90, 0, 0, &g.c_auto_throttle, NULL }, + {"auto_throttle", "if LSM inserts are throttled", /* 90% */ + C_BOOL, 90, 0, 0, &g.c_auto_throttle, NULL}, - { "backups", - "if backups are enabled", /* 20% */ - C_BOOL, 20, 0, 0, &g.c_backups, NULL }, + {"backups", "if backups are enabled", /* 20% */ + C_BOOL, 20, 0, 0, &g.c_backups, NULL}, - { "bitcnt", - "number of bits for fixed-length column-store files", - 0x0, 1, 8, 8, &g.c_bitcnt, NULL }, + {"bitcnt", "number of bits for fixed-length column-store files", 0x0, 1, 8, 8, &g.c_bitcnt, NULL}, - { "bloom", - "if bloom filters are configured", /* 95% */ - C_BOOL, 95, 0, 0, &g.c_bloom, NULL }, + {"bloom", "if bloom filters are configured", /* 95% */ + C_BOOL, 95, 0, 0, &g.c_bloom, NULL}, - { "bloom_bit_count", - "number of bits per item for LSM bloom filters", - 0x0, 4, 64, 1000, &g.c_bloom_bit_count, NULL }, + {"bloom_bit_count", "number of bits per item for LSM bloom filters", 0x0, 4, 64, 1000, + &g.c_bloom_bit_count, NULL}, - { "bloom_hash_count", - "number of hash values per item for LSM bloom filters", - 0x0, 4, 32, 100, &g.c_bloom_hash_count, NULL }, + {"bloom_hash_count", "number of hash values per item for LSM bloom filters", 0x0, 4, 32, 100, + &g.c_bloom_hash_count, NULL}, - { "bloom_oldest", - "if bloom_oldest=true", /* 10% */ - C_BOOL, 10, 0, 0, &g.c_bloom_oldest, NULL }, + {"bloom_oldest", "if bloom_oldest=true", /* 10% */ + C_BOOL, 10, 0, 0, &g.c_bloom_oldest, NULL}, - { "cache", - "size of the cache in MB", - 0x0, 1, 100, 100 * 1024, &g.c_cache, NULL }, + {"cache", "size of the cache in MB", 0x0, 1, 100, 100 * 1024, &g.c_cache, NULL}, - { "cache_minimum", - "minimum size of the cache in MB", - C_IGNORE, 0, 0, 100 * 1024, &g.c_cache_minimum, NULL }, + {"cache_minimum", "minimum size of the cache in MB", C_IGNORE, 0, 0, 100 * 1024, + &g.c_cache_minimum, NULL}, - { "checkpoints", - "type of checkpoints (on | off | wiredtiger)", - C_IGNORE|C_STRING, 0, 0, 0, NULL, &g.c_checkpoint}, + {"checkpoints", "type of checkpoints (on | off | wiredtiger)", C_IGNORE | C_STRING, 0, 0, 0, NULL, + &g.c_checkpoint}, - { "checkpoint_log_size", - "MB of log to wait if wiredtiger checkpoints configured", - 0x0, 20, 200, 1024, &g.c_checkpoint_log_size, NULL}, + {"checkpoint_log_size", "MB of log to wait if wiredtiger checkpoints configured", 0x0, 20, 200, + 1024, &g.c_checkpoint_log_size, NULL}, - { "checkpoint_wait", - "seconds to wait if wiredtiger checkpoints configured", - 0x0, 5, 100, 3600, &g.c_checkpoint_wait, NULL}, + {"checkpoint_wait", "seconds to wait if wiredtiger checkpoints configured", 0x0, 5, 100, 3600, + &g.c_checkpoint_wait, NULL}, - { "checksum", - "type of checksums (on | off | uncompressed)", - C_IGNORE|C_STRING, 0, 0, 0, NULL, &g.c_checksum }, + {"checksum", "type of checksums (on | off | uncompressed)", C_IGNORE | C_STRING, 0, 0, 0, NULL, + &g.c_checksum}, - { "chunk_size", - "LSM chunk size in MB", - 0x0, 1, 10, 100, &g.c_chunk_size, NULL }, + {"chunk_size", "LSM chunk size in MB", 0x0, 1, 10, 100, &g.c_chunk_size, NULL}, - { "compaction", - "if compaction is running", /* 10% */ - C_BOOL, 10, 0, 0, &g.c_compact, NULL }, + {"compaction", "if compaction is running", /* 10% */ + C_BOOL, 10, 0, 0, &g.c_compact, NULL}, - { "compression", - "type of compression " COMPRESSION_LIST, - C_IGNORE|C_STRING, 0, 0, 0, NULL, &g.c_compression }, + {"compression", "type of compression " COMPRESSION_LIST, C_IGNORE | C_STRING, 0, 0, 0, NULL, + &g.c_compression}, - { "data_extend", - "if data files are extended", /* 5% */ - C_BOOL, 5, 0, 0, &g.c_data_extend, NULL }, + {"data_extend", "if data files are extended", /* 5% */ + C_BOOL, 5, 0, 0, &g.c_data_extend, NULL}, - { "data_source", - "data source (file | lsm | table)", - C_IGNORE|C_STRING, 0, 0, 0, NULL, &g.c_data_source }, + {"data_source", "data source (file | lsm | table)", C_IGNORE | C_STRING, 0, 0, 0, NULL, + &g.c_data_source}, - { "delete_pct", - "percent operations that are deletes", - C_IGNORE, 0, 0, 100, &g.c_delete_pct, NULL }, + {"delete_pct", "percent operations that are deletes", C_IGNORE, 0, 0, 100, &g.c_delete_pct, NULL}, - { "dictionary", - "if values are dictionary compressed", /* 20% */ - C_BOOL, 20, 0, 0, &g.c_dictionary, NULL }, + {"dictionary", "if values are dictionary compressed", /* 20% */ + C_BOOL, 20, 0, 0, &g.c_dictionary, NULL}, - { "direct_io", - "if direct I/O is configured for data objects", /* 0% */ - C_IGNORE|C_BOOL, 0, 0, 1, &g.c_direct_io, NULL }, - - { "encryption", - "type of encryption (none | rotn-7)", - C_IGNORE|C_STRING, 0, 0, 0, NULL, &g.c_encryption }, + {"direct_io", "if direct I/O is configured for data objects", /* 0% */ + C_IGNORE | C_BOOL, 0, 0, 1, &g.c_direct_io, NULL}, - { "evict_max", - "the maximum number of eviction workers", - 0x0, 0, 5, 100, &g.c_evict_max, NULL }, + {"encryption", "type of encryption (none | rotn-7)", C_IGNORE | C_STRING, 0, 0, 0, NULL, + &g.c_encryption}, - { "file_type", - "type of store to create (fix | var | row)", - C_IGNORE|C_STRING, 0, 0, 0, NULL, &g.c_file_type }, + {"evict_max", "the maximum number of eviction workers", 0x0, 0, 5, 100, &g.c_evict_max, NULL}, - { "firstfit", - "if allocation is firstfit", /* 10% */ - C_BOOL, 10, 0, 0, &g.c_firstfit, NULL }, + {"file_type", "type of store to create (fix | var | row)", C_IGNORE | C_STRING, 0, 0, 0, NULL, + &g.c_file_type}, - { "huffman_key", - "if keys are huffman encoded", /* 20% */ - C_BOOL, 20, 0, 0, &g.c_huffman_key, NULL }, + {"firstfit", "if allocation is firstfit", /* 10% */ + C_BOOL, 10, 0, 0, &g.c_firstfit, NULL}, - { "huffman_value", - "if values are huffman encoded", /* 20% */ - C_BOOL, 20, 0, 0, &g.c_huffman_value, NULL }, + {"huffman_key", "if keys are huffman encoded", /* 20% */ + C_BOOL, 20, 0, 0, &g.c_huffman_key, NULL}, - { "independent_thread_rng", - "if thread RNG space is independent", /* 75% */ - C_BOOL, 75, 0, 0, &g.c_independent_thread_rng, NULL }, + {"huffman_value", "if values are huffman encoded", /* 20% */ + C_BOOL, 20, 0, 0, &g.c_huffman_value, NULL}, - { "in_memory", - "if in-memory configured", - C_IGNORE|C_BOOL, 0, 0, 1, &g.c_in_memory, NULL }, + {"independent_thread_rng", "if thread RNG space is independent", /* 75% */ + C_BOOL, 75, 0, 0, &g.c_independent_thread_rng, NULL}, - { "insert_pct", - "percent operations that are inserts", - C_IGNORE, 0, 0, 100, &g.c_insert_pct, NULL }, + {"in_memory", "if in-memory configured", C_IGNORE | C_BOOL, 0, 0, 1, &g.c_in_memory, NULL}, - { "internal_key_truncation", - "if internal keys are truncated", /* 95% */ - C_BOOL, 95, 0, 0, &g.c_internal_key_truncation, NULL }, + {"insert_pct", "percent operations that are inserts", C_IGNORE, 0, 0, 100, &g.c_insert_pct, NULL}, - { "internal_page_max", - "maximum size of Btree internal nodes", - 0x0, 9, 17, 27, &g.c_intl_page_max, NULL }, + {"internal_key_truncation", "if internal keys are truncated", /* 95% */ + C_BOOL, 95, 0, 0, &g.c_internal_key_truncation, NULL}, - { "isolation", - "isolation level " - "(random | read-uncommitted | read-committed | snapshot)", - C_IGNORE|C_STRING, 0, 0, 0, NULL, &g.c_isolation }, + {"internal_page_max", "maximum size of Btree internal nodes", 0x0, 9, 17, 27, &g.c_intl_page_max, + NULL}, - { "key_gap", - "gap between instantiated keys on a Btree page", - 0x0, 0, 20, 20, &g.c_key_gap, NULL }, + {"isolation", + "isolation level " + "(random | read-uncommitted | read-committed | snapshot)", + C_IGNORE | C_STRING, 0, 0, 0, NULL, &g.c_isolation}, - { "key_max", - "maximum size of keys", - 0x0, 20, 128, MEGABYTE(10), &g.c_key_max, NULL }, + {"key_gap", "gap between instantiated keys on a Btree page", 0x0, 0, 20, 20, &g.c_key_gap, NULL}, - { "key_min", - "minimum size of keys", - 0x0, 10, 32, 256, &g.c_key_min, NULL }, + {"key_max", "maximum size of keys", 0x0, 20, 128, MEGABYTE(10), &g.c_key_max, NULL}, - { "leaf_page_max", - "maximum size of Btree leaf nodes", - 0x0, 9, 17, 27, &g.c_leaf_page_max, NULL }, + {"key_min", "minimum size of keys", 0x0, 10, 32, 256, &g.c_key_min, NULL}, - { "leak_memory", - "if memory should be leaked on close", - C_BOOL, 0, 0, 0, &g.c_leak_memory, NULL }, + {"leaf_page_max", "maximum size of Btree leaf nodes", 0x0, 9, 17, 27, &g.c_leaf_page_max, NULL}, - { "logging", - "if logging configured", /* 50% */ - C_BOOL, 50, 0, 0, &g.c_logging, NULL }, + {"leak_memory", "if memory should be leaked on close", C_BOOL, 0, 0, 0, &g.c_leak_memory, NULL}, - { "logging_archive", - "if log file archival configured", /* 50% */ - C_BOOL, 50, 0, 0, &g.c_logging_archive, NULL }, + {"logging", "if logging configured", /* 50% */ + C_BOOL, 50, 0, 0, &g.c_logging, NULL}, - { "logging_compression", - "type of logging compression " COMPRESSION_LIST, - C_IGNORE|C_STRING, 0, 0, 0, NULL, &g.c_logging_compression }, + {"logging_archive", "if log file archival configured", /* 50% */ + C_BOOL, 50, 0, 0, &g.c_logging_archive, NULL}, - { "logging_file_max", - "maximum log file size in KB", - 0x0, 100, 512000, 2097152, &g.c_logging_file_max, NULL }, + {"logging_compression", "type of logging compression " COMPRESSION_LIST, C_IGNORE | C_STRING, 0, + 0, 0, NULL, &g.c_logging_compression}, - { "logging_prealloc", - "if log file pre-allocation configured", /* 50% */ - C_BOOL, 50, 0, 0, &g.c_logging_prealloc, NULL }, + {"logging_file_max", "maximum log file size in KB", 0x0, 100, 512000, 2097152, + &g.c_logging_file_max, NULL}, - { "long_running_txn", - "if a long-running transaction configured", /* 0% */ - C_BOOL, 0, 0, 0, &g.c_long_running_txn, NULL }, + {"logging_prealloc", "if log file pre-allocation configured", /* 50% */ + C_BOOL, 50, 0, 0, &g.c_logging_prealloc, NULL}, - { "lsm_worker_threads", - "the number of LSM worker threads", - 0x0, 3, 4, 20, &g.c_lsm_worker_threads, NULL }, + {"long_running_txn", "if a long-running transaction configured", /* 0% */ + C_BOOL, 0, 0, 0, &g.c_long_running_txn, NULL}, - { "memory_page_max", - "maximum size of in-memory pages", - 0x0, 1, 10, 128, &g.c_memory_page_max, NULL }, + {"lsm_worker_threads", "the number of LSM worker threads", 0x0, 3, 4, 20, &g.c_lsm_worker_threads, + NULL}, - { "merge_max", - "the maximum number of chunks to include in a merge operation", - 0x0, 4, 20, 100, &g.c_merge_max, NULL }, + {"memory_page_max", "maximum size of in-memory pages", 0x0, 1, 10, 128, &g.c_memory_page_max, + NULL}, - { "mmap", - "configure for mmap operations", /* 90% */ - C_BOOL, 90, 0, 0, &g.c_mmap, NULL }, + {"merge_max", "the maximum number of chunks to include in a merge operation", 0x0, 4, 20, 100, + &g.c_merge_max, NULL}, - { "modify_pct", - "percent operations that are value modifications", - C_IGNORE, 0, 0, 100, &g.c_modify_pct, NULL }, + {"mmap", "configure for mmap operations", /* 90% */ + C_BOOL, 90, 0, 0, &g.c_mmap, NULL}, - { "ops", - "the number of modification operations done per run", - 0x0, 0, M(2), M(100), &g.c_ops, NULL }, + {"modify_pct", "percent operations that are value modifications", C_IGNORE, 0, 0, 100, + &g.c_modify_pct, NULL}, - { "prefix_compression", - "if keys are prefix compressed", /* 80% */ - C_BOOL, 80, 0, 0, &g.c_prefix_compression, NULL }, + {"ops", "the number of modification operations done per run", 0x0, 0, M(2), M(100), &g.c_ops, + NULL}, - { "prefix_compression_min", - "minimum gain before prefix compression is used", - 0x0, 0, 8, 256, &g.c_prefix_compression_min, NULL }, + {"prefix_compression", "if keys are prefix compressed", /* 80% */ + C_BOOL, 80, 0, 0, &g.c_prefix_compression, NULL}, - { "prepare", - "configure transaction prepare", /* 5% */ - C_BOOL, 5, 0, 0, &g.c_prepare, NULL }, + {"prefix_compression_min", "minimum gain before prefix compression is used", 0x0, 0, 8, 256, + &g.c_prefix_compression_min, NULL}, - { "quiet", - "quiet run (same as -q)", - C_IGNORE|C_BOOL, 0, 0, 1, &g.c_quiet, NULL }, + {"prepare", "configure transaction prepare", /* 5% */ + C_BOOL, 5, 0, 0, &g.c_prepare, NULL}, - { "read_pct", - "percent operations that are reads", - C_IGNORE, 0, 0, 100, &g.c_read_pct, NULL }, + {"quiet", "quiet run (same as -q)", C_IGNORE | C_BOOL, 0, 0, 1, &g.c_quiet, NULL}, - { "rebalance", - "rebalance testing", /* 100% */ - C_BOOL, 100, 1, 0, &g.c_rebalance, NULL }, + {"read_pct", "percent operations that are reads", C_IGNORE, 0, 0, 100, &g.c_read_pct, NULL}, - { "repeat_data_pct", - "percent duplicate values in row- or var-length column-stores", - 0x0, 0, 90, 90, &g.c_repeat_data_pct, NULL }, + {"rebalance", "rebalance testing", /* 100% */ + C_BOOL, 100, 1, 0, &g.c_rebalance, NULL}, - { "reverse", - "collate in reverse order", /* 10% */ - C_BOOL, 10, 0, 0, &g.c_reverse, NULL }, + {"repeat_data_pct", "percent duplicate values in row- or var-length column-stores", 0x0, 0, 90, + 90, &g.c_repeat_data_pct, NULL}, - { "rows", - "the number of rows to create", - 0x0, 10, M(1), M(100), &g.c_rows, NULL }, + {"reverse", "collate in reverse order", /* 10% */ + C_BOOL, 10, 0, 0, &g.c_reverse, NULL}, - { "runs", - "the number of runs", - C_IGNORE, 0, 0, UINT_MAX, &g.c_runs, NULL }, + {"rows", "the number of rows to create", 0x0, 10, M(1), M(100), &g.c_rows, NULL}, - { "salvage", - "salvage testing", /* 100% */ - C_BOOL, 100, 1, 0, &g.c_salvage, NULL }, + {"runs", "the number of runs", C_IGNORE, 0, 0, UINT_MAX, &g.c_runs, NULL}, - { "split_pct", - "page split size as a percentage of the maximum page size", - 0x0, 50, 100, 100, &g.c_split_pct, NULL }, + {"salvage", "salvage testing", /* 100% */ + C_BOOL, 100, 1, 0, &g.c_salvage, NULL}, - { "statistics", - "maintain statistics", /* 20% */ - C_BOOL, 20, 0, 0, &g.c_statistics, NULL }, + {"split_pct", "page split size as a percentage of the maximum page size", 0x0, 50, 100, 100, + &g.c_split_pct, NULL}, - { "statistics_server", - "run the statistics server thread", /* 5% */ - C_BOOL, 5, 0, 0, &g.c_statistics_server, NULL }, + {"statistics", "maintain statistics", /* 20% */ + C_BOOL, 20, 0, 0, &g.c_statistics, NULL}, - { "threads", - "the number of worker threads", - 0x0, 1, 32, 128, &g.c_threads, NULL }, + {"statistics_server", "run the statistics server thread", /* 5% */ + C_BOOL, 5, 0, 0, &g.c_statistics_server, NULL}, - { "timer", - "maximum time to run in minutes", - C_IGNORE, 0, 0, UINT_MAX, &g.c_timer, NULL }, + {"threads", "the number of worker threads", 0x0, 1, 32, 128, &g.c_threads, NULL}, - { "timing_stress_aggressive_sweep", - "stress aggressive sweep", /* 2% */ - C_BOOL, 2, 0, 0, &g.c_timing_stress_aggressive_sweep, NULL }, + {"timer", "maximum time to run in minutes", C_IGNORE, 0, 0, UINT_MAX, &g.c_timer, NULL}, - { "timing_stress_checkpoint", - "stress checkpoints", /* 2% */ - C_BOOL, 2, 0, 0, &g.c_timing_stress_checkpoint, NULL }, + {"timing_stress_aggressive_sweep", "stress aggressive sweep", /* 2% */ + C_BOOL, 2, 0, 0, &g.c_timing_stress_aggressive_sweep, NULL}, - { "timing_stress_lookaside_sweep", - "stress lookaside sweep", /* 2% */ - C_BOOL, 2, 0, 0, &g.c_timing_stress_lookaside_sweep, NULL }, + {"timing_stress_checkpoint", "stress checkpoints", /* 2% */ + C_BOOL, 2, 0, 0, &g.c_timing_stress_checkpoint, NULL}, - { "timing_stress_split_1", - "stress splits (#1)", /* 2% */ - C_BOOL, 2, 0, 0, &g.c_timing_stress_split_1, NULL }, + {"timing_stress_lookaside_sweep", "stress lookaside sweep", /* 2% */ + C_BOOL, 2, 0, 0, &g.c_timing_stress_lookaside_sweep, NULL}, - { "timing_stress_split_2", - "stress splits (#2)", /* 2% */ - C_BOOL, 2, 0, 0, &g.c_timing_stress_split_2, NULL }, + {"timing_stress_split_1", "stress splits (#1)", /* 2% */ + C_BOOL, 2, 0, 0, &g.c_timing_stress_split_1, NULL}, - { "timing_stress_split_3", - "stress splits (#3)", /* 2% */ - C_BOOL, 2, 0, 0, &g.c_timing_stress_split_3, NULL }, + {"timing_stress_split_2", "stress splits (#2)", /* 2% */ + C_BOOL, 2, 0, 0, &g.c_timing_stress_split_2, NULL}, - { "timing_stress_split_4", - "stress splits (#4)", /* 2% */ - C_BOOL, 2, 0, 0, &g.c_timing_stress_split_4, NULL }, + {"timing_stress_split_3", "stress splits (#3)", /* 2% */ + C_BOOL, 2, 0, 0, &g.c_timing_stress_split_3, NULL}, - { "timing_stress_split_5", - "stress splits (#5)", /* 2% */ - C_BOOL, 2, 0, 0, &g.c_timing_stress_split_5, NULL }, + {"timing_stress_split_4", "stress splits (#4)", /* 2% */ + C_BOOL, 2, 0, 0, &g.c_timing_stress_split_4, NULL}, - { "timing_stress_split_6", - "stress splits (#6)", /* 2% */ - C_BOOL, 2, 0, 0, &g.c_timing_stress_split_6, NULL }, + {"timing_stress_split_5", "stress splits (#5)", /* 2% */ + C_BOOL, 2, 0, 0, &g.c_timing_stress_split_5, NULL}, - { "timing_stress_split_7", - "stress splits (#7)", /* 2% */ - C_BOOL, 2, 0, 0, &g.c_timing_stress_split_7, NULL }, + {"timing_stress_split_6", "stress splits (#6)", /* 2% */ + C_BOOL, 2, 0, 0, &g.c_timing_stress_split_6, NULL}, - { "timing_stress_split_8", - "stress splits (#8)", /* 2% */ - C_BOOL, 2, 0, 0, &g.c_timing_stress_split_8, NULL }, + {"timing_stress_split_7", "stress splits (#7)", /* 2% */ + C_BOOL, 2, 0, 0, &g.c_timing_stress_split_7, NULL}, - { "transaction_timestamps", /* 70% */ - "enable transaction timestamp support", - C_BOOL, 70, 0, 0, &g.c_txn_timestamps, NULL }, + {"timing_stress_split_8", "stress splits (#8)", /* 2% */ + C_BOOL, 2, 0, 0, &g.c_timing_stress_split_8, NULL}, - { "transaction-frequency", - "percent operations done inside an explicit transaction", - 0x0, 1, 100, 100, &g.c_txn_freq, NULL }, + {"transaction_timestamps", /* 70% */ + "enable transaction timestamp support", C_BOOL, 70, 0, 0, &g.c_txn_timestamps, NULL}, - { "truncate", /* 100% */ - "enable truncation", - C_BOOL, 100, 0, 0, &g.c_truncate, NULL }, + {"transaction-frequency", "percent operations done inside an explicit transaction", 0x0, 1, 100, + 100, &g.c_txn_freq, NULL}, - { "value_max", - "maximum size of values", - 0x0, 32, 4096, MEGABYTE(10), &g.c_value_max, NULL }, + {"truncate", /* 100% */ + "enable truncation", C_BOOL, 100, 0, 0, &g.c_truncate, NULL}, - { "value_min", - "minimum size of values", - 0x0, 0, 20, 4096, &g.c_value_min, NULL }, + {"value_max", "maximum size of values", 0x0, 32, 4096, MEGABYTE(10), &g.c_value_max, NULL}, - { "verify", - "to regularly verify during a run", /* 100% */ - C_BOOL, 100, 1, 0, &g.c_verify, NULL }, + {"value_min", "minimum size of values", 0x0, 0, 20, 4096, &g.c_value_min, NULL}, - { "wiredtiger_config", - "configuration string used to wiredtiger_open", - C_IGNORE|C_STRING, 0, 0, 0, NULL, &g.c_config_open }, + {"verify", "to regularly verify during a run", /* 100% */ + C_BOOL, 100, 1, 0, &g.c_verify, NULL}, - { "write_pct", - "percent operations that are value updates", - C_IGNORE, 0, 0, 100, &g.c_write_pct, NULL }, + {"wiredtiger_config", "configuration string used to wiredtiger_open", C_IGNORE | C_STRING, 0, 0, + 0, NULL, &g.c_config_open}, - { NULL, NULL, 0x0, 0, 0, 0, NULL, NULL } -}; + {"write_pct", "percent operations that are value updates", C_IGNORE, 0, 0, 100, &g.c_write_pct, + NULL}, + + {NULL, NULL, 0x0, 0, 0, 0, NULL, NULL}}; diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 9d97a2d0428..e90bbf86998 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -30,232 +30,225 @@ #include <signal.h> -#define EXTPATH "../../ext/" /* Extensions path */ +#define EXTPATH "../../ext/" /* Extensions path */ -#define LZ4_PATH \ - EXTPATH "compressors/lz4/.libs/libwiredtiger_lz4.so" -#define SNAPPY_PATH \ - EXTPATH "compressors/snappy/.libs/libwiredtiger_snappy.so" -#define ZLIB_PATH \ - EXTPATH "compressors/zlib/.libs/libwiredtiger_zlib.so" -#define ZSTD_PATH \ - EXTPATH "compressors/zstd/.libs/libwiredtiger_zstd.so" +#define LZ4_PATH EXTPATH "compressors/lz4/.libs/libwiredtiger_lz4.so" +#define SNAPPY_PATH EXTPATH "compressors/snappy/.libs/libwiredtiger_snappy.so" +#define ZLIB_PATH EXTPATH "compressors/zlib/.libs/libwiredtiger_zlib.so" +#define ZSTD_PATH EXTPATH "compressors/zstd/.libs/libwiredtiger_zstd.so" -#define REVERSE_PATH \ - EXTPATH "collators/reverse/.libs/libwiredtiger_reverse_collator.so" +#define REVERSE_PATH EXTPATH "collators/reverse/.libs/libwiredtiger_reverse_collator.so" -#define ROTN_PATH \ - EXTPATH "encryptors/rotn/.libs/libwiredtiger_rotn.so" +#define ROTN_PATH EXTPATH "encryptors/rotn/.libs/libwiredtiger_rotn.so" -#undef M -#define M(v) ((v) * WT_MILLION) /* Million */ -#undef KILOBYTE -#define KILOBYTE(v) ((v) * WT_KILOBYTE) -#undef MEGABYTE -#define MEGABYTE(v) ((v) * WT_MEGABYTE) +#undef M +#define M(v) ((v)*WT_MILLION) /* Million */ +#undef KILOBYTE +#define KILOBYTE(v) ((v)*WT_KILOBYTE) +#undef MEGABYTE +#define MEGABYTE(v) ((v)*WT_MEGABYTE) -#define WT_NAME "wt" /* Object name */ +#define WT_NAME "wt" /* Object name */ -#define DATASOURCE(v) (strcmp(v, g.c_data_source) == 0 ? 1 : 0) -#define SINGLETHREADED (g.c_threads == 1) +#define DATASOURCE(v) (strcmp(v, g.c_data_source) == 0 ? 1 : 0) +#define SINGLETHREADED (g.c_threads == 1) -#define FORMAT_OPERATION_REPS 3 /* 3 thread operations sets */ +#define FORMAT_OPERATION_REPS 3 /* 3 thread operations sets */ -#define MAX_MODIFY_ENTRIES 5 /* maximum change vectors */ +#define MAX_MODIFY_ENTRIES 5 /* maximum change vectors */ typedef struct { - char *home; /* Home directory */ - char *home_backup; /* Hot-backup directory */ - char *home_backup_init; /* Initialize backup command */ - char *home_config; /* Run CONFIG file path */ - char *home_init; /* Initialize home command */ - char *home_log; /* Operation log file path */ - char *home_pagedump; /* Page dump filename */ - char *home_rand; /* RNG log file path */ - char *home_salvage_copy; /* Salvage copy command */ - char *home_stats; /* Statistics file path */ - - char wiredtiger_open_config[8 * 1024]; /* Database open config */ - - WT_CONNECTION *wts_conn; - WT_EXTENSION_API *wt_api; - - bool rand_log_stop; /* Logging turned off */ - FILE *randfp; /* Random number log */ - - uint32_t run_cnt; /* Run counter */ - - bool logging; /* log operations */ - FILE *logfp; /* log file */ - - bool replay; /* Replaying a run. */ - bool workers_finished; /* Operations completed */ - - pthread_rwlock_t backup_lock; /* Backup running */ - - WT_RAND_STATE rnd; /* Global RNG state */ - - /* - * Prepare will return an error if the prepare timestamp is less than - * any active read timestamp. Lock across allocating prepare and read - * timestamps. - * - * We get the last committed timestamp periodically in order to update - * the oldest timestamp, that requires locking out transactional ops - * that set a timestamp. - */ - pthread_rwlock_t ts_lock; - - uint64_t timestamp; /* Counter for timestamps */ - - uint64_t truncate_cnt; /* Counter for truncation */ - - /* - * We have a list of records that are appended, but not yet "resolved", - * that is, we haven't yet incremented the g.rows value to reflect the - * new records. - */ - uint64_t *append; /* Appended records */ - size_t append_max; /* Maximum unresolved records */ - size_t append_cnt; /* Current unresolved records */ - pthread_rwlock_t append_lock; /* Single-thread resolution */ - - pthread_rwlock_t death_lock; /* Single-thread failure */ - - char *uri; /* Object name */ - - char *config_open; /* Command-line configuration */ - - uint32_t c_abort; /* Config values */ - uint32_t c_alter; - uint32_t c_assert_commit_timestamp; - uint32_t c_assert_read_timestamp; - uint32_t c_auto_throttle; - uint32_t c_backups; - uint32_t c_bitcnt; - uint32_t c_bloom; - uint32_t c_bloom_bit_count; - uint32_t c_bloom_hash_count; - uint32_t c_bloom_oldest; - uint32_t c_cache; - uint32_t c_cache_minimum; - char *c_checkpoint; - uint32_t c_checkpoint_log_size; - uint32_t c_checkpoint_wait; - char *c_checksum; - uint32_t c_chunk_size; - uint32_t c_compact; - char *c_compression; - char *c_config_open; - uint32_t c_data_extend; - char *c_data_source; - uint32_t c_delete_pct; - uint32_t c_dictionary; - uint32_t c_direct_io; - char *c_encryption; - uint32_t c_evict_max; - char *c_file_type; - uint32_t c_firstfit; - uint32_t c_huffman_key; - uint32_t c_huffman_value; - uint32_t c_in_memory; - uint32_t c_independent_thread_rng; - uint32_t c_insert_pct; - uint32_t c_internal_key_truncation; - uint32_t c_intl_page_max; - char *c_isolation; - uint32_t c_key_gap; - uint32_t c_key_max; - uint32_t c_key_min; - uint32_t c_leaf_page_max; - uint32_t c_leak_memory; - uint32_t c_logging; - uint32_t c_logging_archive; - char *c_logging_compression; - uint32_t c_logging_file_max; - uint32_t c_logging_prealloc; - uint32_t c_long_running_txn; - uint32_t c_lsm_worker_threads; - uint32_t c_memory_page_max; - uint32_t c_merge_max; - uint32_t c_mmap; - uint32_t c_modify_pct; - uint32_t c_ops; - uint32_t c_prefix_compression; - uint32_t c_prefix_compression_min; - uint32_t c_prepare; - uint32_t c_quiet; - uint32_t c_read_pct; - uint32_t c_rebalance; - uint32_t c_repeat_data_pct; - uint32_t c_reverse; - uint32_t c_rows; - uint32_t c_runs; - uint32_t c_salvage; - uint32_t c_split_pct; - uint32_t c_statistics; - uint32_t c_statistics_server; - uint32_t c_threads; - uint32_t c_timer; - uint32_t c_timing_stress_aggressive_sweep; - uint32_t c_timing_stress_checkpoint; - uint32_t c_timing_stress_lookaside_sweep; - uint32_t c_timing_stress_split_1; - uint32_t c_timing_stress_split_2; - uint32_t c_timing_stress_split_3; - uint32_t c_timing_stress_split_4; - uint32_t c_timing_stress_split_5; - uint32_t c_timing_stress_split_6; - uint32_t c_timing_stress_split_7; - uint32_t c_timing_stress_split_8; - uint32_t c_truncate; - uint32_t c_txn_freq; - uint32_t c_txn_timestamps; - uint32_t c_value_max; - uint32_t c_value_min; - uint32_t c_verify; - uint32_t c_write_pct; - -#define FIX 1 -#define ROW 2 -#define VAR 3 - u_int type; /* File type's flag value */ - -#define CHECKPOINT_OFF 1 -#define CHECKPOINT_ON 2 -#define CHECKPOINT_WIREDTIGER 3 - u_int c_checkpoint_flag; /* Checkpoint flag value */ - -#define CHECKSUM_OFF 1 -#define CHECKSUM_ON 2 -#define CHECKSUM_UNCOMPRESSED 3 - u_int c_checksum_flag; /* Checksum flag value */ - -#define COMPRESS_NONE 1 -#define COMPRESS_LZ4 2 -#define COMPRESS_SNAPPY 3 -#define COMPRESS_ZLIB 4 -#define COMPRESS_ZSTD 5 - u_int c_compression_flag; /* Compression flag value */ - u_int c_logging_compression_flag; /* Log compression flag value */ - -#define ENCRYPT_NONE 1 -#define ENCRYPT_ROTN_7 2 - u_int c_encryption_flag; /* Encryption flag value */ - -#define ISOLATION_RANDOM 1 -#define ISOLATION_READ_UNCOMMITTED 2 -#define ISOLATION_READ_COMMITTED 3 -#define ISOLATION_SNAPSHOT 4 - u_int c_isolation_flag; /* Isolation flag value */ - - uint32_t intl_page_max; /* Maximum page sizes */ - uint32_t leaf_page_max; - - uint64_t key_cnt; /* Keys loaded so far */ - uint64_t rows; /* Total rows */ - - uint32_t key_rand_len[1031]; /* Key lengths */ + char *home; /* Home directory */ + char *home_backup; /* Hot-backup directory */ + char *home_backup_init; /* Initialize backup command */ + char *home_config; /* Run CONFIG file path */ + char *home_init; /* Initialize home command */ + char *home_log; /* Operation log file path */ + char *home_pagedump; /* Page dump filename */ + char *home_rand; /* RNG log file path */ + char *home_salvage_copy; /* Salvage copy command */ + char *home_stats; /* Statistics file path */ + + char wiredtiger_open_config[8 * 1024]; /* Database open config */ + + WT_CONNECTION *wts_conn; + WT_EXTENSION_API *wt_api; + + bool rand_log_stop; /* Logging turned off */ + FILE *randfp; /* Random number log */ + + uint32_t run_cnt; /* Run counter */ + + bool logging; /* log operations */ + FILE *logfp; /* log file */ + + bool replay; /* Replaying a run. */ + bool workers_finished; /* Operations completed */ + + pthread_rwlock_t backup_lock; /* Backup running */ + + WT_RAND_STATE rnd; /* Global RNG state */ + + /* + * Prepare will return an error if the prepare timestamp is less than + * any active read timestamp. Lock across allocating prepare and read + * timestamps. + * + * We get the last committed timestamp periodically in order to update + * the oldest timestamp, that requires locking out transactional ops + * that set a timestamp. + */ + pthread_rwlock_t ts_lock; + + uint64_t timestamp; /* Counter for timestamps */ + + uint64_t truncate_cnt; /* Counter for truncation */ + + /* + * We have a list of records that are appended, but not yet "resolved", that is, we haven't yet + * incremented the g.rows value to reflect the new records. + */ + uint64_t *append; /* Appended records */ + size_t append_max; /* Maximum unresolved records */ + size_t append_cnt; /* Current unresolved records */ + pthread_rwlock_t append_lock; /* Single-thread resolution */ + + pthread_rwlock_t death_lock; /* Single-thread failure */ + + char *uri; /* Object name */ + + char *config_open; /* Command-line configuration */ + + uint32_t c_abort; /* Config values */ + uint32_t c_alter; + uint32_t c_assert_commit_timestamp; + uint32_t c_assert_read_timestamp; + uint32_t c_auto_throttle; + uint32_t c_backups; + uint32_t c_bitcnt; + uint32_t c_bloom; + uint32_t c_bloom_bit_count; + uint32_t c_bloom_hash_count; + uint32_t c_bloom_oldest; + uint32_t c_cache; + uint32_t c_cache_minimum; + char *c_checkpoint; + uint32_t c_checkpoint_log_size; + uint32_t c_checkpoint_wait; + char *c_checksum; + uint32_t c_chunk_size; + uint32_t c_compact; + char *c_compression; + char *c_config_open; + uint32_t c_data_extend; + char *c_data_source; + uint32_t c_delete_pct; + uint32_t c_dictionary; + uint32_t c_direct_io; + char *c_encryption; + uint32_t c_evict_max; + char *c_file_type; + uint32_t c_firstfit; + uint32_t c_huffman_key; + uint32_t c_huffman_value; + uint32_t c_in_memory; + uint32_t c_independent_thread_rng; + uint32_t c_insert_pct; + uint32_t c_internal_key_truncation; + uint32_t c_intl_page_max; + char *c_isolation; + uint32_t c_key_gap; + uint32_t c_key_max; + uint32_t c_key_min; + uint32_t c_leaf_page_max; + uint32_t c_leak_memory; + uint32_t c_logging; + uint32_t c_logging_archive; + char *c_logging_compression; + uint32_t c_logging_file_max; + uint32_t c_logging_prealloc; + uint32_t c_long_running_txn; + uint32_t c_lsm_worker_threads; + uint32_t c_memory_page_max; + uint32_t c_merge_max; + uint32_t c_mmap; + uint32_t c_modify_pct; + uint32_t c_ops; + uint32_t c_prefix_compression; + uint32_t c_prefix_compression_min; + uint32_t c_prepare; + uint32_t c_quiet; + uint32_t c_read_pct; + uint32_t c_rebalance; + uint32_t c_repeat_data_pct; + uint32_t c_reverse; + uint32_t c_rows; + uint32_t c_runs; + uint32_t c_salvage; + uint32_t c_split_pct; + uint32_t c_statistics; + uint32_t c_statistics_server; + uint32_t c_threads; + uint32_t c_timer; + uint32_t c_timing_stress_aggressive_sweep; + uint32_t c_timing_stress_checkpoint; + uint32_t c_timing_stress_lookaside_sweep; + uint32_t c_timing_stress_split_1; + uint32_t c_timing_stress_split_2; + uint32_t c_timing_stress_split_3; + uint32_t c_timing_stress_split_4; + uint32_t c_timing_stress_split_5; + uint32_t c_timing_stress_split_6; + uint32_t c_timing_stress_split_7; + uint32_t c_timing_stress_split_8; + uint32_t c_truncate; + uint32_t c_txn_freq; + uint32_t c_txn_timestamps; + uint32_t c_value_max; + uint32_t c_value_min; + uint32_t c_verify; + uint32_t c_write_pct; + +#define FIX 1 +#define ROW 2 +#define VAR 3 + u_int type; /* File type's flag value */ + +#define CHECKPOINT_OFF 1 +#define CHECKPOINT_ON 2 +#define CHECKPOINT_WIREDTIGER 3 + u_int c_checkpoint_flag; /* Checkpoint flag value */ + +#define CHECKSUM_OFF 1 +#define CHECKSUM_ON 2 +#define CHECKSUM_UNCOMPRESSED 3 + u_int c_checksum_flag; /* Checksum flag value */ + +#define COMPRESS_NONE 1 +#define COMPRESS_LZ4 2 +#define COMPRESS_SNAPPY 3 +#define COMPRESS_ZLIB 4 +#define COMPRESS_ZSTD 5 + u_int c_compression_flag; /* Compression flag value */ + u_int c_logging_compression_flag; /* Log compression flag value */ + +#define ENCRYPT_NONE 1 +#define ENCRYPT_ROTN_7 2 + u_int c_encryption_flag; /* Encryption flag value */ + +#define ISOLATION_RANDOM 1 +#define ISOLATION_READ_UNCOMMITTED 2 +#define ISOLATION_READ_COMMITTED 3 +#define ISOLATION_SNAPSHOT 4 + u_int c_isolation_flag; /* Isolation flag value */ + + uint32_t intl_page_max; /* Maximum page sizes */ + uint32_t leaf_page_max; + + uint64_t key_cnt; /* Keys loaded so far */ + uint64_t rows; /* Total rows */ + + uint32_t key_rand_len[1031]; /* Key lengths */ } GLOBAL; extern GLOBAL g; @@ -266,118 +259,118 @@ typedef enum { INSERT, MODIFY, READ, REMOVE, TRUNCATE, UPDATE } thread_op; typedef enum { NEXT, PREV, SEARCH, SEARCH_NEAR } read_operation; typedef struct { - thread_op op; /* Operation */ - uint64_t opid; /* Operation ID */ + thread_op op; /* Operation */ + uint64_t opid; /* Operation ID */ - uint64_t keyno; /* Row number */ + uint64_t keyno; /* Row number */ - uint64_t ts; /* Read/commit timestamp */ - bool repeatable; /* Operation can be repeated */ + uint64_t ts; /* Read/commit timestamp */ + bool repeatable; /* Operation can be repeated */ - uint64_t last; /* Inclusive end of a truncate range */ + uint64_t last; /* Inclusive end of a truncate range */ - void *kdata; /* If an insert, the generated key */ - size_t ksize; - size_t kmemsize; + void *kdata; /* If an insert, the generated key */ + size_t ksize; + size_t kmemsize; - void *vdata; /* If not a delete, the value */ - size_t vsize; - size_t vmemsize; + void *vdata; /* If not a delete, the value */ + size_t vsize; + size_t vmemsize; } SNAP_OPS; typedef struct { - int id; /* simple thread ID */ - wt_thread_t tid; /* thread ID */ - - WT_RAND_STATE rnd; /* thread RNG state */ - - volatile bool quit; /* thread should quit */ - - uint64_t ops; /* total operations */ - uint64_t commit; /* operation counts */ - uint64_t insert; - uint64_t prepare; - uint64_t remove; - uint64_t rollback; - uint64_t search; - uint64_t truncate; - uint64_t update; - - WT_SESSION *session; /* WiredTiger session */ - WT_CURSOR *cursor; /* WiredTiger cursor */ - - uint64_t keyno; /* key */ - WT_ITEM *key, _key; /* key, value */ - WT_ITEM *value, _value; - - uint64_t last; /* truncate range */ - WT_ITEM *lastkey, _lastkey; - - bool repeatable_reads; /* if read ops repeatable */ - bool repeatable_wrap; /* if circular buffer wrapped */ - uint64_t opid; /* Operation ID */ - uint64_t read_ts; /* read timestamp */ - uint64_t commit_ts; /* commit timestamp */ - SNAP_OPS *snap, *snap_first, snap_list[512]; - - WT_ITEM *tbuf, _tbuf; /* temporary buffer */ - -#define TINFO_RUNNING 1 /* Running */ -#define TINFO_COMPLETE 2 /* Finished */ -#define TINFO_JOINED 3 /* Resolved */ - volatile int state; /* state */ + int id; /* simple thread ID */ + wt_thread_t tid; /* thread ID */ + + WT_RAND_STATE rnd; /* thread RNG state */ + + volatile bool quit; /* thread should quit */ + + uint64_t ops; /* total operations */ + uint64_t commit; /* operation counts */ + uint64_t insert; + uint64_t prepare; + uint64_t remove; + uint64_t rollback; + uint64_t search; + uint64_t truncate; + uint64_t update; + + WT_SESSION *session; /* WiredTiger session */ + WT_CURSOR *cursor; /* WiredTiger cursor */ + + uint64_t keyno; /* key */ + WT_ITEM *key, _key; /* key, value */ + WT_ITEM *value, _value; + + uint64_t last; /* truncate range */ + WT_ITEM *lastkey, _lastkey; + + bool repeatable_reads; /* if read ops repeatable */ + bool repeatable_wrap; /* if circular buffer wrapped */ + uint64_t opid; /* Operation ID */ + uint64_t read_ts; /* read timestamp */ + uint64_t commit_ts; /* commit timestamp */ + SNAP_OPS *snap, *snap_first, snap_list[512]; + + WT_ITEM *tbuf, _tbuf; /* temporary buffer */ + +#define TINFO_RUNNING 1 /* Running */ +#define TINFO_COMPLETE 2 /* Finished */ +#define TINFO_JOINED 3 /* Resolved */ + volatile int state; /* state */ } TINFO; extern TINFO **tinfo_list; -#define logop(wt_session, fmt, ...) do { \ - if (g.logging) \ - testutil_check(g.wt_api->msg_printf( \ - g.wt_api, wt_session, fmt, __VA_ARGS__)); \ -} while (0) +#define logop(wt_session, fmt, ...) \ + do { \ + if (g.logging) \ + testutil_check(g.wt_api->msg_printf(g.wt_api, wt_session, fmt, __VA_ARGS__)); \ + } while (0) WT_THREAD_RET alter(void *); WT_THREAD_RET backup(void *); WT_THREAD_RET checkpoint(void *); WT_THREAD_RET compact(void *); -void config_clear(void); -void config_error(void); -void config_file(const char *); -void config_print(bool); -void config_setup(void); -void config_single(const char *, bool); -void fclose_and_clear(FILE **); -void key_gen(WT_ITEM *, uint64_t); -void key_gen_init(WT_ITEM *); -void key_gen_insert(WT_RAND_STATE *, WT_ITEM *, uint64_t); -void key_gen_teardown(WT_ITEM *); -void key_init(void); +void config_clear(void); +void config_error(void); +void config_file(const char *); +void config_print(bool); +void config_setup(void); +void config_single(const char *, bool); +void fclose_and_clear(FILE **); +void key_gen(WT_ITEM *, uint64_t); +void key_gen_init(WT_ITEM *); +void key_gen_insert(WT_RAND_STATE *, WT_ITEM *, uint64_t); +void key_gen_teardown(WT_ITEM *); +void key_init(void); WT_THREAD_RET lrt(void *); -void path_setup(const char *); -int read_row_worker(WT_CURSOR *, uint64_t, WT_ITEM *, WT_ITEM *, bool); +void path_setup(const char *); +int read_row_worker(WT_CURSOR *, uint64_t, WT_ITEM *, WT_ITEM *, bool); uint32_t rng(WT_RAND_STATE *); -void snap_init(TINFO *, uint64_t, bool); -void snap_repeat_single(WT_CURSOR *, TINFO *); -int snap_repeat_txn(WT_CURSOR *, TINFO *); -void snap_repeat_update(TINFO *, bool); -void snap_track(TINFO *, thread_op); +void snap_init(TINFO *, uint64_t, bool); +void snap_repeat_single(WT_CURSOR *, TINFO *); +int snap_repeat_txn(WT_CURSOR *, TINFO *); +void snap_repeat_update(TINFO *, bool); +void snap_track(TINFO *, thread_op); WT_THREAD_RET timestamp(void *); -void track(const char *, uint64_t, TINFO *); -void val_gen(WT_RAND_STATE *, WT_ITEM *, uint64_t); -void val_gen_init(WT_ITEM *); -void val_gen_teardown(WT_ITEM *); -void val_init(void); -void val_teardown(void); -void wts_close(void); -void wts_dump(const char *, bool); -void wts_init(void); -void wts_load(void); -void wts_open(const char *, bool, WT_CONNECTION **); -void wts_ops(bool); -void wts_read_scan(void); -void wts_rebalance(void); -void wts_reopen(void); -void wts_salvage(void); -void wts_stats(void); -void wts_verify(const char *); +void track(const char *, uint64_t, TINFO *); +void val_gen(WT_RAND_STATE *, WT_ITEM *, uint64_t); +void val_gen_init(WT_ITEM *); +void val_gen_teardown(WT_ITEM *); +void val_init(void); +void val_teardown(void); +void wts_close(void); +void wts_dump(const char *, bool); +void wts_init(void); +void wts_load(void); +void wts_open(const char *, bool, WT_CONNECTION **); +void wts_ops(bool); +void wts_read_scan(void); +void wts_rebalance(void); +void wts_reopen(void); +void wts_salvage(void); +void wts_stats(void); +void wts_verify(const char *); #include "format.i" diff --git a/src/third_party/wiredtiger/test/format/format.i b/src/third_party/wiredtiger/test/format/format.i index a359a5c3492..fe3711bb1a6 100644 --- a/src/third_party/wiredtiger/test/format/format.i +++ b/src/third_party/wiredtiger/test/format/format.i @@ -28,99 +28,95 @@ /* * read_op -- - * Perform a read operation, waiting out prepare conflicts. + * Perform a read operation, waiting out prepare conflicts. */ static inline int read_op(WT_CURSOR *cursor, read_operation op, int *exactp) { - WT_DECL_RET; + WT_DECL_RET; - /* - * Read operations wait out prepare-conflicts. (As part of the snapshot - * isolation checks, we repeat reads that succeeded before, they should - * be repeatable.) - */ - switch (op) { - case NEXT: - while ((ret = cursor->next(cursor)) == WT_PREPARE_CONFLICT) - __wt_yield(); - break; - case PREV: - while ((ret = cursor->prev(cursor)) == WT_PREPARE_CONFLICT) - __wt_yield(); - break; - case SEARCH: - while ((ret = cursor->search(cursor)) == WT_PREPARE_CONFLICT) - __wt_yield(); - break; - case SEARCH_NEAR: - while ((ret = - cursor->search_near(cursor, exactp)) == WT_PREPARE_CONFLICT) - __wt_yield(); - break; - } - return (ret); + /* + * Read operations wait out prepare-conflicts. (As part of the snapshot isolation checks, we + * repeat reads that succeeded before, they should be repeatable.) + */ + switch (op) { + case NEXT: + while ((ret = cursor->next(cursor)) == WT_PREPARE_CONFLICT) + __wt_yield(); + break; + case PREV: + while ((ret = cursor->prev(cursor)) == WT_PREPARE_CONFLICT) + __wt_yield(); + break; + case SEARCH: + while ((ret = cursor->search(cursor)) == WT_PREPARE_CONFLICT) + __wt_yield(); + break; + case SEARCH_NEAR: + while ((ret = cursor->search_near(cursor, exactp)) == WT_PREPARE_CONFLICT) + __wt_yield(); + break; + } + return (ret); } /* * mmrand -- - * Return a random value between a min/max pair, inclusive. + * Return a random value between a min/max pair, inclusive. */ static inline uint32_t mmrand(WT_RAND_STATE *rnd, u_int min, u_int max) { - uint32_t v; - u_int range; + uint32_t v; + u_int range; - /* - * Test runs with small row counts can easily pass a max of 0 (for - * example, "g.rows / 20"). Avoid the problem. - */ - if (max <= min) - return (min); + /* + * Test runs with small row counts can easily pass a max of 0 (for example, "g.rows / 20"). + * Avoid the problem. + */ + if (max <= min) + return (min); - v = rng(rnd); - range = (max - min) + 1; - v %= range; - v += min; - return (v); + v = rng(rnd); + range = (max - min) + 1; + v %= range; + v += min; + return (v); } static inline void random_sleep(WT_RAND_STATE *rnd, u_int max_seconds) { - uint64_t i, micro_seconds; + uint64_t i, micro_seconds; - /* - * We need a fast way to choose a sleep time. We want to sleep a short - * period most of the time, but occasionally wait longer. Divide the - * maximum period of time into 10 buckets (where bucket 0 doesn't sleep - * at all), and roll dice, advancing to the next bucket 50% of the time. - * That means we'll hit the maximum roughly every 1K calls. - */ - for (i = 0;;) - if (rng(rnd) & 0x1 || ++i > 9) - break; + /* + * We need a fast way to choose a sleep time. We want to sleep a short period most of the time, + * but occasionally wait longer. Divide the maximum period of time into 10 buckets (where bucket + * 0 doesn't sleep at all), and roll dice, advancing to the next bucket 50% of the time. That + * means we'll hit the maximum roughly every 1K calls. + */ + for (i = 0;;) + if (rng(rnd) & 0x1 || ++i > 9) + break; - if (i == 0) - __wt_yield(); - else { - micro_seconds = (uint64_t)max_seconds * WT_MILLION; - __wt_sleep(0, i * (micro_seconds / 10)); - } + if (i == 0) + __wt_yield(); + else { + micro_seconds = (uint64_t)max_seconds * WT_MILLION; + __wt_sleep(0, i * (micro_seconds / 10)); + } } static inline void wiredtiger_begin_transaction(WT_SESSION *session, const char *config) { - WT_DECL_RET; + WT_DECL_RET; - /* - * Keep trying to start a new transaction if it's timing out. - * There are no resources pinned, it should succeed eventually. - */ - while ((ret = - session->begin_transaction(session, config)) == WT_CACHE_FULL) - __wt_yield(); - testutil_check(ret); + /* + * Keep trying to start a new transaction if it's timing out. There are no resources pinned, it + * should succeed eventually. + */ + while ((ret = session->begin_transaction(session, config)) == WT_CACHE_FULL) + __wt_yield(); + testutil_check(ret); } diff --git a/src/third_party/wiredtiger/test/format/lrt.c b/src/third_party/wiredtiger/test/format/lrt.c index 58adfc11216..472a8a0d877 100644 --- a/src/third_party/wiredtiger/test/format/lrt.c +++ b/src/third_party/wiredtiger/test/format/lrt.c @@ -30,166 +30,146 @@ /* * lrt -- - * Start a long-running transaction. + * Start a long-running transaction. */ WT_THREAD_RET lrt(void *arg) { - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_ITEM key, value; - WT_SESSION *session; - size_t buf_len, buf_size; - uint64_t keyno, saved_keyno; - uint8_t bitfield; - u_int period; - int pinned, ret; - void *buf; - - (void)(arg); /* Unused parameter */ - - saved_keyno = 0; /* [-Werror=maybe-uninitialized] */ - - key_gen_init(&key); - val_gen_init(&value); - - buf = NULL; - buf_len = buf_size = 0; - - /* Open a session and cursor. */ - conn = g.wts_conn; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - /* - * open_cursor can return EBUSY if concurrent with a metadata - * operation, retry in that case. - */ - while ((ret = session->open_cursor( - session, g.uri, NULL, NULL, &cursor)) == EBUSY) - __wt_yield(); - testutil_check(ret); - - for (pinned = 0;;) { - if (pinned) { - /* Re-read the record at the end of the table. */ - while ((ret = read_row_worker(cursor, - saved_keyno, &key, &value, false)) == WT_ROLLBACK) - ; - if (ret != 0) - testutil_die(ret, - "read_row_worker %" PRIu64, saved_keyno); - - /* Compare the previous value with the current one. */ - if (g.type == FIX) { - ret = cursor->get_value(cursor, &bitfield); - value.data = &bitfield; - value.size = 1; - } else - ret = cursor->get_value(cursor, &value); - if (ret != 0) - testutil_die(ret, - "cursor.get_value: %" PRIu64, saved_keyno); - - if (buf_size != value.size || - memcmp(buf, value.data, value.size) != 0) - testutil_die(0, "mismatched start/stop values"); - - /* End the transaction. */ - testutil_check( - session->commit_transaction(session, NULL)); - - /* Reset the cursor, releasing our pin. */ - testutil_check(cursor->reset(cursor)); - pinned = 0; - } else { - /* - * Test named snapshots: create a snapshot, wait to - * give the transaction state time to move forward, - * then start a transaction with the named snapshot, - * drop it, then commit the transaction. This exercises - * most of the named snapshot logic under load. - */ - testutil_check(session->snapshot(session, "name=test")); - __wt_sleep(1, 0); - wiredtiger_begin_transaction(session, "snapshot=test"); - testutil_check(session->snapshot( - session, "drop=(all)")); - testutil_check(session->commit_transaction( - session, NULL)); - - /* - * Begin transaction: without an explicit transaction, - * the snapshot is only kept around while a cursor is - * positioned. As soon as the cursor loses its position - * a new snapshot will be allocated. - */ - while ((ret = session->begin_transaction( - session, "snapshot=snapshot")) == WT_CACHE_FULL) - ; - testutil_check(ret); - - /* Read a record at the end of the table. */ - do { - saved_keyno = mmrand(NULL, - (u_int)(g.key_cnt - g.key_cnt / 10), - (u_int)g.key_cnt); - while ((ret = read_row_worker(cursor, - saved_keyno, - &key, &value, false)) == WT_ROLLBACK) - ; - } while (ret == WT_NOTFOUND); - if (ret != 0) - testutil_die(ret, - "read_row_worker %" PRIu64, saved_keyno); - - /* Copy the cursor's value. */ - if (g.type == FIX) { - ret = cursor->get_value(cursor, &bitfield); - value.data = &bitfield; - value.size = 1; - } else - ret = cursor->get_value(cursor, &value); - if (ret != 0) - testutil_die(ret, - "cursor.get_value: %" PRIu64, saved_keyno); - if (buf_len < value.size) - buf = drealloc(buf, buf_len = value.size); - memcpy(buf, value.data, buf_size = value.size); - - /* - * Move the cursor to an early record in the table, - * hopefully allowing the page with the record just - * retrieved to be evicted from memory. - */ - do { - keyno = mmrand(NULL, 1, (u_int)g.key_cnt / 5); - while ((ret = read_row_worker(cursor, - keyno, &key, &value, false)) == WT_ROLLBACK) - ; - } while (ret == WT_NOTFOUND); - if (ret != 0) - testutil_die(ret, - "read_row_worker %" PRIu64, keyno); - - pinned = 1; - } - - /* Sleep for some number of seconds. */ - period = mmrand(NULL, 1, 10); - - /* Sleep for short periods so we don't make the run wait. */ - while (period > 0 && !g.workers_finished) { - --period; - __wt_sleep(1, 0); - } - if (g.workers_finished) - break; - } - - testutil_check(session->close(session, NULL)); - - key_gen_teardown(&key); - val_gen_teardown(&value); - free(buf); - - return (WT_THREAD_RET_VALUE); + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_ITEM key, value; + WT_SESSION *session; + size_t buf_len, buf_size; + uint64_t keyno, saved_keyno; + uint8_t bitfield; + u_int period; + int pinned, ret; + void *buf; + + (void)(arg); /* Unused parameter */ + + saved_keyno = 0; /* [-Werror=maybe-uninitialized] */ + + key_gen_init(&key); + val_gen_init(&value); + + buf = NULL; + buf_len = buf_size = 0; + + /* Open a session and cursor. */ + conn = g.wts_conn; + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + /* + * open_cursor can return EBUSY if concurrent with a metadata operation, retry in that case. + */ + while ((ret = session->open_cursor(session, g.uri, NULL, NULL, &cursor)) == EBUSY) + __wt_yield(); + testutil_check(ret); + + for (pinned = 0;;) { + if (pinned) { + /* Re-read the record at the end of the table. */ + while ((ret = read_row_worker(cursor, saved_keyno, &key, &value, false)) == WT_ROLLBACK) + ; + if (ret != 0) + testutil_die(ret, "read_row_worker %" PRIu64, saved_keyno); + + /* Compare the previous value with the current one. */ + if (g.type == FIX) { + ret = cursor->get_value(cursor, &bitfield); + value.data = &bitfield; + value.size = 1; + } else + ret = cursor->get_value(cursor, &value); + if (ret != 0) + testutil_die(ret, "cursor.get_value: %" PRIu64, saved_keyno); + + if (buf_size != value.size || memcmp(buf, value.data, value.size) != 0) + testutil_die(0, "mismatched start/stop values"); + + /* End the transaction. */ + testutil_check(session->commit_transaction(session, NULL)); + + /* Reset the cursor, releasing our pin. */ + testutil_check(cursor->reset(cursor)); + pinned = 0; + } else { + /* + * Test named snapshots: create a snapshot, wait to give the transaction state time to + * move forward, then start a transaction with the named snapshot, drop it, then commit + * the transaction. This exercises most of the named snapshot logic under load. + */ + testutil_check(session->snapshot(session, "name=test")); + __wt_sleep(1, 0); + wiredtiger_begin_transaction(session, "snapshot=test"); + testutil_check(session->snapshot(session, "drop=(all)")); + testutil_check(session->commit_transaction(session, NULL)); + + /* + * Begin transaction: without an explicit transaction, the snapshot is only kept around + * while a cursor is positioned. As soon as the cursor loses its position a new snapshot + * will be allocated. + */ + while ( + (ret = session->begin_transaction(session, "snapshot=snapshot")) == WT_CACHE_FULL) + ; + testutil_check(ret); + + /* Read a record at the end of the table. */ + do { + saved_keyno = mmrand(NULL, (u_int)(g.key_cnt - g.key_cnt / 10), (u_int)g.key_cnt); + while ( + (ret = read_row_worker(cursor, saved_keyno, &key, &value, false)) == WT_ROLLBACK) + ; + } while (ret == WT_NOTFOUND); + if (ret != 0) + testutil_die(ret, "read_row_worker %" PRIu64, saved_keyno); + + /* Copy the cursor's value. */ + if (g.type == FIX) { + ret = cursor->get_value(cursor, &bitfield); + value.data = &bitfield; + value.size = 1; + } else + ret = cursor->get_value(cursor, &value); + if (ret != 0) + testutil_die(ret, "cursor.get_value: %" PRIu64, saved_keyno); + if (buf_len < value.size) + buf = drealloc(buf, buf_len = value.size); + memcpy(buf, value.data, buf_size = value.size); + + /* + * Move the cursor to an early record in the table, hopefully allowing the page with the + * record just retrieved to be evicted from memory. + */ + do { + keyno = mmrand(NULL, 1, (u_int)g.key_cnt / 5); + while ((ret = read_row_worker(cursor, keyno, &key, &value, false)) == WT_ROLLBACK) + ; + } while (ret == WT_NOTFOUND); + if (ret != 0) + testutil_die(ret, "read_row_worker %" PRIu64, keyno); + + pinned = 1; + } + + /* Sleep for some number of seconds. */ + period = mmrand(NULL, 1, 10); + + /* Sleep for short periods so we don't make the run wait. */ + while (period > 0 && !g.workers_finished) { + --period; + __wt_sleep(1, 0); + } + if (g.workers_finished) + break; + } + + testutil_check(session->close(session, NULL)); + + key_gen_teardown(&key); + val_gen_teardown(&value); + free(buf); + + return (WT_THREAD_RET_VALUE); } diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 7adfb795694..a03b42e427b 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -28,50 +28,50 @@ #include "format.h" -static int col_insert(TINFO *, WT_CURSOR *); -static int col_modify(TINFO *, WT_CURSOR *, bool); -static int col_remove(TINFO *, WT_CURSOR *, bool); -static int col_reserve(TINFO *, WT_CURSOR *, bool); -static int col_truncate(TINFO *, WT_CURSOR *); -static int col_update(TINFO *, WT_CURSOR *, bool); -static int nextprev(TINFO *, WT_CURSOR *, bool); +static int col_insert(TINFO *, WT_CURSOR *); +static int col_modify(TINFO *, WT_CURSOR *, bool); +static int col_remove(TINFO *, WT_CURSOR *, bool); +static int col_reserve(TINFO *, WT_CURSOR *, bool); +static int col_truncate(TINFO *, WT_CURSOR *); +static int col_update(TINFO *, WT_CURSOR *, bool); +static int nextprev(TINFO *, WT_CURSOR *, bool); static WT_THREAD_RET ops(void *); -static int read_row(TINFO *, WT_CURSOR *); -static int row_insert(TINFO *, WT_CURSOR *, bool); -static int row_modify(TINFO *, WT_CURSOR *, bool); -static int row_remove(TINFO *, WT_CURSOR *, bool); -static int row_reserve(TINFO *, WT_CURSOR *, bool); -static int row_truncate(TINFO *, WT_CURSOR *); -static int row_update(TINFO *, WT_CURSOR *, bool); -static void table_append_init(void); +static int read_row(TINFO *, WT_CURSOR *); +static int row_insert(TINFO *, WT_CURSOR *, bool); +static int row_modify(TINFO *, WT_CURSOR *, bool); +static int row_remove(TINFO *, WT_CURSOR *, bool); +static int row_reserve(TINFO *, WT_CURSOR *, bool); +static int row_truncate(TINFO *, WT_CURSOR *); +static int row_update(TINFO *, WT_CURSOR *, bool); +static void table_append_init(void); static char modify_repl[256]; /* * modify_repl_init -- - * Initialize the replacement information. + * Initialize the replacement information. */ static void modify_repl_init(void) { - size_t i; + size_t i; - for (i = 0; i < sizeof(modify_repl); ++i) - modify_repl[i] = "zyxwvutsrqponmlkjihgfedcba"[i % 26]; + for (i = 0; i < sizeof(modify_repl); ++i) + modify_repl[i] = "zyxwvutsrqponmlkjihgfedcba"[i % 26]; } static void set_alarm(void) { #ifdef HAVE_TIMER_CREATE - struct itimerspec timer_val; - timer_t timer_id; - - testutil_check(timer_create(CLOCK_REALTIME, NULL, &timer_id)); - memset(&timer_val, 0, sizeof(timer_val)); - timer_val.it_value.tv_sec = 60 * 2; - timer_val.it_value.tv_nsec = 0; - testutil_check(timer_settime(timer_id, 0, &timer_val, NULL)); + struct itimerspec timer_val; + timer_t timer_id; + + testutil_check(timer_create(CLOCK_REALTIME, NULL, &timer_id)); + memset(&timer_val, 0, sizeof(timer_val)); + timer_val.it_value.tv_sec = 60 * 2; + timer_val.it_value.tv_nsec = 0; + testutil_check(timer_settime(timer_id, 0, &timer_val, NULL)); #endif } @@ -79,341 +79,319 @@ TINFO **tinfo_list; /* * wts_ops -- - * Perform a number of operations in a set of threads. + * Perform a number of operations in a set of threads. */ void wts_ops(bool lastrun) { - TINFO *tinfo, total; - WT_CONNECTION *conn; - WT_SESSION *session; - wt_thread_t alter_tid, backup_tid, checkpoint_tid, compact_tid, lrt_tid; - wt_thread_t timestamp_tid; - int64_t fourths, quit_fourths, thread_ops; - uint32_t i; - bool running; - - conn = g.wts_conn; - - session = NULL; /* -Wconditional-uninitialized */ - memset(&alter_tid, 0, sizeof(alter_tid)); - memset(&backup_tid, 0, sizeof(backup_tid)); - memset(&checkpoint_tid, 0, sizeof(checkpoint_tid)); - memset(&compact_tid, 0, sizeof(compact_tid)); - memset(&lrt_tid, 0, sizeof(lrt_tid)); - memset(×tamp_tid, 0, sizeof(timestamp_tid)); - - modify_repl_init(); - - /* - * There are two mechanisms to specify the length of the run, a number - * of operations and a timer, when either expire the run terminates. - * - * Each thread does an equal share of the total operations (and make - * sure that it's not 0). - * - * Calculate how many fourth-of-a-second sleeps until the timer expires. - * If the timer expires and threads don't return in 15 minutes, assume - * there is something hung, and force the quit. - */ - if (g.c_ops == 0) - thread_ops = -1; - else { - if (g.c_ops < g.c_threads) - g.c_ops = g.c_threads; - thread_ops = g.c_ops / g.c_threads; - } - if (g.c_timer == 0) - fourths = quit_fourths = -1; - else { - fourths = ((int64_t)g.c_timer * 4 * 60) / FORMAT_OPERATION_REPS; - quit_fourths = fourths + 15 * 4 * 60; - } - - /* Initialize the table extension code. */ - table_append_init(); - - /* - * We support replay of threaded runs, but don't log random numbers - * after threaded operations start, there's no point. - */ - if (!SINGLETHREADED) - g.rand_log_stop = true; - - /* Logging requires a session. */ - if (g.logging) - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - logop(session, "%s", "=============== thread ops start"); - - /* - * Create the per-thread structures and start the worker threads. - * Allocate the thread structures separately to minimize false sharing. - */ - tinfo_list = dcalloc((size_t)g.c_threads + 1, sizeof(TINFO *)); - for (i = 0; i < g.c_threads; ++i) { - tinfo_list[i] = tinfo = dcalloc(1, sizeof(TINFO)); - - tinfo->id = (int)i + 1; - - /* - * Characterize the per-thread random number generator. Normally - * we want independent behavior so threads start in different - * parts of the RNG space, but we've found bugs by having the - * threads pound on the same key/value pairs, that is, by making - * them traverse the same RNG space. 75% of the time we run in - * independent RNG space. - */ - if (g.c_independent_thread_rng) - __wt_random_init_seed( - (WT_SESSION_IMPL *)session, &tinfo->rnd); - else - __wt_random_init(&tinfo->rnd); - - tinfo->state = TINFO_RUNNING; - testutil_check( - __wt_thread_create(NULL, &tinfo->tid, ops, tinfo)); - } - - /* - * If a multi-threaded run, start optional backup, compaction and - * long-running reader threads. - */ - if (g.c_alter) - testutil_check( - __wt_thread_create(NULL, &alter_tid, alter, NULL)); - if (g.c_backups) - testutil_check( - __wt_thread_create(NULL, &backup_tid, backup, NULL)); - if (g.c_checkpoint_flag == CHECKPOINT_ON) - testutil_check(__wt_thread_create( - NULL, &checkpoint_tid, checkpoint, NULL)); - if (g.c_compact) - testutil_check( - __wt_thread_create(NULL, &compact_tid, compact, NULL)); - if (!SINGLETHREADED && g.c_long_running_txn) - testutil_check(__wt_thread_create(NULL, &lrt_tid, lrt, NULL)); - if (g.c_txn_timestamps) - testutil_check(__wt_thread_create( - NULL, ×tamp_tid, timestamp, tinfo_list)); - - /* Spin on the threads, calculating the totals. */ - for (;;) { - /* Clear out the totals each pass. */ - memset(&total, 0, sizeof(total)); - for (i = 0, running = false; i < g.c_threads; ++i) { - tinfo = tinfo_list[i]; - total.commit += tinfo->commit; - total.insert += tinfo->insert; - total.prepare += tinfo->prepare; - total.remove += tinfo->remove; - total.rollback += tinfo->rollback; - total.search += tinfo->search; - total.truncate += tinfo->truncate; - total.update += tinfo->update; - - switch (tinfo->state) { - case TINFO_RUNNING: - running = true; - break; - case TINFO_COMPLETE: - tinfo->state = TINFO_JOINED; - testutil_check( - __wt_thread_join(NULL, &tinfo->tid)); - break; - case TINFO_JOINED: - break; - } - - /* - * If the timer has expired or this thread has completed - * its operations, notify the thread it should quit. - */ - if (fourths == 0 || - (thread_ops != -1 && - tinfo->ops >= (uint64_t)thread_ops)) { - /* - * On the last execution, optionally drop core - * for recovery testing. - */ - if (lastrun && g.c_abort) { - static char *core = NULL; - *core = 0; - } - tinfo->quit = true; - } - } - track("ops", 0ULL, &total); - if (!running) - break; - __wt_sleep(0, 250000); /* 1/4th of a second */ - if (fourths != -1) - --fourths; - if (quit_fourths != -1 && --quit_fourths == 0) { - fprintf(stderr, "%s\n", - "format run more than 15 minutes past the maximum " - "time"); - fprintf(stderr, "%s\n", - "format run dumping cache and transaction state, " - "then aborting the process"); - - /* - * If the library is deadlocked, we might just join the - * mess, set a timer to limit our exposure. - */ - set_alarm(); - - (void)conn->debug_info(conn, "txn"); - (void)conn->debug_info(conn, "cache"); - - __wt_abort(NULL); - } - } - - /* Wait for the other threads. */ - g.workers_finished = true; - if (g.c_alter) - testutil_check(__wt_thread_join(NULL, &alter_tid)); - if (g.c_backups) - testutil_check(__wt_thread_join(NULL, &backup_tid)); - if (g.c_checkpoint_flag == CHECKPOINT_ON) - testutil_check(__wt_thread_join(NULL, &checkpoint_tid)); - if (g.c_compact) - testutil_check(__wt_thread_join(NULL, &compact_tid)); - if (!SINGLETHREADED && g.c_long_running_txn) - testutil_check(__wt_thread_join(NULL, &lrt_tid)); - if (g.c_txn_timestamps) - testutil_check(__wt_thread_join(NULL, ×tamp_tid)); - g.workers_finished = false; - - logop(session, "%s", "=============== thread ops stop"); - if (g.logging) - testutil_check(session->close(session, NULL)); - - for (i = 0; i < g.c_threads; ++i) - free(tinfo_list[i]); - free(tinfo_list); + TINFO *tinfo, total; + WT_CONNECTION *conn; + WT_SESSION *session; + wt_thread_t alter_tid, backup_tid, checkpoint_tid, compact_tid, lrt_tid; + wt_thread_t timestamp_tid; + int64_t fourths, quit_fourths, thread_ops; + uint32_t i; + bool running; + + conn = g.wts_conn; + + session = NULL; /* -Wconditional-uninitialized */ + memset(&alter_tid, 0, sizeof(alter_tid)); + memset(&backup_tid, 0, sizeof(backup_tid)); + memset(&checkpoint_tid, 0, sizeof(checkpoint_tid)); + memset(&compact_tid, 0, sizeof(compact_tid)); + memset(&lrt_tid, 0, sizeof(lrt_tid)); + memset(×tamp_tid, 0, sizeof(timestamp_tid)); + + modify_repl_init(); + + /* + * There are two mechanisms to specify the length of the run, a number + * of operations and a timer, when either expire the run terminates. + * + * Each thread does an equal share of the total operations (and make + * sure that it's not 0). + * + * Calculate how many fourth-of-a-second sleeps until the timer expires. + * If the timer expires and threads don't return in 15 minutes, assume + * there is something hung, and force the quit. + */ + if (g.c_ops == 0) + thread_ops = -1; + else { + if (g.c_ops < g.c_threads) + g.c_ops = g.c_threads; + thread_ops = g.c_ops / g.c_threads; + } + if (g.c_timer == 0) + fourths = quit_fourths = -1; + else { + fourths = ((int64_t)g.c_timer * 4 * 60) / FORMAT_OPERATION_REPS; + quit_fourths = fourths + 15 * 4 * 60; + } + + /* Initialize the table extension code. */ + table_append_init(); + + /* + * We support replay of threaded runs, but don't log random numbers after threaded operations + * start, there's no point. + */ + if (!SINGLETHREADED) + g.rand_log_stop = true; + + /* Logging requires a session. */ + if (g.logging) + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + logop(session, "%s", "=============== thread ops start"); + + /* + * Create the per-thread structures and start the worker threads. Allocate the thread structures + * separately to minimize false sharing. + */ + tinfo_list = dcalloc((size_t)g.c_threads + 1, sizeof(TINFO *)); + for (i = 0; i < g.c_threads; ++i) { + tinfo_list[i] = tinfo = dcalloc(1, sizeof(TINFO)); + + tinfo->id = (int)i + 1; + + /* + * Characterize the per-thread random number generator. Normally we want independent + * behavior so threads start in different parts of the RNG space, but we've found bugs by + * having the threads pound on the same key/value pairs, that is, by making them traverse + * the same RNG space. 75% of the time we run in independent RNG space. + */ + if (g.c_independent_thread_rng) + __wt_random_init_seed((WT_SESSION_IMPL *)session, &tinfo->rnd); + else + __wt_random_init(&tinfo->rnd); + + tinfo->state = TINFO_RUNNING; + testutil_check(__wt_thread_create(NULL, &tinfo->tid, ops, tinfo)); + } + + /* + * If a multi-threaded run, start optional backup, compaction and long-running reader threads. + */ + if (g.c_alter) + testutil_check(__wt_thread_create(NULL, &alter_tid, alter, NULL)); + if (g.c_backups) + testutil_check(__wt_thread_create(NULL, &backup_tid, backup, NULL)); + if (g.c_checkpoint_flag == CHECKPOINT_ON) + testutil_check(__wt_thread_create(NULL, &checkpoint_tid, checkpoint, NULL)); + if (g.c_compact) + testutil_check(__wt_thread_create(NULL, &compact_tid, compact, NULL)); + if (!SINGLETHREADED && g.c_long_running_txn) + testutil_check(__wt_thread_create(NULL, &lrt_tid, lrt, NULL)); + if (g.c_txn_timestamps) + testutil_check(__wt_thread_create(NULL, ×tamp_tid, timestamp, tinfo_list)); + + /* Spin on the threads, calculating the totals. */ + for (;;) { + /* Clear out the totals each pass. */ + memset(&total, 0, sizeof(total)); + for (i = 0, running = false; i < g.c_threads; ++i) { + tinfo = tinfo_list[i]; + total.commit += tinfo->commit; + total.insert += tinfo->insert; + total.prepare += tinfo->prepare; + total.remove += tinfo->remove; + total.rollback += tinfo->rollback; + total.search += tinfo->search; + total.truncate += tinfo->truncate; + total.update += tinfo->update; + + switch (tinfo->state) { + case TINFO_RUNNING: + running = true; + break; + case TINFO_COMPLETE: + tinfo->state = TINFO_JOINED; + testutil_check(__wt_thread_join(NULL, &tinfo->tid)); + break; + case TINFO_JOINED: + break; + } + + /* + * If the timer has expired or this thread has completed its operations, notify the + * thread it should quit. + */ + if (fourths == 0 || (thread_ops != -1 && tinfo->ops >= (uint64_t)thread_ops)) { + /* + * On the last execution, optionally drop core for recovery testing. + */ + if (lastrun && g.c_abort) { + static char *core = NULL; + *core = 0; + } + tinfo->quit = true; + } + } + track("ops", 0ULL, &total); + if (!running) + break; + __wt_sleep(0, 250000); /* 1/4th of a second */ + if (fourths != -1) + --fourths; + if (quit_fourths != -1 && --quit_fourths == 0) { + fprintf(stderr, "%s\n", + "format run more than 15 minutes past the maximum " + "time"); + fprintf(stderr, "%s\n", + "format run dumping cache and transaction state, " + "then aborting the process"); + + /* + * If the library is deadlocked, we might just join the mess, set a timer to limit our + * exposure. + */ + set_alarm(); + + (void)conn->debug_info(conn, "txn"); + (void)conn->debug_info(conn, "cache"); + + __wt_abort(NULL); + } + } + + /* Wait for the other threads. */ + g.workers_finished = true; + if (g.c_alter) + testutil_check(__wt_thread_join(NULL, &alter_tid)); + if (g.c_backups) + testutil_check(__wt_thread_join(NULL, &backup_tid)); + if (g.c_checkpoint_flag == CHECKPOINT_ON) + testutil_check(__wt_thread_join(NULL, &checkpoint_tid)); + if (g.c_compact) + testutil_check(__wt_thread_join(NULL, &compact_tid)); + if (!SINGLETHREADED && g.c_long_running_txn) + testutil_check(__wt_thread_join(NULL, &lrt_tid)); + if (g.c_txn_timestamps) + testutil_check(__wt_thread_join(NULL, ×tamp_tid)); + g.workers_finished = false; + + logop(session, "%s", "=============== thread ops stop"); + if (g.logging) + testutil_check(session->close(session, NULL)); + + for (i = 0; i < g.c_threads; ++i) + free(tinfo_list[i]); + free(tinfo_list); } /* * begin_transaction_ts -- - * Begin a timestamped transaction. + * Begin a timestamped transaction. */ static void begin_transaction_ts(TINFO *tinfo, u_int *iso_configp) { - TINFO **tlp; - WT_DECL_RET; - WT_SESSION *session; - uint64_t ts; - const char *config; - char buf[64]; - - session = tinfo->session; - - config = "isolation=snapshot"; - *iso_configp = ISOLATION_SNAPSHOT; - - /* - * Transaction reads are normally repeatable, but WiredTiger timestamps - * allow rewriting commits, that is, applications can specify at commit - * time the timestamp at which the commit happens. If that happens, our - * read might no longer be repeatable. Test in both modes: pick a read - * timestamp we know is repeatable (because it's at least as old as the - * oldest resolved commit timestamp in any thread), and pick a current - * timestamp, 50% of the time. - */ - ts = 0; - if (mmrand(&tinfo->rnd, 1, 2) == 1) - for (ts = UINT64_MAX, tlp = tinfo_list; *tlp != NULL; ++tlp) - ts = WT_MIN(ts, (*tlp)->commit_ts); - if (ts != 0) { - wiredtiger_begin_transaction(session, config); - - /* - * If the timestamp has aged out of the system, we'll get EINVAL - * when we try and set it. That kills the transaction, we have - * to restart. - */ - testutil_check(__wt_snprintf( - buf, sizeof(buf), "read_timestamp=%" PRIx64, ts)); - ret = session->timestamp_transaction(session, buf); - if (ret == 0) { - snap_init(tinfo, ts, true); - logop(session, - "begin snapshot read-ts=%" PRIu64 " (repeatable)", - ts); - return; - } - if (ret != EINVAL) - testutil_check(ret); - - testutil_check(session->rollback_transaction(session, NULL)); - } - - wiredtiger_begin_transaction(session, config); - - /* - * Otherwise, pick a current timestamp. - * - * Prepare returns an error if the prepare timestamp is less - * than any active read timestamp, single-thread transaction - * prepare and begin. - * - * Lock out the oldest timestamp update. - */ - testutil_check(pthread_rwlock_wrlock(&g.ts_lock)); - - ts = __wt_atomic_addv64(&g.timestamp, 1); - testutil_check(__wt_snprintf( - buf, sizeof(buf), "read_timestamp=%" PRIx64, ts)); - testutil_check(session->timestamp_transaction(session, buf)); - - testutil_check(pthread_rwlock_unlock(&g.ts_lock)); - - snap_init(tinfo, ts, false); - logop(session, - "begin snapshot read-ts=%" PRIu64 " (not repeatable)", ts); + TINFO **tlp; + WT_DECL_RET; + WT_SESSION *session; + uint64_t ts; + const char *config; + char buf[64]; + + session = tinfo->session; + + config = "isolation=snapshot"; + *iso_configp = ISOLATION_SNAPSHOT; + + /* + * Transaction reads are normally repeatable, but WiredTiger timestamps allow rewriting commits, + * that is, applications can specify at commit time the timestamp at which the commit happens. + * If that happens, our read might no longer be repeatable. Test in both modes: pick a read + * timestamp we know is repeatable (because it's at least as old as the oldest resolved commit + * timestamp in any thread), and pick a current timestamp, 50% of the time. + */ + ts = 0; + if (mmrand(&tinfo->rnd, 1, 2) == 1) + for (ts = UINT64_MAX, tlp = tinfo_list; *tlp != NULL; ++tlp) + ts = WT_MIN(ts, (*tlp)->commit_ts); + if (ts != 0) { + wiredtiger_begin_transaction(session, config); + + /* + * If the timestamp has aged out of the system, we'll get EINVAL when we try and set it. + * That kills the transaction, we have to restart. + */ + testutil_check(__wt_snprintf(buf, sizeof(buf), "read_timestamp=%" PRIx64, ts)); + ret = session->timestamp_transaction(session, buf); + if (ret == 0) { + snap_init(tinfo, ts, true); + logop(session, "begin snapshot read-ts=%" PRIu64 " (repeatable)", ts); + return; + } + if (ret != EINVAL) + testutil_check(ret); + + testutil_check(session->rollback_transaction(session, NULL)); + } + + wiredtiger_begin_transaction(session, config); + + /* + * Otherwise, pick a current timestamp. + * + * Prepare returns an error if the prepare timestamp is less + * than any active read timestamp, single-thread transaction + * prepare and begin. + * + * Lock out the oldest timestamp update. + */ + testutil_check(pthread_rwlock_wrlock(&g.ts_lock)); + + ts = __wt_atomic_addv64(&g.timestamp, 1); + testutil_check(__wt_snprintf(buf, sizeof(buf), "read_timestamp=%" PRIx64, ts)); + testutil_check(session->timestamp_transaction(session, buf)); + + testutil_check(pthread_rwlock_unlock(&g.ts_lock)); + + snap_init(tinfo, ts, false); + logop(session, "begin snapshot read-ts=%" PRIu64 " (not repeatable)", ts); } /* * begin_transaction -- - * Choose an isolation configuration and begin a transaction. + * Choose an isolation configuration and begin a transaction. */ static void begin_transaction(TINFO *tinfo, u_int *iso_configp) { - WT_SESSION *session; - u_int v; - const char *config, *log; - - session = tinfo->session; - - if ((v = g.c_isolation_flag) == ISOLATION_RANDOM) - v = mmrand(&tinfo->rnd, 1, 3); - switch (v) { - case 1: - v = ISOLATION_READ_UNCOMMITTED; - log = "read-uncommitted"; - config = "isolation=read-uncommitted"; - break; - case 2: - v = ISOLATION_READ_COMMITTED; - log = "read-committed"; - config = "isolation=read-committed"; - break; - case 3: - default: - v = ISOLATION_SNAPSHOT; - log = "snapshot"; - config = "isolation=snapshot"; - break; - } - *iso_configp = v; - - wiredtiger_begin_transaction(session, config); - - snap_init(tinfo, WT_TS_NONE, false); - logop(session, "begin %s", log); + WT_SESSION *session; + u_int v; + const char *config, *log; + + session = tinfo->session; + + if ((v = g.c_isolation_flag) == ISOLATION_RANDOM) + v = mmrand(&tinfo->rnd, 1, 3); + switch (v) { + case 1: + v = ISOLATION_READ_UNCOMMITTED; + log = "read-uncommitted"; + config = "isolation=read-uncommitted"; + break; + case 2: + v = ISOLATION_READ_COMMITTED; + log = "read-committed"; + config = "isolation=read-committed"; + break; + case 3: + default: + v = ISOLATION_SNAPSHOT; + log = "snapshot"; + config = "isolation=snapshot"; + break; + } + *iso_configp = v; + + wiredtiger_begin_transaction(session, config); + + snap_init(tinfo, WT_TS_NONE, false); + logop(session, "begin %s", log); } /* @@ -423,41 +401,37 @@ begin_transaction(TINFO *tinfo, u_int *iso_configp) static void commit_transaction(TINFO *tinfo, bool prepared) { - WT_SESSION *session; - uint64_t ts; - char buf[64]; + WT_SESSION *session; + uint64_t ts; + char buf[64]; - ++tinfo->commit; + ++tinfo->commit; - session = tinfo->session; + session = tinfo->session; - ts = 0; /* -Wconditional-uninitialized */ - if (g.c_txn_timestamps) { - /* Lock out the oldest timestamp update. */ - testutil_check(pthread_rwlock_wrlock(&g.ts_lock)); + ts = 0; /* -Wconditional-uninitialized */ + if (g.c_txn_timestamps) { + /* Lock out the oldest timestamp update. */ + testutil_check(pthread_rwlock_wrlock(&g.ts_lock)); - ts = __wt_atomic_addv64(&g.timestamp, 1); - testutil_check(__wt_snprintf( - buf, sizeof(buf), "commit_timestamp=%" PRIx64, ts)); - testutil_check(session->timestamp_transaction(session, buf)); + ts = __wt_atomic_addv64(&g.timestamp, 1); + testutil_check(__wt_snprintf(buf, sizeof(buf), "commit_timestamp=%" PRIx64, ts)); + testutil_check(session->timestamp_transaction(session, buf)); - if (prepared) { - testutil_check(__wt_snprintf(buf, sizeof(buf), - "durable_timestamp=%" PRIx64, ts)); - testutil_check( - session->timestamp_transaction(session, buf)); - } + if (prepared) { + testutil_check(__wt_snprintf(buf, sizeof(buf), "durable_timestamp=%" PRIx64, ts)); + testutil_check(session->timestamp_transaction(session, buf)); + } - testutil_check(pthread_rwlock_unlock(&g.ts_lock)); - } - testutil_check(session->commit_transaction(session, NULL)); + testutil_check(pthread_rwlock_unlock(&g.ts_lock)); + } + testutil_check(session->commit_transaction(session, NULL)); - /* Remember our oldest commit timestamp. */ - tinfo->commit_ts = ts; + /* Remember our oldest commit timestamp. */ + tinfo->commit_ts = ts; - logop(session, - "commit read-ts=%" PRIu64 ", commit-ts=%" PRIu64, - tinfo->read_ts, tinfo->commit_ts); + logop( + session, "commit read-ts=%" PRIu64 ", commit-ts=%" PRIu64, tinfo->read_ts, tinfo->commit_ts); } /* @@ -467,16 +441,15 @@ commit_transaction(TINFO *tinfo, bool prepared) static void rollback_transaction(TINFO *tinfo) { - WT_SESSION *session; + WT_SESSION *session; - session = tinfo->session; + session = tinfo->session; - ++tinfo->rollback; + ++tinfo->rollback; - testutil_check(session->rollback_transaction(session, NULL)); + testutil_check(session->rollback_transaction(session, NULL)); - logop(session, - "abort read-ts=%" PRIu64, tinfo->read_ts); + logop(session, "abort read-ts=%" PRIu64, tinfo->read_ts); } /* @@ -486,131 +459,130 @@ rollback_transaction(TINFO *tinfo) static int prepare_transaction(TINFO *tinfo) { - WT_DECL_RET; - WT_SESSION *session; - uint64_t ts; - char buf[64]; - - session = tinfo->session; - - ++tinfo->prepare; - - /* - * Prepare timestamps must be less than or equal to the eventual commit - * timestamp. Set the prepare timestamp to whatever the global value is - * now. The subsequent commit will increment it, ensuring correctness. - * - * Prepare returns an error if the prepare timestamp is less than any - * active read timestamp, single-thread transaction prepare and begin. - * - * Lock out the oldest timestamp update. - */ - testutil_check(pthread_rwlock_wrlock(&g.ts_lock)); - - ts = __wt_atomic_addv64(&g.timestamp, 1); - testutil_check(__wt_snprintf( - buf, sizeof(buf), "prepare_timestamp=%" PRIx64, ts)); - ret = session->prepare_transaction(session, buf); - - testutil_check(pthread_rwlock_unlock(&g.ts_lock)); - - return (ret); + WT_DECL_RET; + WT_SESSION *session; + uint64_t ts; + char buf[64]; + + session = tinfo->session; + + ++tinfo->prepare; + + /* + * Prepare timestamps must be less than or equal to the eventual commit + * timestamp. Set the prepare timestamp to whatever the global value is + * now. The subsequent commit will increment it, ensuring correctness. + * + * Prepare returns an error if the prepare timestamp is less than any + * active read timestamp, single-thread transaction prepare and begin. + * + * Lock out the oldest timestamp update. + */ + testutil_check(pthread_rwlock_wrlock(&g.ts_lock)); + + ts = __wt_atomic_addv64(&g.timestamp, 1); + testutil_check(__wt_snprintf(buf, sizeof(buf), "prepare_timestamp=%" PRIx64, ts)); + ret = session->prepare_transaction(session, buf); + + testutil_check(pthread_rwlock_unlock(&g.ts_lock)); + + return (ret); } /* * OP_FAILED -- * General error handling. */ -#define OP_FAILED(notfound_ok) do { \ - positioned = false; \ - if (intxn && (ret == WT_CACHE_FULL || ret == WT_ROLLBACK)) \ - goto rollback; \ - testutil_assert((notfound_ok && ret == WT_NOTFOUND) || \ - ret == WT_CACHE_FULL || ret == WT_ROLLBACK); \ -} while (0) +#define OP_FAILED(notfound_ok) \ + do { \ + positioned = false; \ + if (intxn && (ret == WT_CACHE_FULL || ret == WT_ROLLBACK)) \ + goto rollback; \ + testutil_assert( \ + (notfound_ok && ret == WT_NOTFOUND) || ret == WT_CACHE_FULL || ret == WT_ROLLBACK); \ + } while (0) /* - * Rollback updates returning prepare-conflict, they're unlikely to succeed - * unless the prepare aborts. Reads wait out the error, so it's unexpected. + * Rollback updates returning prepare-conflict, they're unlikely to succeed unless the prepare + * aborts. Reads wait out the error, so it's unexpected. */ -#define READ_OP_FAILED(notfound_ok) \ - OP_FAILED(notfound_ok) -#define WRITE_OP_FAILED(notfound_ok) do { \ - if (ret == WT_PREPARE_CONFLICT) \ - ret = WT_ROLLBACK; \ - OP_FAILED(notfound_ok); \ -} while (0) +#define READ_OP_FAILED(notfound_ok) OP_FAILED(notfound_ok) +#define WRITE_OP_FAILED(notfound_ok) \ + do { \ + if (ret == WT_PREPARE_CONFLICT) \ + ret = WT_ROLLBACK; \ + OP_FAILED(notfound_ok); \ + } while (0) /* - * When in a transaction on the live table with snapshot isolation, track - * operations for later repetition. + * When in a transaction on the live table with snapshot isolation, track operations for later + * repetition. */ -#define SNAP_TRACK(tinfo, op) do { \ - if (intxn && !ckpt_handle && iso_config == ISOLATION_SNAPSHOT) \ - snap_track(tinfo, op); \ -} while (0) +#define SNAP_TRACK(tinfo, op) \ + do { \ + if (intxn && !ckpt_handle && iso_config == ISOLATION_SNAPSHOT) \ + snap_track(tinfo, op); \ + } while (0) /* * ops_open_session -- - * Create a new session/cursor pair for the thread. + * Create a new session/cursor pair for the thread. */ static void ops_open_session(TINFO *tinfo, bool *ckpt_handlep) { - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_DECL_RET; - WT_SESSION *session; - - conn = g.wts_conn; - - /* Close any open session/cursor. */ - if ((session = tinfo->session) != NULL) - testutil_check(session->close(session, NULL)); - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - /* - * 10% of the time, perform some read-only operations from a checkpoint. - * Skip if we are using data-sources or LSM, they don't support reading - * from checkpoints. - */ - cursor = NULL; - if (!DATASOURCE("lsm") && mmrand(&tinfo->rnd, 1, 10) == 1) { - /* - * WT_SESSION.open_cursor can return EBUSY if concurrent with a - * metadata operation, retry. - */ - while ((ret = session->open_cursor(session, g.uri, NULL, - "checkpoint=WiredTigerCheckpoint", &cursor)) == EBUSY) - __wt_yield(); - - /* - * If the checkpoint hasn't been created yet, ignore the error. - */ - if (ret != ENOENT) { - testutil_check(ret); - *ckpt_handlep = true; - } - } - if (cursor == NULL) { - /* - * Configure "append", in the case of column stores, we append - * when inserting new rows. - * - * WT_SESSION.open_cursor can return EBUSY if concurrent with a - * metadata operation, retry. - */ - while ((ret = session->open_cursor(session, - g.uri, NULL, "append", &cursor)) == EBUSY) - __wt_yield(); - - testutil_check(ret); - *ckpt_handlep = false; - } - - tinfo->session = session; - tinfo->cursor = cursor; + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_DECL_RET; + WT_SESSION *session; + + conn = g.wts_conn; + + /* Close any open session/cursor. */ + if ((session = tinfo->session) != NULL) + testutil_check(session->close(session, NULL)); + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + /* + * 10% of the time, perform some read-only operations from a checkpoint. + * Skip if we are using data-sources or LSM, they don't support reading + * from checkpoints. + */ + cursor = NULL; + if (!DATASOURCE("lsm") && mmrand(&tinfo->rnd, 1, 10) == 1) { + /* + * WT_SESSION.open_cursor can return EBUSY if concurrent with a metadata operation, retry. + */ + while ((ret = session->open_cursor( + session, g.uri, NULL, "checkpoint=WiredTigerCheckpoint", &cursor)) == EBUSY) + __wt_yield(); + + /* + * If the checkpoint hasn't been created yet, ignore the error. + */ + if (ret != ENOENT) { + testutil_check(ret); + *ckpt_handlep = true; + } + } + if (cursor == NULL) { + /* + * Configure "append", in the case of column stores, we append + * when inserting new rows. + * + * WT_SESSION.open_cursor can return EBUSY if concurrent with a + * metadata operation, retry. + */ + while ((ret = session->open_cursor(session, g.uri, NULL, "append", &cursor)) == EBUSY) + __wt_yield(); + + testutil_check(ret); + *ckpt_handlep = false; + } + + tinfo->session = session; + tinfo->cursor = cursor; } /* @@ -620,1238 +592,1171 @@ ops_open_session(TINFO *tinfo, bool *ckpt_handlep) static WT_THREAD_RET ops(void *arg) { - TINFO *tinfo; - WT_CURSOR *cursor; - WT_DECL_RET; - WT_SESSION *session; - thread_op op; - uint64_t reset_op, session_op, truncate_op; - uint32_t range, rnd; - u_int i, j, iso_config; - bool ckpt_handle, greater_than, intxn, next, positioned, prepared; - - tinfo = arg; - - iso_config = ISOLATION_RANDOM; /* -Wconditional-uninitialized */ - ckpt_handle = false; /* -Wconditional-uninitialized */ - - /* Tracking of transactional snapshot isolation operations. */ - tinfo->snap = tinfo->snap_first = tinfo->snap_list; - - /* Set up the default key and value buffers. */ - tinfo->key = &tinfo->_key; - key_gen_init(tinfo->key); - tinfo->value = &tinfo->_value; - val_gen_init(tinfo->value); - tinfo->lastkey = &tinfo->_lastkey; - key_gen_init(tinfo->lastkey); - tinfo->tbuf = &tinfo->_tbuf; - - /* Set the first operation where we'll create sessions and cursors. */ - cursor = NULL; - session = NULL; - session_op = 0; - - /* Set the first operation where we'll reset the session. */ - reset_op = mmrand(&tinfo->rnd, 100, 10000); - /* Set the first operation where we'll truncate a range. */ - truncate_op = g.c_truncate == 0 ? - UINT64_MAX : mmrand(&tinfo->rnd, 100, 10000); - - for (intxn = false; !tinfo->quit; ++tinfo->ops) { - /* Periodically open up a new session and cursors. */ - if (tinfo->ops > session_op || - session == NULL || cursor == NULL) { - /* - * We can't swap sessions/cursors if in a transaction, - * resolve any running transaction. - */ - if (intxn) { - commit_transaction(tinfo, false); - intxn = false; - } - - ops_open_session(tinfo, &ckpt_handle); - - /* Pick the next session/cursor close/open. */ - session_op += mmrand(&tinfo->rnd, 100, 5000); - - session = tinfo->session; - cursor = tinfo->cursor; - } - - /* - * If not in a transaction, reset the session now and then, just - * to make sure that operation gets tested. The test is not for - * equality, we have to do the reset outside of a transaction so - * we aren't likely to get an exact match. - */ - if (!intxn && tinfo->ops > reset_op) { - testutil_check(session->reset(session)); - - /* Pick the next reset operation. */ - reset_op += mmrand(&tinfo->rnd, 20000, 50000); - } - - /* - * If not in a transaction, have a live handle and running in a - * timestamp world, occasionally repeat a timestamped operation. - */ - if (!intxn && !ckpt_handle && - g.c_txn_timestamps && mmrand(&tinfo->rnd, 1, 15) == 1) { - ++tinfo->search; - snap_repeat_single(cursor, tinfo); - } - - /* - * If not in a transaction and have a live handle, choose an - * isolation level and start a transaction some percentage of - * the time. - */ - if (!intxn && (g.c_txn_timestamps || - mmrand(&tinfo->rnd, 1, 100) <= g.c_txn_freq)) { - if (g.c_txn_timestamps) - begin_transaction_ts(tinfo, &iso_config); - else - begin_transaction(tinfo, &iso_config); - intxn = true; - } - - /* Select an operation. */ - op = READ; - if (!ckpt_handle) { - i = mmrand(&tinfo->rnd, 1, 100); - if (i < g.c_delete_pct && tinfo->ops > truncate_op) { - op = TRUNCATE; - - /* Pick the next truncate operation. */ - truncate_op += - mmrand(&tinfo->rnd, 20000, 100000); - } else if (i < g.c_delete_pct) - op = REMOVE; - else if (i < g.c_delete_pct + g.c_insert_pct) - op = INSERT; - else if (i < g.c_delete_pct + - g.c_insert_pct + g.c_modify_pct) - op = MODIFY; - else if (i < g.c_delete_pct + - g.c_insert_pct + g.c_modify_pct + g.c_write_pct) - op = UPDATE; - } - - /* Select a row. */ - tinfo->keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows); - - /* - * Inserts, removes and updates can be done following a cursor - * set-key, or based on a cursor position taken from a previous - * search. If not already doing a read, position the cursor at - * an existing point in the tree 20% of the time. - */ - positioned = false; - if (op != READ && mmrand(&tinfo->rnd, 1, 5) == 1) { - ++tinfo->search; - ret = read_row(tinfo, cursor); - if (ret == 0) { - positioned = true; - SNAP_TRACK(tinfo, READ); - } else - READ_OP_FAILED(true); - } - - /* - * Optionally reserve a row. Reserving a row before a read isn't - * all that sensible, but not unexpected, either. - */ - if (intxn && !ckpt_handle && mmrand(&tinfo->rnd, 0, 20) == 1) { - switch (g.type) { - case ROW: - ret = row_reserve(tinfo, cursor, positioned); - break; - case FIX: - case VAR: - ret = col_reserve(tinfo, cursor, positioned); - break; - } - if (ret == 0) { - positioned = true; - - __wt_yield(); /* Let other threads proceed. */ - } else - WRITE_OP_FAILED(true); - } - - /* Perform the operation. */ - switch (op) { - case INSERT: - switch (g.type) { - case ROW: - ret = row_insert(tinfo, cursor, positioned); - break; - case FIX: - case VAR: - /* - * We can only append so many new records, once - * we reach that limit, update a record instead - * of inserting. - */ - if (g.append_cnt >= g.append_max) - goto update_instead_of_chosen_op; - - ret = col_insert(tinfo, cursor); - break; - } - - /* Insert never leaves the cursor positioned. */ - positioned = false; - if (ret == 0) { - ++tinfo->insert; - SNAP_TRACK(tinfo, INSERT); - } else - WRITE_OP_FAILED(false); - break; - case MODIFY: - /* - * Change modify into update if not part of a snapshot - * isolation transaction, modify isn't supported in - * those cases. - */ - if (!intxn || iso_config != ISOLATION_SNAPSHOT) - goto update_instead_of_chosen_op; - - ++tinfo->update; - switch (g.type) { - case ROW: - ret = row_modify(tinfo, cursor, positioned); - break; - case VAR: - ret = col_modify(tinfo, cursor, positioned); - break; - } - if (ret == 0) { - positioned = true; - SNAP_TRACK(tinfo, MODIFY); - } else - WRITE_OP_FAILED(true); - break; - case READ: - ++tinfo->search; - ret = read_row(tinfo, cursor); - if (ret == 0) { - positioned = true; - SNAP_TRACK(tinfo, READ); - } else - READ_OP_FAILED(true); - break; - case REMOVE: -remove_instead_of_truncate: - switch (g.type) { - case ROW: - ret = row_remove(tinfo, cursor, positioned); - break; - case FIX: - case VAR: - ret = col_remove(tinfo, cursor, positioned); - break; - } - if (ret == 0) { - ++tinfo->remove; - /* - * Don't set positioned: it's unchanged from the - * previous state, but not necessarily set. - */ - SNAP_TRACK(tinfo, REMOVE); - } else - WRITE_OP_FAILED(true); - break; - case TRUNCATE: - /* - * A maximum of 2 truncation operations at a time, more - * than that can lead to serious thrashing. - */ - if (__wt_atomic_addv64(&g.truncate_cnt, 1) > 2) { - (void)__wt_atomic_subv64(&g.truncate_cnt, 1); - goto remove_instead_of_truncate; - } - - if (!positioned) - tinfo->keyno = - mmrand(&tinfo->rnd, 1, (u_int)g.rows); - - /* - * Truncate up to 5% of the table. If the range overlaps - * the beginning/end of the table, set the key to 0 (the - * truncate function then sets a cursor to NULL so that - * code is tested). - * - * This gets tricky: there are 2 directions (truncating - * from lower keys to the current position or from - * the current position to higher keys), and collation - * order (truncating from lower keys to higher keys or - * vice-versa). - */ - greater_than = mmrand(&tinfo->rnd, 0, 1) == 1; - range = g.rows < 20 ? - 0 : mmrand(&tinfo->rnd, 0, (u_int)g.rows / 20); - tinfo->last = tinfo->keyno; - if (greater_than) { - if (g.c_reverse) { - if (tinfo->keyno <= range) - tinfo->last = 0; - else - tinfo->last -= range; - } else { - tinfo->last += range; - if (tinfo->last > g.rows) - tinfo->last = 0; - } - } else { - if (g.c_reverse) { - tinfo->keyno += range; - if (tinfo->keyno > g.rows) - tinfo->keyno = 0; - } else { - if (tinfo->keyno <= range) - tinfo->keyno = 0; - else - tinfo->keyno -= range; - } - } - switch (g.type) { - case ROW: - ret = row_truncate(tinfo, cursor); - break; - case FIX: - case VAR: - ret = col_truncate(tinfo, cursor); - break; - } - (void)__wt_atomic_subv64(&g.truncate_cnt, 1); - - /* Truncate never leaves the cursor positioned. */ - positioned = false; - if (ret == 0) { - ++tinfo->truncate; - SNAP_TRACK(tinfo, TRUNCATE); - } else - WRITE_OP_FAILED(false); - break; - case UPDATE: -update_instead_of_chosen_op: - ++tinfo->update; - switch (g.type) { - case ROW: - ret = row_update(tinfo, cursor, positioned); - break; - case FIX: - case VAR: - ret = col_update(tinfo, cursor, positioned); - break; - } - if (ret == 0) { - positioned = true; - SNAP_TRACK(tinfo, UPDATE); - } else - WRITE_OP_FAILED(false); - break; - } - - /* - * The cursor is positioned if we did any operation other than - * insert, do a small number of next/prev cursor operations in - * a random direction. - */ - if (positioned) { - next = mmrand(&tinfo->rnd, 0, 1) == 1; - j = mmrand(&tinfo->rnd, 1, 100); - for (i = 0; i < j; ++i) { - if ((ret = nextprev(tinfo, cursor, next)) == 0) - continue; - - READ_OP_FAILED(true); - break; - } - } - - /* Reset the cursor: there is no reason to keep pages pinned. */ - testutil_check(cursor->reset(cursor)); - - /* - * Continue if not in a transaction, else add more operations - * to the transaction half the time. - */ - if (!intxn || (rnd = mmrand(&tinfo->rnd, 1, 10)) > 5) - continue; - - /* - * Ending a transaction. If on a live handle and the transaction - * was configured for snapshot isolation, repeat the operations - * and confirm the results are unchanged. - */ - if (intxn && !ckpt_handle && iso_config == ISOLATION_SNAPSHOT) { - __wt_yield(); /* Encourage races */ - - ret = snap_repeat_txn(cursor, tinfo); - testutil_assert(ret == 0 || ret == WT_ROLLBACK); - if (ret == WT_ROLLBACK) - goto rollback; - } - - /* - * If prepare configured, prepare the transaction 10% of the - * time. - */ - prepared = false; - if (g.c_prepare && mmrand(&tinfo->rnd, 1, 10) == 1) { - ret = prepare_transaction(tinfo); - if (ret != 0) - WRITE_OP_FAILED(false); - - __wt_yield(); /* Encourage races */ - prepared = true; - } - - /* - * If we're in a transaction, commit 40% of the time and - * rollback 10% of the time. - */ - switch (rnd) { - case 1: case 2: case 3: case 4: /* 40% */ - commit_transaction(tinfo, prepared); - snap_repeat_update(tinfo, true); - break; - case 5: /* 10% */ -rollback: rollback_transaction(tinfo); - snap_repeat_update(tinfo, false); - break; - } - - intxn = false; - } - - if (session != NULL) - testutil_check(session->close(session, NULL)); - - for (i = 0; i < WT_ELEMENTS(tinfo->snap_list); ++i) { - free(tinfo->snap_list[i].kdata); - free(tinfo->snap_list[i].vdata); - } - key_gen_teardown(tinfo->key); - val_gen_teardown(tinfo->value); - key_gen_teardown(tinfo->lastkey); - free(tinfo->tbuf->mem); - - tinfo->state = TINFO_COMPLETE; - return (WT_THREAD_RET_VALUE); + TINFO *tinfo; + WT_CURSOR *cursor; + WT_DECL_RET; + WT_SESSION *session; + thread_op op; + uint64_t reset_op, session_op, truncate_op; + uint32_t range, rnd; + u_int i, j, iso_config; + bool ckpt_handle, greater_than, intxn, next, positioned, prepared; + + tinfo = arg; + + iso_config = ISOLATION_RANDOM; /* -Wconditional-uninitialized */ + ckpt_handle = false; /* -Wconditional-uninitialized */ + + /* Tracking of transactional snapshot isolation operations. */ + tinfo->snap = tinfo->snap_first = tinfo->snap_list; + + /* Set up the default key and value buffers. */ + tinfo->key = &tinfo->_key; + key_gen_init(tinfo->key); + tinfo->value = &tinfo->_value; + val_gen_init(tinfo->value); + tinfo->lastkey = &tinfo->_lastkey; + key_gen_init(tinfo->lastkey); + tinfo->tbuf = &tinfo->_tbuf; + + /* Set the first operation where we'll create sessions and cursors. */ + cursor = NULL; + session = NULL; + session_op = 0; + + /* Set the first operation where we'll reset the session. */ + reset_op = mmrand(&tinfo->rnd, 100, 10000); + /* Set the first operation where we'll truncate a range. */ + truncate_op = g.c_truncate == 0 ? UINT64_MAX : mmrand(&tinfo->rnd, 100, 10000); + + for (intxn = false; !tinfo->quit; ++tinfo->ops) { + /* Periodically open up a new session and cursors. */ + if (tinfo->ops > session_op || session == NULL || cursor == NULL) { + /* + * We can't swap sessions/cursors if in a transaction, resolve any running transaction. + */ + if (intxn) { + commit_transaction(tinfo, false); + intxn = false; + } + + ops_open_session(tinfo, &ckpt_handle); + + /* Pick the next session/cursor close/open. */ + session_op += mmrand(&tinfo->rnd, 100, 5000); + + session = tinfo->session; + cursor = tinfo->cursor; + } + + /* + * If not in a transaction, reset the session now and then, just to make sure that operation + * gets tested. The test is not for equality, we have to do the reset outside of a + * transaction so we aren't likely to get an exact match. + */ + if (!intxn && tinfo->ops > reset_op) { + testutil_check(session->reset(session)); + + /* Pick the next reset operation. */ + reset_op += mmrand(&tinfo->rnd, 20000, 50000); + } + + /* + * If not in a transaction, have a live handle and running in a timestamp world, + * occasionally repeat a timestamped operation. + */ + if (!intxn && !ckpt_handle && g.c_txn_timestamps && mmrand(&tinfo->rnd, 1, 15) == 1) { + ++tinfo->search; + snap_repeat_single(cursor, tinfo); + } + + /* + * If not in a transaction and have a live handle, choose an isolation level and start a + * transaction some percentage of the time. + */ + if (!intxn && (g.c_txn_timestamps || mmrand(&tinfo->rnd, 1, 100) <= g.c_txn_freq)) { + if (g.c_txn_timestamps) + begin_transaction_ts(tinfo, &iso_config); + else + begin_transaction(tinfo, &iso_config); + intxn = true; + } + + /* Select an operation. */ + op = READ; + if (!ckpt_handle) { + i = mmrand(&tinfo->rnd, 1, 100); + if (i < g.c_delete_pct && tinfo->ops > truncate_op) { + op = TRUNCATE; + + /* Pick the next truncate operation. */ + truncate_op += mmrand(&tinfo->rnd, 20000, 100000); + } else if (i < g.c_delete_pct) + op = REMOVE; + else if (i < g.c_delete_pct + g.c_insert_pct) + op = INSERT; + else if (i < g.c_delete_pct + g.c_insert_pct + g.c_modify_pct) + op = MODIFY; + else if (i < g.c_delete_pct + g.c_insert_pct + g.c_modify_pct + g.c_write_pct) + op = UPDATE; + } + + /* Select a row. */ + tinfo->keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows); + + /* + * Inserts, removes and updates can be done following a cursor set-key, or based on a cursor + * position taken from a previous search. If not already doing a read, position the cursor + * at an existing point in the tree 20% of the time. + */ + positioned = false; + if (op != READ && mmrand(&tinfo->rnd, 1, 5) == 1) { + ++tinfo->search; + ret = read_row(tinfo, cursor); + if (ret == 0) { + positioned = true; + SNAP_TRACK(tinfo, READ); + } else + READ_OP_FAILED(true); + } + + /* + * Optionally reserve a row. Reserving a row before a read isn't all that sensible, but not + * unexpected, either. + */ + if (intxn && !ckpt_handle && mmrand(&tinfo->rnd, 0, 20) == 1) { + switch (g.type) { + case ROW: + ret = row_reserve(tinfo, cursor, positioned); + break; + case FIX: + case VAR: + ret = col_reserve(tinfo, cursor, positioned); + break; + } + if (ret == 0) { + positioned = true; + + __wt_yield(); /* Let other threads proceed. */ + } else + WRITE_OP_FAILED(true); + } + + /* Perform the operation. */ + switch (op) { + case INSERT: + switch (g.type) { + case ROW: + ret = row_insert(tinfo, cursor, positioned); + break; + case FIX: + case VAR: + /* + * We can only append so many new records, once we reach that limit, update a record + * instead of inserting. + */ + if (g.append_cnt >= g.append_max) + goto update_instead_of_chosen_op; + + ret = col_insert(tinfo, cursor); + break; + } + + /* Insert never leaves the cursor positioned. */ + positioned = false; + if (ret == 0) { + ++tinfo->insert; + SNAP_TRACK(tinfo, INSERT); + } else + WRITE_OP_FAILED(false); + break; + case MODIFY: + /* + * Change modify into update if not part of a snapshot isolation transaction, modify + * isn't supported in those cases. + */ + if (!intxn || iso_config != ISOLATION_SNAPSHOT) + goto update_instead_of_chosen_op; + + ++tinfo->update; + switch (g.type) { + case ROW: + ret = row_modify(tinfo, cursor, positioned); + break; + case VAR: + ret = col_modify(tinfo, cursor, positioned); + break; + } + if (ret == 0) { + positioned = true; + SNAP_TRACK(tinfo, MODIFY); + } else + WRITE_OP_FAILED(true); + break; + case READ: + ++tinfo->search; + ret = read_row(tinfo, cursor); + if (ret == 0) { + positioned = true; + SNAP_TRACK(tinfo, READ); + } else + READ_OP_FAILED(true); + break; + case REMOVE: + remove_instead_of_truncate: + switch (g.type) { + case ROW: + ret = row_remove(tinfo, cursor, positioned); + break; + case FIX: + case VAR: + ret = col_remove(tinfo, cursor, positioned); + break; + } + if (ret == 0) { + ++tinfo->remove; + /* + * Don't set positioned: it's unchanged from the previous state, but not necessarily + * set. + */ + SNAP_TRACK(tinfo, REMOVE); + } else + WRITE_OP_FAILED(true); + break; + case TRUNCATE: + /* + * A maximum of 2 truncation operations at a time, more than that can lead to serious + * thrashing. + */ + if (__wt_atomic_addv64(&g.truncate_cnt, 1) > 2) { + (void)__wt_atomic_subv64(&g.truncate_cnt, 1); + goto remove_instead_of_truncate; + } + + if (!positioned) + tinfo->keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows); + + /* + * Truncate up to 5% of the table. If the range overlaps + * the beginning/end of the table, set the key to 0 (the + * truncate function then sets a cursor to NULL so that + * code is tested). + * + * This gets tricky: there are 2 directions (truncating + * from lower keys to the current position or from + * the current position to higher keys), and collation + * order (truncating from lower keys to higher keys or + * vice-versa). + */ + greater_than = mmrand(&tinfo->rnd, 0, 1) == 1; + range = g.rows < 20 ? 0 : mmrand(&tinfo->rnd, 0, (u_int)g.rows / 20); + tinfo->last = tinfo->keyno; + if (greater_than) { + if (g.c_reverse) { + if (tinfo->keyno <= range) + tinfo->last = 0; + else + tinfo->last -= range; + } else { + tinfo->last += range; + if (tinfo->last > g.rows) + tinfo->last = 0; + } + } else { + if (g.c_reverse) { + tinfo->keyno += range; + if (tinfo->keyno > g.rows) + tinfo->keyno = 0; + } else { + if (tinfo->keyno <= range) + tinfo->keyno = 0; + else + tinfo->keyno -= range; + } + } + switch (g.type) { + case ROW: + ret = row_truncate(tinfo, cursor); + break; + case FIX: + case VAR: + ret = col_truncate(tinfo, cursor); + break; + } + (void)__wt_atomic_subv64(&g.truncate_cnt, 1); + + /* Truncate never leaves the cursor positioned. */ + positioned = false; + if (ret == 0) { + ++tinfo->truncate; + SNAP_TRACK(tinfo, TRUNCATE); + } else + WRITE_OP_FAILED(false); + break; + case UPDATE: + update_instead_of_chosen_op: + ++tinfo->update; + switch (g.type) { + case ROW: + ret = row_update(tinfo, cursor, positioned); + break; + case FIX: + case VAR: + ret = col_update(tinfo, cursor, positioned); + break; + } + if (ret == 0) { + positioned = true; + SNAP_TRACK(tinfo, UPDATE); + } else + WRITE_OP_FAILED(false); + break; + } + + /* + * The cursor is positioned if we did any operation other than insert, do a small number of + * next/prev cursor operations in a random direction. + */ + if (positioned) { + next = mmrand(&tinfo->rnd, 0, 1) == 1; + j = mmrand(&tinfo->rnd, 1, 100); + for (i = 0; i < j; ++i) { + if ((ret = nextprev(tinfo, cursor, next)) == 0) + continue; + + READ_OP_FAILED(true); + break; + } + } + + /* Reset the cursor: there is no reason to keep pages pinned. */ + testutil_check(cursor->reset(cursor)); + + /* + * Continue if not in a transaction, else add more operations to the transaction half the + * time. + */ + if (!intxn || (rnd = mmrand(&tinfo->rnd, 1, 10)) > 5) + continue; + + /* + * Ending a transaction. If on a live handle and the transaction was configured for snapshot + * isolation, repeat the operations and confirm the results are unchanged. + */ + if (intxn && !ckpt_handle && iso_config == ISOLATION_SNAPSHOT) { + __wt_yield(); /* Encourage races */ + + ret = snap_repeat_txn(cursor, tinfo); + testutil_assert(ret == 0 || ret == WT_ROLLBACK); + if (ret == WT_ROLLBACK) + goto rollback; + } + + /* + * If prepare configured, prepare the transaction 10% of the time. + */ + prepared = false; + if (g.c_prepare && mmrand(&tinfo->rnd, 1, 10) == 1) { + ret = prepare_transaction(tinfo); + if (ret != 0) + WRITE_OP_FAILED(false); + + __wt_yield(); /* Encourage races */ + prepared = true; + } + + /* + * If we're in a transaction, commit 40% of the time and rollback 10% of the time. + */ + switch (rnd) { + case 1: + case 2: + case 3: + case 4: /* 40% */ + commit_transaction(tinfo, prepared); + snap_repeat_update(tinfo, true); + break; + case 5: /* 10% */ +rollback: + rollback_transaction(tinfo); + snap_repeat_update(tinfo, false); + break; + } + + intxn = false; + } + + if (session != NULL) + testutil_check(session->close(session, NULL)); + + for (i = 0; i < WT_ELEMENTS(tinfo->snap_list); ++i) { + free(tinfo->snap_list[i].kdata); + free(tinfo->snap_list[i].vdata); + } + key_gen_teardown(tinfo->key); + val_gen_teardown(tinfo->value); + key_gen_teardown(tinfo->lastkey); + free(tinfo->tbuf->mem); + + tinfo->state = TINFO_COMPLETE; + return (WT_THREAD_RET_VALUE); } /* * wts_read_scan -- - * Read and verify a subset of the elements in a file. + * Read and verify a subset of the elements in a file. */ void wts_read_scan(void) { - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_DECL_RET; - WT_ITEM key, value; - WT_SESSION *session; - uint64_t keyno, last_keyno; - - conn = g.wts_conn; - - /* - * We're not configuring transactions or read timestamps, if there's a - * diagnostic check, skip the scan. - */ - if (g.c_assert_read_timestamp) - return; - - /* Set up the default key/value buffers. */ - key_gen_init(&key); - val_gen_init(&value); - - /* Open a session and cursor pair. */ - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - /* - * open_cursor can return EBUSY if concurrent with a metadata - * operation, retry in that case. - */ - while ((ret = session->open_cursor( - session, g.uri, NULL, NULL, &cursor)) == EBUSY) - __wt_yield(); - testutil_check(ret); - - /* Check a random subset of the records using the key. */ - for (last_keyno = keyno = 0; keyno < g.key_cnt;) { - keyno += mmrand(NULL, 1, 17); - if (keyno > g.rows) - keyno = g.rows; - if (keyno - last_keyno > 1000) { - track("read row scan", keyno, NULL); - last_keyno = keyno; - } - - switch (ret = read_row_worker( - cursor, keyno, &key, &value, false)) { - case 0: - case WT_NOTFOUND: - case WT_ROLLBACK: - case WT_PREPARE_CONFLICT: - break; - default: - testutil_die( - ret, "wts_read_scan: read row %" PRIu64, keyno); - } - } - - testutil_check(session->close(session, NULL)); - - key_gen_teardown(&key); - val_gen_teardown(&value); + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_DECL_RET; + WT_ITEM key, value; + WT_SESSION *session; + uint64_t keyno, last_keyno; + + conn = g.wts_conn; + + /* + * We're not configuring transactions or read timestamps, if there's a diagnostic check, skip + * the scan. + */ + if (g.c_assert_read_timestamp) + return; + + /* Set up the default key/value buffers. */ + key_gen_init(&key); + val_gen_init(&value); + + /* Open a session and cursor pair. */ + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + /* + * open_cursor can return EBUSY if concurrent with a metadata operation, retry in that case. + */ + while ((ret = session->open_cursor(session, g.uri, NULL, NULL, &cursor)) == EBUSY) + __wt_yield(); + testutil_check(ret); + + /* Check a random subset of the records using the key. */ + for (last_keyno = keyno = 0; keyno < g.key_cnt;) { + keyno += mmrand(NULL, 1, 17); + if (keyno > g.rows) + keyno = g.rows; + if (keyno - last_keyno > 1000) { + track("read row scan", keyno, NULL); + last_keyno = keyno; + } + + switch (ret = read_row_worker(cursor, keyno, &key, &value, false)) { + case 0: + case WT_NOTFOUND: + case WT_ROLLBACK: + case WT_PREPARE_CONFLICT: + break; + default: + testutil_die(ret, "wts_read_scan: read row %" PRIu64, keyno); + } + } + + testutil_check(session->close(session, NULL)); + + key_gen_teardown(&key); + val_gen_teardown(&value); } /* * read_row_worker -- - * Read and verify a single element in a row- or column-store file. + * Read and verify a single element in a row- or column-store file. */ int -read_row_worker( - WT_CURSOR *cursor, uint64_t keyno, WT_ITEM *key, WT_ITEM *value, bool sn) +read_row_worker(WT_CURSOR *cursor, uint64_t keyno, WT_ITEM *key, WT_ITEM *value, bool sn) { - WT_SESSION *session; - uint8_t bitfield; - int exact, ret; - - session = cursor->session; - - /* Retrieve the key/value pair by key. */ - switch (g.type) { - case FIX: - case VAR: - cursor->set_key(cursor, keyno); - break; - case ROW: - key_gen(key, keyno); - cursor->set_key(cursor, key); - break; - } - - if (sn) { - ret = read_op(cursor, SEARCH_NEAR, &exact); - if (ret == 0 && exact != 0) - ret = WT_NOTFOUND; - } else - ret = read_op(cursor, SEARCH, NULL); - switch (ret) { - case 0: - if (g.type == FIX) { - testutil_check(cursor->get_value(cursor, &bitfield)); - *(uint8_t *)(value->data) = bitfield; - value->size = 1; - } else - testutil_check(cursor->get_value(cursor, value)); - break; - case WT_NOTFOUND: - /* - * In fixed length stores, zero values at the end of the key - * space are returned as not-found. Treat this the same as - * a zero value in the key space, to match BDB's behavior. - * The WiredTiger cursor has lost its position though, so - * we return not-found, the cursor movement can't continue. - */ - if (g.type == FIX) { - *(uint8_t *)(value->data) = 0; - value->size = 1; - } - break; - default: - return (ret); - } - - /* Log the operation */ - if (ret == 0) - switch (g.type) { - case FIX: - logop(session, "%-10s%" PRIu64 " {0x%02x}", - "read", keyno, ((char *)value->data)[0]); - break; - case ROW: - case VAR: - logop(session, "%-10s%" PRIu64 " {%.*s}", - "read", keyno, - (int)value->size, (char *)value->data); - break; - } - - return (ret); + WT_SESSION *session; + uint8_t bitfield; + int exact, ret; + + session = cursor->session; + + /* Retrieve the key/value pair by key. */ + switch (g.type) { + case FIX: + case VAR: + cursor->set_key(cursor, keyno); + break; + case ROW: + key_gen(key, keyno); + cursor->set_key(cursor, key); + break; + } + + if (sn) { + ret = read_op(cursor, SEARCH_NEAR, &exact); + if (ret == 0 && exact != 0) + ret = WT_NOTFOUND; + } else + ret = read_op(cursor, SEARCH, NULL); + switch (ret) { + case 0: + if (g.type == FIX) { + testutil_check(cursor->get_value(cursor, &bitfield)); + *(uint8_t *)(value->data) = bitfield; + value->size = 1; + } else + testutil_check(cursor->get_value(cursor, value)); + break; + case WT_NOTFOUND: + /* + * In fixed length stores, zero values at the end of the key space are returned as + * not-found. Treat this the same as a zero value in the key space, to match BDB's behavior. + * The WiredTiger cursor has lost its position though, so we return not-found, the cursor + * movement can't continue. + */ + if (g.type == FIX) { + *(uint8_t *)(value->data) = 0; + value->size = 1; + } + break; + default: + return (ret); + } + + /* Log the operation */ + if (ret == 0) + switch (g.type) { + case FIX: + logop(session, "%-10s%" PRIu64 " {0x%02x}", "read", keyno, ((char *)value->data)[0]); + break; + case ROW: + case VAR: + logop(session, "%-10s%" PRIu64 " {%.*s}", "read", keyno, (int)value->size, + (char *)value->data); + break; + } + + return (ret); } /* * read_row -- - * Read and verify a single element in a row- or column-store file. + * Read and verify a single element in a row- or column-store file. */ static int read_row(TINFO *tinfo, WT_CURSOR *cursor) { - /* 25% of the time we call search-near. */ - return (read_row_worker(cursor, tinfo->keyno, - tinfo->key, tinfo->value, mmrand(&tinfo->rnd, 0, 3) == 1)); + /* 25% of the time we call search-near. */ + return (read_row_worker( + cursor, tinfo->keyno, tinfo->key, tinfo->value, mmrand(&tinfo->rnd, 0, 3) == 1)); } /* * nextprev -- - * Read and verify the next/prev element in a row- or column-store file. + * Read and verify the next/prev element in a row- or column-store file. */ static int nextprev(TINFO *tinfo, WT_CURSOR *cursor, bool next) { - WT_DECL_RET; - WT_ITEM key, value; - uint64_t keyno, keyno_prev; - uint8_t bitfield; - int cmp; - const char *which; - bool incrementing, record_gaps; - - keyno = 0; - which = next ? "next" : "prev"; - - switch (ret = read_op(cursor, next ? NEXT : PREV, NULL)) { - case 0: - switch (g.type) { - case FIX: - if ((ret = cursor->get_key(cursor, &keyno)) == 0 && - (ret = cursor->get_value(cursor, &bitfield)) == 0) { - value.data = &bitfield; - value.size = 1; - } - break; - case ROW: - if ((ret = cursor->get_key(cursor, &key)) == 0) - ret = cursor->get_value(cursor, &value); - break; - case VAR: - if ((ret = cursor->get_key(cursor, &keyno)) == 0) - ret = cursor->get_value(cursor, &value); - break; - } - if (ret != 0) - testutil_die(ret, "nextprev: get_key/get_value"); - - /* Check that keys are never returned out-of-order. */ - /* - * XXX - * WT-3889 - * LSM has a bug that prevents cursor order checks from - * working, skip the test for now. - */ - if (DATASOURCE("lsm")) - break; - - /* - * Compare the returned key with the previously returned key, - * and assert the order is correct. If not deleting keys, and - * the rows aren't in the column-store insert name space, also - * assert we don't skip groups of records (that's a page-split - * bug symptom). - */ - record_gaps = g.c_delete_pct != 0; - switch (g.type) { - case FIX: - case VAR: - if (tinfo->keyno > g.c_rows || keyno > g.c_rows) - record_gaps = true; - if (!next) { - if (tinfo->keyno < keyno || - (!record_gaps && keyno != tinfo->keyno - 1)) - goto order_error_col; - } else - if (tinfo->keyno > keyno || - (!record_gaps && keyno != tinfo->keyno + 1)) - goto order_error_col; - if (0) { -order_error_col: - testutil_die(0, - "%s returned %" PRIu64 " then %" PRIu64, - which, tinfo->keyno, keyno); - } - - tinfo->keyno = keyno; - break; - case ROW: - incrementing = - (next && !g.c_reverse) || (!next && g.c_reverse); - cmp = memcmp(tinfo->key->data, key.data, - WT_MIN(tinfo->key->size, key.size)); - if (incrementing) { - if (cmp > 0 || - (cmp == 0 && tinfo->key->size < key.size)) - goto order_error_row; - } else - if (cmp < 0 || - (cmp == 0 && tinfo->key->size > key.size)) - goto order_error_row; - if (!record_gaps) { - /* - * Convert the keys to record numbers and then - * compare less-than-or-equal. (Not less-than, - * row-store inserts new rows in-between rows - * by append a new suffix to the row's key.) - */ - testutil_check(__wt_buf_fmt( - (WT_SESSION_IMPL *)cursor->session, - tinfo->tbuf, "%.*s", - (int)tinfo->key->size, - (char *)tinfo->key->data)); - keyno_prev = - strtoul(tinfo->tbuf->data, NULL, 10); - testutil_check(__wt_buf_fmt( - (WT_SESSION_IMPL *)cursor->session, - tinfo->tbuf, "%.*s", - (int)key.size, (char *)key.data)); - keyno = strtoul(tinfo->tbuf->data, NULL, 10); - if (incrementing) { - if (keyno_prev != keyno && - keyno_prev + 1 != keyno) - goto order_error_row; - } else - if (keyno_prev != keyno && - keyno_prev - 1 != keyno) - goto order_error_row; - } - if (0) { -order_error_row: - testutil_die(0, - "%s returned {%.*s} then {%.*s}", - which, - (int)tinfo->key->size, - (char *)tinfo->key->data, - (int)key.size, (char *)key.data); - } - - testutil_check(__wt_buf_set((WT_SESSION_IMPL *) - cursor->session, tinfo->key, key.data, key.size)); - break; - } - break; - case WT_NOTFOUND: - break; - default: - return (ret); - } - - if (ret == 0) - switch (g.type) { - case FIX: - logop(cursor->session, "%-10s%" PRIu64 " {0x%02x}", - which, keyno, ((char *)value.data)[0]); - break; - case ROW: - logop(cursor->session, - "%-10s%" PRIu64 " {%.*s}, {%.*s}", which, keyno, - (int)key.size, (char *)key.data, - (int)value.size, (char *)value.data); - break; - case VAR: - logop(cursor->session, "%-10s%" PRIu64 " {%.*s}", - which, keyno, (int)value.size, (char *)value.data); - break; - } - - return (ret); + WT_DECL_RET; + WT_ITEM key, value; + uint64_t keyno, keyno_prev; + uint8_t bitfield; + int cmp; + const char *which; + bool incrementing, record_gaps; + + keyno = 0; + which = next ? "next" : "prev"; + + switch (ret = read_op(cursor, next ? NEXT : PREV, NULL)) { + case 0: + switch (g.type) { + case FIX: + if ((ret = cursor->get_key(cursor, &keyno)) == 0 && + (ret = cursor->get_value(cursor, &bitfield)) == 0) { + value.data = &bitfield; + value.size = 1; + } + break; + case ROW: + if ((ret = cursor->get_key(cursor, &key)) == 0) + ret = cursor->get_value(cursor, &value); + break; + case VAR: + if ((ret = cursor->get_key(cursor, &keyno)) == 0) + ret = cursor->get_value(cursor, &value); + break; + } + if (ret != 0) + testutil_die(ret, "nextprev: get_key/get_value"); + + /* Check that keys are never returned out-of-order. */ + /* + * XXX WT-3889 LSM has a bug that prevents cursor order checks from working, skip the test + * for now. + */ + if (DATASOURCE("lsm")) + break; + + /* + * Compare the returned key with the previously returned key, and assert the order is + * correct. If not deleting keys, and the rows aren't in the column-store insert name space, + * also assert we don't skip groups of records (that's a page-split bug symptom). + */ + record_gaps = g.c_delete_pct != 0; + switch (g.type) { + case FIX: + case VAR: + if (tinfo->keyno > g.c_rows || keyno > g.c_rows) + record_gaps = true; + if (!next) { + if (tinfo->keyno < keyno || (!record_gaps && keyno != tinfo->keyno - 1)) + goto order_error_col; + } else if (tinfo->keyno > keyno || (!record_gaps && keyno != tinfo->keyno + 1)) + goto order_error_col; + if (0) { + order_error_col: + testutil_die( + 0, "%s returned %" PRIu64 " then %" PRIu64, which, tinfo->keyno, keyno); + } + + tinfo->keyno = keyno; + break; + case ROW: + incrementing = (next && !g.c_reverse) || (!next && g.c_reverse); + cmp = memcmp(tinfo->key->data, key.data, WT_MIN(tinfo->key->size, key.size)); + if (incrementing) { + if (cmp > 0 || (cmp == 0 && tinfo->key->size < key.size)) + goto order_error_row; + } else if (cmp < 0 || (cmp == 0 && tinfo->key->size > key.size)) + goto order_error_row; + if (!record_gaps) { + /* + * Convert the keys to record numbers and then compare less-than-or-equal. (Not + * less-than, row-store inserts new rows in-between rows by append a new suffix to + * the row's key.) + */ + testutil_check(__wt_buf_fmt((WT_SESSION_IMPL *)cursor->session, tinfo->tbuf, "%.*s", + (int)tinfo->key->size, (char *)tinfo->key->data)); + keyno_prev = strtoul(tinfo->tbuf->data, NULL, 10); + testutil_check(__wt_buf_fmt((WT_SESSION_IMPL *)cursor->session, tinfo->tbuf, "%.*s", + (int)key.size, (char *)key.data)); + keyno = strtoul(tinfo->tbuf->data, NULL, 10); + if (incrementing) { + if (keyno_prev != keyno && keyno_prev + 1 != keyno) + goto order_error_row; + } else if (keyno_prev != keyno && keyno_prev - 1 != keyno) + goto order_error_row; + } + if (0) { + order_error_row: + testutil_die(0, "%s returned {%.*s} then {%.*s}", which, (int)tinfo->key->size, + (char *)tinfo->key->data, (int)key.size, (char *)key.data); + } + + testutil_check( + __wt_buf_set((WT_SESSION_IMPL *)cursor->session, tinfo->key, key.data, key.size)); + break; + } + break; + case WT_NOTFOUND: + break; + default: + return (ret); + } + + if (ret == 0) + switch (g.type) { + case FIX: + logop( + cursor->session, "%-10s%" PRIu64 " {0x%02x}", which, keyno, ((char *)value.data)[0]); + break; + case ROW: + logop(cursor->session, "%-10s%" PRIu64 " {%.*s}, {%.*s}", which, keyno, (int)key.size, + (char *)key.data, (int)value.size, (char *)value.data); + break; + case VAR: + logop(cursor->session, "%-10s%" PRIu64 " {%.*s}", which, keyno, (int)value.size, + (char *)value.data); + break; + } + + return (ret); } /* * row_reserve -- - * Reserve a row in a row-store file. + * Reserve a row in a row-store file. */ static int row_reserve(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) { - WT_DECL_RET; + WT_DECL_RET; - if (!positioned) { - key_gen(tinfo->key, tinfo->keyno); - cursor->set_key(cursor, tinfo->key); - } + if (!positioned) { + key_gen(tinfo->key, tinfo->keyno); + cursor->set_key(cursor, tinfo->key); + } - if ((ret = cursor->reserve(cursor)) != 0) - return (ret); + if ((ret = cursor->reserve(cursor)) != 0) + return (ret); - logop(cursor->session, - "%-10s%" PRIu64 " {%.*s}", "reserve", - tinfo->keyno, (int)tinfo->key->size, (char *)tinfo->key->data); + logop(cursor->session, "%-10s%" PRIu64 " {%.*s}", "reserve", tinfo->keyno, + (int)tinfo->key->size, (char *)tinfo->key->data); - return (0); + return (0); } /* * col_reserve -- - * Reserve a row in a column-store file. + * Reserve a row in a column-store file. */ static int col_reserve(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) { - WT_DECL_RET; + WT_DECL_RET; - if (!positioned) - cursor->set_key(cursor, tinfo->keyno); + if (!positioned) + cursor->set_key(cursor, tinfo->keyno); - if ((ret = cursor->reserve(cursor)) != 0) - return (ret); + if ((ret = cursor->reserve(cursor)) != 0) + return (ret); - logop(cursor->session, "%-10s%" PRIu64, "reserve", tinfo->keyno); + logop(cursor->session, "%-10s%" PRIu64, "reserve", tinfo->keyno); - return (0); + return (0); } /* * modify_build -- - * Generate a set of modify vectors. + * Generate a set of modify vectors. */ static void modify_build(TINFO *tinfo, WT_MODIFY *entries, int *nentriesp) { - int i, nentries; - - /* Randomly select a number of byte changes, offsets and lengths. */ - nentries = (int)mmrand(&tinfo->rnd, 1, MAX_MODIFY_ENTRIES); - for (i = 0; i < nentries; ++i) { - entries[i].data.data = modify_repl + - mmrand(&tinfo->rnd, 1, sizeof(modify_repl) - 10); - entries[i].data.size = (size_t)mmrand(&tinfo->rnd, 0, 10); - /* - * Start at least 11 bytes into the buffer so we skip leading - * key information. - */ - entries[i].offset = (size_t)mmrand(&tinfo->rnd, 20, 40); - entries[i].size = (size_t)mmrand(&tinfo->rnd, 0, 10); - } - - *nentriesp = (int)nentries; + int i, nentries; + + /* Randomly select a number of byte changes, offsets and lengths. */ + nentries = (int)mmrand(&tinfo->rnd, 1, MAX_MODIFY_ENTRIES); + for (i = 0; i < nentries; ++i) { + entries[i].data.data = modify_repl + mmrand(&tinfo->rnd, 1, sizeof(modify_repl) - 10); + entries[i].data.size = (size_t)mmrand(&tinfo->rnd, 0, 10); + /* + * Start at least 11 bytes into the buffer so we skip leading key information. + */ + entries[i].offset = (size_t)mmrand(&tinfo->rnd, 20, 40); + entries[i].size = (size_t)mmrand(&tinfo->rnd, 0, 10); + } + + *nentriesp = (int)nentries; } /* * row_modify -- - * Modify a row in a row-store file. + * Modify a row in a row-store file. */ static int row_modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) { - WT_DECL_RET; - WT_MODIFY entries[MAX_MODIFY_ENTRIES]; - int nentries; + WT_DECL_RET; + WT_MODIFY entries[MAX_MODIFY_ENTRIES]; + int nentries; - if (!positioned) { - key_gen(tinfo->key, tinfo->keyno); - cursor->set_key(cursor, tinfo->key); - } + if (!positioned) { + key_gen(tinfo->key, tinfo->keyno); + cursor->set_key(cursor, tinfo->key); + } - modify_build(tinfo, entries, &nentries); - if ((ret = cursor->modify(cursor, entries, nentries)) != 0) - return (ret); + modify_build(tinfo, entries, &nentries); + if ((ret = cursor->modify(cursor, entries, nentries)) != 0) + return (ret); - testutil_check(cursor->get_value(cursor, tinfo->value)); + testutil_check(cursor->get_value(cursor, tinfo->value)); - logop(cursor->session, "%-10s%" PRIu64 " {%.*s}, {%.*s}", "modify", - tinfo->keyno, - (int)tinfo->key->size, (char *)tinfo->key->data, - (int)tinfo->value->size, (char *)tinfo->value->data); + logop(cursor->session, "%-10s%" PRIu64 " {%.*s}, {%.*s}", "modify", tinfo->keyno, + (int)tinfo->key->size, (char *)tinfo->key->data, (int)tinfo->value->size, + (char *)tinfo->value->data); - return (0); + return (0); } /* * col_modify -- - * Modify a row in a column-store file. + * Modify a row in a column-store file. */ static int col_modify(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) { - WT_DECL_RET; - WT_MODIFY entries[MAX_MODIFY_ENTRIES]; - int nentries; + WT_DECL_RET; + WT_MODIFY entries[MAX_MODIFY_ENTRIES]; + int nentries; - if (!positioned) - cursor->set_key(cursor, tinfo->keyno); + if (!positioned) + cursor->set_key(cursor, tinfo->keyno); - modify_build(tinfo, entries, &nentries); - if ((ret = cursor->modify(cursor, entries, nentries)) != 0) - return (ret); + modify_build(tinfo, entries, &nentries); + if ((ret = cursor->modify(cursor, entries, nentries)) != 0) + return (ret); - testutil_check(cursor->get_value(cursor, tinfo->value)); + testutil_check(cursor->get_value(cursor, tinfo->value)); - logop(cursor->session, "%-10s%" PRIu64 ", {%.*s}", "modify", - tinfo->keyno, (int)tinfo->value->size, (char *)tinfo->value->data); + logop(cursor->session, "%-10s%" PRIu64 ", {%.*s}", "modify", tinfo->keyno, + (int)tinfo->value->size, (char *)tinfo->value->data); - return (0); + return (0); } /* * row_truncate -- - * Truncate rows in a row-store file. + * Truncate rows in a row-store file. */ static int row_truncate(TINFO *tinfo, WT_CURSOR *cursor) { - WT_CURSOR *c2; - WT_DECL_RET; - WT_SESSION *session; - - session = cursor->session; - - /* - * The code assumes we're never truncating the entire object, assert - * that fact. - */ - testutil_assert(tinfo->keyno != 0 || tinfo->last != 0); - - c2 = NULL; - if (tinfo->keyno == 0) { - key_gen(tinfo->key, tinfo->last); - cursor->set_key(cursor, tinfo->key); - ret = session->truncate(session, NULL, NULL, cursor, NULL); - } else if (tinfo->last == 0) { - key_gen(tinfo->key, tinfo->keyno); - cursor->set_key(cursor, tinfo->key); - ret = session->truncate(session, NULL, cursor, NULL, NULL); - } else { - key_gen(tinfo->key, tinfo->keyno); - cursor->set_key(cursor, tinfo->key); - - testutil_check( - session->open_cursor(session, g.uri, NULL, NULL, &c2)); - key_gen(tinfo->lastkey, tinfo->last); - cursor->set_key(c2, tinfo->lastkey); - - ret = session->truncate(session, NULL, cursor, c2, NULL); - testutil_check(c2->close(c2)); - } - - if (ret != 0) - return (ret); - - logop(session, "%-10s%" PRIu64 ", %" PRIu64, - "truncate", tinfo->keyno, tinfo->last); - - return (0); + WT_CURSOR *c2; + WT_DECL_RET; + WT_SESSION *session; + + session = cursor->session; + + /* + * The code assumes we're never truncating the entire object, assert that fact. + */ + testutil_assert(tinfo->keyno != 0 || tinfo->last != 0); + + c2 = NULL; + if (tinfo->keyno == 0) { + key_gen(tinfo->key, tinfo->last); + cursor->set_key(cursor, tinfo->key); + ret = session->truncate(session, NULL, NULL, cursor, NULL); + } else if (tinfo->last == 0) { + key_gen(tinfo->key, tinfo->keyno); + cursor->set_key(cursor, tinfo->key); + ret = session->truncate(session, NULL, cursor, NULL, NULL); + } else { + key_gen(tinfo->key, tinfo->keyno); + cursor->set_key(cursor, tinfo->key); + + testutil_check(session->open_cursor(session, g.uri, NULL, NULL, &c2)); + key_gen(tinfo->lastkey, tinfo->last); + cursor->set_key(c2, tinfo->lastkey); + + ret = session->truncate(session, NULL, cursor, c2, NULL); + testutil_check(c2->close(c2)); + } + + if (ret != 0) + return (ret); + + logop(session, "%-10s%" PRIu64 ", %" PRIu64, "truncate", tinfo->keyno, tinfo->last); + + return (0); } /* * col_truncate -- - * Truncate rows in a column-store file. + * Truncate rows in a column-store file. */ static int col_truncate(TINFO *tinfo, WT_CURSOR *cursor) { - WT_CURSOR *c2; - WT_DECL_RET; - WT_SESSION *session; - - session = cursor->session; - - /* - * The code assumes we're never truncating the entire object, assert - * that fact. - */ - testutil_assert(tinfo->keyno != 0 || tinfo->last != 0); - - c2 = NULL; - if (tinfo->keyno == 0) { - cursor->set_key(cursor, tinfo->last); - ret = session->truncate(session, NULL, NULL, cursor, NULL); - } else if (tinfo->last == 0) { - cursor->set_key(cursor, tinfo->keyno); - ret = session->truncate(session, NULL, cursor, NULL, NULL); - } else { - cursor->set_key(cursor, tinfo->keyno); - - testutil_check( - session->open_cursor(session, g.uri, NULL, NULL, &c2)); - cursor->set_key(c2, tinfo->last); - - ret = session->truncate(session, NULL, cursor, c2, NULL); - testutil_check(c2->close(c2)); - } - if (ret != 0) - return (ret); - - logop(session, - "%-10s%" PRIu64 "-%" PRIu64, "truncate", tinfo->keyno, tinfo->last); - - return (0); + WT_CURSOR *c2; + WT_DECL_RET; + WT_SESSION *session; + + session = cursor->session; + + /* + * The code assumes we're never truncating the entire object, assert that fact. + */ + testutil_assert(tinfo->keyno != 0 || tinfo->last != 0); + + c2 = NULL; + if (tinfo->keyno == 0) { + cursor->set_key(cursor, tinfo->last); + ret = session->truncate(session, NULL, NULL, cursor, NULL); + } else if (tinfo->last == 0) { + cursor->set_key(cursor, tinfo->keyno); + ret = session->truncate(session, NULL, cursor, NULL, NULL); + } else { + cursor->set_key(cursor, tinfo->keyno); + + testutil_check(session->open_cursor(session, g.uri, NULL, NULL, &c2)); + cursor->set_key(c2, tinfo->last); + + ret = session->truncate(session, NULL, cursor, c2, NULL); + testutil_check(c2->close(c2)); + } + if (ret != 0) + return (ret); + + logop(session, "%-10s%" PRIu64 "-%" PRIu64, "truncate", tinfo->keyno, tinfo->last); + + return (0); } /* * row_update -- - * Update a row in a row-store file. + * Update a row in a row-store file. */ static int row_update(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) { - WT_DECL_RET; + WT_DECL_RET; - if (!positioned) { - key_gen(tinfo->key, tinfo->keyno); - cursor->set_key(cursor, tinfo->key); - } - val_gen(&tinfo->rnd, tinfo->value, tinfo->keyno); - cursor->set_value(cursor, tinfo->value); + if (!positioned) { + key_gen(tinfo->key, tinfo->keyno); + cursor->set_key(cursor, tinfo->key); + } + val_gen(&tinfo->rnd, tinfo->value, tinfo->keyno); + cursor->set_value(cursor, tinfo->value); - if ((ret = cursor->update(cursor)) != 0) - return (ret); + if ((ret = cursor->update(cursor)) != 0) + return (ret); - logop(cursor->session, "%-10s%" PRIu64 " {%.*s}, {%.*s}", "update", - tinfo->keyno, - (int)tinfo->key->size, (char *)tinfo->key->data, - (int)tinfo->value->size, (char *)tinfo->value->data); + logop(cursor->session, "%-10s%" PRIu64 " {%.*s}, {%.*s}", "update", tinfo->keyno, + (int)tinfo->key->size, (char *)tinfo->key->data, (int)tinfo->value->size, + (char *)tinfo->value->data); - return (0); + return (0); } /* * col_update -- - * Update a row in a column-store file. + * Update a row in a column-store file. */ static int col_update(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) { - WT_DECL_RET; - - if (!positioned) - cursor->set_key(cursor, tinfo->keyno); - val_gen(&tinfo->rnd, tinfo->value, tinfo->keyno); - if (g.type == FIX) - cursor->set_value(cursor, *(uint8_t *)tinfo->value->data); - else - cursor->set_value(cursor, tinfo->value); - - if ((ret = cursor->update(cursor)) != 0) - return (ret); - - if (g.type == FIX) - logop(cursor->session, "%-10s%" PRIu64 " {0x%02" PRIx8 "}", - "update", tinfo->keyno, ((uint8_t *)tinfo->value->data)[0]); - else - logop(cursor->session, "%-10s%" PRIu64 " {%.*s}", - "update", tinfo->keyno, - (int)tinfo->value->size, (char *)tinfo->value->data); - - return (0); + WT_DECL_RET; + + if (!positioned) + cursor->set_key(cursor, tinfo->keyno); + val_gen(&tinfo->rnd, tinfo->value, tinfo->keyno); + if (g.type == FIX) + cursor->set_value(cursor, *(uint8_t *)tinfo->value->data); + else + cursor->set_value(cursor, tinfo->value); + + if ((ret = cursor->update(cursor)) != 0) + return (ret); + + if (g.type == FIX) + logop(cursor->session, "%-10s%" PRIu64 " {0x%02" PRIx8 "}", "update", tinfo->keyno, + ((uint8_t *)tinfo->value->data)[0]); + else + logop(cursor->session, "%-10s%" PRIu64 " {%.*s}", "update", tinfo->keyno, + (int)tinfo->value->size, (char *)tinfo->value->data); + + return (0); } /* * table_append_init -- - * Re-initialize the appended records list. + * Re-initialize the appended records list. */ static void table_append_init(void) { - /* Append up to 10 records per thread before waiting on resolution. */ - g.append_max = (size_t)g.c_threads * 10; - g.append_cnt = 0; + /* Append up to 10 records per thread before waiting on resolution. */ + g.append_max = (size_t)g.c_threads * 10; + g.append_cnt = 0; - free(g.append); - g.append = dcalloc(g.append_max, sizeof(uint64_t)); + free(g.append); + g.append = dcalloc(g.append_max, sizeof(uint64_t)); } /* * table_append -- - * Resolve the appended records. + * Resolve the appended records. */ static void table_append(uint64_t keyno) { - uint64_t *ep, *p; - int done; - - ep = g.append + g.append_max; - - /* - * We don't want to ignore records we append, which requires we update - * the "last row" as we insert new records. Threads allocating record - * numbers can race with other threads, so the thread allocating record - * N may return after the thread allocating N + 1. We can't update a - * record before it's been inserted, and so we can't leave gaps when the - * count of records in the table is incremented. - * - * The solution is the append table, which contains an unsorted list of - * appended records. Every time we finish appending a record, process - * the table, trying to update the total records in the object. - * - * First, enter the new key into the append list. - * - * It's technically possible to race: we allocated space for 10 records - * per thread, but the check for the maximum number of records being - * appended doesn't lock. If a thread allocated a new record and went - * to sleep (so the append table fills up), then N threads of control - * used the same g.append_cnt value to decide there was an available - * slot in the append table and both allocated new records, we could run - * out of space in the table. It's unfortunately not even unlikely in - * the case of a large number of threads all inserting as fast as they - * can and a single thread going to sleep for an unexpectedly long time. - * If it happens, sleep and retry until earlier records are resolved - * and we find a slot. - */ - for (done = 0;;) { - testutil_check(pthread_rwlock_wrlock(&g.append_lock)); - - /* - * If this is the thread we've been waiting for, and its record - * won't fit, we'd loop infinitely. If there are many append - * operations and a thread goes to sleep for a little too long, - * it can happen. - */ - if (keyno == g.rows + 1) { - g.rows = keyno; - done = 1; - - /* - * Clean out the table, incrementing the total count of - * records until we don't find the next key. - */ - for (;;) { - for (p = g.append; p < ep; ++p) - if (*p == g.rows + 1) { - g.rows = *p; - *p = 0; - --g.append_cnt; - break; - } - if (p == ep) - break; - } - } else - /* Enter the key into the table. */ - for (p = g.append; p < ep; ++p) - if (*p == 0) { - *p = keyno; - ++g.append_cnt; - done = 1; - break; - } - - testutil_check(pthread_rwlock_unlock(&g.append_lock)); - - if (done) - break; - __wt_sleep(1, 0); - } + uint64_t *ep, *p; + int done; + + ep = g.append + g.append_max; + + /* + * We don't want to ignore records we append, which requires we update + * the "last row" as we insert new records. Threads allocating record + * numbers can race with other threads, so the thread allocating record + * N may return after the thread allocating N + 1. We can't update a + * record before it's been inserted, and so we can't leave gaps when the + * count of records in the table is incremented. + * + * The solution is the append table, which contains an unsorted list of + * appended records. Every time we finish appending a record, process + * the table, trying to update the total records in the object. + * + * First, enter the new key into the append list. + * + * It's technically possible to race: we allocated space for 10 records + * per thread, but the check for the maximum number of records being + * appended doesn't lock. If a thread allocated a new record and went + * to sleep (so the append table fills up), then N threads of control + * used the same g.append_cnt value to decide there was an available + * slot in the append table and both allocated new records, we could run + * out of space in the table. It's unfortunately not even unlikely in + * the case of a large number of threads all inserting as fast as they + * can and a single thread going to sleep for an unexpectedly long time. + * If it happens, sleep and retry until earlier records are resolved + * and we find a slot. + */ + for (done = 0;;) { + testutil_check(pthread_rwlock_wrlock(&g.append_lock)); + + /* + * If this is the thread we've been waiting for, and its record won't fit, we'd loop + * infinitely. If there are many append operations and a thread goes to sleep for a little + * too long, it can happen. + */ + if (keyno == g.rows + 1) { + g.rows = keyno; + done = 1; + + /* + * Clean out the table, incrementing the total count of records until we don't find the + * next key. + */ + for (;;) { + for (p = g.append; p < ep; ++p) + if (*p == g.rows + 1) { + g.rows = *p; + *p = 0; + --g.append_cnt; + break; + } + if (p == ep) + break; + } + } else + /* Enter the key into the table. */ + for (p = g.append; p < ep; ++p) + if (*p == 0) { + *p = keyno; + ++g.append_cnt; + done = 1; + break; + } + + testutil_check(pthread_rwlock_unlock(&g.append_lock)); + + if (done) + break; + __wt_sleep(1, 0); + } } /* * row_insert -- - * Insert a row in a row-store file. + * Insert a row in a row-store file. */ static int row_insert(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) { - WT_DECL_RET; - - /* - * If we positioned the cursor already, it's a test of an update using - * the insert method. Otherwise, generate a unique key and insert. - */ - if (!positioned) { - key_gen_insert(&tinfo->rnd, tinfo->key, tinfo->keyno); - cursor->set_key(cursor, tinfo->key); - } - val_gen(&tinfo->rnd, tinfo->value, tinfo->keyno); - cursor->set_value(cursor, tinfo->value); - - if ((ret = cursor->insert(cursor)) != 0) - return (ret); - - /* Log the operation */ - logop(cursor->session, "%-10s%" PRIu64 " {%.*s}, {%.*s}", "insert", - tinfo->keyno, - (int)tinfo->key->size, (char *)tinfo->key->data, - (int)tinfo->value->size, (char *)tinfo->value->data); - - return (0); + WT_DECL_RET; + + /* + * If we positioned the cursor already, it's a test of an update using the insert method. + * Otherwise, generate a unique key and insert. + */ + if (!positioned) { + key_gen_insert(&tinfo->rnd, tinfo->key, tinfo->keyno); + cursor->set_key(cursor, tinfo->key); + } + val_gen(&tinfo->rnd, tinfo->value, tinfo->keyno); + cursor->set_value(cursor, tinfo->value); + + if ((ret = cursor->insert(cursor)) != 0) + return (ret); + + /* Log the operation */ + logop(cursor->session, "%-10s%" PRIu64 " {%.*s}, {%.*s}", "insert", tinfo->keyno, + (int)tinfo->key->size, (char *)tinfo->key->data, (int)tinfo->value->size, + (char *)tinfo->value->data); + + return (0); } /* * col_insert -- - * Insert an element in a column-store file. + * Insert an element in a column-store file. */ static int col_insert(TINFO *tinfo, WT_CURSOR *cursor) { - WT_DECL_RET; + WT_DECL_RET; - val_gen(&tinfo->rnd, tinfo->value, g.rows + 1); - if (g.type == FIX) - cursor->set_value(cursor, *(uint8_t *)tinfo->value->data); - else - cursor->set_value(cursor, tinfo->value); + val_gen(&tinfo->rnd, tinfo->value, g.rows + 1); + if (g.type == FIX) + cursor->set_value(cursor, *(uint8_t *)tinfo->value->data); + else + cursor->set_value(cursor, tinfo->value); - if ((ret = cursor->insert(cursor)) != 0) - return (ret); + if ((ret = cursor->insert(cursor)) != 0) + return (ret); - testutil_check(cursor->get_key(cursor, &tinfo->keyno)); + testutil_check(cursor->get_key(cursor, &tinfo->keyno)); - table_append(tinfo->keyno); /* Extend the object. */ + table_append(tinfo->keyno); /* Extend the object. */ - if (g.type == FIX) - logop(cursor->session, "%-10s%" PRIu64 " {0x%02" PRIx8 "}", - "insert", tinfo->keyno, ((uint8_t *)tinfo->value->data)[0]); - else - logop(cursor->session, "%-10s%" PRIu64 " {%.*s}", - "insert", tinfo->keyno, - (int)tinfo->value->size, (char *)tinfo->value->data); + if (g.type == FIX) + logop(cursor->session, "%-10s%" PRIu64 " {0x%02" PRIx8 "}", "insert", tinfo->keyno, + ((uint8_t *)tinfo->value->data)[0]); + else + logop(cursor->session, "%-10s%" PRIu64 " {%.*s}", "insert", tinfo->keyno, + (int)tinfo->value->size, (char *)tinfo->value->data); - return (0); + return (0); } /* * row_remove -- - * Remove an row from a row-store file. + * Remove an row from a row-store file. */ static int row_remove(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) { - WT_DECL_RET; + WT_DECL_RET; - if (!positioned) { - key_gen(tinfo->key, tinfo->keyno); - cursor->set_key(cursor, tinfo->key); - } + if (!positioned) { + key_gen(tinfo->key, tinfo->keyno); + cursor->set_key(cursor, tinfo->key); + } - /* We use the cursor in overwrite mode, check for existence. */ - if ((ret = read_op(cursor, SEARCH, NULL)) == 0) - ret = cursor->remove(cursor); + /* We use the cursor in overwrite mode, check for existence. */ + if ((ret = read_op(cursor, SEARCH, NULL)) == 0) + ret = cursor->remove(cursor); - if (ret != 0 && ret != WT_NOTFOUND) - return (ret); + if (ret != 0 && ret != WT_NOTFOUND) + return (ret); - logop(cursor->session, "%-10s%" PRIu64, "remove", tinfo->keyno); + logop(cursor->session, "%-10s%" PRIu64, "remove", tinfo->keyno); - return (ret); + return (ret); } /* * col_remove -- - * Remove a row from a column-store file. + * Remove a row from a column-store file. */ static int col_remove(TINFO *tinfo, WT_CURSOR *cursor, bool positioned) { - WT_DECL_RET; + WT_DECL_RET; - if (!positioned) - cursor->set_key(cursor, tinfo->keyno); + if (!positioned) + cursor->set_key(cursor, tinfo->keyno); - /* We use the cursor in overwrite mode, check for existence. */ - if ((ret = read_op(cursor, SEARCH, NULL)) == 0) - ret = cursor->remove(cursor); + /* We use the cursor in overwrite mode, check for existence. */ + if ((ret = read_op(cursor, SEARCH, NULL)) == 0) + ret = cursor->remove(cursor); - if (ret != 0 && ret != WT_NOTFOUND) - return (ret); + if (ret != 0 && ret != WT_NOTFOUND) + return (ret); - logop(cursor->session, "%-10s%" PRIu64, "remove", tinfo->keyno); + logop(cursor->session, "%-10s%" PRIu64, "remove", tinfo->keyno); - return (ret); + return (ret); } diff --git a/src/third_party/wiredtiger/test/format/rebalance.c b/src/third_party/wiredtiger/test/format/rebalance.c index 0a845e1b2fb..94a992644ae 100644 --- a/src/third_party/wiredtiger/test/format/rebalance.c +++ b/src/third_party/wiredtiger/test/format/rebalance.c @@ -31,52 +31,47 @@ void wts_rebalance(void) { - WT_CONNECTION *conn; - WT_SESSION *session; - char cmd[1024]; + WT_CONNECTION *conn; + WT_SESSION *session; + char cmd[1024]; - if (g.c_rebalance == 0) - return; + if (g.c_rebalance == 0) + return; - track("rebalance", 0ULL, NULL); + track("rebalance", 0ULL, NULL); - /* Dump the current object. */ - testutil_check(__wt_snprintf(cmd, sizeof(cmd), - ".." DIR_DELIM_STR ".." DIR_DELIM_STR "wt" - " -h %s dump -f %s/rebalance.orig %s", - g.home, g.home, g.uri)); - testutil_checkfmt(system(cmd), "command failed: %s", cmd); + /* Dump the current object. */ + testutil_check(__wt_snprintf(cmd, sizeof(cmd), ".." DIR_DELIM_STR ".." DIR_DELIM_STR "wt" + " -h %s dump -f %s/rebalance.orig %s", + g.home, g.home, g.uri)); + testutil_checkfmt(system(cmd), "command failed: %s", cmd); - /* Rebalance, then verify the object. */ - wts_reopen(); - conn = g.wts_conn; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - logop(session, "%s", "=============== rebalance start"); + /* Rebalance, then verify the object. */ + wts_reopen(); + conn = g.wts_conn; + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + logop(session, "%s", "=============== rebalance start"); - testutil_checkfmt( - session->rebalance(session, g.uri, NULL), "%s", g.uri); + testutil_checkfmt(session->rebalance(session, g.uri, NULL), "%s", g.uri); - logop(session, "%s", "=============== rebalance stop"); - testutil_check(session->close(session, NULL)); + logop(session, "%s", "=============== rebalance stop"); + testutil_check(session->close(session, NULL)); - wts_verify("post-rebalance verify"); - wts_close(); + wts_verify("post-rebalance verify"); + wts_close(); - testutil_check(__wt_snprintf(cmd, sizeof(cmd), - ".." DIR_DELIM_STR ".." DIR_DELIM_STR "wt" - " -h %s dump -f %s/rebalance.new %s", - g.home, g.home, g.uri)); - testutil_checkfmt(system(cmd), "command failed: %s", cmd); + testutil_check(__wt_snprintf(cmd, sizeof(cmd), ".." DIR_DELIM_STR ".." DIR_DELIM_STR "wt" + " -h %s dump -f %s/rebalance.new %s", + g.home, g.home, g.uri)); + testutil_checkfmt(system(cmd), "command failed: %s", cmd); - /* Compare the old/new versions of the object. */ +/* Compare the old/new versions of the object. */ #ifdef _WIN32 - testutil_check(__wt_snprintf(cmd, sizeof(cmd), - "fc /b %s\\rebalance.orig %s\\rebalance.new > NUL", - g.home, g.home)); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "fc /b %s\\rebalance.orig %s\\rebalance.new > NUL", g.home, g.home)); #else - testutil_check(__wt_snprintf(cmd, sizeof(cmd), - "cmp %s/rebalance.orig %s/rebalance.new > /dev/null", - g.home, g.home)); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "cmp %s/rebalance.orig %s/rebalance.new > /dev/null", g.home, g.home)); #endif - testutil_checkfmt(system(cmd), "command failed: %s", cmd); + testutil_checkfmt(system(cmd), "command failed: %s", cmd); } diff --git a/src/third_party/wiredtiger/test/format/salvage.c b/src/third_party/wiredtiger/test/format/salvage.c index f6ce1d3ca5c..efe2e0162a4 100644 --- a/src/third_party/wiredtiger/test/format/salvage.c +++ b/src/third_party/wiredtiger/test/format/salvage.c @@ -30,141 +30,132 @@ /* * salvage -- - * A single salvage. + * A single salvage. */ static void salvage(void) { - WT_CONNECTION *conn; - WT_SESSION *session; + WT_CONNECTION *conn; + WT_SESSION *session; - conn = g.wts_conn; - track("salvage", 0ULL, NULL); + conn = g.wts_conn; + track("salvage", 0ULL, NULL); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->salvage(session, g.uri, "force=true")); - testutil_check(session->close(session, NULL)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->salvage(session, g.uri, "force=true")); + testutil_check(session->close(session, NULL)); } /* * corrupt -- - * Corrupt the file in a random way. + * Corrupt the file in a random way. */ static int corrupt(void) { - struct stat sb; - FILE *fp; - wt_off_t offset; - size_t len, nw; - int fd, ret; - char buf[8 * 1024], copycmd[2 * 1024]; - - /* - * If it's a single Btree file (not LSM), open the file, and corrupt - * roughly 2% of the file at a random spot, including the beginning - * of the file and overlapping the end. - * - * It's a little tricky: if the data source is a file, we're looking - * for "wt", if the data source is a table, we're looking for "wt.wt". - */ - testutil_check(__wt_snprintf( - buf, sizeof(buf), "%s/%s", g.home, WT_NAME)); - if ((fd = open(buf, O_RDWR)) != -1) { + struct stat sb; + FILE *fp; + wt_off_t offset; + size_t len, nw; + int fd, ret; + char buf[8 * 1024], copycmd[2 * 1024]; + + /* + * If it's a single Btree file (not LSM), open the file, and corrupt + * roughly 2% of the file at a random spot, including the beginning + * of the file and overlapping the end. + * + * It's a little tricky: if the data source is a file, we're looking + * for "wt", if the data source is a table, we're looking for "wt.wt". + */ + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s/%s", g.home, WT_NAME)); + if ((fd = open(buf, O_RDWR)) != -1) { #ifdef _WIN32 - testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), - "copy %s\\%s %s\\slvg.copy\\%s.corrupted", - g.home, WT_NAME, g.home, WT_NAME)); + testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), + "copy %s\\%s %s\\slvg.copy\\%s.corrupted", g.home, WT_NAME, g.home, WT_NAME)); #else - testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), - "cp %s/%s %s/slvg.copy/%s.corrupted", - g.home, WT_NAME, g.home, WT_NAME)); + testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), "cp %s/%s %s/slvg.copy/%s.corrupted", + g.home, WT_NAME, g.home, WT_NAME)); #endif - goto found; - } - testutil_check(__wt_snprintf( - buf, sizeof(buf), "%s/%s.wt", g.home, WT_NAME)); - if ((fd = open(buf, O_RDWR)) != -1) { + goto found; + } + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s/%s.wt", g.home, WT_NAME)); + if ((fd = open(buf, O_RDWR)) != -1) { #ifdef _WIN32 - testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), - "copy %s\\%s.wt %s\\slvg.copy\\%s.wt.corrupted", - g.home, WT_NAME, g.home, WT_NAME)); + testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), + "copy %s\\%s.wt %s\\slvg.copy\\%s.wt.corrupted", g.home, WT_NAME, g.home, WT_NAME)); #else - testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), - "cp %s/%s.wt %s/slvg.copy/%s.wt.corrupted", - g.home, WT_NAME, g.home, WT_NAME)); + testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), + "cp %s/%s.wt %s/slvg.copy/%s.wt.corrupted", g.home, WT_NAME, g.home, WT_NAME)); #endif - goto found; - } - return (0); - -found: if (fstat(fd, &sb) == -1) - testutil_die(errno, "salvage-corrupt: fstat"); - - offset = mmrand(NULL, 0, (u_int)sb.st_size); - len = (size_t)(20 + (sb.st_size / 100) * 2); - testutil_check(__wt_snprintf( - buf, sizeof(buf), "%s/slvg.corrupt", g.home)); - if ((fp = fopen(buf, "w")) == NULL) - testutil_die(errno, "salvage-corrupt: open: %s", buf); - (void)fprintf(fp, - "salvage-corrupt: offset %" PRIuMAX ", length %" WT_SIZET_FMT "\n", - (uintmax_t)offset, len); - fclose_and_clear(&fp); - - if (lseek(fd, offset, SEEK_SET) == -1) - testutil_die(errno, "salvage-corrupt: lseek"); - - memset(buf, 'z', sizeof(buf)); - for (; len > 0; len -= nw) { - nw = (size_t)(len > sizeof(buf) ? sizeof(buf) : len); - if (write(fd, buf, nw) == -1) - testutil_die(errno, "salvage-corrupt: write"); - } - - if (close(fd) == -1) - testutil_die(errno, "salvage-corrupt: close"); - - /* - * Save a copy of the corrupted file so we can replay the salvage step - * as necessary. - */ - if ((ret = system(copycmd)) != 0) - testutil_die(ret, "salvage corrupt copy step failed"); - - return (1); + goto found; + } + return (0); + +found: + if (fstat(fd, &sb) == -1) + testutil_die(errno, "salvage-corrupt: fstat"); + + offset = mmrand(NULL, 0, (u_int)sb.st_size); + len = (size_t)(20 + (sb.st_size / 100) * 2); + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s/slvg.corrupt", g.home)); + if ((fp = fopen(buf, "w")) == NULL) + testutil_die(errno, "salvage-corrupt: open: %s", buf); + (void)fprintf(fp, "salvage-corrupt: offset %" PRIuMAX ", length %" WT_SIZET_FMT "\n", + (uintmax_t)offset, len); + fclose_and_clear(&fp); + + if (lseek(fd, offset, SEEK_SET) == -1) + testutil_die(errno, "salvage-corrupt: lseek"); + + memset(buf, 'z', sizeof(buf)); + for (; len > 0; len -= nw) { + nw = (size_t)(len > sizeof(buf) ? sizeof(buf) : len); + if (write(fd, buf, nw) == -1) + testutil_die(errno, "salvage-corrupt: write"); + } + + if (close(fd) == -1) + testutil_die(errno, "salvage-corrupt: close"); + + /* + * Save a copy of the corrupted file so we can replay the salvage step as necessary. + */ + if ((ret = system(copycmd)) != 0) + testutil_die(ret, "salvage corrupt copy step failed"); + + return (1); } /* * wts_salvage -- - * Salvage testing. + * Salvage testing. */ void wts_salvage(void) { - WT_DECL_RET; - - if (g.c_salvage == 0) - return; - - /* - * Save a copy of the interesting files so we can replay the salvage - * step as necessary. - */ - if ((ret = system(g.home_salvage_copy)) != 0) - testutil_die(ret, "salvage copy step failed"); - - /* Salvage, then verify. */ - wts_open(g.home, true, &g.wts_conn); - salvage(); - wts_verify("post-salvage verify"); - wts_close(); - - /* Corrupt the file randomly, salvage, then verify. */ - if (corrupt()) { - wts_open(g.home, true, &g.wts_conn); - salvage(); - wts_verify("post-corrupt-salvage verify"); - wts_close(); - } + WT_DECL_RET; + + if (g.c_salvage == 0) + return; + + /* + * Save a copy of the interesting files so we can replay the salvage step as necessary. + */ + if ((ret = system(g.home_salvage_copy)) != 0) + testutil_die(ret, "salvage copy step failed"); + + /* Salvage, then verify. */ + wts_open(g.home, true, &g.wts_conn); + salvage(); + wts_verify("post-salvage verify"); + wts_close(); + + /* Corrupt the file randomly, salvage, then verify. */ + if (corrupt()) { + wts_open(g.home, true, &g.wts_conn); + salvage(); + wts_verify("post-corrupt-salvage verify"); + wts_close(); + } } diff --git a/src/third_party/wiredtiger/test/format/snap.c b/src/third_party/wiredtiger/test/format/snap.c index b38f6958f1c..e68309c0149 100644 --- a/src/third_party/wiredtiger/test/format/snap.c +++ b/src/third_party/wiredtiger/test/format/snap.c @@ -30,18 +30,18 @@ /* * snap_init -- - * Initialize the repeatable operation tracking. + * Initialize the repeatable operation tracking. */ void snap_init(TINFO *tinfo, uint64_t read_ts, bool repeatable_reads) { - ++tinfo->opid; + ++tinfo->opid; - tinfo->snap_first = tinfo->snap; + tinfo->snap_first = tinfo->snap; - tinfo->read_ts = read_ts; - tinfo->repeatable_reads = repeatable_reads; - tinfo->repeatable_wrap = false; + tinfo->read_ts = read_ts; + tinfo->repeatable_reads = repeatable_reads; + tinfo->repeatable_wrap = false; } /* @@ -51,483 +51,450 @@ snap_init(TINFO *tinfo, uint64_t read_ts, bool repeatable_reads) void snap_track(TINFO *tinfo, thread_op op) { - WT_ITEM *ip; - SNAP_OPS *snap; - - snap = tinfo->snap; - snap->op = op; - snap->opid = tinfo->opid; - snap->keyno = tinfo->keyno; - snap->ts = WT_TS_NONE; - snap->repeatable = false; - snap->last = op == TRUNCATE ? tinfo->last : 0; - snap->ksize = snap->vsize = 0; - - if (op == INSERT && g.type == ROW) { - ip = tinfo->key; - if (snap->kmemsize < ip->size) { - snap->kdata = drealloc(snap->kdata, ip->size); - snap->kmemsize = ip->size; - } - memcpy(snap->kdata, ip->data, snap->ksize = ip->size); - } - - if (op != REMOVE && op != TRUNCATE) { - ip = tinfo->value; - if (snap->vmemsize < ip->size) { - snap->vdata = drealloc(snap->vdata, ip->size); - snap->vmemsize = ip->size; - } - memcpy(snap->vdata, ip->data, snap->vsize = ip->size); - } - - /* Move to the next slot, wrap at the end of the circular buffer. */ - if (++tinfo->snap >= &tinfo->snap_list[WT_ELEMENTS(tinfo->snap_list)]) - tinfo->snap = tinfo->snap_list; - - /* - * It's possible to pass this transaction's buffer starting point and - * start replacing our own entries. If that happens, we can't repeat - * operations because we don't know which ones were previously modified. - */ - if (tinfo->snap->opid == tinfo->opid) - tinfo->repeatable_wrap = true; + WT_ITEM *ip; + SNAP_OPS *snap; + + snap = tinfo->snap; + snap->op = op; + snap->opid = tinfo->opid; + snap->keyno = tinfo->keyno; + snap->ts = WT_TS_NONE; + snap->repeatable = false; + snap->last = op == TRUNCATE ? tinfo->last : 0; + snap->ksize = snap->vsize = 0; + + if (op == INSERT && g.type == ROW) { + ip = tinfo->key; + if (snap->kmemsize < ip->size) { + snap->kdata = drealloc(snap->kdata, ip->size); + snap->kmemsize = ip->size; + } + memcpy(snap->kdata, ip->data, snap->ksize = ip->size); + } + + if (op != REMOVE && op != TRUNCATE) { + ip = tinfo->value; + if (snap->vmemsize < ip->size) { + snap->vdata = drealloc(snap->vdata, ip->size); + snap->vmemsize = ip->size; + } + memcpy(snap->vdata, ip->data, snap->vsize = ip->size); + } + + /* Move to the next slot, wrap at the end of the circular buffer. */ + if (++tinfo->snap >= &tinfo->snap_list[WT_ELEMENTS(tinfo->snap_list)]) + tinfo->snap = tinfo->snap_list; + + /* + * It's possible to pass this transaction's buffer starting point and start replacing our own + * entries. If that happens, we can't repeat operations because we don't know which ones were + * previously modified. + */ + if (tinfo->snap->opid == tinfo->opid) + tinfo->repeatable_wrap = true; } /* * print_item_data -- - * Display a single data/size pair, with a tag. + * Display a single data/size pair, with a tag. */ static void print_item_data(const char *tag, const uint8_t *data, size_t size) { - static const char hex[] = "0123456789abcdef"; - u_char ch; - - fprintf(stderr, "%s {", tag); - if (g.type == FIX) - fprintf(stderr, "0x%02x", data[0]); - else - for (; size > 0; --size, ++data) { - ch = data[0]; - if (__wt_isprint(ch)) - fprintf(stderr, "%c", (int)ch); - else - fprintf(stderr, "%x%x", - (u_int)hex[(data[0] & 0xf0) >> 4], - (u_int)hex[data[0] & 0x0f]); - } - fprintf(stderr, "}\n"); + static const char hex[] = "0123456789abcdef"; + u_char ch; + + fprintf(stderr, "%s {", tag); + if (g.type == FIX) + fprintf(stderr, "0x%02x", data[0]); + else + for (; size > 0; --size, ++data) { + ch = data[0]; + if (__wt_isprint(ch)) + fprintf(stderr, "%c", (int)ch); + else + fprintf( + stderr, "%x%x", (u_int)hex[(data[0] & 0xf0) >> 4], (u_int)hex[data[0] & 0x0f]); + } + fprintf(stderr, "}\n"); } /* * snap_verify -- - * Repeat a read and verify the contents. + * Repeat a read and verify the contents. */ static int snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) { - WT_DECL_RET; - WT_ITEM *key, *value; - uint64_t keyno; - uint8_t bitfield; - - testutil_assert(snap->op != TRUNCATE); - - key = tinfo->key; - value = tinfo->value; - keyno = snap->keyno; - - /* - * Retrieve the key/value pair by key. Row-store inserts have a unique - * generated key we saved, else generate the key from the key number. - */ - if (snap->op == INSERT && g.type == ROW) { - key->data = snap->kdata; - key->size = snap->ksize; - cursor->set_key(cursor, key); - } else { - switch (g.type) { - case FIX: - case VAR: - cursor->set_key(cursor, keyno); - break; - case ROW: - key_gen(key, keyno); - cursor->set_key(cursor, key); - break; - } - } - - switch (ret = read_op(cursor, SEARCH, NULL)) { - case 0: - if (g.type == FIX) { - testutil_check(cursor->get_value(cursor, &bitfield)); - *(uint8_t *)(value->data) = bitfield; - value->size = 1; - } else - testutil_check(cursor->get_value(cursor, value)); - break; - case WT_NOTFOUND: - break; - default: - return (ret); - } - - /* Check for simple matches. */ - if (ret == 0 && snap->op != REMOVE && - value->size == snap->vsize && - memcmp(value->data, snap->vdata, value->size) == 0) - return (0); - if (ret == WT_NOTFOUND && snap->op == REMOVE) - return (0); - - /* - * In fixed length stores, zero values at the end of the key space are - * returned as not-found, and not-found row reads are saved as zero - * values. Map back-and-forth for simplicity. - */ - if (g.type == FIX) { - if (ret == WT_NOTFOUND && - snap->vsize == 1 && *(uint8_t *)snap->vdata == 0) - return (0); - if (snap->op == REMOVE && - value->size == 1 && *(uint8_t *)value->data == 0) - return (0); - } - - /* Things went pear-shaped. */ + WT_DECL_RET; + WT_ITEM *key, *value; + uint64_t keyno; + uint8_t bitfield; + + testutil_assert(snap->op != TRUNCATE); + + key = tinfo->key; + value = tinfo->value; + keyno = snap->keyno; + + /* + * Retrieve the key/value pair by key. Row-store inserts have a unique generated key we saved, + * else generate the key from the key number. + */ + if (snap->op == INSERT && g.type == ROW) { + key->data = snap->kdata; + key->size = snap->ksize; + cursor->set_key(cursor, key); + } else { + switch (g.type) { + case FIX: + case VAR: + cursor->set_key(cursor, keyno); + break; + case ROW: + key_gen(key, keyno); + cursor->set_key(cursor, key); + break; + } + } + + switch (ret = read_op(cursor, SEARCH, NULL)) { + case 0: + if (g.type == FIX) { + testutil_check(cursor->get_value(cursor, &bitfield)); + *(uint8_t *)(value->data) = bitfield; + value->size = 1; + } else + testutil_check(cursor->get_value(cursor, value)); + break; + case WT_NOTFOUND: + break; + default: + return (ret); + } + + /* Check for simple matches. */ + if (ret == 0 && snap->op != REMOVE && value->size == snap->vsize && + memcmp(value->data, snap->vdata, value->size) == 0) + return (0); + if (ret == WT_NOTFOUND && snap->op == REMOVE) + return (0); + + /* + * In fixed length stores, zero values at the end of the key space are returned as not-found, + * and not-found row reads are saved as zero values. Map back-and-forth for simplicity. + */ + if (g.type == FIX) { + if (ret == WT_NOTFOUND && snap->vsize == 1 && *(uint8_t *)snap->vdata == 0) + return (0); + if (snap->op == REMOVE && value->size == 1 && *(uint8_t *)value->data == 0) + return (0); + } + +/* Things went pear-shaped. */ #ifdef HAVE_DIAGNOSTIC - fprintf(stderr, - "snapshot-isolation error: Dumping page to %s\n", g.home_pagedump); - testutil_check(__wt_debug_cursor_page(cursor, g.home_pagedump)); + fprintf(stderr, "snapshot-isolation error: Dumping page to %s\n", g.home_pagedump); + testutil_check(__wt_debug_cursor_page(cursor, g.home_pagedump)); #endif - switch (g.type) { - case FIX: - testutil_die(ret, - "snapshot-isolation: %" PRIu64 " search: " - "expected {0x%02x}, found {0x%02x}", - keyno, - snap->op == REMOVE ? 0 : *(uint8_t *)snap->vdata, - ret == WT_NOTFOUND ? 0 : *(uint8_t *)value->data); - /* NOTREACHED */ - case ROW: - fprintf(stderr, - "snapshot-isolation %.*s search mismatch\n", - (int)key->size, (char *)key->data); - - if (snap->op == REMOVE) - fprintf(stderr, "expected {deleted}\n"); - else - print_item_data("expected", snap->vdata, snap->vsize); - if (ret == WT_NOTFOUND) - fprintf(stderr, " found {deleted}\n"); - else - print_item_data(" found", value->data, value->size); - - testutil_die(ret, - "snapshot-isolation: %.*s search mismatch", - (int)key->size, (char *)key->data); - /* NOTREACHED */ - case VAR: - fprintf(stderr, - "snapshot-isolation %" PRIu64 " search mismatch\n", keyno); - - if (snap->op == REMOVE) - fprintf(stderr, "expected {deleted}\n"); - else - print_item_data("expected", snap->vdata, snap->vsize); - if (ret == WT_NOTFOUND) - fprintf(stderr, " found {deleted}\n"); - else - print_item_data(" found", value->data, value->size); - - testutil_die(ret, - "snapshot-isolation: %" PRIu64 " search mismatch", keyno); - /* NOTREACHED */ - } - - /* NOTREACHED */ - return (1); + switch (g.type) { + case FIX: + testutil_die(ret, "snapshot-isolation: %" PRIu64 + " search: " + "expected {0x%02x}, found {0x%02x}", + keyno, snap->op == REMOVE ? 0 : *(uint8_t *)snap->vdata, + ret == WT_NOTFOUND ? 0 : *(uint8_t *)value->data); + /* NOTREACHED */ + case ROW: + fprintf( + stderr, "snapshot-isolation %.*s search mismatch\n", (int)key->size, (char *)key->data); + + if (snap->op == REMOVE) + fprintf(stderr, "expected {deleted}\n"); + else + print_item_data("expected", snap->vdata, snap->vsize); + if (ret == WT_NOTFOUND) + fprintf(stderr, " found {deleted}\n"); + else + print_item_data(" found", value->data, value->size); + + testutil_die( + ret, "snapshot-isolation: %.*s search mismatch", (int)key->size, (char *)key->data); + /* NOTREACHED */ + case VAR: + fprintf(stderr, "snapshot-isolation %" PRIu64 " search mismatch\n", keyno); + + if (snap->op == REMOVE) + fprintf(stderr, "expected {deleted}\n"); + else + print_item_data("expected", snap->vdata, snap->vsize); + if (ret == WT_NOTFOUND) + fprintf(stderr, " found {deleted}\n"); + else + print_item_data(" found", value->data, value->size); + + testutil_die(ret, "snapshot-isolation: %" PRIu64 " search mismatch", keyno); + /* NOTREACHED */ + } + + /* NOTREACHED */ + return (1); } /* * snap_ts_clear -- - * Clear snapshots at or before a specified timestamp. + * Clear snapshots at or before a specified timestamp. */ static void snap_ts_clear(TINFO *tinfo, uint64_t ts) { - SNAP_OPS *snap; - int count; - - /* Check from the first slot to the last. */ - for (snap = tinfo->snap_list, - count = WT_ELEMENTS(tinfo->snap_list); count > 0; --count, ++snap) - if (snap->repeatable && snap->ts <= ts) - snap->repeatable = false; + SNAP_OPS *snap; + int count; + + /* Check from the first slot to the last. */ + for (snap = tinfo->snap_list, count = WT_ELEMENTS(tinfo->snap_list); count > 0; --count, ++snap) + if (snap->repeatable && snap->ts <= ts) + snap->repeatable = false; } /* * snap_repeat_ok_match -- - * Compare two operations and see if they modified the same record. + * Compare two operations and see if they modified the same record. */ static bool snap_repeat_ok_match(SNAP_OPS *current, SNAP_OPS *a) { - /* Reads are never a problem, there's no modification. */ - if (a->op == READ) - return (true); - - /* Check for a matching single record modification. */ - if (a->keyno == current->keyno) - return (false); - - /* Truncates are slightly harder, make sure the ranges don't overlap. */ - if (a->op == TRUNCATE) { - if (g.c_reverse && - (a->keyno == 0 || a->keyno >= current->keyno) && - (a->last == 0 || a->last <= current->keyno)) - return (false); - if (!g.c_reverse && - (a->keyno == 0 || a->keyno <= current->keyno) && - (a->last == 0 || a->last >= current->keyno)) - return (false); - } - - return (true); + /* Reads are never a problem, there's no modification. */ + if (a->op == READ) + return (true); + + /* Check for a matching single record modification. */ + if (a->keyno == current->keyno) + return (false); + + /* Truncates are slightly harder, make sure the ranges don't overlap. */ + if (a->op == TRUNCATE) { + if (g.c_reverse && (a->keyno == 0 || a->keyno >= current->keyno) && + (a->last == 0 || a->last <= current->keyno)) + return (false); + if (!g.c_reverse && (a->keyno == 0 || a->keyno <= current->keyno) && + (a->last == 0 || a->last >= current->keyno)) + return (false); + } + + return (true); } /* * snap_repeat_ok_commit -- - * Return if an operation in the transaction can be repeated, where the - * transaction isn't yet committed (so all locks are in place), or has already - * committed successfully. + * Return if an operation in the transaction can be repeated, where the transaction isn't yet + * committed (so all locks are in place), or has already committed successfully. */ static bool snap_repeat_ok_commit(TINFO *tinfo, SNAP_OPS *current) { - SNAP_OPS *p; - - /* - * Truncates can't be repeated, we don't know the exact range of records - * that were removed (if any). - */ - if (current->op == TRUNCATE) - return (false); - - /* - * For updates, check for subsequent changes to the record and don't - * repeat the read. For reads, check for either subsequent or previous - * changes to the record and don't repeat the read. (The reads are - * repeatable, but only at the commit timestamp, and the update will - * do the repeatable read in that case.) - */ - for (p = current;;) { - /* Wrap at the end of the circular buffer. */ - if (++p >= &tinfo->snap_list[WT_ELEMENTS(tinfo->snap_list)]) - p = tinfo->snap_list; - if (p->opid != tinfo->opid) - break; - - if (!snap_repeat_ok_match(current, p)) - return (false); - } - - if (current->op != READ) - return (true); - for (p = current;;) { - /* Wrap at the beginning of the circular buffer. */ - if (--p < tinfo->snap_list) - p = &tinfo->snap_list[ - WT_ELEMENTS(tinfo->snap_list) - 1]; - if (p->opid != tinfo->opid) - break; - - if (!snap_repeat_ok_match(current, p)) - return (false); - - } - return (true); + SNAP_OPS *p; + + /* + * Truncates can't be repeated, we don't know the exact range of records that were removed (if + * any). + */ + if (current->op == TRUNCATE) + return (false); + + /* + * For updates, check for subsequent changes to the record and don't repeat the read. For reads, + * check for either subsequent or previous changes to the record and don't repeat the read. (The + * reads are repeatable, but only at the commit timestamp, and the update will do the repeatable + * read in that case.) + */ + for (p = current;;) { + /* Wrap at the end of the circular buffer. */ + if (++p >= &tinfo->snap_list[WT_ELEMENTS(tinfo->snap_list)]) + p = tinfo->snap_list; + if (p->opid != tinfo->opid) + break; + + if (!snap_repeat_ok_match(current, p)) + return (false); + } + + if (current->op != READ) + return (true); + for (p = current;;) { + /* Wrap at the beginning of the circular buffer. */ + if (--p < tinfo->snap_list) + p = &tinfo->snap_list[WT_ELEMENTS(tinfo->snap_list) - 1]; + if (p->opid != tinfo->opid) + break; + + if (!snap_repeat_ok_match(current, p)) + return (false); + } + return (true); } /* * snap_repeat_ok_rollback -- - * Return if an operation in the transaction can be repeated, after a - * transaction has rolled back. + * Return if an operation in the transaction can be repeated, after a transaction has rolled + * back. */ static bool snap_repeat_ok_rollback(TINFO *tinfo, SNAP_OPS *current) { - SNAP_OPS *p; - - /* Ignore update operations, they can't be repeated after rollback. */ - if (current->op != READ) - return (false); - - /* - * Check for previous changes to the record and don't attempt to repeat - * the read in that case. - */ - for (p = current;;) { - /* Wrap at the beginning of the circular buffer. */ - if (--p < tinfo->snap_list) - p = &tinfo->snap_list[ - WT_ELEMENTS(tinfo->snap_list) - 1]; - if (p->opid != tinfo->opid) - break; - - if (!snap_repeat_ok_match(current, p)) - return (false); - - } - return (true); + SNAP_OPS *p; + + /* Ignore update operations, they can't be repeated after rollback. */ + if (current->op != READ) + return (false); + + /* + * Check for previous changes to the record and don't attempt to repeat the read in that case. + */ + for (p = current;;) { + /* Wrap at the beginning of the circular buffer. */ + if (--p < tinfo->snap_list) + p = &tinfo->snap_list[WT_ELEMENTS(tinfo->snap_list) - 1]; + if (p->opid != tinfo->opid) + break; + + if (!snap_repeat_ok_match(current, p)) + return (false); + } + return (true); } /* * snap_repeat_txn -- - * Repeat each operation done within a snapshot isolation transaction. + * Repeat each operation done within a snapshot isolation transaction. */ int snap_repeat_txn(WT_CURSOR *cursor, TINFO *tinfo) { - SNAP_OPS *current; + SNAP_OPS *current; - /* If we wrapped the buffer, we can't repeat operations. */ - if (tinfo->repeatable_wrap) - return (0); + /* If we wrapped the buffer, we can't repeat operations. */ + if (tinfo->repeatable_wrap) + return (0); - /* Check from the first operation we saved to the last. */ - for (current = tinfo->snap_first;; ++current) { - /* Wrap at the end of the circular buffer. */ - if (current >= &tinfo->snap_list[WT_ELEMENTS(tinfo->snap_list)]) - current = tinfo->snap_list; - if (current->opid != tinfo->opid) - break; + /* Check from the first operation we saved to the last. */ + for (current = tinfo->snap_first;; ++current) { + /* Wrap at the end of the circular buffer. */ + if (current >= &tinfo->snap_list[WT_ELEMENTS(tinfo->snap_list)]) + current = tinfo->snap_list; + if (current->opid != tinfo->opid) + break; - if (snap_repeat_ok_commit(tinfo, current)) - WT_RET(snap_verify(cursor, tinfo, current)); - } + if (snap_repeat_ok_commit(tinfo, current)) + WT_RET(snap_verify(cursor, tinfo, current)); + } - return (0); + return (0); } /* * snap_repeat_update -- - * Update the list of snapshot operations based on final transaction - * resolution. + * Update the list of snapshot operations based on final transaction resolution. */ void snap_repeat_update(TINFO *tinfo, bool committed) { - SNAP_OPS *current; - - /* If we wrapped the buffer, we can't repeat operations. */ - if (tinfo->repeatable_wrap) - return; - - /* Check from the first operation we saved to the last. */ - for (current = tinfo->snap_first;; ++current) { - /* Wrap at the end of the circular buffer. */ - if (current >= &tinfo->snap_list[WT_ELEMENTS(tinfo->snap_list)]) - current = tinfo->snap_list; - if (current->opid != tinfo->opid) - break; - - /* - * First, reads may simply not be repeatable because the read - * timestamp chosen wasn't older than all concurrently running - * uncommitted updates. - */ - if (!tinfo->repeatable_reads && current->op == READ) - continue; - - /* - * Second, check based on the transaction resolution (the rules - * are different if the transaction committed or rolled back). - */ - current->repeatable = committed ? - snap_repeat_ok_commit(tinfo, current) : - snap_repeat_ok_rollback(tinfo, current); - - /* - * Repeat reads at the transaction's read timestamp and updates - * at the commit timestamp. - */ - if (current->repeatable) - current->ts = current->op == READ ? - tinfo->read_ts : tinfo->commit_ts; - } + SNAP_OPS *current; + + /* If we wrapped the buffer, we can't repeat operations. */ + if (tinfo->repeatable_wrap) + return; + + /* Check from the first operation we saved to the last. */ + for (current = tinfo->snap_first;; ++current) { + /* Wrap at the end of the circular buffer. */ + if (current >= &tinfo->snap_list[WT_ELEMENTS(tinfo->snap_list)]) + current = tinfo->snap_list; + if (current->opid != tinfo->opid) + break; + + /* + * First, reads may simply not be repeatable because the read timestamp chosen wasn't older + * than all concurrently running uncommitted updates. + */ + if (!tinfo->repeatable_reads && current->op == READ) + continue; + + /* + * Second, check based on the transaction resolution (the rules are different if the + * transaction committed or rolled back). + */ + current->repeatable = committed ? snap_repeat_ok_commit(tinfo, current) : + snap_repeat_ok_rollback(tinfo, current); + + /* + * Repeat reads at the transaction's read timestamp and updates at the commit timestamp. + */ + if (current->repeatable) + current->ts = current->op == READ ? tinfo->read_ts : tinfo->commit_ts; + } } /* * snap_repeat_single -- - * Repeat an historic operation. + * Repeat an historic operation. */ void snap_repeat_single(WT_CURSOR *cursor, TINFO *tinfo) { - SNAP_OPS *snap; - WT_DECL_RET; - WT_SESSION *session; - int count; - u_int v; - char buf[64]; - - session = cursor->session; - - /* - * Start at a random spot in the list of operations and look for a read - * to retry. Stop when we've walked the entire list or found one. - */ - v = mmrand(&tinfo->rnd, 1, WT_ELEMENTS(tinfo->snap_list)) - 1; - for (snap = &tinfo->snap_list[v], - count = WT_ELEMENTS(tinfo->snap_list); count > 0; --count, ++snap) { - /* Wrap at the end of the circular buffer. */ - if (snap >= &tinfo->snap_list[WT_ELEMENTS(tinfo->snap_list)]) - snap = tinfo->snap_list; - - if (snap->repeatable) - break; - } - - if (count == 0) - return; - - /* - * Start a new transaction. - * Set the read timestamp. - * Verify the record. - * Discard the transaction. - */ - while ((ret = session->begin_transaction( - session, "isolation=snapshot")) == WT_CACHE_FULL) - __wt_yield(); - testutil_check(ret); - - /* - * If the timestamp has aged out of the system, we'll get EINVAL when we - * try and set it. - */ - testutil_check(__wt_snprintf( - buf, sizeof(buf), "read_timestamp=%" PRIx64, snap->ts)); - - ret = session->timestamp_transaction(session, buf); - if (ret == 0) { - logop(session, "%-10s%" PRIu64 " ts=%" PRIu64 " {%.*s}", - "repeat", snap->keyno, snap->ts, - (int)snap->vsize, (char *)snap->vdata); - - /* The only expected error is rollback. */ - ret = snap_verify(cursor, tinfo, snap); - - if (ret != 0 && ret != WT_ROLLBACK) - testutil_check(ret); - } else if (ret == EINVAL) - snap_ts_clear(tinfo, snap->ts); - else - testutil_check(ret); - - /* Discard the transaction. */ - testutil_check(session->rollback_transaction(session, NULL)); + SNAP_OPS *snap; + WT_DECL_RET; + WT_SESSION *session; + int count; + u_int v; + char buf[64]; + + session = cursor->session; + + /* + * Start at a random spot in the list of operations and look for a read to retry. Stop when + * we've walked the entire list or found one. + */ + v = mmrand(&tinfo->rnd, 1, WT_ELEMENTS(tinfo->snap_list)) - 1; + for (snap = &tinfo->snap_list[v], count = WT_ELEMENTS(tinfo->snap_list); count > 0; + --count, ++snap) { + /* Wrap at the end of the circular buffer. */ + if (snap >= &tinfo->snap_list[WT_ELEMENTS(tinfo->snap_list)]) + snap = tinfo->snap_list; + + if (snap->repeatable) + break; + } + + if (count == 0) + return; + + /* + * Start a new transaction. Set the read timestamp. Verify the record. Discard the transaction. + */ + while ((ret = session->begin_transaction(session, "isolation=snapshot")) == WT_CACHE_FULL) + __wt_yield(); + testutil_check(ret); + + /* + * If the timestamp has aged out of the system, we'll get EINVAL when we try and set it. + */ + testutil_check(__wt_snprintf(buf, sizeof(buf), "read_timestamp=%" PRIx64, snap->ts)); + + ret = session->timestamp_transaction(session, buf); + if (ret == 0) { + logop(session, "%-10s%" PRIu64 " ts=%" PRIu64 " {%.*s}", "repeat", snap->keyno, snap->ts, + (int)snap->vsize, (char *)snap->vdata); + + /* The only expected error is rollback. */ + ret = snap_verify(cursor, tinfo, snap); + + if (ret != 0 && ret != WT_ROLLBACK) + testutil_check(ret); + } else if (ret == EINVAL) + snap_ts_clear(tinfo, snap->ts); + else + testutil_check(ret); + + /* Discard the transaction. */ + testutil_check(session->rollback_transaction(session, NULL)); } diff --git a/src/third_party/wiredtiger/test/format/t.c b/src/third_party/wiredtiger/test/format/t.c index 84175ba53d6..c46a12f45b2 100644 --- a/src/third_party/wiredtiger/test/format/t.c +++ b/src/third_party/wiredtiger/test/format/t.c @@ -32,50 +32,47 @@ GLOBAL g; static void format_die(void); static void startup(void); -static void usage(void) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); extern int __wt_optind; extern char *__wt_optarg; /* * signal_handler -- - * Handle signals. + * Handle signals. */ static void signal_handler(int signo) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void signal_handler(int signo) { - fprintf(stderr, - "format caught signal %d, aborting the process\n", signo); - __wt_abort(NULL); + fprintf(stderr, "format caught signal %d, aborting the process\n", signo); + __wt_abort(NULL); } int main(int argc, char *argv[]) { - time_t start; - int ch, onerun, reps; - const char *config, *home; + time_t start; + int ch, onerun, reps; + const char *config, *home; - custom_die = format_die; /* Local death handler. */ + custom_die = format_die; /* Local death handler. */ - config = NULL; + config = NULL; - (void)testutil_set_progname(argv); + (void)testutil_set_progname(argv); - /* - * Windows and Linux support different sets of signals, be conservative - * about installing handlers. - */ +/* + * Windows and Linux support different sets of signals, be conservative about installing handlers. + */ #ifdef SIGALRM - (void)signal(SIGALRM, signal_handler); + (void)signal(SIGALRM, signal_handler); #endif #ifdef SIGHUP - (void)signal(SIGHUP, signal_handler); + (void)signal(SIGHUP, signal_handler); #endif #ifdef SIGTERM - (void)signal(SIGTERM, signal_handler); + (void)signal(SIGTERM, signal_handler); #endif #if 0 @@ -87,274 +84,266 @@ main(int argc, char *argv[]) (void)setenv("MALLOC_OPTIONS", "AJ", 1); #endif - /* Track progress unless we're re-directing output to a file. */ - g.c_quiet = isatty(1) ? 0 : 1; - - /* Set values from the command line. */ - home = NULL; - onerun = 0; - while ((ch = __wt_getopt( - progname, argc, argv, "1C:c:h:lqrt:")) != EOF) - switch (ch) { - case '1': /* One run */ - onerun = 1; - break; - case 'C': /* wiredtiger_open config */ - g.config_open = __wt_optarg; - break; - case 'c': /* Configuration from a file */ - config = __wt_optarg; - break; - case 'h': - home = __wt_optarg; - break; - case 'l': /* Log operations to a file */ - g.logging = true; - break; - case 'q': /* Quiet */ - g.c_quiet = 1; - break; - case 'r': /* Replay a run */ - g.replay = true; - break; - default: - usage(); - } - argv += __wt_optind; - - /* Initialize the global RNG. */ - __wt_random_init_seed(NULL, &g.rnd); - - /* Set up paths. */ - path_setup(home); - - /* If it's a replay, use the home directory's CONFIG file. */ - if (g.replay) { - if (config != NULL) - testutil_die(EINVAL, "-c incompatible with -r"); - if (access(g.home_config, R_OK) != 0) - testutil_die(ENOENT, "%s", g.home_config); - config = g.home_config; - } - - /* - * If we weren't given a configuration file, set values from "CONFIG", - * if it exists. - * - * Small hack to ignore any CONFIG file named ".", that just makes it - * possible to ignore any local CONFIG file, used when running checks. - */ - if (config == NULL && access("CONFIG", R_OK) == 0) - config = "CONFIG"; - if (config != NULL && strcmp(config, ".") != 0) - config_file(config); - - /* - * The rest of the arguments are individual configurations that modify - * the base configuration. - */ - for (; *argv != NULL; ++argv) - config_single(*argv, true); - - /* - * Multithreaded runs can be replayed: it's useful and we'll get the - * configuration correct. Obviously the order of operations changes, - * warn the user. - */ - if (g.replay && !SINGLETHREADED) - printf("Warning: replaying a threaded run\n"); - - /* - * Single-threaded runs historically exited after a single replay, which - * makes sense when you're debugging, leave that semantic in place. - */ - if (g.replay && SINGLETHREADED) - g.c_runs = 1; - - /* - * Let the command line -1 flag override runs configured from other - * sources. - */ - if (onerun) - g.c_runs = 1; - - /* - * Initialize locks to single-thread named checkpoints and backups, last - * last-record updates, and failures. - */ - testutil_check(pthread_rwlock_init(&g.append_lock, NULL)); - testutil_check(pthread_rwlock_init(&g.backup_lock, NULL)); - testutil_check(pthread_rwlock_init(&g.death_lock, NULL)); - testutil_check(pthread_rwlock_init(&g.ts_lock, NULL)); - - printf("%s: process %" PRIdMAX "\n", progname, (intmax_t)getpid()); - while (++g.run_cnt <= g.c_runs || g.c_runs == 0 ) { - startup(); /* Start a run */ - - config_setup(); /* Run configuration */ - config_print(false); /* Dump run configuration */ - key_init(); /* Setup keys/values */ - val_init(); - - start = time(NULL); - track("starting up", 0ULL, NULL); - - wts_open(g.home, true, &g.wts_conn); - wts_init(); - - wts_load(); /* Load initial records */ - wts_verify("post-bulk verify"); /* Verify */ - - /* - * If we're not doing any operations, scan the bulk-load, copy - * the statistics and we're done. Otherwise, loop reading and - * operations, with a verify after each set. - */ - if (g.c_timer == 0 && g.c_ops == 0) { - wts_read_scan(); /* Read scan */ - wts_stats(); /* Statistics */ - } else - for (reps = 1; reps <= FORMAT_OPERATION_REPS; ++reps) { - wts_read_scan(); /* Read scan */ - - /* Operations */ - wts_ops(reps == FORMAT_OPERATION_REPS); - - /* - * Copy out the run's statistics after the last - * set of operations. - * - * XXX - * Verify closes the underlying handle and - * discards the statistics, read them first. - */ - if (reps == FORMAT_OPERATION_REPS) - wts_stats(); - - /* Verify */ - wts_verify("post-ops verify"); - } - - track("shutting down", 0ULL, NULL); - wts_close(); - - /* - * Rebalance testing. - */ - wts_rebalance(); - - /* - * Salvage testing. - */ - wts_salvage(); - - /* Overwrite the progress line with a completion line. */ - if (!g.c_quiet) - printf("\r%78s\r", " "); - printf("%4" PRIu32 ": %s, %s (%.0f seconds)\n", - g.run_cnt, g.c_data_source, - g.c_file_type, difftime(time(NULL), start)); - fflush(stdout); - - val_teardown(); /* Teardown keys/values */ - } - - /* Flush/close any logging information. */ - fclose_and_clear(&g.logfp); - fclose_and_clear(&g.randfp); - - config_print(false); - - testutil_check(pthread_rwlock_destroy(&g.append_lock)); - testutil_check(pthread_rwlock_destroy(&g.backup_lock)); - testutil_check(pthread_rwlock_destroy(&g.death_lock)); - testutil_check(pthread_rwlock_destroy(&g.ts_lock)); - - config_clear(); - - return (EXIT_SUCCESS); + /* Track progress unless we're re-directing output to a file. */ + g.c_quiet = isatty(1) ? 0 : 1; + + /* Set values from the command line. */ + home = NULL; + onerun = 0; + while ((ch = __wt_getopt(progname, argc, argv, "1C:c:h:lqrt:")) != EOF) + switch (ch) { + case '1': /* One run */ + onerun = 1; + break; + case 'C': /* wiredtiger_open config */ + g.config_open = __wt_optarg; + break; + case 'c': /* Configuration from a file */ + config = __wt_optarg; + break; + case 'h': + home = __wt_optarg; + break; + case 'l': /* Log operations to a file */ + g.logging = true; + break; + case 'q': /* Quiet */ + g.c_quiet = 1; + break; + case 'r': /* Replay a run */ + g.replay = true; + break; + default: + usage(); + } + argv += __wt_optind; + + /* Initialize the global RNG. */ + __wt_random_init_seed(NULL, &g.rnd); + + /* Set up paths. */ + path_setup(home); + + /* If it's a replay, use the home directory's CONFIG file. */ + if (g.replay) { + if (config != NULL) + testutil_die(EINVAL, "-c incompatible with -r"); + if (access(g.home_config, R_OK) != 0) + testutil_die(ENOENT, "%s", g.home_config); + config = g.home_config; + } + + /* + * If we weren't given a configuration file, set values from "CONFIG", + * if it exists. + * + * Small hack to ignore any CONFIG file named ".", that just makes it + * possible to ignore any local CONFIG file, used when running checks. + */ + if (config == NULL && access("CONFIG", R_OK) == 0) + config = "CONFIG"; + if (config != NULL && strcmp(config, ".") != 0) + config_file(config); + + /* + * The rest of the arguments are individual configurations that modify the base configuration. + */ + for (; *argv != NULL; ++argv) + config_single(*argv, true); + + /* + * Multithreaded runs can be replayed: it's useful and we'll get the configuration correct. + * Obviously the order of operations changes, warn the user. + */ + if (g.replay && !SINGLETHREADED) + printf("Warning: replaying a threaded run\n"); + + /* + * Single-threaded runs historically exited after a single replay, which makes sense when you're + * debugging, leave that semantic in place. + */ + if (g.replay && SINGLETHREADED) + g.c_runs = 1; + + /* + * Let the command line -1 flag override runs configured from other sources. + */ + if (onerun) + g.c_runs = 1; + + /* + * Initialize locks to single-thread named checkpoints and backups, last last-record updates, + * and failures. + */ + testutil_check(pthread_rwlock_init(&g.append_lock, NULL)); + testutil_check(pthread_rwlock_init(&g.backup_lock, NULL)); + testutil_check(pthread_rwlock_init(&g.death_lock, NULL)); + testutil_check(pthread_rwlock_init(&g.ts_lock, NULL)); + + printf("%s: process %" PRIdMAX "\n", progname, (intmax_t)getpid()); + while (++g.run_cnt <= g.c_runs || g.c_runs == 0) { + startup(); /* Start a run */ + + config_setup(); /* Run configuration */ + config_print(false); /* Dump run configuration */ + key_init(); /* Setup keys/values */ + val_init(); + + start = time(NULL); + track("starting up", 0ULL, NULL); + + wts_open(g.home, true, &g.wts_conn); + wts_init(); + + wts_load(); /* Load initial records */ + wts_verify("post-bulk verify"); /* Verify */ + + /* + * If we're not doing any operations, scan the bulk-load, copy the statistics and we're + * done. Otherwise, loop reading and operations, with a verify after each set. + */ + if (g.c_timer == 0 && g.c_ops == 0) { + wts_read_scan(); /* Read scan */ + wts_stats(); /* Statistics */ + } else + for (reps = 1; reps <= FORMAT_OPERATION_REPS; ++reps) { + wts_read_scan(); /* Read scan */ + + /* Operations */ + wts_ops(reps == FORMAT_OPERATION_REPS); + + /* + * Copy out the run's statistics after the last + * set of operations. + * + * XXX + * Verify closes the underlying handle and + * discards the statistics, read them first. + */ + if (reps == FORMAT_OPERATION_REPS) + wts_stats(); + + /* Verify */ + wts_verify("post-ops verify"); + } + + track("shutting down", 0ULL, NULL); + wts_close(); + + /* + * Rebalance testing. + */ + wts_rebalance(); + + /* + * Salvage testing. + */ + wts_salvage(); + + /* Overwrite the progress line with a completion line. */ + if (!g.c_quiet) + printf("\r%78s\r", " "); + printf("%4" PRIu32 ": %s, %s (%.0f seconds)\n", g.run_cnt, g.c_data_source, g.c_file_type, + difftime(time(NULL), start)); + fflush(stdout); + + val_teardown(); /* Teardown keys/values */ + } + + /* Flush/close any logging information. */ + fclose_and_clear(&g.logfp); + fclose_and_clear(&g.randfp); + + config_print(false); + + testutil_check(pthread_rwlock_destroy(&g.append_lock)); + testutil_check(pthread_rwlock_destroy(&g.backup_lock)); + testutil_check(pthread_rwlock_destroy(&g.death_lock)); + testutil_check(pthread_rwlock_destroy(&g.ts_lock)); + + config_clear(); + + return (EXIT_SUCCESS); } /* * startup -- - * Initialize for a run. + * Initialize for a run. */ static void startup(void) { - WT_DECL_RET; + WT_DECL_RET; - /* Flush/close any logging information. */ - fclose_and_clear(&g.logfp); - fclose_and_clear(&g.randfp); + /* Flush/close any logging information. */ + fclose_and_clear(&g.logfp); + fclose_and_clear(&g.randfp); - /* Create or initialize the home and data-source directories. */ - if ((ret = system(g.home_init)) != 0) - testutil_die(ret, "home directory initialization failed"); + /* Create or initialize the home and data-source directories. */ + if ((ret = system(g.home_init)) != 0) + testutil_die(ret, "home directory initialization failed"); - /* Open/truncate the logging file. */ - if (g.logging && (g.logfp = fopen(g.home_log, "w")) == NULL) - testutil_die(errno, "fopen: %s", g.home_log); + /* Open/truncate the logging file. */ + if (g.logging && (g.logfp = fopen(g.home_log, "w")) == NULL) + testutil_die(errno, "fopen: %s", g.home_log); - /* Open/truncate the random number logging file. */ - if ((g.randfp = fopen(g.home_rand, g.replay ? "r" : "w")) == NULL) - testutil_die(errno, "%s", g.home_rand); + /* Open/truncate the random number logging file. */ + if ((g.randfp = fopen(g.home_rand, g.replay ? "r" : "w")) == NULL) + testutil_die(errno, "%s", g.home_rand); } /* * die -- - * Report an error, dumping the configuration. + * Report an error, dumping the configuration. */ static void format_die(void) { - /* - * Turn off tracking and logging so we don't obscure the error message. - * The lock we're about to acquire will act as a barrier to flush the - * writes. This is really a "best effort" more than a guarantee, there's - * too much stuff in flight to be sure. - */ - g.c_quiet = 1; - g.logging = false; - - /* - * Single-thread error handling, our caller exits after calling us (we - * never release the lock). - */ - (void)pthread_rwlock_wrlock(&g.death_lock); - - /* Flush/close any logging information. */ - fclose_and_clear(&g.logfp); - fclose_and_clear(&g.randfp); - - fprintf(stderr, "\n"); - - /* Display the configuration that failed. */ - if (g.run_cnt) - config_print(true); + /* + * Turn off tracking and logging so we don't obscure the error message. The lock we're about to + * acquire will act as a barrier to flush the writes. This is really a "best effort" more than a + * guarantee, there's too much stuff in flight to be sure. + */ + g.c_quiet = 1; + g.logging = false; + + /* + * Single-thread error handling, our caller exits after calling us (we never release the lock). + */ + (void)pthread_rwlock_wrlock(&g.death_lock); + + /* Flush/close any logging information. */ + fclose_and_clear(&g.logfp); + fclose_and_clear(&g.randfp); + + fprintf(stderr, "\n"); + + /* Display the configuration that failed. */ + if (g.run_cnt) + config_print(true); } /* * usage -- - * Display usage statement and exit failure. + * Display usage statement and exit failure. */ static void usage(void) { - fprintf(stderr, - "usage: %s [-1lqr] [-C wiredtiger-config]\n " - "[-c config-file] [-h home] [name=value ...]\n", - progname); - fprintf(stderr, "%s", - "\t-1 run once\n" - "\t-C specify wiredtiger_open configuration arguments\n" - "\t-c read test program configuration from a file\n" - "\t-h home (default 'RUNDIR')\n" - "\t-l log operations to a file\n" - "\t-q run quietly\n" - "\t-r replay the last run\n"); - - config_error(); - exit(EXIT_FAILURE); + fprintf(stderr, + "usage: %s [-1lqr] [-C wiredtiger-config]\n " + "[-c config-file] [-h home] [name=value ...]\n", + progname); + fprintf(stderr, "%s", + "\t-1 run once\n" + "\t-C specify wiredtiger_open configuration arguments\n" + "\t-c read test program configuration from a file\n" + "\t-h home (default 'RUNDIR')\n" + "\t-l log operations to a file\n" + "\t-q run quietly\n" + "\t-r replay the last run\n"); + + config_error(); + exit(EXIT_FAILURE); } diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index 91d9bf51697..88c5afd8e06 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -29,655 +29,639 @@ #include "format.h" #ifndef MAX -#define MAX(a, b) (((a) > (b)) ? (a) : (b)) +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) #endif void key_init(void) { - size_t i; - uint32_t max; - - /* - * The key is a variable length item with a leading 10-digit value. - * Since we have to be able re-construct it from the record number - * (when doing row lookups), we pre-load a set of random lengths in - * a lookup table, and then use the record number to choose one of - * the pre-loaded lengths. - * - * Fill in the random key lengths. - * - * Focus on relatively small items, admitting the possibility of larger - * items. Pick a size close to the minimum most of the time, only create - * a larger item 1 in 20 times. - */ - for (i = 0; - i < sizeof(g.key_rand_len) / sizeof(g.key_rand_len[0]); ++i) { - max = g.c_key_max; - if (i % 20 != 0 && max > g.c_key_min + 20) - max = g.c_key_min + 20; - g.key_rand_len[i] = mmrand(NULL, g.c_key_min, max); - } + size_t i; + uint32_t max; + + /* + * The key is a variable length item with a leading 10-digit value. + * Since we have to be able re-construct it from the record number + * (when doing row lookups), we pre-load a set of random lengths in + * a lookup table, and then use the record number to choose one of + * the pre-loaded lengths. + * + * Fill in the random key lengths. + * + * Focus on relatively small items, admitting the possibility of larger + * items. Pick a size close to the minimum most of the time, only create + * a larger item 1 in 20 times. + */ + for (i = 0; i < sizeof(g.key_rand_len) / sizeof(g.key_rand_len[0]); ++i) { + max = g.c_key_max; + if (i % 20 != 0 && max > g.c_key_min + 20) + max = g.c_key_min + 20; + g.key_rand_len[i] = mmrand(NULL, g.c_key_min, max); + } } void key_gen_init(WT_ITEM *key) { - size_t i, len; - char *p; - - len = MAX(KILOBYTE(100), g.c_key_max); - p = dmalloc(len); - for (i = 0; i < len; ++i) - p[i] = "abcdefghijklmnopqrstuvwxyz"[i % 26]; - - key->mem = p; - key->memsize = len; - key->data = key->mem; - key->size = 0; + size_t i, len; + char *p; + + len = MAX(KILOBYTE(100), g.c_key_max); + p = dmalloc(len); + for (i = 0; i < len; ++i) + p[i] = "abcdefghijklmnopqrstuvwxyz"[i % 26]; + + key->mem = p; + key->memsize = len; + key->data = key->mem; + key->size = 0; } void key_gen_teardown(WT_ITEM *key) { - free(key->mem); - memset(key, 0, sizeof(*key)); + free(key->mem); + memset(key, 0, sizeof(*key)); } static void -key_gen_common(WT_ITEM *key, uint64_t keyno, const char * const suffix) +key_gen_common(WT_ITEM *key, uint64_t keyno, const char *const suffix) { - int len; - char *p; - - p = key->mem; - - /* - * The key always starts with a 10-digit string (the specified row) - * followed by two digits, a random number between 1 and 15 if it's - * an insert, otherwise 00. - */ - u64_to_string_zf(keyno, key->mem, 11); - p[10] = '.'; - p[11] = suffix[0]; - p[12] = suffix[1]; - len = 13; - - /* - * In a column-store, the key isn't used, it doesn't need a random - * length. - */ - if (g.type == ROW) { - p[len] = '/'; - - /* - * Because we're doing table lookup for key sizes, we weren't - * able to set really big keys sizes in the table, the table - * isn't big enough to keep our hash from selecting too many - * big keys and blowing out the cache. Handle that here, use a - * really big key 1 in 2500 times. - */ - len = keyno % 2500 == 0 && g.c_key_max < KILOBYTE(80) ? - KILOBYTE(80) : - (int)g.key_rand_len[keyno % WT_ELEMENTS(g.key_rand_len)]; - } - - key->data = key->mem; - key->size = (size_t)len; + int len; + char *p; + + p = key->mem; + + /* + * The key always starts with a 10-digit string (the specified row) followed by two digits, a + * random number between 1 and 15 if it's an insert, otherwise 00. + */ + u64_to_string_zf(keyno, key->mem, 11); + p[10] = '.'; + p[11] = suffix[0]; + p[12] = suffix[1]; + len = 13; + + /* + * In a column-store, the key isn't used, it doesn't need a random length. + */ + if (g.type == ROW) { + p[len] = '/'; + + /* + * Because we're doing table lookup for key sizes, we weren't able to set really big keys + * sizes in the table, the table isn't big enough to keep our hash from selecting too many + * big keys and blowing out the cache. Handle that here, use a really big key 1 in 2500 + * times. + */ + len = keyno % 2500 == 0 && g.c_key_max < KILOBYTE(80) ? + KILOBYTE(80) : + (int)g.key_rand_len[keyno % WT_ELEMENTS(g.key_rand_len)]; + } + + key->data = key->mem; + key->size = (size_t)len; } void key_gen(WT_ITEM *key, uint64_t keyno) { - key_gen_common(key, keyno, "00"); + key_gen_common(key, keyno, "00"); } void key_gen_insert(WT_RAND_STATE *rnd, WT_ITEM *key, uint64_t keyno) { - static const char * const suffix[15] = { - "01", "02", "03", "04", "05", - "06", "07", "08", "09", "10", - "11", "12", "13", "14", "15" - }; + static const char *const suffix[15] = { + "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15"}; - key_gen_common(key, keyno, suffix[mmrand(rnd, 0, 14)]); + key_gen_common(key, keyno, suffix[mmrand(rnd, 0, 14)]); } -static char *val_base; /* Base/original value */ -static uint32_t val_dup_data_len; /* Length of duplicate data items */ -static uint32_t val_len; /* Length of data items */ +static char *val_base; /* Base/original value */ +static uint32_t val_dup_data_len; /* Length of duplicate data items */ +static uint32_t val_len; /* Length of data items */ static inline uint32_t value_len(WT_RAND_STATE *rnd, uint64_t keyno, uint32_t min, uint32_t max) { - /* - * Focus on relatively small items, admitting the possibility of larger - * items. Pick a size close to the minimum most of the time, only create - * a larger item 1 in 20 times, and a really big item 1 in somewhere - * around 2500 items. - */ - if (keyno % 2500 == 0 && max < KILOBYTE(80)) { - min = KILOBYTE(80); - max = KILOBYTE(100); - } else if (keyno % 20 != 0 && max > min + 20) - max = min + 20; - return (mmrand(rnd, min, max)); + /* + * Focus on relatively small items, admitting the possibility of larger items. Pick a size close + * to the minimum most of the time, only create a larger item 1 in 20 times, and a really big + * item 1 in somewhere around 2500 items. + */ + if (keyno % 2500 == 0 && max < KILOBYTE(80)) { + min = KILOBYTE(80); + max = KILOBYTE(100); + } else if (keyno % 20 != 0 && max > min + 20) + max = min + 20; + return (mmrand(rnd, min, max)); } void val_init(void) { - size_t i; - - /* - * Set initial buffer contents to recognizable text. - * - * Add a few extra bytes in order to guarantee we can always offset - * into the buffer by a few extra bytes, used to generate different - * data for column-store run-length encoded files. - */ - val_len = MAX(KILOBYTE(100), g.c_value_max) + 20; - val_base = dmalloc(val_len); - for (i = 0; i < val_len; ++i) - val_base[i] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"[i % 26]; - - val_dup_data_len = value_len(NULL, - (uint64_t)mmrand(NULL, 1, 20), g.c_value_min, g.c_value_max); + size_t i; + + /* + * Set initial buffer contents to recognizable text. + * + * Add a few extra bytes in order to guarantee we can always offset + * into the buffer by a few extra bytes, used to generate different + * data for column-store run-length encoded files. + */ + val_len = MAX(KILOBYTE(100), g.c_value_max) + 20; + val_base = dmalloc(val_len); + for (i = 0; i < val_len; ++i) + val_base[i] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"[i % 26]; + + val_dup_data_len = value_len(NULL, (uint64_t)mmrand(NULL, 1, 20), g.c_value_min, g.c_value_max); } void val_teardown(void) { - free(val_base); - val_base = NULL; - val_dup_data_len = val_len = 0; + free(val_base); + val_base = NULL; + val_dup_data_len = val_len = 0; } void val_gen_init(WT_ITEM *value) { - value->mem = dmalloc(val_len); - value->memsize = val_len; - value->data = value->mem; - value->size = 0; + value->mem = dmalloc(val_len); + value->memsize = val_len; + value->data = value->mem; + value->size = 0; } void val_gen_teardown(WT_ITEM *value) { - free(value->mem); - memset(value, 0, sizeof(*value)); + free(value->mem); + memset(value, 0, sizeof(*value)); } void val_gen(WT_RAND_STATE *rnd, WT_ITEM *value, uint64_t keyno) { - char *p; - - p = value->mem; - value->data = value->mem; - - /* - * Fixed-length records: take the low N bits from the last digit of - * the record number. - */ - if (g.type == FIX) { - switch (g.c_bitcnt) { - case 8: p[0] = (char)mmrand(rnd, 1, 0xff); break; - case 7: p[0] = (char)mmrand(rnd, 1, 0x7f); break; - case 6: p[0] = (char)mmrand(rnd, 1, 0x3f); break; - case 5: p[0] = (char)mmrand(rnd, 1, 0x1f); break; - case 4: p[0] = (char)mmrand(rnd, 1, 0x0f); break; - case 3: p[0] = (char)mmrand(rnd, 1, 0x07); break; - case 2: p[0] = (char)mmrand(rnd, 1, 0x03); break; - case 1: p[0] = 1; break; - } - value->size = 1; - return; - } - - /* - * WiredTiger doesn't store zero-length data items in row-store files, - * test that by inserting a zero-length data item every so often. - */ - if (keyno % 63 == 0) { - p[0] = '\0'; - value->size = 0; - return; - } - - /* - * Data items have unique leading numbers by default and random lengths; - * variable-length column-stores use a duplicate data value to test RLE. - */ - if (g.type == VAR && mmrand(rnd, 1, 100) < g.c_repeat_data_pct) { - value->size = val_dup_data_len; - memcpy(p, val_base, value->size); - (void)strcpy(p, "DUPLICATEV"); - p[10] = '/'; - } else { - value->size = - value_len(rnd, keyno, g.c_value_min, g.c_value_max); - memcpy(p, val_base, value->size); - u64_to_string_zf(keyno, p, 11); - p[10] = '/'; - } + char *p; + + p = value->mem; + value->data = value->mem; + + /* + * Fixed-length records: take the low N bits from the last digit of the record number. + */ + if (g.type == FIX) { + switch (g.c_bitcnt) { + case 8: + p[0] = (char)mmrand(rnd, 1, 0xff); + break; + case 7: + p[0] = (char)mmrand(rnd, 1, 0x7f); + break; + case 6: + p[0] = (char)mmrand(rnd, 1, 0x3f); + break; + case 5: + p[0] = (char)mmrand(rnd, 1, 0x1f); + break; + case 4: + p[0] = (char)mmrand(rnd, 1, 0x0f); + break; + case 3: + p[0] = (char)mmrand(rnd, 1, 0x07); + break; + case 2: + p[0] = (char)mmrand(rnd, 1, 0x03); + break; + case 1: + p[0] = 1; + break; + } + value->size = 1; + return; + } + + /* + * WiredTiger doesn't store zero-length data items in row-store files, test that by inserting a + * zero-length data item every so often. + */ + if (keyno % 63 == 0) { + p[0] = '\0'; + value->size = 0; + return; + } + + /* + * Data items have unique leading numbers by default and random lengths; variable-length + * column-stores use a duplicate data value to test RLE. + */ + if (g.type == VAR && mmrand(rnd, 1, 100) < g.c_repeat_data_pct) { + value->size = val_dup_data_len; + memcpy(p, val_base, value->size); + (void)strcpy(p, "DUPLICATEV"); + p[10] = '/'; + } else { + value->size = value_len(rnd, keyno, g.c_value_min, g.c_value_max); + memcpy(p, val_base, value->size); + u64_to_string_zf(keyno, p, 11); + p[10] = '/'; + } } void track(const char *tag, uint64_t cnt, TINFO *tinfo) { - static size_t lastlen = 0; - size_t len; - char msg[128]; - - if (g.c_quiet || tag == NULL) - return; - - if (tinfo == NULL && cnt == 0) - testutil_check(__wt_snprintf_len_set( - msg, sizeof(msg), &len, - "%4" PRIu32 ": %s", g.run_cnt, tag)); - else if (tinfo == NULL) - testutil_check(__wt_snprintf_len_set( - msg, sizeof(msg), &len, - "%4" PRIu32 ": %s: %" PRIu64, g.run_cnt, tag, cnt)); - else - testutil_check(__wt_snprintf_len_set( - msg, sizeof(msg), &len, - "%4" PRIu32 ": %s: " - "search %" PRIu64 "%s, " - "insert %" PRIu64 "%s, " - "update %" PRIu64 "%s, " - "remove %" PRIu64 "%s", - g.run_cnt, tag, - tinfo->search > M(9) ? tinfo->search / M(1) : tinfo->search, - tinfo->search > M(9) ? "M" : "", - tinfo->insert > M(9) ? tinfo->insert / M(1) : tinfo->insert, - tinfo->insert > M(9) ? "M" : "", - tinfo->update > M(9) ? tinfo->update / M(1) : tinfo->update, - tinfo->update > M(9) ? "M" : "", - tinfo->remove > M(9) ? tinfo->remove / M(1) : tinfo->remove, - tinfo->remove > M(9) ? "M" : "")); - - if (lastlen > len) { - memset(msg + len, ' ', (size_t)(lastlen - len)); - msg[lastlen] = '\0'; - } - lastlen = len; - - if (printf("%s\r", msg) < 0) - testutil_die(EIO, "printf"); - if (fflush(stdout) == EOF) - testutil_die(errno, "fflush"); + static size_t lastlen = 0; + size_t len; + char msg[128]; + + if (g.c_quiet || tag == NULL) + return; + + if (tinfo == NULL && cnt == 0) + testutil_check( + __wt_snprintf_len_set(msg, sizeof(msg), &len, "%4" PRIu32 ": %s", g.run_cnt, tag)); + else if (tinfo == NULL) + testutil_check(__wt_snprintf_len_set( + msg, sizeof(msg), &len, "%4" PRIu32 ": %s: %" PRIu64, g.run_cnt, tag, cnt)); + else + testutil_check(__wt_snprintf_len_set(msg, sizeof(msg), &len, "%4" PRIu32 ": %s: " + "search %" PRIu64 "%s, " + "insert %" PRIu64 "%s, " + "update %" PRIu64 "%s, " + "remove %" PRIu64 "%s", + g.run_cnt, tag, tinfo->search > M(9) ? tinfo->search / M(1) : tinfo->search, + tinfo->search > M(9) ? "M" : "", + tinfo->insert > M(9) ? tinfo->insert / M(1) : tinfo->insert, + tinfo->insert > M(9) ? "M" : "", + tinfo->update > M(9) ? tinfo->update / M(1) : tinfo->update, + tinfo->update > M(9) ? "M" : "", + tinfo->remove > M(9) ? tinfo->remove / M(1) : tinfo->remove, + tinfo->remove > M(9) ? "M" : "")); + + if (lastlen > len) { + memset(msg + len, ' ', (size_t)(lastlen - len)); + msg[lastlen] = '\0'; + } + lastlen = len; + + if (printf("%s\r", msg) < 0) + testutil_die(EIO, "printf"); + if (fflush(stdout) == EOF) + testutil_die(errno, "fflush"); } /* * path_setup -- - * Build the standard paths and shell commands we use. + * Build the standard paths and shell commands we use. */ void path_setup(const char *home) { - size_t len; - - /* Home directory. */ - g.home = dstrdup(home == NULL ? "RUNDIR" : home); - - /* Log file. */ - len = strlen(g.home) + strlen("log") + 2; - g.home_log = dmalloc(len); - testutil_check(__wt_snprintf(g.home_log, len, "%s/%s", g.home, "log")); - - /* Page dump file. */ - len = strlen(g.home) + strlen("pagedump") + 2; - g.home_pagedump = dmalloc(len); - testutil_check(__wt_snprintf( - g.home_pagedump, len, "%s/%s", g.home, "pagedump")); - - /* RNG log file. */ - len = strlen(g.home) + strlen("rand") + 2; - g.home_rand = dmalloc(len); - testutil_check(__wt_snprintf( - g.home_rand, len, "%s/%s", g.home, "rand")); - - /* Run file. */ - len = strlen(g.home) + strlen("CONFIG") + 2; - g.home_config = dmalloc(len); - testutil_check(__wt_snprintf( - g.home_config, len, "%s/%s", g.home, "CONFIG")); - - /* Statistics file. */ - len = strlen(g.home) + strlen("stats") + 2; - g.home_stats = dmalloc(len); - testutil_check(__wt_snprintf( - g.home_stats, len, "%s/%s", g.home, "stats")); - - /* - * Home directory initialize command: create the directory if it doesn't - * exist, else remove everything except the RNG log file. - * - * Redirect the "cd" command to /dev/null so chatty cd implementations - * don't add the new working directory to our output. - */ -#undef CMD + size_t len; + + /* Home directory. */ + g.home = dstrdup(home == NULL ? "RUNDIR" : home); + + /* Log file. */ + len = strlen(g.home) + strlen("log") + 2; + g.home_log = dmalloc(len); + testutil_check(__wt_snprintf(g.home_log, len, "%s/%s", g.home, "log")); + + /* Page dump file. */ + len = strlen(g.home) + strlen("pagedump") + 2; + g.home_pagedump = dmalloc(len); + testutil_check(__wt_snprintf(g.home_pagedump, len, "%s/%s", g.home, "pagedump")); + + /* RNG log file. */ + len = strlen(g.home) + strlen("rand") + 2; + g.home_rand = dmalloc(len); + testutil_check(__wt_snprintf(g.home_rand, len, "%s/%s", g.home, "rand")); + + /* Run file. */ + len = strlen(g.home) + strlen("CONFIG") + 2; + g.home_config = dmalloc(len); + testutil_check(__wt_snprintf(g.home_config, len, "%s/%s", g.home, "CONFIG")); + + /* Statistics file. */ + len = strlen(g.home) + strlen("stats") + 2; + g.home_stats = dmalloc(len); + testutil_check(__wt_snprintf(g.home_stats, len, "%s/%s", g.home, "stats")); + +/* + * Home directory initialize command: create the directory if it doesn't + * exist, else remove everything except the RNG log file. + * + * Redirect the "cd" command to /dev/null so chatty cd implementations + * don't add the new working directory to our output. + */ +#undef CMD #ifdef _WIN32 -#define CMD "del /q rand.copy & " \ - "(IF EXIST %s\\rand copy /y %s\\rand rand.copy) & " \ - "(IF EXIST %s rd /s /q %s) & mkdir %s & " \ - "(IF EXIST rand.copy copy rand.copy %s\\rand)" - len = strlen(g.home) * 7 + strlen(CMD) + 1; - g.home_init = dmalloc(len); - testutil_check(__wt_snprintf(g.home_init, len, CMD, - g.home, g.home, g.home, g.home, g.home, g.home, g.home)); +#define CMD \ + "del /q rand.copy & " \ + "(IF EXIST %s\\rand copy /y %s\\rand rand.copy) & " \ + "(IF EXIST %s rd /s /q %s) & mkdir %s & " \ + "(IF EXIST rand.copy copy rand.copy %s\\rand)" + len = strlen(g.home) * 7 + strlen(CMD) + 1; + g.home_init = dmalloc(len); + testutil_check( + __wt_snprintf(g.home_init, len, CMD, g.home, g.home, g.home, g.home, g.home, g.home, g.home)); #else -#define CMD "test -e %s || mkdir %s; " \ - "cd %s > /dev/null && rm -rf `ls | sed /rand/d`" - len = strlen(g.home) * 3 + strlen(CMD) + 1; - g.home_init = dmalloc(len); - testutil_check(__wt_snprintf( - g.home_init, len, CMD, g.home, g.home, g.home)); +#define CMD \ + "test -e %s || mkdir %s; " \ + "cd %s > /dev/null && rm -rf `ls | sed /rand/d`" + len = strlen(g.home) * 3 + strlen(CMD) + 1; + g.home_init = dmalloc(len); + testutil_check(__wt_snprintf(g.home_init, len, CMD, g.home, g.home, g.home)); #endif - /* Primary backup directory. */ - len = strlen(g.home) + strlen("BACKUP") + 2; - g.home_backup = dmalloc(len); - testutil_check(__wt_snprintf( - g.home_backup, len, "%s/%s", g.home, "BACKUP")); - - /* - * Backup directory initialize command, remove and re-create the primary - * backup directory, plus a copy we maintain for recovery testing. - */ -#undef CMD + /* Primary backup directory. */ + len = strlen(g.home) + strlen("BACKUP") + 2; + g.home_backup = dmalloc(len); + testutil_check(__wt_snprintf(g.home_backup, len, "%s/%s", g.home, "BACKUP")); + +/* + * Backup directory initialize command, remove and re-create the primary backup directory, plus a + * copy we maintain for recovery testing. + */ +#undef CMD #ifdef _WIN32 -#define CMD "rd /s /q %s\\%s %s\\%s & mkdir %s\\%s %s\\%s" +#define CMD "rd /s /q %s\\%s %s\\%s & mkdir %s\\%s %s\\%s" #else -#define CMD "rm -rf %s/%s %s/%s && mkdir %s/%s %s/%s" +#define CMD "rm -rf %s/%s %s/%s && mkdir %s/%s %s/%s" #endif - len = strlen(g.home) * 4 + - strlen("BACKUP") * 2 + strlen("BACKUP_COPY") * 2 + strlen(CMD) + 1; - g.home_backup_init = dmalloc(len); - testutil_check(__wt_snprintf(g.home_backup_init, len, CMD, - g.home, "BACKUP", g.home, "BACKUP_COPY", - g.home, "BACKUP", g.home, "BACKUP_COPY")); - - /* - * Salvage command, save the interesting files so we can replay the - * salvage command as necessary. - * - * Redirect the "cd" command to /dev/null so chatty cd implementations - * don't add the new working directory to our output. - */ -#undef CMD + len = strlen(g.home) * 4 + strlen("BACKUP") * 2 + strlen("BACKUP_COPY") * 2 + strlen(CMD) + 1; + g.home_backup_init = dmalloc(len); + testutil_check(__wt_snprintf(g.home_backup_init, len, CMD, g.home, "BACKUP", g.home, + "BACKUP_COPY", g.home, "BACKUP", g.home, "BACKUP_COPY")); + +/* + * Salvage command, save the interesting files so we can replay the + * salvage command as necessary. + * + * Redirect the "cd" command to /dev/null so chatty cd implementations + * don't add the new working directory to our output. + */ +#undef CMD #ifdef _WIN32 -#define CMD \ - "cd %s && " \ - "rd /q /s slvg.copy & mkdir slvg.copy && " \ - "copy WiredTiger* slvg.copy\\ >:nul && copy wt* slvg.copy\\ >:nul" +#define CMD \ + "cd %s && " \ + "rd /q /s slvg.copy & mkdir slvg.copy && " \ + "copy WiredTiger* slvg.copy\\ >:nul && copy wt* slvg.copy\\ >:nul" #else -#define CMD \ - "cd %s > /dev/null && " \ - "rm -rf slvg.copy && mkdir slvg.copy && " \ - "cp WiredTiger* wt* slvg.copy/" +#define CMD \ + "cd %s > /dev/null && " \ + "rm -rf slvg.copy && mkdir slvg.copy && " \ + "cp WiredTiger* wt* slvg.copy/" #endif - len = strlen(g.home) + strlen(CMD) + 1; - g.home_salvage_copy = dmalloc(len); - testutil_check(__wt_snprintf(g.home_salvage_copy, len, CMD, g.home)); + len = strlen(g.home) + strlen(CMD) + 1; + g.home_salvage_copy = dmalloc(len); + testutil_check(__wt_snprintf(g.home_salvage_copy, len, CMD, g.home)); } /* * rng -- - * Return a random number. + * Return a random number. */ uint32_t rng(WT_RAND_STATE *rnd) { - u_long ulv; - uint32_t v; - char *endptr, buf[64]; - - /* - * Threaded operations have their own RNG information, otherwise we - * use the default. - */ - if (rnd == NULL) - rnd = &g.rnd; - - /* - * We can reproduce a single-threaded run based on the random numbers - * used in the initial run, plus the configuration files. - * - * Check g.replay and g.rand_log_stop: multithreaded runs log/replay - * until they get to the operations phase, then turn off log/replay, - * threaded operation order can't be replayed. - */ - if (g.rand_log_stop) - return (__wt_random(rnd)); - - if (g.replay) { - if (fgets(buf, sizeof(buf), g.randfp) == NULL) { - if (feof(g.randfp)) { - fprintf(stderr, - "\n" "end of random number log reached\n"); - exit(EXIT_SUCCESS); - } - testutil_die(errno, "random number log"); - } - - errno = 0; - ulv = strtoul(buf, &endptr, 10); - testutil_assert(errno == 0 && endptr[0] == '\n'); - testutil_assert(ulv <= UINT32_MAX); - return ((uint32_t)ulv); - } - - v = __wt_random(rnd); - - /* Save and flush the random number so we're up-to-date on error. */ - (void)fprintf(g.randfp, "%" PRIu32 "\n", v); - (void)fflush(g.randfp); - - return (v); + u_long ulv; + uint32_t v; + char *endptr, buf[64]; + + /* + * Threaded operations have their own RNG information, otherwise we use the default. + */ + if (rnd == NULL) + rnd = &g.rnd; + + /* + * We can reproduce a single-threaded run based on the random numbers + * used in the initial run, plus the configuration files. + * + * Check g.replay and g.rand_log_stop: multithreaded runs log/replay + * until they get to the operations phase, then turn off log/replay, + * threaded operation order can't be replayed. + */ + if (g.rand_log_stop) + return (__wt_random(rnd)); + + if (g.replay) { + if (fgets(buf, sizeof(buf), g.randfp) == NULL) { + if (feof(g.randfp)) { + fprintf(stderr, + "\n" + "end of random number log reached\n"); + exit(EXIT_SUCCESS); + } + testutil_die(errno, "random number log"); + } + + errno = 0; + ulv = strtoul(buf, &endptr, 10); + testutil_assert(errno == 0 && endptr[0] == '\n'); + testutil_assert(ulv <= UINT32_MAX); + return ((uint32_t)ulv); + } + + v = __wt_random(rnd); + + /* Save and flush the random number so we're up-to-date on error. */ + (void)fprintf(g.randfp, "%" PRIu32 "\n", v); + (void)fflush(g.randfp); + + return (v); } /* * fclose_and_clear -- - * Close a file and clear the handle so we don't close twice. + * Close a file and clear the handle so we don't close twice. */ void fclose_and_clear(FILE **fpp) { - FILE *fp; - - if ((fp = *fpp) == NULL) - return; - *fpp = NULL; - if (fclose(fp) != 0) - testutil_die(errno, "fclose"); - return; + FILE *fp; + + if ((fp = *fpp) == NULL) + return; + *fpp = NULL; + if (fclose(fp) != 0) + testutil_die(errno, "fclose"); + return; } /* * checkpoint -- - * Periodically take a checkpoint + * Periodically take a checkpoint */ WT_THREAD_RET checkpoint(void *arg) { - WT_CONNECTION *conn; - WT_DECL_RET; - WT_SESSION *session; - u_int secs; - const char *ckpt_config; - char config_buf[64]; - bool backup_locked; - - (void)arg; - conn = g.wts_conn; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - for (secs = mmrand(NULL, 1, 10); !g.workers_finished;) { - if (secs > 0) { - __wt_sleep(1, 0); - --secs; - continue; - } - - /* - * LSM and data-sources don't support named checkpoints. Also, - * don't attempt named checkpoints during a hot backup. It's - * OK to create named checkpoints during a hot backup, but we - * can't delete them, so repeating an already existing named - * checkpoint will fail when we can't drop the previous one. - */ - ckpt_config = NULL; - backup_locked = false; - if (!DATASOURCE("lsm")) - switch (mmrand(NULL, 1, 20)) { - case 1: - /* - * 5% create a named snapshot. Rotate between a - * few names to test multiple named snapshots in - * the system. - */ - ret = pthread_rwlock_trywrlock(&g.backup_lock); - if (ret == 0) { - backup_locked = true; - testutil_check(__wt_snprintf( - config_buf, sizeof(config_buf), - "name=mine.%" PRIu32, - mmrand(NULL, 1, 4))); - ckpt_config = config_buf; - } else if (ret != EBUSY) - testutil_check(ret); - break; - case 2: - /* - * 5% drop all named snapshots. - */ - ret = pthread_rwlock_trywrlock(&g.backup_lock); - if (ret == 0) { - backup_locked = true; - ckpt_config = "drop=(all)"; - } else if (ret != EBUSY) - testutil_check(ret); - break; - } - - testutil_check(session->checkpoint(session, ckpt_config)); - - if (backup_locked) - testutil_check(pthread_rwlock_unlock(&g.backup_lock)); - - secs = mmrand(NULL, 5, 40); - } - - testutil_check(session->close(session, NULL)); - return (WT_THREAD_RET_VALUE); + WT_CONNECTION *conn; + WT_DECL_RET; + WT_SESSION *session; + u_int secs; + char config_buf[64]; + const char *ckpt_config; + bool backup_locked; + + (void)arg; + conn = g.wts_conn; + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + for (secs = mmrand(NULL, 1, 10); !g.workers_finished;) { + if (secs > 0) { + __wt_sleep(1, 0); + --secs; + continue; + } + + /* + * LSM and data-sources don't support named checkpoints. Also, don't attempt named + * checkpoints during a hot backup. It's OK to create named checkpoints during a hot backup, + * but we can't delete them, so repeating an already existing named checkpoint will fail + * when we can't drop the previous one. + */ + ckpt_config = NULL; + backup_locked = false; + if (!DATASOURCE("lsm")) + switch (mmrand(NULL, 1, 20)) { + case 1: + /* + * 5% create a named snapshot. Rotate between a + * few names to test multiple named snapshots in + * the system. + */ + ret = pthread_rwlock_trywrlock(&g.backup_lock); + if (ret == 0) { + backup_locked = true; + testutil_check(__wt_snprintf( + config_buf, sizeof(config_buf), "name=mine.%" PRIu32, mmrand(NULL, 1, 4))); + ckpt_config = config_buf; + } else if (ret != EBUSY) + testutil_check(ret); + break; + case 2: + /* + * 5% drop all named snapshots. + */ + ret = pthread_rwlock_trywrlock(&g.backup_lock); + if (ret == 0) { + backup_locked = true; + ckpt_config = "drop=(all)"; + } else if (ret != EBUSY) + testutil_check(ret); + break; + } + + testutil_check(session->checkpoint(session, ckpt_config)); + + if (backup_locked) + testutil_check(pthread_rwlock_unlock(&g.backup_lock)); + + secs = mmrand(NULL, 5, 40); + } + + testutil_check(session->close(session, NULL)); + return (WT_THREAD_RET_VALUE); } /* * timestamp -- - * Periodically update the oldest timestamp. + * Periodically update the oldest timestamp. */ WT_THREAD_RET timestamp(void *arg) { - WT_CONNECTION *conn; - WT_DECL_RET; - WT_SESSION *session; - char buf[WT_TS_HEX_STRING_SIZE + 64]; - bool done; - - (void)(arg); - conn = g.wts_conn; - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - testutil_check( - __wt_snprintf(buf, sizeof(buf), "%s", "oldest_timestamp=")); - - /* Update the oldest timestamp at least once every 15 seconds. */ - done = false; - do { - /* - * Do a final bump of the oldest timestamp as part of shutting - * down the worker threads, otherwise recent operations can - * prevent verify from running. - */ - if (g.workers_finished) - done = true; - else - random_sleep(&g.rnd, 15); - - /* - * Lock out transaction timestamp operations. The lock acts as a - * barrier ensuring we've checked if the workers have finished, - * we don't want that line reordered. - */ - testutil_check(pthread_rwlock_wrlock(&g.ts_lock)); - - ret = conn->query_timestamp(conn, - buf + strlen("oldest_timestamp="), "get=all_committed"); - testutil_assert(ret == 0 || ret == WT_NOTFOUND); - if (ret == 0) - testutil_check(conn->set_timestamp(conn, buf)); - - testutil_check(pthread_rwlock_unlock(&g.ts_lock)); - } while (!done); - - testutil_check(session->close(session, NULL)); - return (WT_THREAD_RET_VALUE); + WT_CONNECTION *conn; + WT_DECL_RET; + WT_SESSION *session; + char buf[WT_TS_HEX_STRING_SIZE + 64]; + bool done; + + (void)(arg); + conn = g.wts_conn; + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s", "oldest_timestamp=")); + + /* Update the oldest timestamp at least once every 15 seconds. */ + done = false; + do { + /* + * Do a final bump of the oldest timestamp as part of shutting down the worker threads, + * otherwise recent operations can prevent verify from running. + */ + if (g.workers_finished) + done = true; + else + random_sleep(&g.rnd, 15); + + /* + * Lock out transaction timestamp operations. The lock acts as a barrier ensuring we've + * checked if the workers have finished, we don't want that line reordered. + */ + testutil_check(pthread_rwlock_wrlock(&g.ts_lock)); + + ret = conn->query_timestamp(conn, buf + strlen("oldest_timestamp="), "get=all_committed"); + testutil_assert(ret == 0 || ret == WT_NOTFOUND); + if (ret == 0) + testutil_check(conn->set_timestamp(conn, buf)); + + testutil_check(pthread_rwlock_unlock(&g.ts_lock)); + } while (!done); + + testutil_check(session->close(session, NULL)); + return (WT_THREAD_RET_VALUE); } /* * alter -- - * Periodically alter a table's metadata. + * Periodically alter a table's metadata. */ WT_THREAD_RET alter(void *arg) { - WT_CONNECTION *conn; - WT_DECL_RET; - WT_SESSION *session; - u_int period; - char buf[32]; - bool access_value; - - (void)(arg); - conn = g.wts_conn; - - /* - * Only alter the access pattern hint. If we alter the cache resident - * setting we may end up with a setting that fills cache and doesn't - * allow it to be evicted. - */ - access_value = false; - - /* Open a session */ - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - while (!g.workers_finished) { - period = mmrand(NULL, 1, 10); - - testutil_check(__wt_snprintf(buf, sizeof(buf), - "access_pattern_hint=%s", - access_value ? "random" : "none")); - access_value = !access_value; - /* - * Alter can return EBUSY if concurrent with other operations. - */ - while ((ret = session->alter(session, g.uri, buf)) != 0 && - ret != EBUSY) - testutil_die(ret, "session.alter"); - while (period > 0 && !g.workers_finished) { - --period; - __wt_sleep(1, 0); - } - } - - testutil_check(session->close(session, NULL)); - return (WT_THREAD_RET_VALUE); + WT_CONNECTION *conn; + WT_DECL_RET; + WT_SESSION *session; + u_int period; + char buf[32]; + bool access_value; + + (void)(arg); + conn = g.wts_conn; + + /* + * Only alter the access pattern hint. If we alter the cache resident setting we may end up with + * a setting that fills cache and doesn't allow it to be evicted. + */ + access_value = false; + + /* Open a session */ + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + while (!g.workers_finished) { + period = mmrand(NULL, 1, 10); + + testutil_check(__wt_snprintf( + buf, sizeof(buf), "access_pattern_hint=%s", access_value ? "random" : "none")); + access_value = !access_value; + /* + * Alter can return EBUSY if concurrent with other operations. + */ + while ((ret = session->alter(session, g.uri, buf)) != 0 && ret != EBUSY) + testutil_die(ret, "session.alter"); + while (period > 0 && !g.workers_finished) { + --period; + __wt_sleep(1, 0); + } + } + + testutil_check(session->close(session, NULL)); + return (WT_THREAD_RET_VALUE); } diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index 9e0a69aa433..89a72f090e7 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -30,547 +30,508 @@ /* * compressor -- - * Configure compression. + * Configure compression. */ static const char * compressor(uint32_t compress_flag) { - const char *p; - - p = "unrecognized compressor flag"; - switch (compress_flag) { - case COMPRESS_NONE: - p ="none"; - break; - case COMPRESS_LZ4: - p ="lz4"; - break; - case COMPRESS_SNAPPY: - p ="snappy"; - break; - case COMPRESS_ZLIB: - p ="zlib"; - break; - case COMPRESS_ZSTD: - p ="zstd"; - break; - default: - testutil_die(EINVAL, - "illegal compression flag: %#" PRIx32, compress_flag); - /* NOTREACHED */ - } - return (p); + const char *p; + + p = "unrecognized compressor flag"; + switch (compress_flag) { + case COMPRESS_NONE: + p = "none"; + break; + case COMPRESS_LZ4: + p = "lz4"; + break; + case COMPRESS_SNAPPY: + p = "snappy"; + break; + case COMPRESS_ZLIB: + p = "zlib"; + break; + case COMPRESS_ZSTD: + p = "zstd"; + break; + default: + testutil_die(EINVAL, "illegal compression flag: %#" PRIx32, compress_flag); + /* NOTREACHED */ + } + return (p); } /* * encryptor -- - * Configure encryption. + * Configure encryption. */ static const char * encryptor(uint32_t encrypt_flag) { - const char *p; - - p = "unrecognized encryptor flag"; - switch (encrypt_flag) { - case ENCRYPT_NONE: - p = "none"; - break; - case ENCRYPT_ROTN_7: - p = "rotn,keyid=7"; - break; - default: - testutil_die(EINVAL, - "illegal encryption flag: %#" PRIx32, encrypt_flag); - /* NOTREACHED */ - } - return (p); + const char *p; + + p = "unrecognized encryptor flag"; + switch (encrypt_flag) { + case ENCRYPT_NONE: + p = "none"; + break; + case ENCRYPT_ROTN_7: + p = "rotn,keyid=7"; + break; + default: + testutil_die(EINVAL, "illegal encryption flag: %#" PRIx32, encrypt_flag); + /* NOTREACHED */ + } + return (p); } static int -handle_message(WT_EVENT_HANDLER *handler, - WT_SESSION *session, const char *message) +handle_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message) { - int out; - - (void)(handler); - (void)(session); - - /* - * WiredTiger logs a verbose message when the read timestamp is set to a - * value older than the oldest timestamp. Ignore the message, it happens - * when repeating operations to confirm timestamped values don't change - * underneath us. - */ - if (strstr(message, "less than the oldest timestamp") != NULL) - return (0); - - /* Write and flush the message so we're up-to-date on error. */ - if (g.logfp == NULL) { - out = printf("%p:%s\n", (void *)session, message); - (void)fflush(stdout); - } else { - out = fprintf(g.logfp, "%p:%s\n", (void *)session, message); - (void)fflush(g.logfp); - } - return (out < 0 ? EIO : 0); + int out; + + (void)(handler); + (void)(session); + + /* + * WiredTiger logs a verbose message when the read timestamp is set to a value older than the + * oldest timestamp. Ignore the message, it happens when repeating operations to confirm + * timestamped values don't change underneath us. + */ + if (strstr(message, "less than the oldest timestamp") != NULL) + return (0); + + /* Write and flush the message so we're up-to-date on error. */ + if (g.logfp == NULL) { + out = printf("%p:%s\n", (void *)session, message); + (void)fflush(stdout); + } else { + out = fprintf(g.logfp, "%p:%s\n", (void *)session, message); + (void)fflush(g.logfp); + } + return (out < 0 ? EIO : 0); } /* * __handle_progress_default -- - * Default WT_EVENT_HANDLER->handle_progress implementation: ignore. + * Default WT_EVENT_HANDLER->handle_progress implementation: ignore. */ static int -handle_progress(WT_EVENT_HANDLER *handler, - WT_SESSION *session, const char *operation, uint64_t progress) +handle_progress( + WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *operation, uint64_t progress) { - (void)(handler); - (void)(session); + (void)(handler); + (void)(session); - track(operation, progress, NULL); - return (0); + track(operation, progress, NULL); + return (0); } static WT_EVENT_HANDLER event_handler = { - NULL, - handle_message, - handle_progress, - NULL /* Close handler. */ + NULL, handle_message, handle_progress, NULL /* Close handler. */ }; -#define CONFIG_APPEND(p, ...) do { \ - size_t __len; \ - testutil_check( \ - __wt_snprintf_len_set(p, max, &__len, __VA_ARGS__)); \ - if (__len > max) \ - __len = max; \ - p += __len; \ - max -= __len; \ -} while (0) +#define CONFIG_APPEND(p, ...) \ + do { \ + size_t __len; \ + testutil_check(__wt_snprintf_len_set(p, max, &__len, __VA_ARGS__)); \ + if (__len > max) \ + __len = max; \ + p += __len; \ + max -= __len; \ + } while (0) /* * wts_open -- - * Open a connection to a WiredTiger database. + * Open a connection to a WiredTiger database. */ void wts_open(const char *home, bool set_api, WT_CONNECTION **connp) { - WT_CONNECTION *conn; - size_t max; - char *config, *p; - - *connp = NULL; - - config = p = g.wiredtiger_open_config; - max = sizeof(g.wiredtiger_open_config); - - CONFIG_APPEND(p, - "create=true" - ",cache_size=%" PRIu32 "MB" - ",checkpoint_sync=false" - ",error_prefix=\"%s\"", - g.c_cache, progname); - - /* In-memory configuration. */ - if (g.c_in_memory != 0) - CONFIG_APPEND(p, ",in_memory=1"); - - /* LSM configuration. */ - if (DATASOURCE("lsm")) - CONFIG_APPEND(p, - ",lsm_manager=(worker_thread_max=%" PRIu32 "),", - g.c_lsm_worker_threads); - - if (DATASOURCE("lsm") || g.c_cache < 20) - CONFIG_APPEND(p, ",eviction_dirty_trigger=95"); - - /* Checkpoints. */ - if (g.c_checkpoint_flag == CHECKPOINT_WIREDTIGER) - CONFIG_APPEND(p, - ",checkpoint=(wait=%" PRIu32 ",log_size=%" PRIu32 ")", - g.c_checkpoint_wait, MEGABYTE(g.c_checkpoint_log_size)); - - /* Eviction worker configuration. */ - if (g.c_evict_max != 0) - CONFIG_APPEND(p, - ",eviction=(threads_max=%" PRIu32 ")", g.c_evict_max); - - /* Logging configuration. */ - if (g.c_logging) - CONFIG_APPEND(p, - ",log=(enabled=true,archive=%d," - "prealloc=%d,file_max=%" PRIu32 ",compressor=\"%s\")", - g.c_logging_archive ? 1 : 0, - g.c_logging_prealloc ? 1 : 0, - KILOBYTE(g.c_logging_file_max), - compressor(g.c_logging_compression_flag)); - - /* Encryption. */ - if (g.c_encryption) - CONFIG_APPEND(p, - ",encryption=(name=%s)", encryptor(g.c_encryption_flag)); - - /* Miscellaneous. */ + WT_CONNECTION *conn; + size_t max; + char *config, *p; + + *connp = NULL; + + config = p = g.wiredtiger_open_config; + max = sizeof(g.wiredtiger_open_config); + + CONFIG_APPEND(p, + "create=true" + ",cache_size=%" PRIu32 + "MB" + ",checkpoint_sync=false" + ",error_prefix=\"%s\"", + g.c_cache, progname); + + /* In-memory configuration. */ + if (g.c_in_memory != 0) + CONFIG_APPEND(p, ",in_memory=1"); + + /* LSM configuration. */ + if (DATASOURCE("lsm")) + CONFIG_APPEND(p, ",lsm_manager=(worker_thread_max=%" PRIu32 "),", g.c_lsm_worker_threads); + + if (DATASOURCE("lsm") || g.c_cache < 20) + CONFIG_APPEND(p, ",eviction_dirty_trigger=95"); + + /* Checkpoints. */ + if (g.c_checkpoint_flag == CHECKPOINT_WIREDTIGER) + CONFIG_APPEND(p, ",checkpoint=(wait=%" PRIu32 ",log_size=%" PRIu32 ")", g.c_checkpoint_wait, + MEGABYTE(g.c_checkpoint_log_size)); + + /* Eviction worker configuration. */ + if (g.c_evict_max != 0) + CONFIG_APPEND(p, ",eviction=(threads_max=%" PRIu32 ")", g.c_evict_max); + + /* Logging configuration. */ + if (g.c_logging) + CONFIG_APPEND(p, + ",log=(enabled=true,archive=%d," + "prealloc=%d,file_max=%" PRIu32 ",compressor=\"%s\")", + g.c_logging_archive ? 1 : 0, g.c_logging_prealloc ? 1 : 0, KILOBYTE(g.c_logging_file_max), + compressor(g.c_logging_compression_flag)); + + /* Encryption. */ + if (g.c_encryption) + CONFIG_APPEND(p, ",encryption=(name=%s)", encryptor(g.c_encryption_flag)); + +/* Miscellaneous. */ #ifdef HAVE_POSIX_MEMALIGN - CONFIG_APPEND(p, ",buffer_alignment=512"); + CONFIG_APPEND(p, ",buffer_alignment=512"); #endif - CONFIG_APPEND(p, ",mmap=%d", g.c_mmap ? 1 : 0); - - if (g.c_direct_io) - CONFIG_APPEND(p, ",direct_io=(data)"); - - if (g.c_data_extend) - CONFIG_APPEND(p, ",file_extend=(data=8MB)"); - - /* - * Run the statistics server and/or maintain statistics in the engine. - * Sometimes specify a set of sources just to exercise that code. - */ - if (g.c_statistics_server) { - if (mmrand(NULL, 0, 5) == 1 && - memcmp(g.uri, "file:", strlen("file:")) == 0) - CONFIG_APPEND(p, - ",statistics=(fast),statistics_log=" - "(json,on_close,wait=5,sources=(\"file:\"))"); - else - CONFIG_APPEND(p, - ",statistics=(fast),statistics_log=" - "(json,on_close,wait=5)"); - } else - CONFIG_APPEND(p, - ",statistics=(%s)", g.c_statistics ? "fast" : "none"); - - /* Optionally stress operations. */ - CONFIG_APPEND(p, ",timing_stress_for_test=["); - if (g.c_timing_stress_aggressive_sweep) - CONFIG_APPEND(p, ",aggressive_sweep"); - if (g.c_timing_stress_checkpoint) - CONFIG_APPEND(p, ",checkpoint_slow"); - if (g.c_timing_stress_lookaside_sweep) - CONFIG_APPEND(p, ",lookaside_sweep_race"); - if (g.c_timing_stress_split_1) - CONFIG_APPEND(p, ",split_1"); - if (g.c_timing_stress_split_2) - CONFIG_APPEND(p, ",split_2"); - if (g.c_timing_stress_split_3) - CONFIG_APPEND(p, ",split_3"); - if (g.c_timing_stress_split_4) - CONFIG_APPEND(p, ",split_4"); - if (g.c_timing_stress_split_5) - CONFIG_APPEND(p, ",split_5"); - if (g.c_timing_stress_split_6) - CONFIG_APPEND(p, ",split_6"); - if (g.c_timing_stress_split_7) - CONFIG_APPEND(p, ",split_7"); - if (g.c_timing_stress_split_8) - CONFIG_APPEND(p, ",split_8"); - CONFIG_APPEND(p, "]"); - - /* Extensions. */ - CONFIG_APPEND(p, - ",extensions=[" - "\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],", - g.c_reverse ? REVERSE_PATH : "", - access(LZ4_PATH, R_OK) == 0 ? LZ4_PATH : "", - access(ROTN_PATH, R_OK) == 0 ? ROTN_PATH : "", - access(SNAPPY_PATH, R_OK) == 0 ? SNAPPY_PATH : "", - access(ZLIB_PATH, R_OK) == 0 ? ZLIB_PATH : "", - access(ZSTD_PATH, R_OK) == 0 ? ZSTD_PATH : ""); - - /* - * Put configuration file configuration options second to last. Put - * command line configuration options at the end. Do this so they - * override the standard configuration. - */ - if (g.c_config_open != NULL) - CONFIG_APPEND(p, ",%s", g.c_config_open); - if (g.config_open != NULL) - CONFIG_APPEND(p, ",%s", g.config_open); - - if (max == 0) - testutil_die(ENOMEM, - "wiredtiger_open configuration buffer too small"); - - /* - * Direct I/O may not work with backups, doing copies through the buffer - * cache after configuring direct I/O in Linux won't work. If direct - * I/O is configured, turn off backups. This isn't a great place to do - * this check, but it's only here we have the configuration string. - */ - if (strstr(config, "direct_io") != NULL) - g.c_backups = 0; - - testutil_checkfmt( - wiredtiger_open(home, &event_handler, config, &conn), "%s", home); - - if (set_api) - g.wt_api = conn->get_extension_api(conn); - - *connp = conn; + CONFIG_APPEND(p, ",mmap=%d", g.c_mmap ? 1 : 0); + + if (g.c_direct_io) + CONFIG_APPEND(p, ",direct_io=(data)"); + + if (g.c_data_extend) + CONFIG_APPEND(p, ",file_extend=(data=8MB)"); + + /* + * Run the statistics server and/or maintain statistics in the engine. Sometimes specify a set + * of sources just to exercise that code. + */ + if (g.c_statistics_server) { + if (mmrand(NULL, 0, 5) == 1 && memcmp(g.uri, "file:", strlen("file:")) == 0) + CONFIG_APPEND(p, + ",statistics=(fast),statistics_log=" + "(json,on_close,wait=5,sources=(\"file:\"))"); + else + CONFIG_APPEND(p, + ",statistics=(fast),statistics_log=" + "(json,on_close,wait=5)"); + } else + CONFIG_APPEND(p, ",statistics=(%s)", g.c_statistics ? "fast" : "none"); + + /* Optionally stress operations. */ + CONFIG_APPEND(p, ",timing_stress_for_test=["); + if (g.c_timing_stress_aggressive_sweep) + CONFIG_APPEND(p, ",aggressive_sweep"); + if (g.c_timing_stress_checkpoint) + CONFIG_APPEND(p, ",checkpoint_slow"); + if (g.c_timing_stress_lookaside_sweep) + CONFIG_APPEND(p, ",lookaside_sweep_race"); + if (g.c_timing_stress_split_1) + CONFIG_APPEND(p, ",split_1"); + if (g.c_timing_stress_split_2) + CONFIG_APPEND(p, ",split_2"); + if (g.c_timing_stress_split_3) + CONFIG_APPEND(p, ",split_3"); + if (g.c_timing_stress_split_4) + CONFIG_APPEND(p, ",split_4"); + if (g.c_timing_stress_split_5) + CONFIG_APPEND(p, ",split_5"); + if (g.c_timing_stress_split_6) + CONFIG_APPEND(p, ",split_6"); + if (g.c_timing_stress_split_7) + CONFIG_APPEND(p, ",split_7"); + if (g.c_timing_stress_split_8) + CONFIG_APPEND(p, ",split_8"); + CONFIG_APPEND(p, "]"); + + /* Extensions. */ + CONFIG_APPEND(p, + ",extensions=[" + "\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],", + g.c_reverse ? REVERSE_PATH : "", access(LZ4_PATH, R_OK) == 0 ? LZ4_PATH : "", + access(ROTN_PATH, R_OK) == 0 ? ROTN_PATH : "", + access(SNAPPY_PATH, R_OK) == 0 ? SNAPPY_PATH : "", + access(ZLIB_PATH, R_OK) == 0 ? ZLIB_PATH : "", access(ZSTD_PATH, R_OK) == 0 ? ZSTD_PATH : ""); + + /* + * Put configuration file configuration options second to last. Put command line configuration + * options at the end. Do this so they override the standard configuration. + */ + if (g.c_config_open != NULL) + CONFIG_APPEND(p, ",%s", g.c_config_open); + if (g.config_open != NULL) + CONFIG_APPEND(p, ",%s", g.config_open); + + if (max == 0) + testutil_die(ENOMEM, "wiredtiger_open configuration buffer too small"); + + /* + * Direct I/O may not work with backups, doing copies through the buffer cache after configuring + * direct I/O in Linux won't work. If direct I/O is configured, turn off backups. This isn't a + * great place to do this check, but it's only here we have the configuration string. + */ + if (strstr(config, "direct_io") != NULL) + g.c_backups = 0; + + testutil_checkfmt(wiredtiger_open(home, &event_handler, config, &conn), "%s", home); + + if (set_api) + g.wt_api = conn->get_extension_api(conn); + + *connp = conn; } /* * wts_reopen -- - * Re-open a connection to a WiredTiger database. + * Re-open a connection to a WiredTiger database. */ void wts_reopen(void) { - WT_CONNECTION *conn; + WT_CONNECTION *conn; - testutil_checkfmt(wiredtiger_open(g.home, &event_handler, - g.wiredtiger_open_config, &conn), "%s", g.home); + testutil_checkfmt( + wiredtiger_open(g.home, &event_handler, g.wiredtiger_open_config, &conn), "%s", g.home); - g.wt_api = conn->get_extension_api(conn); - g.wts_conn = conn; + g.wt_api = conn->get_extension_api(conn); + g.wts_conn = conn; } /* * wts_create -- - * Create the underlying store. + * Create the underlying store. */ void wts_init(void) { - WT_CONNECTION *conn; - WT_SESSION *session; - size_t max; - uint32_t maxintlkey, maxleafkey, maxleafvalue; - char config[4096], *p; - - conn = g.wts_conn; - p = config; - max = sizeof(config); - - CONFIG_APPEND(p, - "key_format=%s" - ",allocation_size=512" - ",%s" - ",internal_page_max=%" PRIu32 - ",leaf_page_max=%" PRIu32 - ",memory_page_max=%" PRIu32, - (g.type == ROW) ? "u" : "r", - g.c_firstfit ? "block_allocation=first" : "", - g.intl_page_max, g.leaf_page_max, MEGABYTE(g.c_memory_page_max)); - - /* - * Configure the maximum key/value sizes, but leave it as the default - * if we come up with something crazy. - */ - maxintlkey = mmrand(NULL, g.intl_page_max / 50, g.intl_page_max / 40); - if (maxintlkey > 20) - CONFIG_APPEND(p, ",internal_key_max=%" PRIu32, maxintlkey); - maxleafkey = mmrand(NULL, g.leaf_page_max / 50, g.leaf_page_max / 40); - if (maxleafkey > 20) - CONFIG_APPEND(p, ",leaf_key_max=%" PRIu32, maxleafkey); - maxleafvalue = mmrand(NULL, g.leaf_page_max * 10, g.leaf_page_max / 40); - if (maxleafvalue > 40 && maxleafvalue < 100 * 1024) - CONFIG_APPEND(p, ",leaf_value_max=%" PRIu32, maxleafvalue); - - switch (g.type) { - case FIX: - CONFIG_APPEND(p, ",value_format=%" PRIu32 "t", g.c_bitcnt); - break; - case ROW: - if (g.c_huffman_key) - CONFIG_APPEND(p, ",huffman_key=english"); - if (g.c_prefix_compression) - CONFIG_APPEND(p, - ",prefix_compression_min=%" PRIu32, - g.c_prefix_compression_min); - else - CONFIG_APPEND(p, ",prefix_compression=false"); - if (g.c_reverse) - CONFIG_APPEND(p, ",collator=reverse"); - /* FALLTHROUGH */ - case VAR: - if (g.c_huffman_value) - CONFIG_APPEND(p, ",huffman_value=english"); - if (g.c_dictionary) - CONFIG_APPEND(p, - ",dictionary=%" PRIu32, mmrand(NULL, 123, 517)); - break; - } - - /* Configure checksums. */ - switch (g.c_checksum_flag) { - case CHECKSUM_OFF: - CONFIG_APPEND(p, ",checksum=\"off\""); - break; - case CHECKSUM_ON: - CONFIG_APPEND(p, ",checksum=\"on\""); - break; - case CHECKSUM_UNCOMPRESSED: - CONFIG_APPEND(p, ",checksum=\"uncompressed\""); - break; - } - - /* Configure compression. */ - if (g.c_compression_flag != COMPRESS_NONE) - CONFIG_APPEND(p, ",block_compressor=\"%s\"", - compressor(g.c_compression_flag)); - - /* Configure Btree internal key truncation. */ - CONFIG_APPEND(p, ",internal_key_truncate=%s", - g.c_internal_key_truncation ? "true" : "false"); - - /* Configure Btree page key gap. */ - CONFIG_APPEND(p, ",key_gap=%" PRIu32, g.c_key_gap); - - /* Configure Btree split page percentage. */ - CONFIG_APPEND(p, ",split_pct=%" PRIu32, g.c_split_pct); - - /* - * Assertions. - * Assertions slow down the code for additional diagnostic checking. - */ - if (g.c_txn_timestamps && g.c_assert_commit_timestamp) - CONFIG_APPEND(p, ",assert=(commit_timestamp=key_consistent)"); - if (g.c_txn_timestamps && g.c_assert_read_timestamp) - CONFIG_APPEND(p, ",assert=(read_timestamp=always)"); - - /* Configure LSM. */ - if (DATASOURCE("lsm")) { - CONFIG_APPEND(p, ",type=lsm,lsm=("); - CONFIG_APPEND(p, - "auto_throttle=%s,", g.c_auto_throttle ? "true" : "false"); - CONFIG_APPEND(p, "chunk_size=%" PRIu32 "MB,", g.c_chunk_size); - /* - * We can't set bloom_oldest without bloom, and we want to test - * with Bloom filters on most of the time anyway. - */ - if (g.c_bloom_oldest) - g.c_bloom = 1; - CONFIG_APPEND(p, "bloom=%s,", g.c_bloom ? "true" : "false"); - CONFIG_APPEND(p, - "bloom_bit_count=%" PRIu32 ",", g.c_bloom_bit_count); - CONFIG_APPEND(p, - "bloom_hash_count=%" PRIu32 ",", g.c_bloom_hash_count); - CONFIG_APPEND(p, - "bloom_oldest=%s,", g.c_bloom_oldest ? "true" : "false"); - CONFIG_APPEND(p, "merge_max=%" PRIu32 ",", g.c_merge_max); - CONFIG_APPEND(p, ",)"); - } - - if (max == 0) - testutil_die(ENOMEM, - "WT_SESSION.create configuration buffer too small"); - - /* - * Create the underlying store. - */ - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_checkfmt(session->create(session, g.uri, config), "%s", g.uri); - testutil_check(session->close(session, NULL)); + WT_CONNECTION *conn; + WT_SESSION *session; + size_t max; + uint32_t maxintlkey, maxleafkey, maxleafvalue; + char config[4096], *p; + + conn = g.wts_conn; + p = config; + max = sizeof(config); + + CONFIG_APPEND(p, + "key_format=%s" + ",allocation_size=512" + ",%s" + ",internal_page_max=%" PRIu32 ",leaf_page_max=%" PRIu32 ",memory_page_max=%" PRIu32, + (g.type == ROW) ? "u" : "r", g.c_firstfit ? "block_allocation=first" : "", g.intl_page_max, + g.leaf_page_max, MEGABYTE(g.c_memory_page_max)); + + /* + * Configure the maximum key/value sizes, but leave it as the default if we come up with + * something crazy. + */ + maxintlkey = mmrand(NULL, g.intl_page_max / 50, g.intl_page_max / 40); + if (maxintlkey > 20) + CONFIG_APPEND(p, ",internal_key_max=%" PRIu32, maxintlkey); + maxleafkey = mmrand(NULL, g.leaf_page_max / 50, g.leaf_page_max / 40); + if (maxleafkey > 20) + CONFIG_APPEND(p, ",leaf_key_max=%" PRIu32, maxleafkey); + maxleafvalue = mmrand(NULL, g.leaf_page_max * 10, g.leaf_page_max / 40); + if (maxleafvalue > 40 && maxleafvalue < 100 * 1024) + CONFIG_APPEND(p, ",leaf_value_max=%" PRIu32, maxleafvalue); + + switch (g.type) { + case FIX: + CONFIG_APPEND(p, ",value_format=%" PRIu32 "t", g.c_bitcnt); + break; + case ROW: + if (g.c_huffman_key) + CONFIG_APPEND(p, ",huffman_key=english"); + if (g.c_prefix_compression) + CONFIG_APPEND(p, ",prefix_compression_min=%" PRIu32, g.c_prefix_compression_min); + else + CONFIG_APPEND(p, ",prefix_compression=false"); + if (g.c_reverse) + CONFIG_APPEND(p, ",collator=reverse"); + /* FALLTHROUGH */ + case VAR: + if (g.c_huffman_value) + CONFIG_APPEND(p, ",huffman_value=english"); + if (g.c_dictionary) + CONFIG_APPEND(p, ",dictionary=%" PRIu32, mmrand(NULL, 123, 517)); + break; + } + + /* Configure checksums. */ + switch (g.c_checksum_flag) { + case CHECKSUM_OFF: + CONFIG_APPEND(p, ",checksum=\"off\""); + break; + case CHECKSUM_ON: + CONFIG_APPEND(p, ",checksum=\"on\""); + break; + case CHECKSUM_UNCOMPRESSED: + CONFIG_APPEND(p, ",checksum=\"uncompressed\""); + break; + } + + /* Configure compression. */ + if (g.c_compression_flag != COMPRESS_NONE) + CONFIG_APPEND(p, ",block_compressor=\"%s\"", compressor(g.c_compression_flag)); + + /* Configure Btree internal key truncation. */ + CONFIG_APPEND(p, ",internal_key_truncate=%s", g.c_internal_key_truncation ? "true" : "false"); + + /* Configure Btree page key gap. */ + CONFIG_APPEND(p, ",key_gap=%" PRIu32, g.c_key_gap); + + /* Configure Btree split page percentage. */ + CONFIG_APPEND(p, ",split_pct=%" PRIu32, g.c_split_pct); + + /* + * Assertions. Assertions slow down the code for additional diagnostic checking. + */ + if (g.c_txn_timestamps && g.c_assert_commit_timestamp) + CONFIG_APPEND(p, ",assert=(commit_timestamp=key_consistent)"); + if (g.c_txn_timestamps && g.c_assert_read_timestamp) + CONFIG_APPEND(p, ",assert=(read_timestamp=always)"); + + /* Configure LSM. */ + if (DATASOURCE("lsm")) { + CONFIG_APPEND(p, ",type=lsm,lsm=("); + CONFIG_APPEND(p, "auto_throttle=%s,", g.c_auto_throttle ? "true" : "false"); + CONFIG_APPEND(p, "chunk_size=%" PRIu32 "MB,", g.c_chunk_size); + /* + * We can't set bloom_oldest without bloom, and we want to test with Bloom filters on most + * of the time anyway. + */ + if (g.c_bloom_oldest) + g.c_bloom = 1; + CONFIG_APPEND(p, "bloom=%s,", g.c_bloom ? "true" : "false"); + CONFIG_APPEND(p, "bloom_bit_count=%" PRIu32 ",", g.c_bloom_bit_count); + CONFIG_APPEND(p, "bloom_hash_count=%" PRIu32 ",", g.c_bloom_hash_count); + CONFIG_APPEND(p, "bloom_oldest=%s,", g.c_bloom_oldest ? "true" : "false"); + CONFIG_APPEND(p, "merge_max=%" PRIu32 ",", g.c_merge_max); + CONFIG_APPEND(p, ",)"); + } + + if (max == 0) + testutil_die(ENOMEM, "WT_SESSION.create configuration buffer too small"); + + /* + * Create the underlying store. + */ + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_checkfmt(session->create(session, g.uri, config), "%s", g.uri); + testutil_check(session->close(session, NULL)); } void wts_close(void) { - WT_CONNECTION *conn; - const char *config; + WT_CONNECTION *conn; + const char *config; - conn = g.wts_conn; + conn = g.wts_conn; - config = g.c_leak_memory ? "leak_memory" : NULL; + config = g.c_leak_memory ? "leak_memory" : NULL; - testutil_check(conn->close(conn, config)); - g.wts_conn = NULL; - g.wt_api = NULL; + testutil_check(conn->close(conn, config)); + g.wts_conn = NULL; + g.wt_api = NULL; } void wts_verify(const char *tag) { - WT_CONNECTION *conn; - WT_DECL_RET; - WT_SESSION *session; - - if (g.c_verify == 0) - return; - - conn = g.wts_conn; - track("verify", 0ULL, NULL); - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - logop(session, "%s", "=============== verify start"); - - /* - * Verify can return EBUSY if the handle isn't available. Don't yield - * and retry, in the case of LSM, the handle may not be available for - * a long time. - */ - ret = session->verify(session, g.uri, "strict"); - testutil_assertfmt( - ret == 0 || ret == EBUSY, "session.verify: %s: %s", g.uri, tag); - - logop(session, "%s", "=============== verify stop"); - testutil_check(session->close(session, NULL)); + WT_CONNECTION *conn; + WT_DECL_RET; + WT_SESSION *session; + + if (g.c_verify == 0) + return; + + conn = g.wts_conn; + track("verify", 0ULL, NULL); + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + logop(session, "%s", "=============== verify start"); + + /* + * Verify can return EBUSY if the handle isn't available. Don't yield and retry, in the case of + * LSM, the handle may not be available for a long time. + */ + ret = session->verify(session, g.uri, "strict"); + testutil_assertfmt(ret == 0 || ret == EBUSY, "session.verify: %s: %s", g.uri, tag); + + logop(session, "%s", "=============== verify stop"); + testutil_check(session->close(session, NULL)); } /* * wts_stats -- - * Dump the run's statistics. + * Dump the run's statistics. */ void wts_stats(void) { - FILE *fp; - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_DECL_RET; - WT_SESSION *session; - size_t len; - uint64_t v; - const char *desc, *pval; - char *stat_name; - - /* Ignore statistics if they're not configured. */ - if (g.c_statistics == 0) - return; - - conn = g.wts_conn; - track("stat", 0ULL, NULL); - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - if ((fp = fopen(g.home_stats, "w")) == NULL) - testutil_die(errno, "fopen: %s", g.home_stats); - - /* Connection statistics. */ - fprintf(fp, "====== Connection statistics:\n"); - testutil_check(session->open_cursor( - session, "statistics:", NULL, NULL, &cursor)); - - while ((ret = cursor->next(cursor)) == 0 && - (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) - if (fprintf(fp, "%s=%s\n", desc, pval) < 0) - testutil_die(errno, "fprintf"); - - if (ret != WT_NOTFOUND) - testutil_die(ret, "cursor.next"); - testutil_check(cursor->close(cursor)); - - /* Data source statistics. */ - fprintf(fp, "\n\n====== Data source statistics:\n"); - len = strlen("statistics:") + strlen(g.uri) + 1; - stat_name = dmalloc(len); - testutil_check(__wt_snprintf(stat_name, len, "statistics:%s", g.uri)); - testutil_check(session->open_cursor( - session, stat_name, NULL, NULL, &cursor)); - free(stat_name); - - while ((ret = cursor->next(cursor)) == 0 && - (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) - if (fprintf(fp, "%s=%s\n", desc, pval) < 0) - testutil_die(errno, "fprintf"); - - if (ret != WT_NOTFOUND) - testutil_die(ret, "cursor.next"); - testutil_check(cursor->close(cursor)); - - fclose_and_clear(&fp); - - testutil_check(session->close(session, NULL)); + FILE *fp; + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_DECL_RET; + WT_SESSION *session; + size_t len; + uint64_t v; + char *stat_name; + const char *desc, *pval; + + /* Ignore statistics if they're not configured. */ + if (g.c_statistics == 0) + return; + + conn = g.wts_conn; + track("stat", 0ULL, NULL); + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + if ((fp = fopen(g.home_stats, "w")) == NULL) + testutil_die(errno, "fopen: %s", g.home_stats); + + /* Connection statistics. */ + fprintf(fp, "====== Connection statistics:\n"); + testutil_check(session->open_cursor(session, "statistics:", NULL, NULL, &cursor)); + + while ( + (ret = cursor->next(cursor)) == 0 && (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) + if (fprintf(fp, "%s=%s\n", desc, pval) < 0) + testutil_die(errno, "fprintf"); + + if (ret != WT_NOTFOUND) + testutil_die(ret, "cursor.next"); + testutil_check(cursor->close(cursor)); + + /* Data source statistics. */ + fprintf(fp, "\n\n====== Data source statistics:\n"); + len = strlen("statistics:") + strlen(g.uri) + 1; + stat_name = dmalloc(len); + testutil_check(__wt_snprintf(stat_name, len, "statistics:%s", g.uri)); + testutil_check(session->open_cursor(session, stat_name, NULL, NULL, &cursor)); + free(stat_name); + + while ( + (ret = cursor->next(cursor)) == 0 && (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) + if (fprintf(fp, "%s=%s\n", desc, pval) < 0) + testutil_die(errno, "fprintf"); + + if (ret != WT_NOTFOUND) + testutil_die(ret, "cursor.next"); + testutil_check(cursor->close(cursor)); + + fclose_and_clear(&fp); + + testutil_check(session->close(session, NULL)); } diff --git a/src/third_party/wiredtiger/test/huge/huge.c b/src/third_party/wiredtiger/test/huge/huge.c index a5d651c6d9a..92441d17f70 100644 --- a/src/third_party/wiredtiger/test/huge/huge.c +++ b/src/third_party/wiredtiger/test/huge/huge.c @@ -28,120 +28,107 @@ #include "test_util.h" -static char home[512]; /* Program working dir */ -static uint8_t *big; /* Big key/value buffer */ +static char home[512]; /* Program working dir */ +static uint8_t *big; /* Big key/value buffer */ -#define GIGABYTE (1073741824) -#define MEGABYTE (1048576) +#define GIGABYTE (1073741824) +#define MEGABYTE (1048576) /* * List of configurations we test. */ -typedef struct { - const char *uri; /* Object URI */ - const char *config; /* Object configuration */ - int recno; /* Column-store key */ +typedef struct { + const char *uri; /* Object URI */ + const char *config; /* Object configuration */ + int recno; /* Column-store key */ } CONFIG; -static CONFIG config[] = { - { "file:xxx", "key_format=S,value_format=S", 0 }, - { "file:xxx", "key_format=r,value_format=S", 1 }, - { "lsm:xxx", "key_format=S,value_format=S", 0 }, - { "table:xxx", "key_format=S,value_format=S", 0 }, - { "table:xxx", "key_format=r,value_format=S", 1 }, - { NULL, NULL, 0 } -}; - -#define SMALL_MAX MEGABYTE -static size_t lengths[] = { - 20, /* Check configuration */ - (size_t)1 * MEGABYTE, /* 1MB (largest -s configuration) */ - (size_t)250 * MEGABYTE, /* 250MB */ - (size_t)1 * GIGABYTE, /* 1GB */ - (size_t)2 * GIGABYTE, /* 2GB */ - (size_t)3 * GIGABYTE, /* 3GB */ - ((size_t)4 * GIGABYTE) - MEGABYTE, /* Roughly the max we can handle */ - 0 -}; - -static void usage(void) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static CONFIG config[] = {{"file:xxx", "key_format=S,value_format=S", 0}, + {"file:xxx", "key_format=r,value_format=S", 1}, {"lsm:xxx", "key_format=S,value_format=S", 0}, + {"table:xxx", "key_format=S,value_format=S", 0}, {"table:xxx", "key_format=r,value_format=S", 1}, + {NULL, NULL, 0}}; + +#define SMALL_MAX MEGABYTE +static size_t lengths[] = {20, /* Check configuration */ + (size_t)1 * MEGABYTE, /* 1MB (largest -s configuration) */ + (size_t)250 * MEGABYTE, /* 250MB */ + (size_t)1 * GIGABYTE, /* 1GB */ + (size_t)2 * GIGABYTE, /* 2GB */ + (size_t)3 * GIGABYTE, /* 3GB */ + ((size_t)4 * GIGABYTE) - MEGABYTE, /* Roughly the max we can handle */ + 0}; + +static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void usage(void) { - fprintf(stderr, "usage: %s [-s]\n", progname); - fprintf(stderr, "%s", "\t-s small run, only test up to 1GB\n"); - exit(EXIT_FAILURE); + fprintf(stderr, "usage: %s [-s]\n", progname); + fprintf(stderr, "%s", "\t-s small run, only test up to 1GB\n"); + exit(EXIT_FAILURE); } #ifndef _WIN32 -#define SIZET_FMT "%zu" /* size_t format string */ +#define SIZET_FMT "%zu" /* size_t format string */ #else -#define SIZET_FMT "%Iu" /* size_t format string */ +#define SIZET_FMT "%Iu" /* size_t format string */ #endif static void run(CONFIG *cp, int bigkey, size_t bytes) { - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_SESSION *session; - uint64_t keyno; - void *p; - - big[bytes - 1] = '\0'; - - printf(SIZET_FMT "%s%s: %s %s big %s\n", - bytes < MEGABYTE ? bytes : - (bytes < GIGABYTE ? bytes / MEGABYTE : bytes / GIGABYTE), - bytes < MEGABYTE ? "" : - (bytes < GIGABYTE ? - (bytes % MEGABYTE == 0 ? "" : "+") : - (bytes % GIGABYTE == 0 ? "" : "+")), - bytes < MEGABYTE ? "B" : (bytes < GIGABYTE ? "MB" : "GB"), - cp->uri, cp->config, bigkey ? "key" : "value"); - - testutil_make_work_dir(home); - - /* - * Open/create the database, connection, session and cursor; set the - * cache size large, we don't want to try and evict anything. - */ - testutil_check( - wiredtiger_open(home, NULL, "create,cache_size=10GB", &conn)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->create(session, cp->uri, cp->config)); - testutil_check( - session->open_cursor(session, cp->uri, NULL, NULL, &cursor)); - - /* Set the key/value. */ - if (bigkey) - cursor->set_key(cursor, big); - else if (cp->recno) { - keyno = 1; - cursor->set_key(cursor, keyno); - } else - cursor->set_key(cursor, "key001"); - cursor->set_value(cursor, big); - - /* Insert the record (use update, insert discards the key). */ - testutil_check(cursor->update(cursor)); - - /* Retrieve the record and check it. */ - testutil_check(cursor->search(cursor)); - if (bigkey) - testutil_check(cursor->get_key(cursor, &p)); - testutil_check(cursor->get_value(cursor, &p)); - if (memcmp(p, big, bytes) != 0) - testutil_die(0, - "retrieved big key/value item did not match original"); - - /* Remove the record. */ - testutil_check(cursor->remove(cursor)); - - testutil_check(conn->close(conn, NULL)); - - big[bytes - 1] = 'a'; + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_SESSION *session; + uint64_t keyno; + void *p; + + big[bytes - 1] = '\0'; + + printf(SIZET_FMT "%s%s: %s %s big %s\n", + bytes < MEGABYTE ? bytes : (bytes < GIGABYTE ? bytes / MEGABYTE : bytes / GIGABYTE), + bytes < MEGABYTE ? "" : (bytes < GIGABYTE ? (bytes % MEGABYTE == 0 ? "" : "+") : + (bytes % GIGABYTE == 0 ? "" : "+")), + bytes < MEGABYTE ? "B" : (bytes < GIGABYTE ? "MB" : "GB"), cp->uri, cp->config, + bigkey ? "key" : "value"); + + testutil_make_work_dir(home); + + /* + * Open/create the database, connection, session and cursor; set the cache size large, we don't + * want to try and evict anything. + */ + testutil_check(wiredtiger_open(home, NULL, "create,cache_size=10GB", &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->create(session, cp->uri, cp->config)); + testutil_check(session->open_cursor(session, cp->uri, NULL, NULL, &cursor)); + + /* Set the key/value. */ + if (bigkey) + cursor->set_key(cursor, big); + else if (cp->recno) { + keyno = 1; + cursor->set_key(cursor, keyno); + } else + cursor->set_key(cursor, "key001"); + cursor->set_value(cursor, big); + + /* Insert the record (use update, insert discards the key). */ + testutil_check(cursor->update(cursor)); + + /* Retrieve the record and check it. */ + testutil_check(cursor->search(cursor)); + if (bigkey) + testutil_check(cursor->get_key(cursor, &p)); + testutil_check(cursor->get_value(cursor, &p)); + if (memcmp(p, big, bytes) != 0) + testutil_die(0, "retrieved big key/value item did not match original"); + + /* Remove the record. */ + testutil_check(cursor->remove(cursor)); + + testutil_check(conn->close(conn, NULL)); + + big[bytes - 1] = 'a'; } extern int __wt_optind; @@ -150,50 +137,50 @@ extern char *__wt_optarg; int main(int argc, char *argv[]) { - CONFIG *cp; - size_t len, *lp; - int ch, small; - char *working_dir; - - (void)testutil_set_progname(argv); - - small = 0; - working_dir = NULL; - while ((ch = __wt_getopt(progname, argc, argv, "h:s")) != EOF) - switch (ch) { - case 'h': - working_dir = __wt_optarg; - break; - case 's': /* Gigabytes */ - small = 1; - break; - default: - usage(); - } - argc -= __wt_optind; - if (argc != 0) - usage(); - - testutil_work_dir_from_path(home, 512, working_dir); - - /* Allocate a buffer to use. */ - len = small ? ((size_t)SMALL_MAX) : ((size_t)4 * GIGABYTE); - big = dmalloc(len); - memset(big, 'a', len); - - /* Make sure the configurations all work. */ - for (lp = lengths; *lp != 0; ++lp) { - if (small && *lp > SMALL_MAX) - break; - for (cp = config; cp->uri != NULL; ++cp) { - if (!cp->recno) /* Big key on row-store */ - run(cp, 1, *lp); - run(cp, 0, *lp); /* Big value */ - } - } - free(big); - - testutil_clean_work_dir(home); - - return (EXIT_SUCCESS); + CONFIG *cp; + size_t len, *lp; + int ch, small; + char *working_dir; + + (void)testutil_set_progname(argv); + + small = 0; + working_dir = NULL; + while ((ch = __wt_getopt(progname, argc, argv, "h:s")) != EOF) + switch (ch) { + case 'h': + working_dir = __wt_optarg; + break; + case 's': /* Gigabytes */ + small = 1; + break; + default: + usage(); + } + argc -= __wt_optind; + if (argc != 0) + usage(); + + testutil_work_dir_from_path(home, 512, working_dir); + + /* Allocate a buffer to use. */ + len = small ? ((size_t)SMALL_MAX) : ((size_t)4 * GIGABYTE); + big = dmalloc(len); + memset(big, 'a', len); + + /* Make sure the configurations all work. */ + for (lp = lengths; *lp != 0; ++lp) { + if (small && *lp > SMALL_MAX) + break; + for (cp = config; cp->uri != NULL; ++cp) { + if (!cp->recno) /* Big key on row-store */ + run(cp, 1, *lp); + run(cp, 0, *lp); /* Big value */ + } + } + free(big); + + testutil_clean_work_dir(home); + + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/manydbs/manydbs.c b/src/third_party/wiredtiger/test/manydbs/manydbs.c index 8eedbc03814..6397a27759c 100644 --- a/src/third_party/wiredtiger/test/manydbs/manydbs.c +++ b/src/third_party/wiredtiger/test/manydbs/manydbs.c @@ -28,40 +28,38 @@ #include "test_util.h" -#define HOME_SIZE 512 -#define HOME_BASE "WT_TEST" -static char home[HOME_SIZE]; /* Base home directory */ -static char hometmp[HOME_SIZE]; /* Each conn home directory */ -static const char * const uri = "table:main"; - -#define WTOPEN_CFG_COMMON \ - "create,log=(file_max=10M,archive=false,enabled)," \ +#define HOME_SIZE 512 +#define HOME_BASE "WT_TEST" +static char home[HOME_SIZE]; /* Base home directory */ +static char hometmp[HOME_SIZE]; /* Each conn home directory */ +static const char *const uri = "table:main"; + +#define WTOPEN_CFG_COMMON \ + "create,log=(file_max=10M,archive=false,enabled)," \ "statistics=(fast),statistics_log=(wait=5)," -#define WT_CONFIG0 \ - WTOPEN_CFG_COMMON \ +#define WT_CONFIG0 \ + WTOPEN_CFG_COMMON \ "transaction_sync=(enabled=false)" -#define WT_CONFIG1 \ - WTOPEN_CFG_COMMON \ +#define WT_CONFIG1 \ + WTOPEN_CFG_COMMON \ "transaction_sync=(enabled,method=none)" -#define WT_CONFIG2 \ - WTOPEN_CFG_COMMON \ +#define WT_CONFIG2 \ + WTOPEN_CFG_COMMON \ "transaction_sync=(enabled,method=fsync)" -#define MAX_DBS 10 -#define MAX_IDLE_TIME 30 -#define IDLE_INCR 5 +#define MAX_DBS 10 +#define MAX_IDLE_TIME 30 +#define IDLE_INCR 5 -#define MAX_KV 100 -#define MAX_VAL 128 +#define MAX_KV 100 +#define MAX_VAL 128 -static void usage(void) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void usage(void) { - fprintf(stderr, - "usage: %s [-I] [-D maxdbs] [-h dir]\n", progname); - exit(EXIT_FAILURE); + fprintf(stderr, "usage: %s [-I] [-D maxdbs] [-h dir]\n", progname); + exit(EXIT_FAILURE); } extern int __wt_optind; @@ -75,166 +73,155 @@ static WT_SESSION **sessions = NULL; static int get_stat(WT_SESSION *stat_session, int stat_field, uint64_t *valuep) { - WT_CURSOR *statc; - int ret; - const char *desc, *pvalue; - - testutil_check(stat_session->open_cursor(stat_session, - "statistics:", NULL, NULL, &statc)); - statc->set_key(statc, stat_field); - if ((ret = statc->search(statc)) != 0) - return (ret); - - ret = statc->get_value(statc, &desc, &pvalue, valuep); - testutil_check(statc->close(statc)); - return (ret); + WT_CURSOR *statc; + int ret; + const char *desc, *pvalue; + + testutil_check(stat_session->open_cursor(stat_session, "statistics:", NULL, NULL, &statc)); + statc->set_key(statc, stat_field); + if ((ret = statc->search(statc)) != 0) + return (ret); + + ret = statc->get_value(statc, &desc, &pvalue, valuep); + testutil_check(statc->close(statc)); + return (ret); } static void run_ops(int dbs) { - WT_ITEM data; - uint32_t db; - uint8_t buf[MAX_VAL]; - int db_set, i, key; - - memset(buf, 0, sizeof(buf)); - for (i = 0; i < MAX_VAL; ++i) - buf[i] = (uint8_t)__wt_random(&rnd); - data.data = buf; - /* - * Write a small amount of data into a random subset of the databases. - */ - db_set = dbs / 4; - for (i = 0; i < db_set; ++i) { - db = __wt_random(&rnd) % (uint32_t)dbs; - printf("Write to database %" PRIu32 "\n", db); - for (key = 0; key < MAX_KV; ++key) { - data.size = __wt_random(&rnd) % MAX_VAL; - cursors[db]->set_key(cursors[db], key); - cursors[db]->set_value(cursors[db], &data); - testutil_check(cursors[db]->insert(cursors[db])); - } - } + WT_ITEM data; + uint32_t db; + uint8_t buf[MAX_VAL]; + int db_set, i, key; + + memset(buf, 0, sizeof(buf)); + for (i = 0; i < MAX_VAL; ++i) + buf[i] = (uint8_t)__wt_random(&rnd); + data.data = buf; + /* + * Write a small amount of data into a random subset of the databases. + */ + db_set = dbs / 4; + for (i = 0; i < db_set; ++i) { + db = __wt_random(&rnd) % (uint32_t)dbs; + printf("Write to database %" PRIu32 "\n", db); + for (key = 0; key < MAX_KV; ++key) { + data.size = __wt_random(&rnd) % MAX_VAL; + cursors[db]->set_key(cursors[db], key); + cursors[db]->set_value(cursors[db], &data); + testutil_check(cursors[db]->insert(cursors[db])); + } + } } int main(int argc, char *argv[]) { - uint64_t cond_reset, cond_wait; - uint64_t *cond_reset_orig; - int cfg, ch, dbs, i; - char cmd[128]; - const char *working_dir, *wt_cfg; - bool idle; - - (void)testutil_set_progname(argv); - - dbs = MAX_DBS; - working_dir = HOME_BASE; - idle = false; - while ((ch = __wt_getopt(progname, argc, argv, "D:h:I")) != EOF) - switch (ch) { - case 'D': - dbs = atoi(__wt_optarg); - break; - case 'h': - working_dir = __wt_optarg; - break; - case 'I': - idle = true; - break; - default: - usage(); - } - argc -= __wt_optind; - if (argc != 0) - usage(); - - /* - * Allocate arrays for connection handles, sessions, statistics - * cursors and, if needed, data cursors. - */ - connections = dcalloc((size_t)dbs, sizeof(WT_CONNECTION *)); - sessions = dcalloc((size_t)dbs, sizeof(WT_SESSION *)); - cond_reset_orig = dcalloc((size_t)dbs, sizeof(uint64_t)); - cursors = idle ? NULL : dcalloc((size_t)dbs, sizeof(WT_CURSOR *)); - memset(cmd, 0, sizeof(cmd)); - /* - * Set up all the directory names. - */ - testutil_work_dir_from_path(home, HOME_SIZE, working_dir); - testutil_make_work_dir(home); - __wt_random_init(&rnd); - for (i = 0; i < dbs; ++i) { - testutil_check(__wt_snprintf( - hometmp, HOME_SIZE, "%s/%s.%d", home, HOME_BASE, i)); - testutil_make_work_dir(hometmp); - /* - * Open each database. Rotate different configurations - * among them. Open a session and statistics cursor. - * If writing data, create the table and open a data cursor. - */ - cfg = i % 3; - if (cfg == 0) - wt_cfg = WT_CONFIG0; - else if (cfg == 1) - wt_cfg = WT_CONFIG1; - else - wt_cfg = WT_CONFIG2; - testutil_check(wiredtiger_open( - hometmp, NULL, wt_cfg, &connections[i])); - testutil_check(connections[i]->open_session(connections[i], - NULL, NULL, &sessions[i])); - if (!idle) { - testutil_check(sessions[i]->create(sessions[i], - uri, "key_format=Q,value_format=u")); - testutil_check(sessions[i]->open_cursor(sessions[i], - uri, NULL, NULL, &cursors[i])); - } - } - - sleep(10); - - /* - * Record original reset setting. There could have been some - * activity during the creation period. - */ - for (i = 0; i < dbs; ++i) - testutil_check(get_stat(sessions[i], - WT_STAT_CONN_COND_AUTO_WAIT_RESET, &cond_reset_orig[i])); - for (i = 0; i < MAX_IDLE_TIME; i += IDLE_INCR) { - if (!idle) - run_ops(dbs); - printf("Sleep %d (%d of %d)\n", IDLE_INCR, i, MAX_IDLE_TIME); - sleep(IDLE_INCR); - } - for (i = 0; i < dbs; ++i) { - testutil_check(get_stat(sessions[i], - WT_STAT_CONN_COND_AUTO_WAIT_RESET, &cond_reset)); - testutil_check(get_stat(sessions[i], - WT_STAT_CONN_COND_AUTO_WAIT, &cond_wait)); - /* - * On an idle workload there should be no resets of condition - * variables during the idle period. Even with a light - * workload, resets should not be very common. We look for 5%. - */ - if (idle && cond_reset != cond_reset_orig[i]) - testutil_die(ERANGE, - "condition reset on idle connection %d of %" PRIu64, - i, cond_reset); - if (!idle && cond_reset > cond_wait / 20) - testutil_die(ERANGE, "connection %d condition reset %" - PRIu64 " exceeds 5%% of %" PRIu64, - i, cond_reset, cond_wait); - testutil_check(connections[i]->close(connections[i], NULL)); - } - - /* Cleanup allocated memory. */ - free(connections); - free(sessions); - free(cond_reset_orig); - free(cursors); - - return (EXIT_SUCCESS); + uint64_t cond_reset, cond_wait; + uint64_t *cond_reset_orig; + int cfg, ch, dbs, i; + char cmd[128]; + const char *working_dir, *wt_cfg; + bool idle; + + (void)testutil_set_progname(argv); + + dbs = MAX_DBS; + working_dir = HOME_BASE; + idle = false; + while ((ch = __wt_getopt(progname, argc, argv, "D:h:I")) != EOF) + switch (ch) { + case 'D': + dbs = atoi(__wt_optarg); + break; + case 'h': + working_dir = __wt_optarg; + break; + case 'I': + idle = true; + break; + default: + usage(); + } + argc -= __wt_optind; + if (argc != 0) + usage(); + + /* + * Allocate arrays for connection handles, sessions, statistics cursors and, if needed, data + * cursors. + */ + connections = dcalloc((size_t)dbs, sizeof(WT_CONNECTION *)); + sessions = dcalloc((size_t)dbs, sizeof(WT_SESSION *)); + cond_reset_orig = dcalloc((size_t)dbs, sizeof(uint64_t)); + cursors = idle ? NULL : dcalloc((size_t)dbs, sizeof(WT_CURSOR *)); + memset(cmd, 0, sizeof(cmd)); + /* + * Set up all the directory names. + */ + testutil_work_dir_from_path(home, HOME_SIZE, working_dir); + testutil_make_work_dir(home); + __wt_random_init(&rnd); + for (i = 0; i < dbs; ++i) { + testutil_check(__wt_snprintf(hometmp, HOME_SIZE, "%s/%s.%d", home, HOME_BASE, i)); + testutil_make_work_dir(hometmp); + /* + * Open each database. Rotate different configurations among them. Open a session and + * statistics cursor. If writing data, create the table and open a data cursor. + */ + cfg = i % 3; + if (cfg == 0) + wt_cfg = WT_CONFIG0; + else if (cfg == 1) + wt_cfg = WT_CONFIG1; + else + wt_cfg = WT_CONFIG2; + testutil_check(wiredtiger_open(hometmp, NULL, wt_cfg, &connections[i])); + testutil_check(connections[i]->open_session(connections[i], NULL, NULL, &sessions[i])); + if (!idle) { + testutil_check(sessions[i]->create(sessions[i], uri, "key_format=Q,value_format=u")); + testutil_check(sessions[i]->open_cursor(sessions[i], uri, NULL, NULL, &cursors[i])); + } + } + + sleep(10); + + /* + * Record original reset setting. There could have been some activity during the creation + * period. + */ + for (i = 0; i < dbs; ++i) + testutil_check( + get_stat(sessions[i], WT_STAT_CONN_COND_AUTO_WAIT_RESET, &cond_reset_orig[i])); + for (i = 0; i < MAX_IDLE_TIME; i += IDLE_INCR) { + if (!idle) + run_ops(dbs); + printf("Sleep %d (%d of %d)\n", IDLE_INCR, i, MAX_IDLE_TIME); + sleep(IDLE_INCR); + } + for (i = 0; i < dbs; ++i) { + testutil_check(get_stat(sessions[i], WT_STAT_CONN_COND_AUTO_WAIT_RESET, &cond_reset)); + testutil_check(get_stat(sessions[i], WT_STAT_CONN_COND_AUTO_WAIT, &cond_wait)); + /* + * On an idle workload there should be no resets of condition variables during the idle + * period. Even with a light workload, resets should not be very common. We look for 5%. + */ + if (idle && cond_reset != cond_reset_orig[i]) + testutil_die( + ERANGE, "condition reset on idle connection %d of %" PRIu64, i, cond_reset); + if (!idle && cond_reset > cond_wait / 20) + testutil_die(ERANGE, + "connection %d condition reset %" PRIu64 " exceeds 5%% of %" PRIu64, i, cond_reset, + cond_wait); + testutil_check(connections[i]->close(connections[i], NULL)); + } + + /* Cleanup allocated memory. */ + free(connections); + free(sessions); + free(cond_reset_orig); + free(cursors); + + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/packing/intpack-test.c b/src/third_party/wiredtiger/test/packing/intpack-test.c index 75b7cbd620c..75e8a238e27 100644 --- a/src/third_party/wiredtiger/test/packing/intpack-test.c +++ b/src/third_party/wiredtiger/test/packing/intpack-test.c @@ -31,50 +31,47 @@ int main(void) { - uint64_t ncalls, r, r2, s; - uint8_t *p; - uint8_t buf[WT_INTPACK64_MAXSIZE + 8]; /* -Werror=array-bounds */ - const uint8_t *cp; - size_t used_len; - int i; + uint64_t ncalls, r, r2, s; + uint8_t *p; + uint8_t buf[WT_INTPACK64_MAXSIZE + 8]; /* -Werror=array-bounds */ + const uint8_t *cp; + size_t used_len; + int i; - memset(buf, 0xff, sizeof(buf)); /* -Werror=maybe-uninitialized */ + memset(buf, 0xff, sizeof(buf)); /* -Werror=maybe-uninitialized */ - /* - * Required on some systems to pull in parts of the library - * for which we have data references. - */ - testutil_check(__wt_library_init()); + /* + * Required on some systems to pull in parts of the library for which we have data references. + */ + testutil_check(__wt_library_init()); - for (ncalls = 0, i = 0; i < 10000000; i++) { - for (s = 0; s < 50; s += 5) { - ++ncalls; - r = 1ULL << s; + for (ncalls = 0, i = 0; i < 10000000; i++) { + for (s = 0; s < 50; s += 5) { + ++ncalls; + r = 1ULL << s; #if 1 - p = buf; - testutil_check(__wt_vpack_uint(&p, sizeof(buf), r)); - used_len = (size_t)(p - buf); - testutil_assert(used_len <= WT_INTPACK64_MAXSIZE); - cp = buf; - testutil_check( - __wt_vunpack_uint(&cp, sizeof(buf), &r2)); + p = buf; + testutil_check(__wt_vpack_uint(&p, sizeof(buf), r)); + used_len = (size_t)(p - buf); + testutil_assert(used_len <= WT_INTPACK64_MAXSIZE); + cp = buf; + testutil_check(__wt_vunpack_uint(&cp, sizeof(buf), &r2)); #else - /* - * Note: use memmove for comparison because GCC does - * aggressive optimization of memcpy and it's difficult - * to measure anything. - */ - p = buf; - memmove(p, &r, sizeof(r)); - cp = buf; - memmove(&r2, cp, sizeof(r2)); + /* + * Note: use memmove for comparison because GCC does aggressive optimization of memcpy + * and it's difficult to measure anything. + */ + p = buf; + memmove(p, &r, sizeof(r)); + cp = buf; + memmove(&r2, cp, sizeof(r2)); #endif - testutil_assert(r == r2); - } - } + testutil_assert(r == r2); + } + } - printf("Number of calls: %" PRIu64 "\n", ncalls); + printf("Number of calls: %" PRIu64 "\n", ncalls); - return (0); + return (0); } diff --git a/src/third_party/wiredtiger/test/packing/intpack-test2.c b/src/third_party/wiredtiger/test/packing/intpack-test2.c index 1c0bfec8de6..c3747d1a31f 100644 --- a/src/third_party/wiredtiger/test/packing/intpack-test2.c +++ b/src/third_party/wiredtiger/test/packing/intpack-test2.c @@ -31,39 +31,37 @@ int main(void) { - size_t used_len; - int64_t i; - uint8_t *end, *p; - uint8_t buf[WT_INTPACK64_MAXSIZE + 8]; /* -Werror=array-bounds */ + size_t used_len; + int64_t i; + uint8_t *end, *p; + uint8_t buf[WT_INTPACK64_MAXSIZE + 8]; /* -Werror=array-bounds */ - memset(buf, 0xff, sizeof(buf)); /* -Werror=maybe-uninitialized */ + memset(buf, 0xff, sizeof(buf)); /* -Werror=maybe-uninitialized */ - /* - * Required on some systems to pull in parts of the library - * for which we have data references. - */ - testutil_check(__wt_library_init()); + /* + * Required on some systems to pull in parts of the library for which we have data references. + */ + testutil_check(__wt_library_init()); - for (i = 1; i < 1LL << 60; i <<= 1) { - end = buf; - testutil_check( - __wt_vpack_uint(&end, sizeof(buf), (uint64_t)i)); - used_len = (size_t)(end - buf); - testutil_assert(used_len <= WT_INTPACK64_MAXSIZE); - printf("%" PRId64 " ", i); - for (p = buf; p < end; p++) - printf("%02x", *p); - printf("\n"); + for (i = 1; i < 1LL << 60; i <<= 1) { + end = buf; + testutil_check(__wt_vpack_uint(&end, sizeof(buf), (uint64_t)i)); + used_len = (size_t)(end - buf); + testutil_assert(used_len <= WT_INTPACK64_MAXSIZE); + printf("%" PRId64 " ", i); + for (p = buf; p < end; p++) + printf("%02x", *p); + printf("\n"); - end = buf; - testutil_check(__wt_vpack_int(&end, sizeof(buf), -i)); - used_len = (size_t)(end - buf); - testutil_assert(used_len <= WT_INTPACK64_MAXSIZE); - printf("%" PRId64 " ", -i); - for (p = buf; p < end; p++) - printf("%02x", *p); - printf("\n"); - } + end = buf; + testutil_check(__wt_vpack_int(&end, sizeof(buf), -i)); + used_len = (size_t)(end - buf); + testutil_assert(used_len <= WT_INTPACK64_MAXSIZE); + printf("%" PRId64 " ", -i); + for (p = buf; p < end; p++) + printf("%02x", *p); + printf("\n"); + } - return (0); + return (0); } diff --git a/src/third_party/wiredtiger/test/packing/intpack-test3.c b/src/third_party/wiredtiger/test/packing/intpack-test3.c index ffb5f40b0f7..1a506509add 100644 --- a/src/third_party/wiredtiger/test/packing/intpack-test3.c +++ b/src/third_party/wiredtiger/test/packing/intpack-test3.c @@ -34,102 +34,93 @@ void test_spread(int64_t, int64_t, int64_t); void test_value(int64_t val) { - const uint8_t *cp; - uint8_t buf[WT_INTPACK64_MAXSIZE + 8]; /* -Werror=array-bounds */ - uint8_t *p; - int64_t sinput, soutput; - uint64_t uinput, uoutput; - size_t used_len; + const uint8_t *cp; + uint8_t buf[WT_INTPACK64_MAXSIZE + 8]; /* -Werror=array-bounds */ + uint8_t *p; + int64_t sinput, soutput; + uint64_t uinput, uoutput; + size_t used_len; - memset(buf, 0xff, sizeof(buf)); /* -Werror=maybe-uninitialized */ - sinput = val; - soutput = 0; /* -Werror=maybe-uninitialized */ + memset(buf, 0xff, sizeof(buf)); /* -Werror=maybe-uninitialized */ + sinput = val; + soutput = 0; /* -Werror=maybe-uninitialized */ - /* - * Required on some systems to pull in parts of the library - * for which we have data references. - */ - testutil_check(__wt_library_init()); + /* + * Required on some systems to pull in parts of the library for which we have data references. + */ + testutil_check(__wt_library_init()); - p = buf; - testutil_check(__wt_vpack_int(&p, sizeof(buf), sinput)); - used_len = (size_t)(p - buf); - testutil_assert(used_len <= WT_INTPACK64_MAXSIZE); - cp = buf; - testutil_check(__wt_vunpack_int(&cp, used_len, &soutput)); - /* Ensure we got the correct value back */ - if (sinput != soutput) { - fprintf(stderr, - "mismatch %" PRId64 ", %" PRId64 "\n", sinput, soutput); - abort(); - } - /* Ensure that decoding used the correct amount of buffer */ - if (cp != p) { - fprintf(stderr, - "Unpack consumed wrong size for %" PRId64 - ", expected %" WT_SIZET_FMT ", got %" WT_SIZET_FMT "\n", - sinput, used_len, cp > p ? - used_len + (size_t)(cp - p) : /* More than buf used */ - used_len - (size_t)(p - cp)); /* Less than buf used */ - abort(); - } + p = buf; + testutil_check(__wt_vpack_int(&p, sizeof(buf), sinput)); + used_len = (size_t)(p - buf); + testutil_assert(used_len <= WT_INTPACK64_MAXSIZE); + cp = buf; + testutil_check(__wt_vunpack_int(&cp, used_len, &soutput)); + /* Ensure we got the correct value back */ + if (sinput != soutput) { + fprintf(stderr, "mismatch %" PRId64 ", %" PRId64 "\n", sinput, soutput); + abort(); + } + /* Ensure that decoding used the correct amount of buffer */ + if (cp != p) { + fprintf(stderr, "Unpack consumed wrong size for %" PRId64 ", expected %" WT_SIZET_FMT + ", got %" WT_SIZET_FMT "\n", + sinput, used_len, cp > p ? used_len + (size_t)(cp - p) : /* More than buf used */ + used_len - (size_t)(p - cp)); /* Less than buf used */ + abort(); + } - /* Test unsigned, convert negative into bigger positive values */ - uinput = (uint64_t)val; + /* Test unsigned, convert negative into bigger positive values */ + uinput = (uint64_t)val; - p = buf; - testutil_check(__wt_vpack_uint(&p, sizeof(buf), uinput)); - used_len = (size_t)(p - buf); - testutil_assert(used_len <= WT_INTPACK64_MAXSIZE); - cp = buf; - testutil_check(__wt_vunpack_uint(&cp, sizeof(buf), &uoutput)); - /* Ensure we got the correct value back */ - if (sinput != soutput) { - fprintf(stderr, - "mismatch %" PRId64 ", %" PRId64 "\n", sinput, soutput); - abort(); - } - /* Ensure that decoding used the correct amount of buffer */ - if (cp != p) { - fprintf(stderr, - "Unpack consumed wrong size for %" PRId64 - ", expected %" WT_SIZET_FMT ", got %" WT_SIZET_FMT "\n", - sinput, used_len, cp > p ? - used_len + (size_t)(cp - p) : - used_len - (size_t)(p - cp)); - abort(); - } + p = buf; + testutil_check(__wt_vpack_uint(&p, sizeof(buf), uinput)); + used_len = (size_t)(p - buf); + testutil_assert(used_len <= WT_INTPACK64_MAXSIZE); + cp = buf; + testutil_check(__wt_vunpack_uint(&cp, sizeof(buf), &uoutput)); + /* Ensure we got the correct value back */ + if (sinput != soutput) { + fprintf(stderr, "mismatch %" PRId64 ", %" PRId64 "\n", sinput, soutput); + abort(); + } + /* Ensure that decoding used the correct amount of buffer */ + if (cp != p) { + fprintf(stderr, "Unpack consumed wrong size for %" PRId64 ", expected %" WT_SIZET_FMT + ", got %" WT_SIZET_FMT "\n", + sinput, used_len, cp > p ? used_len + (size_t)(cp - p) : used_len - (size_t)(p - cp)); + abort(); + } } void test_spread(int64_t start, int64_t before, int64_t after) { - int64_t i; + int64_t i; - printf( - "Testing range: %" PRId64 " to %" PRId64 ". Spread: % " PRId64 "\n", - start - before, start + after, before + after); - for (i = start - before; i < start + after; i++) - test_value(i); + printf("Testing range: %" PRId64 " to %" PRId64 ". Spread: % " PRId64 "\n", start - before, + start + after, before + after); + for (i = start - before; i < start + after; i++) + test_value(i); } int main(void) { - int64_t i; + int64_t i; - /* - * Test all values in a range, to ensure pack/unpack of small numbers - * (which most actively use different numbers of bits) works. - */ - test_spread(0, 100000, 100000); - test_spread(INT16_MAX, 1025, 1025); - test_spread(INT32_MAX, 1025, 1025); - test_spread(INT64_MAX, 1025, 1025); - /* Test bigger numbers */ - for (i = INT64_MAX; i > 0; i = i / 2) - test_spread(i, 1025, 1025); - printf("\n"); + /* + * Test all values in a range, to ensure pack/unpack of small numbers + * (which most actively use different numbers of bits) works. + */ + test_spread(0, 100000, 100000); + test_spread(INT16_MAX, 1025, 1025); + test_spread(INT32_MAX, 1025, 1025); + test_spread(INT64_MAX, 1025, 1025); + /* Test bigger numbers */ + for (i = INT64_MAX; i > 0; i = i / 2) + test_spread(i, 1025, 1025); + printf("\n"); - return (0); + return (0); } diff --git a/src/third_party/wiredtiger/test/packing/packing-test.c b/src/third_party/wiredtiger/test/packing/packing-test.c index ecf52c7c337..7f3f8a96e33 100644 --- a/src/third_party/wiredtiger/test/packing/packing-test.c +++ b/src/third_party/wiredtiger/test/packing/packing-test.c @@ -31,47 +31,45 @@ static void check(const char *fmt, ...) { - size_t len; - char buf[200], *end, *p; - va_list ap; + size_t len; + char buf[200], *end, *p; + va_list ap; - len = 0; /* -Werror=maybe-uninitialized */ + len = 0; /* -Werror=maybe-uninitialized */ - va_start(ap, fmt); - testutil_check(__wt_struct_sizev(NULL, &len, fmt, ap)); - va_end(ap); + va_start(ap, fmt); + testutil_check(__wt_struct_sizev(NULL, &len, fmt, ap)); + va_end(ap); - if (len < 1 || len >= sizeof(buf)) - testutil_die(EINVAL, - "Unexpected length from __wt_struct_sizev"); + if (len < 1 || len >= sizeof(buf)) + testutil_die(EINVAL, "Unexpected length from __wt_struct_sizev"); - va_start(ap, fmt); - testutil_check(__wt_struct_packv(NULL, buf, sizeof(buf), fmt, ap)); - va_end(ap); + va_start(ap, fmt); + testutil_check(__wt_struct_packv(NULL, buf, sizeof(buf), fmt, ap)); + va_end(ap); - printf("%s ", fmt); - for (p = buf, end = p + len; p < end; p++) - printf("%02x", (u_char)*p & 0xff); - printf("\n"); + printf("%s ", fmt); + for (p = buf, end = p + len; p < end; p++) + printf("%02x", (u_char)*p & 0xff); + printf("\n"); } int main(void) { - /* - * Required on some systems to pull in parts of the library - * for which we have data references. - */ - testutil_check(__wt_library_init()); + /* + * Required on some systems to pull in parts of the library for which we have data references. + */ + testutil_check(__wt_library_init()); - check("iii", 0, 101, -99); - check("3i", 0, 101, -99); - check("iS", 42, "forty two"); - check("s", "a big string"); + check("iii", 0, 101, -99); + check("3i", 0, 101, -99); + check("iS", 42, "forty two"); + check("s", "a big string"); #if 0 /* TODO: need a WT_ITEM */ check("u", r"\x42" * 20) check("uu", r"\x42" * 10, r"\x42" * 10) #endif - return (0); + return (0); } diff --git a/src/third_party/wiredtiger/test/readonly/readonly.c b/src/third_party/wiredtiger/test/readonly/readonly.c index b29e43171f1..bd6adae429a 100644 --- a/src/third_party/wiredtiger/test/readonly/readonly.c +++ b/src/third_party/wiredtiger/test/readonly/readonly.c @@ -30,124 +30,117 @@ #include <sys/wait.h> -#define HOME_SIZE 512 -static char home[HOME_SIZE]; /* Program working dir lock file */ -#define HOME_WR_SUFFIX ".WRNOLOCK" /* Writable dir copy no lock file */ +#define HOME_SIZE 512 +static char home[HOME_SIZE]; /* Program working dir lock file */ +#define HOME_WR_SUFFIX ".WRNOLOCK" /* Writable dir copy no lock file */ static char home_wr[HOME_SIZE + sizeof(HOME_WR_SUFFIX)]; -#define HOME_RD_SUFFIX ".RD" /* Read-only dir */ +#define HOME_RD_SUFFIX ".RD" /* Read-only dir */ static char home_rd[HOME_SIZE + sizeof(HOME_RD_SUFFIX)]; -#define HOME_RD2_SUFFIX ".RDNOLOCK" /* Read-only dir no lock file */ +#define HOME_RD2_SUFFIX ".RDNOLOCK" /* Read-only dir no lock file */ static char home_rd2[HOME_SIZE + sizeof(HOME_RD2_SUFFIX)]; -static const char *saved_argv0; /* Program command */ -static const char * const uri = "table:main"; +static const char *saved_argv0; /* Program command */ +static const char *const uri = "table:main"; -#define ENV_CONFIG \ - "create,log=(file_max=10M,archive=false,enabled)," \ +#define ENV_CONFIG \ + "create,log=(file_max=10M,archive=false,enabled)," \ "operation_tracking=(enabled=false),transaction_sync=(enabled,method=none)" -#define ENV_CONFIG_RD "operation_tracking=(enabled=false),readonly=true" -#define ENV_CONFIG_WR "operation_tracking=(enabled=false),readonly=false" -#define MAX_VAL 4096 -#define MAX_KV 10000 +#define ENV_CONFIG_RD "operation_tracking=(enabled=false),readonly=true" +#define ENV_CONFIG_WR "operation_tracking=(enabled=false),readonly=false" +#define MAX_VAL 4096 +#define MAX_KV 10000 -#define EXPECT_ERR 1 -#define EXPECT_SUCCESS 0 +#define EXPECT_ERR 1 +#define EXPECT_SUCCESS 0 -#define OP_READ 0 -#define OP_WRITE 1 +#define OP_READ 0 +#define OP_WRITE 1 -static void usage(void) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void usage(void) { - fprintf(stderr, "usage: %s [-h dir]\n", progname); - exit(EXIT_FAILURE); + fprintf(stderr, "usage: %s [-h dir]\n", progname); + exit(EXIT_FAILURE); } static int run_child(const char *homedir, int op, int expect) { - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_SESSION *session; - int i, ret; - const char *cfg; - - /* - * We expect the read-only database will allow the second read-only - * handle to succeed because no one can create or set the lock file. - */ - if (op == OP_READ) - cfg = ENV_CONFIG_RD; - else - cfg = ENV_CONFIG_WR; - if ((ret = wiredtiger_open(homedir, NULL, cfg, &conn)) == 0) { - if (expect == EXPECT_ERR) - testutil_die( - ret, "wiredtiger_open expected error, succeeded"); - } else { - if (expect == EXPECT_SUCCESS) - testutil_die( - ret, "wiredtiger_open expected success, error"); - /* - * If we expect an error and got one, we're done. - */ - return (0); - } - - /* - * Make sure we can read the data. - */ - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); - - i = 0; - while ((ret = cursor->next(cursor)) == 0) - ++i; - if (i != MAX_KV) - testutil_die(ret, "cursor walk"); - testutil_check(conn->close(conn, NULL)); - return (0); + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_SESSION *session; + int i, ret; + const char *cfg; + + /* + * We expect the read-only database will allow the second read-only handle to succeed because no + * one can create or set the lock file. + */ + if (op == OP_READ) + cfg = ENV_CONFIG_RD; + else + cfg = ENV_CONFIG_WR; + if ((ret = wiredtiger_open(homedir, NULL, cfg, &conn)) == 0) { + if (expect == EXPECT_ERR) + testutil_die(ret, "wiredtiger_open expected error, succeeded"); + } else { + if (expect == EXPECT_SUCCESS) + testutil_die(ret, "wiredtiger_open expected success, error"); + /* + * If we expect an error and got one, we're done. + */ + return (0); + } + + /* + * Make sure we can read the data. + */ + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); + + i = 0; + while ((ret = cursor->next(cursor)) == 0) + ++i; + if (i != MAX_KV) + testutil_die(ret, "cursor walk"); + testutil_check(conn->close(conn, NULL)); + return (0); } /* * Child process opens both databases readonly. */ +static void open_dbs(int, const char *, const char *, const char *, const char *) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void -open_dbs(int, const char *, const char *, - const char *, const char *) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); -static void -open_dbs(int op, const char *dir, - const char *dir_wr, const char *dir_rd, const char *dir_rd2) +open_dbs(int op, const char *dir, const char *dir_wr, const char *dir_rd, const char *dir_rd2) { - int expect, ret; - - /* - * The parent has an open connection to all directories. - * We expect opening the writeable homes to return an error. - * It is a failure if the child successfully opens that. - */ - expect = EXPECT_ERR; - if ((ret = run_child(dir, op, expect)) != 0) - testutil_die(ret, "wiredtiger_open readonly allowed"); - if ((ret = run_child(dir_wr, op, expect)) != 0) - testutil_die(ret, "wiredtiger_open readonly allowed"); - - /* - * The parent must have a read-only connection open to the - * read-only databases. If the child is opening read-only - * too, we expect success. Otherwise an error if the child - * attempts to open read/write (permission error). - */ - if (op == OP_READ) - expect = EXPECT_SUCCESS; - if ((ret = run_child(dir_rd, op, expect)) != 0) - testutil_die(ret, "run child 1"); - if ((ret = run_child(dir_rd2, op, expect)) != 0) - testutil_die(ret, "run child 2"); - exit(EXIT_SUCCESS); + int expect, ret; + + /* + * The parent has an open connection to all directories. We expect opening the writeable homes + * to return an error. It is a failure if the child successfully opens that. + */ + expect = EXPECT_ERR; + if ((ret = run_child(dir, op, expect)) != 0) + testutil_die(ret, "wiredtiger_open readonly allowed"); + if ((ret = run_child(dir_wr, op, expect)) != 0) + testutil_die(ret, "wiredtiger_open readonly allowed"); + + /* + * The parent must have a read-only connection open to the read-only databases. If the child is + * opening read-only too, we expect success. Otherwise an error if the child attempts to open + * read/write (permission error). + */ + if (op == OP_READ) + expect = EXPECT_SUCCESS; + if ((ret = run_child(dir_rd, op, expect)) != 0) + testutil_die(ret, "run child 1"); + if ((ret = run_child(dir_rd2, op, expect)) != 0) + testutil_die(ret, "run child 2"); + exit(EXIT_SUCCESS); } extern int __wt_optind; @@ -156,232 +149,214 @@ extern char *__wt_optarg; int main(int argc, char *argv[]) { - WT_CONNECTION *conn, *conn2, *conn3, *conn4; - WT_CURSOR *cursor; - WT_ITEM data; - WT_SESSION *session; - uint64_t i; - uint8_t buf[MAX_VAL]; - int ch, op, ret, status; - char cmd[512]; - const char *working_dir; - bool child; - - (void)testutil_set_progname(argv); - - /* - * Needed unaltered for system command later. - */ - saved_argv0 = argv[0]; - - working_dir = "WT_RD"; - child = false; - op = OP_READ; - while ((ch = __wt_getopt(progname, argc, argv, "Rh:W")) != EOF) - switch (ch) { - case 'R': - child = true; - op = OP_READ; - break; - case 'W': - child = true; - op = OP_WRITE; - break; - case 'h': - working_dir = __wt_optarg; - break; - default: - usage(); - } - argc -= __wt_optind; - if (argc != 0) - usage(); - - /* - * Set up all the directory names. - */ - testutil_work_dir_from_path(home, sizeof(home), working_dir); - testutil_check(__wt_snprintf( - home_wr, sizeof(home_wr), "%s%s", home, HOME_WR_SUFFIX)); - testutil_check(__wt_snprintf( - home_rd, sizeof(home_rd), "%s%s", home, HOME_RD_SUFFIX)); - testutil_check(__wt_snprintf( - home_rd2, sizeof(home_rd2), "%s%s", home, HOME_RD2_SUFFIX)); - if (!child) { - testutil_make_work_dir(home); - testutil_make_work_dir(home_wr); - testutil_make_work_dir(home_rd); - testutil_make_work_dir(home_rd2); - } else - /* - * We are a child process, we just want to call - * the open_dbs with the directories we have. - * The child function will exit. - */ - open_dbs(op, home, home_wr, home_rd, home_rd2); - - /* - * Parent creates a database and table. Then cleanly shuts down. - * Then copy database to read-only directory and chmod. - * Also copy database to read-only directory and remove the lock - * file. One read-only database will have a lock file in the - * file system and the other will not. - * Parent opens all databases with read-only configuration flag. - * Parent forks off child who tries to also open all databases - * with the read-only flag. It should error on the writeable - * directory, but allow it on the read-only directories. - * The child then confirms it can read all the data. - */ - /* - * Run in the home directory and create the table. - */ - testutil_check(wiredtiger_open(home, NULL, ENV_CONFIG, &conn)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check( - session->create(session, uri, "key_format=Q,value_format=u")); - testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); - - /* - * Write data into the table and then cleanly shut down connection. - */ - memset(buf, 0, sizeof(buf)); - data.data = buf; - data.size = MAX_VAL; - for (i = 0; i < MAX_KV; ++i) { - cursor->set_key(cursor, i); - cursor->set_value(cursor, &data); - testutil_check(cursor->insert(cursor)); - } - testutil_check(conn->close(conn, NULL)); - - /* - * Copy the database. Remove any lock file from one copy - * and chmod the copies to be read-only permissions. - */ - testutil_check(__wt_snprintf(cmd, sizeof(cmd), - "cp -rp %s/* %s; rm -f %s/WiredTiger.lock", - home, home_wr, home_wr)); - if ((status = system(cmd)) < 0) - testutil_die(status, "system: %s", cmd); - - testutil_check(__wt_snprintf(cmd, sizeof(cmd), - "cp -rp %s/* %s; chmod 0555 %s; chmod -R 0444 %s/*", - home, home_rd, home_rd, home_rd)); - if ((status = system(cmd)) < 0) - testutil_die(status, "system: %s", cmd); - - testutil_check(__wt_snprintf(cmd, sizeof(cmd), - "cp -rp %s/* %s; rm -f %s/WiredTiger.lock; " - "chmod 0555 %s; chmod -R 0444 %s/*", - home, home_rd2, home_rd2, home_rd2, home_rd2)); - if ((status = system(cmd)) < 0) - testutil_die(status, "system: %s", cmd); - - /* - * Run four scenarios. Sometimes expect errors, sometimes success. - * The writable database directories should always fail to allow the - * child to open due to the lock file. The read-only ones will only - * succeed when the child attempts read-only. - * - * 1. Parent has read-only handle to all databases. Child opens - * read-only also. - * 2. Parent has read-only handle to all databases. Child opens - * read-write. - * 3. Parent has read-write handle to writable databases and - * read-only to read-only databases. Child opens read-only. - * 4. Parent has read-write handle to writable databases and - * read-only to read-only databases. Child opens read-write. - */ - /* - * Open a connection handle to all databases. - */ - fprintf(stderr, " *** Expect several error messages from WT ***\n"); - /* - * Scenario 1. - */ - if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG_RD, &conn)) != 0) - testutil_die(ret, "wiredtiger_open original home"); - if ((ret = wiredtiger_open(home_wr, NULL, ENV_CONFIG_RD, &conn2)) != 0) - testutil_die(ret, "wiredtiger_open write nolock"); - if ((ret = wiredtiger_open(home_rd, NULL, ENV_CONFIG_RD, &conn3)) != 0) - testutil_die(ret, "wiredtiger_open readonly"); - if ((ret = wiredtiger_open(home_rd2, NULL, ENV_CONFIG_RD, &conn4)) != 0) - testutil_die(ret, "wiredtiger_open readonly nolock"); - - /* - * Create a child to also open a connection handle to the databases. - * We cannot use fork here because using fork the child inherits the - * same memory image. Therefore the WT process structure is set in - * the child even though it should not be. So use 'system' to spawn - * an entirely new process. - * - * The child will exit with success if its test passes. - */ - testutil_check(__wt_snprintf( - cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir)); - if ((status = system(cmd)) < 0) - testutil_die(status, "system: %s", cmd); - if (WEXITSTATUS(status) != 0) - testutil_die(WEXITSTATUS(status), "system: %s", cmd); - - /* - * Scenario 2. Run child with writable config. - */ - testutil_check(__wt_snprintf( - cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir)); - if ((status = system(cmd)) < 0) - testutil_die(status, "system: %s", cmd); - if (WEXITSTATUS(status) != 0) - testutil_die(WEXITSTATUS(status), "system: %s", cmd); - - /* - * Reopen the two writable directories and rerun the child. - */ - testutil_check(conn->close(conn, NULL)); - testutil_check(conn2->close(conn2, NULL)); - if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG_RD, &conn)) != 0) - testutil_die(ret, "wiredtiger_open original home"); - if ((ret = wiredtiger_open(home_wr, NULL, ENV_CONFIG_RD, &conn2)) != 0) - testutil_die(ret, "wiredtiger_open write nolock"); - /* - * Scenario 3. Child read-only. - */ - testutil_check(__wt_snprintf( - cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir)); - if ((status = system(cmd)) < 0) - testutil_die(status, "system: %s", cmd); - if (WEXITSTATUS(status) != 0) - testutil_die(WEXITSTATUS(status), "system: %s", cmd); - - /* - * Scenario 4. Run child with writable config. - */ - testutil_check(__wt_snprintf( - cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir)); - if ((status = system(cmd)) < 0) - testutil_die(status, "system: %s", cmd); - if (WEXITSTATUS(status) != 0) - testutil_die(WEXITSTATUS(status), "system: %s", cmd); - - /* - * Clean-up. - */ - testutil_check(conn->close(conn, NULL)); - testutil_check(conn2->close(conn2, NULL)); - testutil_check(conn3->close(conn3, NULL)); - testutil_check(conn4->close(conn4, NULL)); - /* - * We need to chmod the read-only databases back so that they can - * be removed by scripts. - */ - testutil_check(__wt_snprintf( - cmd, sizeof(cmd), "chmod 0777 %s %s", home_rd, home_rd2)); - if ((status = system(cmd)) < 0) - testutil_die(status, "system: %s", cmd); - testutil_check(__wt_snprintf( - cmd, sizeof(cmd), "chmod -R 0666 %s/* %s/*", home_rd, home_rd2)); - if ((status = system(cmd)) < 0) - testutil_die(status, "system: %s", cmd); - printf(" *** Readonly test successful ***\n"); - return (EXIT_SUCCESS); + WT_CONNECTION *conn, *conn2, *conn3, *conn4; + WT_CURSOR *cursor; + WT_ITEM data; + WT_SESSION *session; + uint64_t i; + uint8_t buf[MAX_VAL]; + int ch, op, ret, status; + char cmd[512]; + const char *working_dir; + bool child; + + (void)testutil_set_progname(argv); + + /* + * Needed unaltered for system command later. + */ + saved_argv0 = argv[0]; + + working_dir = "WT_RD"; + child = false; + op = OP_READ; + while ((ch = __wt_getopt(progname, argc, argv, "Rh:W")) != EOF) + switch (ch) { + case 'R': + child = true; + op = OP_READ; + break; + case 'W': + child = true; + op = OP_WRITE; + break; + case 'h': + working_dir = __wt_optarg; + break; + default: + usage(); + } + argc -= __wt_optind; + if (argc != 0) + usage(); + + /* + * Set up all the directory names. + */ + testutil_work_dir_from_path(home, sizeof(home), working_dir); + testutil_check(__wt_snprintf(home_wr, sizeof(home_wr), "%s%s", home, HOME_WR_SUFFIX)); + testutil_check(__wt_snprintf(home_rd, sizeof(home_rd), "%s%s", home, HOME_RD_SUFFIX)); + testutil_check(__wt_snprintf(home_rd2, sizeof(home_rd2), "%s%s", home, HOME_RD2_SUFFIX)); + if (!child) { + testutil_make_work_dir(home); + testutil_make_work_dir(home_wr); + testutil_make_work_dir(home_rd); + testutil_make_work_dir(home_rd2); + } else + /* + * We are a child process, we just want to call the open_dbs with the directories we have. + * The child function will exit. + */ + open_dbs(op, home, home_wr, home_rd, home_rd2); + + /* + * Parent creates a database and table. Then cleanly shuts down. Then copy database to read-only + * directory and chmod. Also copy database to read-only directory and remove the lock file. One + * read-only database will have a lock file in the file system and the other will not. Parent + * opens all databases with read-only configuration flag. Parent forks off child who tries to + * also open all databases with the read-only flag. It should error on the writeable directory, + * but allow it on the read-only directories. The child then confirms it can read all the data. + */ + /* + * Run in the home directory and create the table. + */ + testutil_check(wiredtiger_open(home, NULL, ENV_CONFIG, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->create(session, uri, "key_format=Q,value_format=u")); + testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); + + /* + * Write data into the table and then cleanly shut down connection. + */ + memset(buf, 0, sizeof(buf)); + data.data = buf; + data.size = MAX_VAL; + for (i = 0; i < MAX_KV; ++i) { + cursor->set_key(cursor, i); + cursor->set_value(cursor, &data); + testutil_check(cursor->insert(cursor)); + } + testutil_check(conn->close(conn, NULL)); + + /* + * Copy the database. Remove any lock file from one copy and chmod the copies to be read-only + * permissions. + */ + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "cp -rp %s/* %s; rm -f %s/WiredTiger.lock", home, home_wr, home_wr)); + if ((status = system(cmd)) < 0) + testutil_die(status, "system: %s", cmd); + + testutil_check(__wt_snprintf(cmd, sizeof(cmd), + "cp -rp %s/* %s; chmod 0555 %s; chmod -R 0444 %s/*", home, home_rd, home_rd, home_rd)); + if ((status = system(cmd)) < 0) + testutil_die(status, "system: %s", cmd); + + testutil_check(__wt_snprintf(cmd, sizeof(cmd), + "cp -rp %s/* %s; rm -f %s/WiredTiger.lock; " + "chmod 0555 %s; chmod -R 0444 %s/*", + home, home_rd2, home_rd2, home_rd2, home_rd2)); + if ((status = system(cmd)) < 0) + testutil_die(status, "system: %s", cmd); + + /* + * Run four scenarios. Sometimes expect errors, sometimes success. + * The writable database directories should always fail to allow the + * child to open due to the lock file. The read-only ones will only + * succeed when the child attempts read-only. + * + * 1. Parent has read-only handle to all databases. Child opens + * read-only also. + * 2. Parent has read-only handle to all databases. Child opens + * read-write. + * 3. Parent has read-write handle to writable databases and + * read-only to read-only databases. Child opens read-only. + * 4. Parent has read-write handle to writable databases and + * read-only to read-only databases. Child opens read-write. + */ + /* + * Open a connection handle to all databases. + */ + fprintf(stderr, " *** Expect several error messages from WT ***\n"); + /* + * Scenario 1. + */ + if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG_RD, &conn)) != 0) + testutil_die(ret, "wiredtiger_open original home"); + if ((ret = wiredtiger_open(home_wr, NULL, ENV_CONFIG_RD, &conn2)) != 0) + testutil_die(ret, "wiredtiger_open write nolock"); + if ((ret = wiredtiger_open(home_rd, NULL, ENV_CONFIG_RD, &conn3)) != 0) + testutil_die(ret, "wiredtiger_open readonly"); + if ((ret = wiredtiger_open(home_rd2, NULL, ENV_CONFIG_RD, &conn4)) != 0) + testutil_die(ret, "wiredtiger_open readonly nolock"); + + /* + * Create a child to also open a connection handle to the databases. + * We cannot use fork here because using fork the child inherits the + * same memory image. Therefore the WT process structure is set in + * the child even though it should not be. So use 'system' to spawn + * an entirely new process. + * + * The child will exit with success if its test passes. + */ + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir)); + if ((status = system(cmd)) < 0) + testutil_die(status, "system: %s", cmd); + if (WEXITSTATUS(status) != 0) + testutil_die(WEXITSTATUS(status), "system: %s", cmd); + + /* + * Scenario 2. Run child with writable config. + */ + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir)); + if ((status = system(cmd)) < 0) + testutil_die(status, "system: %s", cmd); + if (WEXITSTATUS(status) != 0) + testutil_die(WEXITSTATUS(status), "system: %s", cmd); + + /* + * Reopen the two writable directories and rerun the child. + */ + testutil_check(conn->close(conn, NULL)); + testutil_check(conn2->close(conn2, NULL)); + if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG_RD, &conn)) != 0) + testutil_die(ret, "wiredtiger_open original home"); + if ((ret = wiredtiger_open(home_wr, NULL, ENV_CONFIG_RD, &conn2)) != 0) + testutil_die(ret, "wiredtiger_open write nolock"); + /* + * Scenario 3. Child read-only. + */ + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir)); + if ((status = system(cmd)) < 0) + testutil_die(status, "system: %s", cmd); + if (WEXITSTATUS(status) != 0) + testutil_die(WEXITSTATUS(status), "system: %s", cmd); + + /* + * Scenario 4. Run child with writable config. + */ + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir)); + if ((status = system(cmd)) < 0) + testutil_die(status, "system: %s", cmd); + if (WEXITSTATUS(status) != 0) + testutil_die(WEXITSTATUS(status), "system: %s", cmd); + + /* + * Clean-up. + */ + testutil_check(conn->close(conn, NULL)); + testutil_check(conn2->close(conn2, NULL)); + testutil_check(conn3->close(conn3, NULL)); + testutil_check(conn4->close(conn4, NULL)); + /* + * We need to chmod the read-only databases back so that they can be removed by scripts. + */ + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "chmod 0777 %s %s", home_rd, home_rd2)); + if ((status = system(cmd)) < 0) + testutil_die(status, "system: %s", cmd); + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "chmod -R 0666 %s/* %s/*", home_rd, home_rd2)); + if ((status = system(cmd)) < 0) + testutil_die(status, "system: %s", cmd); + printf(" *** Readonly test successful ***\n"); + return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/salvage/salvage.c b/src/third_party/wiredtiger/test/salvage/salvage.c index 06386e5e86e..283a2bb3573 100644 --- a/src/third_party/wiredtiger/test/salvage/salvage.c +++ b/src/third_party/wiredtiger/test/salvage/salvage.c @@ -30,16 +30,16 @@ #include <assert.h> -#define HOME "WT_TEST" -#define DUMP "WT_TEST/__slvg.dump" /* Dump file */ -#define LOAD "WT_TEST/__slvg.load" /* Build file */ -#define LOAD_URI "file:__slvg.load" /* Build URI */ -#define RSLT "WT_TEST/__slvg.result" /* Result file */ -#define SLVG "WT_TEST/__slvg.slvg" /* Salvage file */ -#define SLVG_URI "file:__slvg.slvg" /* Salvage URI */ +#define HOME "WT_TEST" +#define DUMP "WT_TEST/__slvg.dump" /* Dump file */ +#define LOAD "WT_TEST/__slvg.load" /* Build file */ +#define LOAD_URI "file:__slvg.load" /* Build URI */ +#define RSLT "WT_TEST/__slvg.result" /* Result file */ +#define SLVG "WT_TEST/__slvg.slvg" /* Salvage file */ +#define SLVG_URI "file:__slvg.slvg" /* Salvage URI */ -#define PSIZE (2 * 1024) -#define OSIZE (PSIZE / 20) +#define PSIZE (2 * 1024) +#define OSIZE (PSIZE / 20) void build(int, int, int); void copy(u_int, u_int); @@ -48,12 +48,12 @@ void print_res(int, int, int); void process(void); void run(int); void t(int, u_int, int); -int usage(void); +int usage(void); -static FILE *res_fp; /* Results file */ -static u_int page_type; /* File types */ -static int value_unique; /* Values are unique */ -static int verbose; /* -v flag */ +static FILE *res_fp; /* Results file */ +static u_int page_type; /* File types */ +static int value_unique; /* Values are unique */ +static int verbose; /* -v flag */ extern int __wt_optind; extern char *__wt_optarg; @@ -61,673 +61,691 @@ extern char *__wt_optarg; int main(int argc, char *argv[]) { - u_int ptype; - int ch, r; - - (void)testutil_set_progname(argv); - - r = 0; - ptype = 0; - while ((ch = __wt_getopt(progname, argc, argv, "r:t:v")) != EOF) - switch (ch) { - case 'r': - r = atoi(__wt_optarg); - if (r == 0) - return (usage()); - break; - case 't': - if (strcmp(__wt_optarg, "fix") == 0) - ptype = WT_PAGE_COL_FIX; - else if (strcmp(__wt_optarg, "var") == 0) - ptype = WT_PAGE_COL_VAR; - else if (strcmp(__wt_optarg, "row") == 0) - ptype = WT_PAGE_ROW_LEAF; - else - return (usage()); - break; - case 'v': - verbose = 1; - break; - case '?': - default: - return (usage()); - } - argc -= __wt_optind; - if (argc != 0) - return (usage()); - - printf("salvage test run started\n"); - - t(r, ptype, 1); - t(r, ptype, 0); - - printf("salvage test run completed\n"); - return (EXIT_SUCCESS); + u_int ptype; + int ch, r; + + (void)testutil_set_progname(argv); + + r = 0; + ptype = 0; + while ((ch = __wt_getopt(progname, argc, argv, "r:t:v")) != EOF) + switch (ch) { + case 'r': + r = atoi(__wt_optarg); + if (r == 0) + return (usage()); + break; + case 't': + if (strcmp(__wt_optarg, "fix") == 0) + ptype = WT_PAGE_COL_FIX; + else if (strcmp(__wt_optarg, "var") == 0) + ptype = WT_PAGE_COL_VAR; + else if (strcmp(__wt_optarg, "row") == 0) + ptype = WT_PAGE_ROW_LEAF; + else + return (usage()); + break; + case 'v': + verbose = 1; + break; + case '?': + default: + return (usage()); + } + argc -= __wt_optind; + if (argc != 0) + return (usage()); + + printf("salvage test run started\n"); + + t(r, ptype, 1); + t(r, ptype, 0); + + printf("salvage test run completed\n"); + return (EXIT_SUCCESS); } void t(int r, u_int ptype, int unique) { - printf("%sunique values\n", unique ? "" : "non-"); - value_unique = unique; - -#define NTESTS 24 - if (r == 0) { - if (ptype == 0) { - page_type = WT_PAGE_COL_FIX; - for (r = 1; r <= NTESTS; ++r) - run(r); - - page_type = WT_PAGE_COL_VAR; - for (r = 1; r <= NTESTS; ++r) - run(r); - - page_type = WT_PAGE_ROW_LEAF; - for (r = 1; r <= NTESTS; ++r) - run(r); - } else { - page_type = ptype; - for (r = 1; r <= NTESTS; ++r) - run(r); - } - } else if (ptype == 0) { - page_type = WT_PAGE_COL_FIX; - run(r); - page_type = WT_PAGE_COL_VAR; - run(r); - page_type = WT_PAGE_ROW_LEAF; - run(r); - } else { - page_type = ptype; - run(r); - } + printf("%sunique values\n", unique ? "" : "non-"); + value_unique = unique; + +#define NTESTS 24 + if (r == 0) { + if (ptype == 0) { + page_type = WT_PAGE_COL_FIX; + for (r = 1; r <= NTESTS; ++r) + run(r); + + page_type = WT_PAGE_COL_VAR; + for (r = 1; r <= NTESTS; ++r) + run(r); + + page_type = WT_PAGE_ROW_LEAF; + for (r = 1; r <= NTESTS; ++r) + run(r); + } else { + page_type = ptype; + for (r = 1; r <= NTESTS; ++r) + run(r); + } + } else if (ptype == 0) { + page_type = WT_PAGE_COL_FIX; + run(r); + page_type = WT_PAGE_COL_VAR; + run(r); + page_type = WT_PAGE_ROW_LEAF; + run(r); + } else { + page_type = ptype; + run(r); + } } int usage(void) { - (void)fprintf(stderr, - "usage: %s [-v] [-r run] [-t fix|var|row]\n", progname); - return (EXIT_FAILURE); + (void)fprintf(stderr, "usage: %s [-v] [-r run] [-t fix|var|row]\n", progname); + return (EXIT_FAILURE); } void run(int r) { - char buf[128]; - - printf("\t%s: run %d\n", __wt_page_type_string(page_type), r); - - testutil_make_work_dir(HOME); - - testutil_checksys((res_fp = fopen(RSLT, "w")) == NULL); - - /* - * Each run builds the LOAD file, and then appends the first page of - * the LOAD file into the SLVG file. The SLVG file is then salvaged, - * verified, and dumped into the DUMP file, which is compared to the - * results file, which are the expected results. - */ - switch (r) { - case 1: - /* - * Smoke test: empty files. - */ - build(0, 0, 0); copy(0, 0); - break; - case 2: - /* - * Smoke test: - * Sequential pages, all pages should be kept. - */ - build(100, 100, 20); copy(6, 1); - build(200, 200, 20); copy(7, 21); - build(300, 300, 20); copy(8, 41); - print_res(100, 100, 20); - print_res(200, 200, 20); - print_res(300, 300, 20); - break; - case 3: - /* - * Smoke test: - * Sequential pages, all pages should be kept. - */ - build(100, 100, 20); copy(8, 1); - build(200, 200, 20); copy(7, 21); - build(300, 300, 20); copy(6, 41); - print_res(100, 100, 20); - print_res(200, 200, 20); - print_res(300, 300, 20); - break; - case 4: - /* - * Case #1: - * 3 pages, each with 20 records starting with the same record - * and sequential LSNs; salvage should leave the page with the - * largest LSN. - */ - build(100, 100, 20); copy(6, 1); - build(100, 200, 20); copy(7, 1); - build(100, 300, 20); copy(8, 1); - print_res(100, 300, 20); - break; - case 5: - /* - * Case #1: - * 3 pages, each with 20 records starting with the same record - * and sequential LSNs; salvage should leave the page with the - * largest LSN. - */ - build(100, 100, 20); copy(6, 1); - build(100, 200, 20); copy(8, 1); - build(100, 300, 20); copy(7, 1); - print_res(100, 200, 20); - break; - case 6: - /* - * Case #1: - * 3 pages, each with 20 records starting with the same record - * and sequential LSNs; salvage should leave the page with the - * largest LSN. - */ - build(100, 100, 20); copy(8, 1); - build(100, 200, 20); copy(7, 1); - build(100, 300, 20); copy(6, 1); - print_res(100, 100, 20); - break; - case 7: - /* - * Case #2: - * The second page overlaps the beginning of the first page, and - * the first page has a higher LSN. - */ - build(110, 100, 20); copy(7, 11); - build(100, 200, 20); copy(6, 1); - print_res(100, 200, 10); - print_res(110, 100, 20); - break; - case 8: - /* - * Case #2: - * The second page overlaps the beginning of the first page, and - * the second page has a higher LSN. - */ - build(110, 100, 20); copy(6, 11); - build(100, 200, 20); copy(7, 1); - print_res(100, 200, 20); - print_res(120, 110, 10); - break; - case 9: - /* - * Case #3: - * The second page overlaps with the end of the first page, and - * the first page has a higher LSN. - */ - build(100, 100, 20); copy(7, 1); - build(110, 200, 20); copy(6, 11); - print_res(100, 100, 20); - print_res(120, 210, 10); - break; - case 10: - /* - * Case #3: - * The second page overlaps with the end of the first page, and - * the second page has a higher LSN. - */ - build(100, 100, 20); copy(6, 1); - build(110, 200, 20); copy(7, 11); - print_res(100, 100, 10); - print_res(110, 200, 20); - break; - case 11: - /* - * Case #4: - * The second page is a prefix of the first page, and the first - * page has a higher LSN. - */ - build(100, 100, 20); copy(7, 1); - build(100, 200, 5); copy(6, 1); - print_res(100, 100, 20); - break; - case 12: - /* - * Case #4: - * The second page is a prefix of the first page, and the second - * page has a higher LSN. - */ - build(100, 100, 20); copy(6, 1); - build(100, 200, 5); copy(7, 1); - print_res(100, 200, 5); - print_res(105, 105, 15); - break; - case 13: - /* - * Case #5: - * The second page is in the middle of the first page, and the - * first page has a higher LSN. - */ - build(100, 100, 40); copy(7, 1); - build(110, 200, 10); copy(6, 11); - print_res(100, 100, 40); - break; - case 14: - /* - * Case #5: - * The second page is in the middle of the first page, and the - * second page has a higher LSN. - */ - build(100, 100, 40); copy(6, 1); - build(110, 200, 10); copy(7, 11); - print_res(100, 100, 10); - print_res(110, 200, 10); - print_res(120, 120, 20); - break; - case 15: - /* - * Case #6: - * The second page is a suffix of the first page, and the first - * page has a higher LSN. - */ - build(100, 100, 40); copy(7, 1); - build(130, 200, 10); copy(6, 31); - print_res(100, 100, 40); - break; - case 16: - /* - * Case #6: - * The second page is a suffix of the first page, and the second - * page has a higher LSN. - */ - build(100, 100, 40); copy(6, 1); - build(130, 200, 10); copy(7, 31); - print_res(100, 100, 30); - print_res(130, 200, 10); - break; - case 17: - /* - * Case #9: - * The first page is a prefix of the second page, and the first - * page has a higher LSN. - */ - build(100, 100, 20); copy(7, 1); - build(100, 200, 40); copy(6, 1); - print_res(100, 100, 20); - print_res(120, 220, 20); - break; - case 18: - /* - * Case #9: - * The first page is a prefix of the second page, and the second - * page has a higher LSN. - */ - build(100, 100, 20); copy(6, 1); - build(100, 200, 40); copy(7, 1); - print_res(100, 200, 40); - break; - case 19: - /* - * Case #10: - * The first page is a suffix of the second page, and the first - * page has a higher LSN. - */ - build(130, 100, 10); copy(7, 31); - build(100, 200, 40); copy(6, 1); - print_res(100, 200, 30); - print_res(130, 100, 10); - break; - case 20: - /* - * Case #10: - * The first page is a suffix of the second page, and the second - * page has a higher LSN. - */ - build(130, 100, 10); copy(6, 31); - build(100, 200, 40); copy(7, 1); - print_res(100, 200, 40); - break; - case 21: - /* - * Case #11: - * The first page is in the middle of the second page, and the - * first page has a higher LSN. - */ - build(110, 100, 10); copy(7, 11); - build(100, 200, 40); copy(6, 1); - print_res(100, 200, 10); - print_res(110, 100, 10); - print_res(120, 220, 20); - break; - case 22: - /* - * Case #11: - * The first page is in the middle of the second page, and the - * second page has a higher LSN. - */ - build(110, 100, 10); copy(6, 11); - build(100, 200, 40); copy(7, 1); - print_res(100, 200, 40); - break; - case 23: - /* - * Column-store only: missing an initial key range of 99 - * records. - */ - build(100, 100, 10); copy(1, 100); - empty(99); - print_res(100, 100, 10); - break; - case 24: - /* - * Column-store only: missing a middle key range of 37 - * records. - */ - build(100, 100, 10); copy(1, 1); - build(138, 138, 10); copy(1, 48); - print_res(100, 100, 10); - empty(37); - print_res(138, 138, 10); - break; - default: - fprintf(stderr, "salvage: %d: no such test\n", r); - exit(EXIT_FAILURE); - } - - testutil_assert(fclose(res_fp) == 0); - - process(); - - testutil_check(__wt_snprintf( - buf, sizeof(buf), "cmp %s %s > /dev/null", DUMP, RSLT)); - if (system(buf)) { - fprintf(stderr, - "check failed, salvage results were incorrect\n"); - exit(EXIT_FAILURE); - } - - testutil_clean_work_dir(HOME); + char buf[128]; + + printf("\t%s: run %d\n", __wt_page_type_string(page_type), r); + + testutil_make_work_dir(HOME); + + testutil_checksys((res_fp = fopen(RSLT, "w")) == NULL); + + /* + * Each run builds the LOAD file, and then appends the first page of the LOAD file into the SLVG + * file. The SLVG file is then salvaged, verified, and dumped into the DUMP file, which is + * compared to the results file, which are the expected results. + */ + switch (r) { + case 1: + /* + * Smoke test: empty files. + */ + build(0, 0, 0); + copy(0, 0); + break; + case 2: + /* + * Smoke test: Sequential pages, all pages should be kept. + */ + build(100, 100, 20); + copy(6, 1); + build(200, 200, 20); + copy(7, 21); + build(300, 300, 20); + copy(8, 41); + print_res(100, 100, 20); + print_res(200, 200, 20); + print_res(300, 300, 20); + break; + case 3: + /* + * Smoke test: Sequential pages, all pages should be kept. + */ + build(100, 100, 20); + copy(8, 1); + build(200, 200, 20); + copy(7, 21); + build(300, 300, 20); + copy(6, 41); + print_res(100, 100, 20); + print_res(200, 200, 20); + print_res(300, 300, 20); + break; + case 4: + /* + * Case #1: + * 3 pages, each with 20 records starting with the same record + * and sequential LSNs; salvage should leave the page with the + * largest LSN. + */ + build(100, 100, 20); + copy(6, 1); + build(100, 200, 20); + copy(7, 1); + build(100, 300, 20); + copy(8, 1); + print_res(100, 300, 20); + break; + case 5: + /* + * Case #1: + * 3 pages, each with 20 records starting with the same record + * and sequential LSNs; salvage should leave the page with the + * largest LSN. + */ + build(100, 100, 20); + copy(6, 1); + build(100, 200, 20); + copy(8, 1); + build(100, 300, 20); + copy(7, 1); + print_res(100, 200, 20); + break; + case 6: + /* + * Case #1: + * 3 pages, each with 20 records starting with the same record + * and sequential LSNs; salvage should leave the page with the + * largest LSN. + */ + build(100, 100, 20); + copy(8, 1); + build(100, 200, 20); + copy(7, 1); + build(100, 300, 20); + copy(6, 1); + print_res(100, 100, 20); + break; + case 7: + /* + * Case #2: The second page overlaps the beginning of the first page, and the first page has + * a higher LSN. + */ + build(110, 100, 20); + copy(7, 11); + build(100, 200, 20); + copy(6, 1); + print_res(100, 200, 10); + print_res(110, 100, 20); + break; + case 8: + /* + * Case #2: The second page overlaps the beginning of the first page, and the second page + * has a higher LSN. + */ + build(110, 100, 20); + copy(6, 11); + build(100, 200, 20); + copy(7, 1); + print_res(100, 200, 20); + print_res(120, 110, 10); + break; + case 9: + /* + * Case #3: The second page overlaps with the end of the first page, and the first page has + * a higher LSN. + */ + build(100, 100, 20); + copy(7, 1); + build(110, 200, 20); + copy(6, 11); + print_res(100, 100, 20); + print_res(120, 210, 10); + break; + case 10: + /* + * Case #3: The second page overlaps with the end of the first page, and the second page has + * a higher LSN. + */ + build(100, 100, 20); + copy(6, 1); + build(110, 200, 20); + copy(7, 11); + print_res(100, 100, 10); + print_res(110, 200, 20); + break; + case 11: + /* + * Case #4: The second page is a prefix of the first page, and the first page has a higher + * LSN. + */ + build(100, 100, 20); + copy(7, 1); + build(100, 200, 5); + copy(6, 1); + print_res(100, 100, 20); + break; + case 12: + /* + * Case #4: The second page is a prefix of the first page, and the second page has a higher + * LSN. + */ + build(100, 100, 20); + copy(6, 1); + build(100, 200, 5); + copy(7, 1); + print_res(100, 200, 5); + print_res(105, 105, 15); + break; + case 13: + /* + * Case #5: The second page is in the middle of the first page, and the first page has a + * higher LSN. + */ + build(100, 100, 40); + copy(7, 1); + build(110, 200, 10); + copy(6, 11); + print_res(100, 100, 40); + break; + case 14: + /* + * Case #5: The second page is in the middle of the first page, and the second page has a + * higher LSN. + */ + build(100, 100, 40); + copy(6, 1); + build(110, 200, 10); + copy(7, 11); + print_res(100, 100, 10); + print_res(110, 200, 10); + print_res(120, 120, 20); + break; + case 15: + /* + * Case #6: The second page is a suffix of the first page, and the first page has a higher + * LSN. + */ + build(100, 100, 40); + copy(7, 1); + build(130, 200, 10); + copy(6, 31); + print_res(100, 100, 40); + break; + case 16: + /* + * Case #6: The second page is a suffix of the first page, and the second page has a higher + * LSN. + */ + build(100, 100, 40); + copy(6, 1); + build(130, 200, 10); + copy(7, 31); + print_res(100, 100, 30); + print_res(130, 200, 10); + break; + case 17: + /* + * Case #9: The first page is a prefix of the second page, and the first page has a higher + * LSN. + */ + build(100, 100, 20); + copy(7, 1); + build(100, 200, 40); + copy(6, 1); + print_res(100, 100, 20); + print_res(120, 220, 20); + break; + case 18: + /* + * Case #9: The first page is a prefix of the second page, and the second page has a higher + * LSN. + */ + build(100, 100, 20); + copy(6, 1); + build(100, 200, 40); + copy(7, 1); + print_res(100, 200, 40); + break; + case 19: + /* + * Case #10: The first page is a suffix of the second page, and the first page has a higher + * LSN. + */ + build(130, 100, 10); + copy(7, 31); + build(100, 200, 40); + copy(6, 1); + print_res(100, 200, 30); + print_res(130, 100, 10); + break; + case 20: + /* + * Case #10: The first page is a suffix of the second page, and the second page has a higher + * LSN. + */ + build(130, 100, 10); + copy(6, 31); + build(100, 200, 40); + copy(7, 1); + print_res(100, 200, 40); + break; + case 21: + /* + * Case #11: The first page is in the middle of the second page, and the first page has a + * higher LSN. + */ + build(110, 100, 10); + copy(7, 11); + build(100, 200, 40); + copy(6, 1); + print_res(100, 200, 10); + print_res(110, 100, 10); + print_res(120, 220, 20); + break; + case 22: + /* + * Case #11: The first page is in the middle of the second page, and the second page has a + * higher LSN. + */ + build(110, 100, 10); + copy(6, 11); + build(100, 200, 40); + copy(7, 1); + print_res(100, 200, 40); + break; + case 23: + /* + * Column-store only: missing an initial key range of 99 records. + */ + build(100, 100, 10); + copy(1, 100); + empty(99); + print_res(100, 100, 10); + break; + case 24: + /* + * Column-store only: missing a middle key range of 37 records. + */ + build(100, 100, 10); + copy(1, 1); + build(138, 138, 10); + copy(1, 48); + print_res(100, 100, 10); + empty(37); + print_res(138, 138, 10); + break; + default: + fprintf(stderr, "salvage: %d: no such test\n", r); + exit(EXIT_FAILURE); + } + + testutil_assert(fclose(res_fp) == 0); + + process(); + + testutil_check(__wt_snprintf(buf, sizeof(buf), "cmp %s %s > /dev/null", DUMP, RSLT)); + if (system(buf)) { + fprintf(stderr, "check failed, salvage results were incorrect\n"); + exit(EXIT_FAILURE); + } + + testutil_clean_work_dir(HOME); } /* * file_exists -- - * Return if the file exists. + * Return if the file exists. */ static int file_exists(const char *path) { - struct stat sb; + struct stat sb; - return (stat(path, &sb) == 0); + return (stat(path, &sb) == 0); } /* * build -- - * Build a row- or column-store page in a file. + * Build a row- or column-store page in a file. */ void build(int ikey, int ivalue, int cnt) { - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_ITEM key, value; - WT_SESSION *session; - int new_slvg; - char config[256], kbuf[64], vbuf[64]; - - /* - * Disable logging: we're modifying files directly, we don't want to - * run recovery. - */ - testutil_check(wiredtiger_open( - HOME, NULL, "create,log=(enabled=false)", &conn)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->drop(session, LOAD_URI, "force")); - - switch (page_type) { - case WT_PAGE_COL_FIX: - testutil_check(__wt_snprintf(config, sizeof(config), - "key_format=r,value_format=7t," - "allocation_size=%d," - "internal_page_max=%d,internal_item_max=%d," - "leaf_page_max=%d,leaf_item_max=%d", - PSIZE, PSIZE, OSIZE, PSIZE, OSIZE)); - break; - case WT_PAGE_COL_VAR: - testutil_check(__wt_snprintf(config, sizeof(config), - "key_format=r," - "allocation_size=%d," - "internal_page_max=%d,internal_item_max=%d," - "leaf_page_max=%d,leaf_item_max=%d", - PSIZE, PSIZE, OSIZE, PSIZE, OSIZE)); - break; - case WT_PAGE_ROW_LEAF: - testutil_check(__wt_snprintf(config, sizeof(config), - "key_format=u," - "allocation_size=%d," - "internal_page_max=%d,internal_item_max=%d," - "leaf_page_max=%d,leaf_item_max=%d", - PSIZE, PSIZE, OSIZE, PSIZE, OSIZE)); - break; - default: - assert(0); - } - testutil_check(session->create(session, LOAD_URI, config)); - testutil_check(session->open_cursor( - session, LOAD_URI, NULL, "bulk,append", &cursor)); - for (; cnt > 0; --cnt, ++ikey, ++ivalue) { - switch (page_type) { /* Build the key. */ - case WT_PAGE_COL_FIX: - case WT_PAGE_COL_VAR: - break; - case WT_PAGE_ROW_LEAF: - testutil_check(__wt_snprintf( - kbuf, sizeof(kbuf), "%010d KEY------", ikey)); - key.data = kbuf; - key.size = 20; - cursor->set_key(cursor, &key); - break; - } - - switch (page_type) { /* Build the value. */ - case WT_PAGE_COL_FIX: - cursor->set_value(cursor, ivalue & 0x7f); - break; - case WT_PAGE_COL_VAR: - case WT_PAGE_ROW_LEAF: - testutil_check(__wt_snprintf(vbuf, sizeof(vbuf), - "%010d VALUE----", value_unique ? ivalue : 37)); - value.data = vbuf; - value.size = 20; - cursor->set_value(cursor, &value); - } - testutil_check(cursor->insert(cursor)); - } - - /* - * The first time through this routine we create the salvage file and - * then remove it (all we want is the appropriate schema entry, we're - * creating the salvage file itself by hand). - */ - new_slvg = !file_exists(SLVG); - if (new_slvg) { - testutil_check(session->drop(session, SLVG_URI, "force")); - testutil_check(session->create(session, SLVG_URI, config)); - } - testutil_check(conn->close(conn, 0)); - if (new_slvg) - (void)remove(SLVG); + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_ITEM key, value; + WT_SESSION *session; + int new_slvg; + char config[256], kbuf[64], vbuf[64]; + + /* + * Disable logging: we're modifying files directly, we don't want to run recovery. + */ + testutil_check(wiredtiger_open(HOME, NULL, "create,log=(enabled=false)", &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->drop(session, LOAD_URI, "force")); + + switch (page_type) { + case WT_PAGE_COL_FIX: + testutil_check(__wt_snprintf(config, sizeof(config), + "key_format=r,value_format=7t," + "allocation_size=%d," + "internal_page_max=%d,internal_item_max=%d," + "leaf_page_max=%d,leaf_item_max=%d", + PSIZE, PSIZE, OSIZE, PSIZE, OSIZE)); + break; + case WT_PAGE_COL_VAR: + testutil_check(__wt_snprintf(config, sizeof(config), + "key_format=r," + "allocation_size=%d," + "internal_page_max=%d,internal_item_max=%d," + "leaf_page_max=%d,leaf_item_max=%d", + PSIZE, PSIZE, OSIZE, PSIZE, OSIZE)); + break; + case WT_PAGE_ROW_LEAF: + testutil_check(__wt_snprintf(config, sizeof(config), + "key_format=u," + "allocation_size=%d," + "internal_page_max=%d,internal_item_max=%d," + "leaf_page_max=%d,leaf_item_max=%d", + PSIZE, PSIZE, OSIZE, PSIZE, OSIZE)); + break; + default: + assert(0); + } + testutil_check(session->create(session, LOAD_URI, config)); + testutil_check(session->open_cursor(session, LOAD_URI, NULL, "bulk,append", &cursor)); + for (; cnt > 0; --cnt, ++ikey, ++ivalue) { + switch (page_type) { /* Build the key. */ + case WT_PAGE_COL_FIX: + case WT_PAGE_COL_VAR: + break; + case WT_PAGE_ROW_LEAF: + testutil_check(__wt_snprintf(kbuf, sizeof(kbuf), "%010d KEY------", ikey)); + key.data = kbuf; + key.size = 20; + cursor->set_key(cursor, &key); + break; + } + + switch (page_type) { /* Build the value. */ + case WT_PAGE_COL_FIX: + cursor->set_value(cursor, ivalue & 0x7f); + break; + case WT_PAGE_COL_VAR: + case WT_PAGE_ROW_LEAF: + testutil_check( + __wt_snprintf(vbuf, sizeof(vbuf), "%010d VALUE----", value_unique ? ivalue : 37)); + value.data = vbuf; + value.size = 20; + cursor->set_value(cursor, &value); + } + testutil_check(cursor->insert(cursor)); + } + + /* + * The first time through this routine we create the salvage file and then remove it (all we + * want is the appropriate schema entry, we're creating the salvage file itself by hand). + */ + new_slvg = !file_exists(SLVG); + if (new_slvg) { + testutil_check(session->drop(session, SLVG_URI, "force")); + testutil_check(session->create(session, SLVG_URI, config)); + } + testutil_check(conn->close(conn, 0)); + if (new_slvg) + (void)remove(SLVG); } /* * copy -- - * Copy the created page to the end of the salvage file. + * Copy the created page to the end of the salvage file. */ void copy(u_int gen, u_int recno) { - FILE *ifp, *ofp; - WT_BLOCK_HEADER *blk; - WT_PAGE_HEADER *dsk; - uint64_t recno64; - uint32_t cksum32, gen32; - char buf[PSIZE]; - - testutil_checksys((ifp = fopen(LOAD, "r")) == NULL); - - /* - * If the salvage file doesn't exist, then we're creating it: - * copy the first sector (the file description). - * Otherwise, we are appending to an existing file. - */ - if (file_exists(SLVG)) - testutil_checksys((ofp = fopen(SLVG, "a")) == NULL); - else { - testutil_checksys((ofp = fopen(SLVG, "w")) == NULL); - testutil_assert(fread(buf, 1, PSIZE, ifp) == PSIZE); - testutil_assert(fwrite(buf, 1, PSIZE, ofp) == PSIZE); - } - - /* - * If there's data, copy/update the first formatted page. - */ - if (gen != 0) { - testutil_assert(fseek(ifp, (long)PSIZE, SEEK_SET) == 0); - testutil_assert(fread(buf, 1, PSIZE, ifp) == PSIZE); - - /* - * Page headers are written in little-endian format, swap before - * calculating the checksum on big-endian hardware. Checksums - * always returned in little-endian format, no swap is required. - */ - gen32 = gen; - recno64 = recno; + FILE *ifp, *ofp; + WT_BLOCK_HEADER *blk; + WT_PAGE_HEADER *dsk; + uint64_t recno64; + uint32_t cksum32, gen32; + char buf[PSIZE]; + + testutil_checksys((ifp = fopen(LOAD, "r")) == NULL); + + /* + * If the salvage file doesn't exist, then we're creating it: copy the first sector (the file + * description). Otherwise, we are appending to an existing file. + */ + if (file_exists(SLVG)) + testutil_checksys((ofp = fopen(SLVG, "a")) == NULL); + else { + testutil_checksys((ofp = fopen(SLVG, "w")) == NULL); + testutil_assert(fread(buf, 1, PSIZE, ifp) == PSIZE); + testutil_assert(fwrite(buf, 1, PSIZE, ofp) == PSIZE); + } + + /* + * If there's data, copy/update the first formatted page. + */ + if (gen != 0) { + testutil_assert(fseek(ifp, (long)PSIZE, SEEK_SET) == 0); + testutil_assert(fread(buf, 1, PSIZE, ifp) == PSIZE); + + /* + * Page headers are written in little-endian format, swap before calculating the checksum on + * big-endian hardware. Checksums always returned in little-endian format, no swap is + * required. + */ + gen32 = gen; + recno64 = recno; #ifdef WORDS_BIGENDIAN - gen32 = __wt_bswap32(gen32); - recno64 = __wt_bswap64(recno64); + gen32 = __wt_bswap32(gen32); + recno64 = __wt_bswap64(recno64); #endif - dsk = (void *)buf; - if (page_type != WT_PAGE_ROW_LEAF) - dsk->recno = recno64; - dsk->write_gen = gen32; - blk = WT_BLOCK_HEADER_REF(buf); - blk->checksum = 0; - cksum32 = __wt_checksum(dsk, PSIZE); + dsk = (void *)buf; + if (page_type != WT_PAGE_ROW_LEAF) + dsk->recno = recno64; + dsk->write_gen = gen32; + blk = WT_BLOCK_HEADER_REF(buf); + blk->checksum = 0; + cksum32 = __wt_checksum(dsk, PSIZE); #ifdef WORDS_BIGENDIAN - cksum32 = __wt_bswap32(cksum32); + cksum32 = __wt_bswap32(cksum32); #endif - blk->checksum = cksum32; - testutil_assert(fwrite(buf, 1, PSIZE, ofp) == PSIZE); - } + blk->checksum = cksum32; + testutil_assert(fwrite(buf, 1, PSIZE, ofp) == PSIZE); + } - testutil_assert(fclose(ifp) == 0); - testutil_assert(fclose(ofp) == 0); + testutil_assert(fclose(ifp) == 0); + testutil_assert(fclose(ofp) == 0); } /* * process -- - * Salvage, verify and dump the created file. + * Salvage, verify and dump the created file. */ void process(void) { - FILE *fp; - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_SESSION *session; - char config[100]; - const char *key, *value; - - /* Salvage. */ - config[0] = '\0'; - if (verbose) - testutil_check(__wt_snprintf(config, sizeof(config), - "error_prefix=\"%s\",verbose=[salvage,verify],", - progname)); - strcat(config, "log=(enabled=false),"); - - testutil_check(wiredtiger_open(HOME, NULL, config, &conn)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->salvage(session, SLVG_URI, 0)); - testutil_check(conn->close(conn, 0)); - - /* Verify. */ - testutil_check(wiredtiger_open(HOME, NULL, config, &conn)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->verify(session, SLVG_URI, 0)); - testutil_check(conn->close(conn, 0)); - - /* Dump. */ - testutil_checksys((fp = fopen(DUMP, "w")) == NULL); - testutil_check(wiredtiger_open(HOME, NULL, config, &conn)); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->open_cursor( - session, SLVG_URI, NULL, "dump=print", &cursor)); - while (cursor->next(cursor) == 0) { - if (page_type == WT_PAGE_ROW_LEAF) { - testutil_check(cursor->get_key(cursor, &key)); - testutil_assert(fputs(key, fp) >= 0); - testutil_assert(fputc('\n', fp) >= 0); - } - testutil_check(cursor->get_value(cursor, &value)); - testutil_assert(fputs(value, fp) >= 0); - testutil_assert(fputc('\n', fp) >= 0); - } - testutil_check(conn->close(conn, 0)); - testutil_assert(fclose(fp) == 0); + FILE *fp; + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_SESSION *session; + char config[100]; + const char *key, *value; + + /* Salvage. */ + config[0] = '\0'; + if (verbose) + testutil_check(__wt_snprintf( + config, sizeof(config), "error_prefix=\"%s\",verbose=[salvage,verify],", progname)); + strcat(config, "log=(enabled=false),"); + + testutil_check(wiredtiger_open(HOME, NULL, config, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->salvage(session, SLVG_URI, 0)); + testutil_check(conn->close(conn, 0)); + + /* Verify. */ + testutil_check(wiredtiger_open(HOME, NULL, config, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->verify(session, SLVG_URI, 0)); + testutil_check(conn->close(conn, 0)); + + /* Dump. */ + testutil_checksys((fp = fopen(DUMP, "w")) == NULL); + testutil_check(wiredtiger_open(HOME, NULL, config, &conn)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->open_cursor(session, SLVG_URI, NULL, "dump=print", &cursor)); + while (cursor->next(cursor) == 0) { + if (page_type == WT_PAGE_ROW_LEAF) { + testutil_check(cursor->get_key(cursor, &key)); + testutil_assert(fputs(key, fp) >= 0); + testutil_assert(fputc('\n', fp) >= 0); + } + testutil_check(cursor->get_value(cursor, &value)); + testutil_assert(fputs(value, fp) >= 0); + testutil_assert(fputc('\n', fp) >= 0); + } + testutil_check(conn->close(conn, 0)); + testutil_assert(fclose(fp) == 0); } /* * empty -- - * Print empty print_res, for fixed-length column-store files. + * Print empty print_res, for fixed-length column-store files. */ void empty(int cnt) { - int i; + int i; - if (page_type == WT_PAGE_COL_FIX) - for (i = 0; i < cnt; ++i) - testutil_assert(fputs("\\00\n", res_fp)); + if (page_type == WT_PAGE_COL_FIX) + for (i = 0; i < cnt; ++i) + testutil_assert(fputs("\\00\n", res_fp)); } /* * print_res -- - * Write results file. + * Write results file. */ void print_res(int key, int value, int cnt) { - static const char hex[] = "0123456789abcdef"; - int ch; - - for (; cnt > 0; ++key, ++value, --cnt) { - switch (page_type) { /* Print key */ - case WT_PAGE_COL_FIX: - case WT_PAGE_COL_VAR: - break; - case WT_PAGE_ROW_LEAF: - fprintf(res_fp, "%010d KEY------\n", key); - break; - } - - switch (page_type) { /* Print value */ - case WT_PAGE_COL_FIX: - ch = value & 0x7f; - if (__wt_isprint((u_char)ch)) { - if (ch == '\\') - fputc('\\', res_fp); - fputc(ch, res_fp); - } else { - fputc('\\', res_fp); - fputc(hex[(ch & 0xf0) >> 4], res_fp); - fputc(hex[ch & 0x0f], res_fp); - } - fputc('\n', res_fp); - break; - case WT_PAGE_COL_VAR: - case WT_PAGE_ROW_LEAF: - fprintf(res_fp, - "%010d VALUE----\n", value_unique ? value : 37); - break; - } - } + static const char hex[] = "0123456789abcdef"; + int ch; + + for (; cnt > 0; ++key, ++value, --cnt) { + switch (page_type) { /* Print key */ + case WT_PAGE_COL_FIX: + case WT_PAGE_COL_VAR: + break; + case WT_PAGE_ROW_LEAF: + fprintf(res_fp, "%010d KEY------\n", key); + break; + } + + switch (page_type) { /* Print value */ + case WT_PAGE_COL_FIX: + ch = value & 0x7f; + if (__wt_isprint((u_char)ch)) { + if (ch == '\\') + fputc('\\', res_fp); + fputc(ch, res_fp); + } else { + fputc('\\', res_fp); + fputc(hex[(ch & 0xf0) >> 4], res_fp); + fputc(hex[ch & 0x0f], res_fp); + } + fputc('\n', res_fp); + break; + case WT_PAGE_COL_VAR: + case WT_PAGE_ROW_LEAF: + fprintf(res_fp, "%010d VALUE----\n", value_unique ? value : 37); + break; + } + } } diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode03.py b/src/third_party/wiredtiger/test/suite/test_debug_mode03.py index feb5c0d904a..ca50b2f83b6 100644 --- a/src/third_party/wiredtiger/test/suite/test_debug_mode03.py +++ b/src/third_party/wiredtiger/test/suite/test_debug_mode03.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -# Public Domain 2034-2039 MongoDB, Inc. -# Public Domain 2008-2034 WiredTiger, Inc. +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. # diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode04.py b/src/third_party/wiredtiger/test/suite/test_debug_mode04.py index 1f5429495e8..b1e2510e728 100644 --- a/src/third_party/wiredtiger/test/suite/test_debug_mode04.py +++ b/src/third_party/wiredtiger/test/suite/test_debug_mode04.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -# Public Domain 2034-2039 MongoDB, Inc. -# Public Domain 2008-2034 WiredTiger, Inc. +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. # diff --git a/src/third_party/wiredtiger/test/suite/test_debug_mode05.py b/src/third_party/wiredtiger/test/suite/test_debug_mode05.py new file mode 100644 index 00000000000..f248a05e646 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_debug_mode05.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest + +def timestamp_str(t): + return '%x' %t + +# test_debug_mode05.py +# As per WT-5046, the debug table logging settings prevent rollback to +# stable in the presence of prepared transactions. +# +# This test is to confirm the fix and prevent similar regressions. +class test_debug_mode05(wttest.WiredTigerTestCase): + conn_config = 'log=(enabled),debug_mode=(table_logging=true)' + session_config = 'isolation=snapshot' + uri = 'file:test_debug_mode05' + + def test_table_logging_rollback_to_stable(self): + self.session.create(self.uri, 'key_format=i,value_format=u') + cursor = self.session.open_cursor(self.uri, None) + + self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100)) + + # Try doing a normal prepared txn and then rollback to stable. + self.session.begin_transaction() + for i in range(1, 50): + cursor[i] = b'a' * 100 + self.session.prepare_transaction( + 'prepare_timestamp=' + timestamp_str(150)) + self.session.timestamp_transaction( + 'commit_timestamp=' + timestamp_str(200)) + self.session.timestamp_transaction( + 'durable_timestamp=' + timestamp_str(250)) + self.session.commit_transaction() + + self.conn.rollback_to_stable() + + # The original bug happened when we had a txn that: + # 1. Was prepared. + # 2. Did not cause anything to be written to the log before committing. + # 3. Was the last txn before the rollback to stable call. + # Therefore, we're specifically not doing any operations here. + self.session.begin_transaction() + self.session.prepare_transaction( + 'prepare_timestamp=' + timestamp_str(300)) + self.session.timestamp_transaction( + 'commit_timestamp=' + timestamp_str(350)) + self.session.timestamp_transaction( + 'durable_timestamp=' + timestamp_str(400)) + self.session.commit_transaction() + + # The aforementioned bug resulted in a failure in rollback to stable. + # This is because we failed to clear out a txn id from our global state + # which caused us to think that we had a running txn. + # Verify that we can rollback to stable without issues. + self.conn.rollback_to_stable() + + self.session.begin_transaction() + for i in range(1, 50): + cursor[i] = b'b' * 100 + self.session.commit_transaction( + 'commit_timestamp=' + timestamp_str(450)) + + self.conn.rollback_to_stable() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/syscall/wt2336_base/main.c b/src/third_party/wiredtiger/test/syscall/wt2336_base/main.c index b04c38bbb1d..8ccd3690920 100644 --- a/src/third_party/wiredtiger/test/syscall/wt2336_base/main.c +++ b/src/third_party/wiredtiger/test/syscall/wt2336_base/main.c @@ -29,69 +29,66 @@ #include <stdlib.h> #include <unistd.h> // TODO -#include <fcntl.h> // TODO +#include <fcntl.h> // TODO #include <wt_internal.h> static void fail(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void -fail(int ret) { - fprintf(stderr, - "%s: %d (%s)\n", - "wt2336_fileop_basic", ret, wiredtiger_strerror(ret)); - exit(ret); +fail(int ret) +{ + fprintf(stderr, "%s: %d (%s)\n", "wt2336_fileop_basic", ret, wiredtiger_strerror(ret)); + exit(ret); } -#define SEPARATOR "--------------" +#define SEPARATOR "--------------" int main(int argc, char *argv[]) { - WT_CONNECTION *conn; - WT_SESSION *session; - int ret; + WT_CONNECTION *conn; + WT_SESSION *session; + int ret; - (void)argc; - (void)argv; - fprintf(stderr, SEPARATOR "wiredtiger_open\n"); - if ((ret = wiredtiger_open(".", NULL, "create", &conn)) != 0) - fail(ret); + (void)argc; + (void)argv; + fprintf(stderr, SEPARATOR "wiredtiger_open\n"); + if ((ret = wiredtiger_open(".", NULL, "create", &conn)) != 0) + fail(ret); - usleep(100); - fflush(stderr); - fprintf(stderr, SEPARATOR "open_session\n"); - fflush(stderr); + usleep(100); + fflush(stderr); + fprintf(stderr, SEPARATOR "open_session\n"); + fflush(stderr); - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - fail(ret); + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + fail(ret); - usleep(100); - fflush(stderr); - fprintf(stderr, SEPARATOR "create\n"); - fflush(stderr); + usleep(100); + fflush(stderr); + fprintf(stderr, SEPARATOR "create\n"); + fflush(stderr); - if ((ret = session->create( - session, "table:hello", "key_format=S,value_format=S")) != 0) - fail(ret); + if ((ret = session->create(session, "table:hello", "key_format=S,value_format=S")) != 0) + fail(ret); - usleep(100); - fprintf(stderr, SEPARATOR "rename\n"); + usleep(100); + fprintf(stderr, SEPARATOR "rename\n"); - if ((ret = session->rename( - session, "table:hello", "table:world", NULL)) != 0) - fail(ret); + if ((ret = session->rename(session, "table:hello", "table:world", NULL)) != 0) + fail(ret); - fflush(stdout); - fprintf(stderr, SEPARATOR "drop\n"); - fflush(stdout); + fflush(stdout); + fprintf(stderr, SEPARATOR "drop\n"); + fflush(stdout); - if ((ret = session->drop(session, "table:world", NULL)) != 0) - fail(ret); + if ((ret = session->drop(session, "table:world", NULL)) != 0) + fail(ret); - fprintf(stderr, SEPARATOR "WT_CONNECTION::close\n"); + fprintf(stderr, SEPARATOR "WT_CONNECTION::close\n"); - if ((ret = conn->close(conn, NULL)) != 0) - fail(ret); + if ((ret = conn->close(conn, NULL)) != 0) + fail(ret); - return (0); + return (0); } diff --git a/src/third_party/wiredtiger/test/thread/file.c b/src/third_party/wiredtiger/test/thread/file.c index fa29f9061ec..6e544bb62e7 100644 --- a/src/third_party/wiredtiger/test/thread/file.c +++ b/src/third_party/wiredtiger/test/thread/file.c @@ -31,80 +31,76 @@ static void file_create(const char *name) { - WT_SESSION *session; - int ret; - char config[128]; + WT_SESSION *session; + int ret; + char config[128]; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(__wt_snprintf(config, sizeof(config), - "key_format=%s," - "internal_page_max=%d," - "leaf_page_max=%d," - "%s", - ftype == ROW ? "u" : "r", 16 * 1024, 128 * 1024, - ftype == FIX ? ",value_format=3t" : "")); + testutil_check(__wt_snprintf(config, sizeof(config), + "key_format=%s," + "internal_page_max=%d," + "leaf_page_max=%d," + "%s", + ftype == ROW ? "u" : "r", 16 * 1024, 128 * 1024, ftype == FIX ? ",value_format=3t" : "")); - if ((ret = session->create(session, name, config)) != 0) - if (ret != EEXIST) - testutil_die(ret, "session.create"); + if ((ret = session->create(session, name, config)) != 0) + if (ret != EEXIST) + testutil_die(ret, "session.create"); - testutil_check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); } void load(const char *name) { - WT_CURSOR *cursor; - WT_ITEM *key, _key, *value, _value; - WT_SESSION *session; - size_t len; - uint64_t keyno; - char keybuf[64], valuebuf[64]; - - file_create(name); - - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - - testutil_check( - session->open_cursor(session, name, NULL, "bulk", &cursor)); - - key = &_key; - value = &_value; - for (keyno = 1; keyno <= nkeys; ++keyno) { - if (ftype == ROW) { - testutil_check(__wt_snprintf_len_set( - keybuf, sizeof(keybuf), - &len, "%017" PRIu64, keyno)); - key->data = keybuf; - key->size = (uint32_t)len; - cursor->set_key(cursor, key); - } else - cursor->set_key(cursor, keyno); - if (ftype == FIX) - cursor->set_value(cursor, 0x01); - else { - testutil_check(__wt_snprintf_len_set( - valuebuf, sizeof(valuebuf), - &len, "%37" PRIu64, keyno)); - value->data = valuebuf; - value->size = (uint32_t)len; - cursor->set_value(cursor, value); - } - testutil_check(cursor->insert(cursor)); - } - - testutil_check(session->close(session, NULL)); + WT_CURSOR *cursor; + WT_ITEM *key, _key, *value, _value; + WT_SESSION *session; + size_t len; + uint64_t keyno; + char keybuf[64], valuebuf[64]; + + file_create(name); + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + testutil_check(session->open_cursor(session, name, NULL, "bulk", &cursor)); + + key = &_key; + value = &_value; + for (keyno = 1; keyno <= nkeys; ++keyno) { + if (ftype == ROW) { + testutil_check( + __wt_snprintf_len_set(keybuf, sizeof(keybuf), &len, "%017" PRIu64, keyno)); + key->data = keybuf; + key->size = (uint32_t)len; + cursor->set_key(cursor, key); + } else + cursor->set_key(cursor, keyno); + if (ftype == FIX) + cursor->set_value(cursor, 0x01); + else { + testutil_check( + __wt_snprintf_len_set(valuebuf, sizeof(valuebuf), &len, "%37" PRIu64, keyno)); + value->data = valuebuf; + value->size = (uint32_t)len; + cursor->set_value(cursor, value); + } + testutil_check(cursor->insert(cursor)); + } + + testutil_check(session->close(session, NULL)); } void verify(const char *name) { - WT_SESSION *session; + WT_SESSION *session; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->verify(session, name, NULL)); + testutil_check(session->verify(session, name, NULL)); - testutil_check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); } diff --git a/src/third_party/wiredtiger/test/thread/rw.c b/src/third_party/wiredtiger/test/thread/rw.c index cf38157f59f..4a1879b786c 100644 --- a/src/third_party/wiredtiger/test/thread/rw.c +++ b/src/third_party/wiredtiger/test/thread/rw.c @@ -28,19 +28,19 @@ #include "thread.h" -static void print_stats(u_int); +static void print_stats(u_int); static WT_THREAD_RET reader(void *); static WT_THREAD_RET writer(void *); typedef struct { - char *name; /* object name */ - u_int nops; /* Thread op count */ + char *name; /* object name */ + u_int nops; /* Thread op count */ - WT_RAND_STATE rnd; /* RNG */ + WT_RAND_STATE rnd; /* RNG */ - int remove; /* cursor.remove */ - int update; /* cursor.update */ - int reads; /* cursor.search */ + int remove; /* cursor.remove */ + int update; /* cursor.update */ + int reads; /* cursor.search */ } INFO; static INFO *run_info; @@ -48,289 +48,269 @@ static INFO *run_info; void rw_start(u_int readers, u_int writers) { - struct timeval start, stop; - wt_thread_t *tids; - double seconds; - u_int i, name_index, offset, total_nops; - - tids = NULL; /* Keep GCC 4.1 happy. */ - total_nops = 0; - - /* Create per-thread structures. */ - run_info = dcalloc((size_t)(readers + writers), sizeof(*run_info)); - tids = dcalloc((size_t)(readers + writers), sizeof(*tids)); - - /* Create the files and load the initial records. */ - for (i = 0; i < writers; ++i) { - if (i == 0 || multiple_files) { - run_info[i].name = dmalloc(64); - testutil_check(__wt_snprintf( - run_info[i].name, 64, FNAME, (int)i)); - - /* Vary by orders of magnitude */ - if (vary_nops) - run_info[i].nops = WT_MAX(1000, max_nops >> i); - load(run_info[i].name); - } else - run_info[i].name = run_info[0].name; - - /* Setup op count if not varying ops. */ - if (run_info[i].nops == 0) - run_info[i].nops = max_nops; - total_nops += run_info[i].nops; - } - - /* Setup the reader configurations */ - for (i = 0; i < readers; ++i) { - offset = i + writers; - if (multiple_files) { - run_info[offset].name = dmalloc(64); - /* Have readers read from tables with writes. */ - name_index = i % writers; - testutil_check(__wt_snprintf( - run_info[offset].name, 64, FNAME, (int)name_index)); - - /* Vary by orders of magnitude */ - if (vary_nops) - run_info[offset].nops = - WT_MAX(1000, max_nops >> name_index); - } else - run_info[offset].name = run_info[0].name; - - /* Setup op count if not varying ops. */ - if (run_info[offset].nops == 0) - run_info[offset].nops = max_nops; - total_nops += run_info[offset].nops; - } - - (void)gettimeofday(&start, NULL); - - /* Create threads. */ - for (i = 0; i < readers; ++i) - testutil_check(__wt_thread_create( - NULL, &tids[i], reader, (void *)(uintptr_t)i)); - for (; i < readers + writers; ++i) - testutil_check(__wt_thread_create( - NULL, &tids[i], writer, (void *)(uintptr_t)i)); - - /* Wait for the threads. */ - for (i = 0; i < readers + writers; ++i) - testutil_check(__wt_thread_join(NULL, &tids[i])); - - (void)gettimeofday(&stop, NULL); - seconds = (stop.tv_sec - start.tv_sec) + - (stop.tv_usec - start.tv_usec) * 1e-6; - fprintf(stderr, "timer: %.2lf seconds (%d ops/second)\n", - seconds, (int)(((readers + writers) * total_nops) / seconds)); - - /* Verify the files. */ - for (i = 0; i < readers + writers; ++i) { - verify(run_info[i].name); - if (!multiple_files) - break; - } - - /* Output run statistics. */ - print_stats(readers + writers); - - /* Free allocated memory. */ - for (i = 0; i < readers + writers; ++i) { - free(run_info[i].name); - if (!multiple_files) - break; - } - - free(run_info); - free(tids); + struct timeval start, stop; + wt_thread_t *tids; + double seconds; + u_int i, name_index, offset, total_nops; + + tids = NULL; /* Keep GCC 4.1 happy. */ + total_nops = 0; + + /* Create per-thread structures. */ + run_info = dcalloc((size_t)(readers + writers), sizeof(*run_info)); + tids = dcalloc((size_t)(readers + writers), sizeof(*tids)); + + /* Create the files and load the initial records. */ + for (i = 0; i < writers; ++i) { + if (i == 0 || multiple_files) { + run_info[i].name = dmalloc(64); + testutil_check(__wt_snprintf(run_info[i].name, 64, FNAME, (int)i)); + + /* Vary by orders of magnitude */ + if (vary_nops) + run_info[i].nops = WT_MAX(1000, max_nops >> i); + load(run_info[i].name); + } else + run_info[i].name = run_info[0].name; + + /* Setup op count if not varying ops. */ + if (run_info[i].nops == 0) + run_info[i].nops = max_nops; + total_nops += run_info[i].nops; + } + + /* Setup the reader configurations */ + for (i = 0; i < readers; ++i) { + offset = i + writers; + if (multiple_files) { + run_info[offset].name = dmalloc(64); + /* Have readers read from tables with writes. */ + name_index = i % writers; + testutil_check(__wt_snprintf(run_info[offset].name, 64, FNAME, (int)name_index)); + + /* Vary by orders of magnitude */ + if (vary_nops) + run_info[offset].nops = WT_MAX(1000, max_nops >> name_index); + } else + run_info[offset].name = run_info[0].name; + + /* Setup op count if not varying ops. */ + if (run_info[offset].nops == 0) + run_info[offset].nops = max_nops; + total_nops += run_info[offset].nops; + } + + (void)gettimeofday(&start, NULL); + + /* Create threads. */ + for (i = 0; i < readers; ++i) + testutil_check(__wt_thread_create(NULL, &tids[i], reader, (void *)(uintptr_t)i)); + for (; i < readers + writers; ++i) + testutil_check(__wt_thread_create(NULL, &tids[i], writer, (void *)(uintptr_t)i)); + + /* Wait for the threads. */ + for (i = 0; i < readers + writers; ++i) + testutil_check(__wt_thread_join(NULL, &tids[i])); + + (void)gettimeofday(&stop, NULL); + seconds = (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec) * 1e-6; + fprintf(stderr, "timer: %.2lf seconds (%d ops/second)\n", seconds, + (int)(((readers + writers) * total_nops) / seconds)); + + /* Verify the files. */ + for (i = 0; i < readers + writers; ++i) { + verify(run_info[i].name); + if (!multiple_files) + break; + } + + /* Output run statistics. */ + print_stats(readers + writers); + + /* Free allocated memory. */ + for (i = 0; i < readers + writers; ++i) { + free(run_info[i].name); + if (!multiple_files) + break; + } + + free(run_info); + free(tids); } /* * reader_op -- - * Read operation. + * Read operation. */ static inline void reader_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) { - WT_ITEM *key, _key; - size_t len; - uint64_t keyno; - int ret; - char keybuf[64]; - - key = &_key; - - keyno = __wt_random(&s->rnd) % nkeys + 1; - if (ftype == ROW) { - testutil_check(__wt_snprintf_len_set( - keybuf, sizeof(keybuf), &len, "%017" PRIu64, keyno)); - key->data = keybuf; - key->size = (uint32_t)len; - cursor->set_key(cursor, key); - } else - cursor->set_key(cursor, keyno); - if ((ret = cursor->search(cursor)) != 0 && ret != WT_NOTFOUND) - testutil_die(ret, "cursor.search"); - if (log_print) - testutil_check(session->log_printf(session, - "Reader Thread %p key %017" PRIu64, pthread_self(), keyno)); + WT_ITEM *key, _key; + size_t len; + uint64_t keyno; + int ret; + char keybuf[64]; + + key = &_key; + + keyno = __wt_random(&s->rnd) % nkeys + 1; + if (ftype == ROW) { + testutil_check(__wt_snprintf_len_set(keybuf, sizeof(keybuf), &len, "%017" PRIu64, keyno)); + key->data = keybuf; + key->size = (uint32_t)len; + cursor->set_key(cursor, key); + } else + cursor->set_key(cursor, keyno); + if ((ret = cursor->search(cursor)) != 0 && ret != WT_NOTFOUND) + testutil_die(ret, "cursor.search"); + if (log_print) + testutil_check( + session->log_printf(session, "Reader Thread %p key %017" PRIu64, pthread_self(), keyno)); } /* * reader -- - * Reader thread start function. + * Reader thread start function. */ static WT_THREAD_RET reader(void *arg) { - INFO *s; - WT_CURSOR *cursor; - WT_SESSION *session; - u_int i; - int id; - char tid[128]; - - id = (int)(uintptr_t)arg; - s = &run_info[id]; - testutil_check(__wt_thread_str(tid, sizeof(tid))); - __wt_random_init(&s->rnd); - - printf(" read thread %2d starting: tid: %s, file: %s\n", - id, tid, s->name); - - __wt_yield(); /* Get all the threads created. */ - - if (session_per_op) { - for (i = 0; i < s->nops; ++i, ++s->reads, __wt_yield()) { - testutil_check( - conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->open_cursor( - session, s->name, NULL, NULL, &cursor)); - reader_op(session, cursor, s); - testutil_check(session->close(session, NULL)); - } - } else { - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->open_cursor( - session, s->name, NULL, NULL, &cursor)); - for (i = 0; i < s->nops; ++i, ++s->reads, __wt_yield()) - reader_op(session, cursor, s); - testutil_check(session->close(session, NULL)); - } - - printf(" read thread %2d stopping: tid: %s, file: %s\n", - id, tid, s->name); - - return (WT_THREAD_RET_VALUE); + INFO *s; + WT_CURSOR *cursor; + WT_SESSION *session; + u_int i; + int id; + char tid[128]; + + id = (int)(uintptr_t)arg; + s = &run_info[id]; + testutil_check(__wt_thread_str(tid, sizeof(tid))); + __wt_random_init(&s->rnd); + + printf(" read thread %2d starting: tid: %s, file: %s\n", id, tid, s->name); + + __wt_yield(); /* Get all the threads created. */ + + if (session_per_op) { + for (i = 0; i < s->nops; ++i, ++s->reads, __wt_yield()) { + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->open_cursor(session, s->name, NULL, NULL, &cursor)); + reader_op(session, cursor, s); + testutil_check(session->close(session, NULL)); + } + } else { + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->open_cursor(session, s->name, NULL, NULL, &cursor)); + for (i = 0; i < s->nops; ++i, ++s->reads, __wt_yield()) + reader_op(session, cursor, s); + testutil_check(session->close(session, NULL)); + } + + printf(" read thread %2d stopping: tid: %s, file: %s\n", id, tid, s->name); + + return (WT_THREAD_RET_VALUE); } /* * writer_op -- - * Write operation. + * Write operation. */ static inline void writer_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) { - WT_ITEM *key, _key, *value, _value; - size_t len; - uint64_t keyno; - int ret; - char keybuf[64], valuebuf[64]; - - key = &_key; - value = &_value; - - keyno = __wt_random(&s->rnd) % nkeys + 1; - if (ftype == ROW) { - testutil_check(__wt_snprintf_len_set( - keybuf, sizeof(keybuf), &len, "%017" PRIu64, keyno)); - key->data = keybuf; - key->size = (uint32_t)len; - cursor->set_key(cursor, key); - } else - cursor->set_key(cursor, keyno); - if (keyno % 5 == 0) { - ++s->remove; - if ((ret = cursor->remove(cursor)) != 0 && ret != WT_NOTFOUND) - testutil_die(ret, "cursor.remove"); - } else { - ++s->update; - value->data = valuebuf; - if (ftype == FIX) - cursor->set_value(cursor, 0x10); - else { - testutil_check(__wt_snprintf_len_set( - valuebuf, sizeof(valuebuf), - &len, "XXX %37" PRIu64, keyno)); - value->size = (uint32_t)len; - cursor->set_value(cursor, value); - } - testutil_check(cursor->update(cursor)); - } - if (log_print) - testutil_check(session->log_printf(session, - "Writer Thread %p key %017" PRIu64, pthread_self(), keyno)); + WT_ITEM *key, _key, *value, _value; + size_t len; + uint64_t keyno; + int ret; + char keybuf[64], valuebuf[64]; + + key = &_key; + value = &_value; + + keyno = __wt_random(&s->rnd) % nkeys + 1; + if (ftype == ROW) { + testutil_check(__wt_snprintf_len_set(keybuf, sizeof(keybuf), &len, "%017" PRIu64, keyno)); + key->data = keybuf; + key->size = (uint32_t)len; + cursor->set_key(cursor, key); + } else + cursor->set_key(cursor, keyno); + if (keyno % 5 == 0) { + ++s->remove; + if ((ret = cursor->remove(cursor)) != 0 && ret != WT_NOTFOUND) + testutil_die(ret, "cursor.remove"); + } else { + ++s->update; + value->data = valuebuf; + if (ftype == FIX) + cursor->set_value(cursor, 0x10); + else { + testutil_check( + __wt_snprintf_len_set(valuebuf, sizeof(valuebuf), &len, "XXX %37" PRIu64, keyno)); + value->size = (uint32_t)len; + cursor->set_value(cursor, value); + } + testutil_check(cursor->update(cursor)); + } + if (log_print) + testutil_check( + session->log_printf(session, "Writer Thread %p key %017" PRIu64, pthread_self(), keyno)); } /* * writer -- - * Writer thread start function. + * Writer thread start function. */ static WT_THREAD_RET writer(void *arg) { - INFO *s; - WT_CURSOR *cursor; - WT_SESSION *session; - u_int i; - int id; - char tid[128]; - - id = (int)(uintptr_t)arg; - s = &run_info[id]; - testutil_check(__wt_thread_str(tid, sizeof(tid))); - __wt_random_init(&s->rnd); - - printf("write thread %2d starting: tid: %s, file: %s\n", - id, tid, s->name); - - __wt_yield(); /* Get all the threads created. */ - - if (session_per_op) { - for (i = 0; i < s->nops; ++i, __wt_yield()) { - testutil_check(conn->open_session( - conn, NULL, NULL, &session)); - testutil_check(session->open_cursor( - session, s->name, NULL, NULL, &cursor)); - writer_op(session, cursor, s); - testutil_check(session->close(session, NULL)); - } - } else { - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->open_cursor( - session, s->name, NULL, NULL, &cursor)); - for (i = 0; i < s->nops; ++i, __wt_yield()) - writer_op(session, cursor, s); - testutil_check(session->close(session, NULL)); - } - - printf("write thread %2d stopping: tid: %s, file: %s\n", - id, tid, s->name); - - return (WT_THREAD_RET_VALUE); + INFO *s; + WT_CURSOR *cursor; + WT_SESSION *session; + u_int i; + int id; + char tid[128]; + + id = (int)(uintptr_t)arg; + s = &run_info[id]; + testutil_check(__wt_thread_str(tid, sizeof(tid))); + __wt_random_init(&s->rnd); + + printf("write thread %2d starting: tid: %s, file: %s\n", id, tid, s->name); + + __wt_yield(); /* Get all the threads created. */ + + if (session_per_op) { + for (i = 0; i < s->nops; ++i, __wt_yield()) { + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->open_cursor(session, s->name, NULL, NULL, &cursor)); + writer_op(session, cursor, s); + testutil_check(session->close(session, NULL)); + } + } else { + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->open_cursor(session, s->name, NULL, NULL, &cursor)); + for (i = 0; i < s->nops; ++i, __wt_yield()) + writer_op(session, cursor, s); + testutil_check(session->close(session, NULL)); + } + + printf("write thread %2d stopping: tid: %s, file: %s\n", id, tid, s->name); + + return (WT_THREAD_RET_VALUE); } /* * print_stats -- - * Display reader/writer thread stats. + * Display reader/writer thread stats. */ static void print_stats(u_int nthreads) { - INFO *s; - u_int id; + INFO *s; + u_int id; - s = run_info; - for (id = 0; id < nthreads; ++id, ++s) - printf("%3u: read %6d, remove %6d, update %6d\n", - id, s->reads, s->remove, s->update); + s = run_info; + for (id = 0; id < nthreads; ++id, ++s) + printf("%3u: read %6d, remove %6d, update %6d\n", id, s->reads, s->remove, s->update); } diff --git a/src/third_party/wiredtiger/test/thread/stats.c b/src/third_party/wiredtiger/test/thread/stats.c index b6c0f817109..a23d40f78ff 100644 --- a/src/third_party/wiredtiger/test/thread/stats.c +++ b/src/third_party/wiredtiger/test/thread/stats.c @@ -35,47 +35,44 @@ void stats(void) { - FILE *fp; - WT_CURSOR *cursor; - WT_SESSION *session; - uint64_t v; - int ret; - const char *desc, *pval; - char name[64]; + FILE *fp; + WT_CURSOR *cursor; + WT_SESSION *session; + uint64_t v; + int ret; + char name[64]; + const char *desc, *pval; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); - if ((fp = fopen(FNAME_STAT, "w")) == NULL) - testutil_die(errno, "fopen " FNAME_STAT); + if ((fp = fopen(FNAME_STAT, "w")) == NULL) + testutil_die(errno, "fopen " FNAME_STAT); - /* Connection statistics. */ - testutil_check(session->open_cursor( - session, "statistics:", NULL, NULL, &cursor)); + /* Connection statistics. */ + testutil_check(session->open_cursor(session, "statistics:", NULL, NULL, &cursor)); - while ((ret = cursor->next(cursor)) == 0 && - (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) - (void)fprintf(fp, "%s=%s\n", desc, pval); + while ( + (ret = cursor->next(cursor)) == 0 && (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) + (void)fprintf(fp, "%s=%s\n", desc, pval); - if (ret != WT_NOTFOUND) - testutil_die(ret, "cursor.next"); - testutil_check(cursor->close(cursor)); + if (ret != WT_NOTFOUND) + testutil_die(ret, "cursor.next"); + testutil_check(cursor->close(cursor)); - /* File statistics. */ - if (!multiple_files) { - testutil_check(__wt_snprintf( - name, sizeof(name), "statistics:" FNAME, 0)); - testutil_check(session->open_cursor( - session, name, NULL, NULL, &cursor)); + /* File statistics. */ + if (!multiple_files) { + testutil_check(__wt_snprintf(name, sizeof(name), "statistics:" FNAME, 0)); + testutil_check(session->open_cursor(session, name, NULL, NULL, &cursor)); - while ((ret = cursor->next(cursor)) == 0 && - (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) - (void)fprintf(fp, "%s=%s\n", desc, pval); + while ((ret = cursor->next(cursor)) == 0 && + (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) + (void)fprintf(fp, "%s=%s\n", desc, pval); - if (ret != WT_NOTFOUND) - testutil_die(ret, "cursor.next"); - testutil_check(cursor->close(cursor)); + if (ret != WT_NOTFOUND) + testutil_die(ret, "cursor.next"); + testutil_check(cursor->close(cursor)); - testutil_check(session->close(session, NULL)); - } - (void)fclose(fp); + testutil_check(session->close(session, NULL)); + } + (void)fclose(fp); } diff --git a/src/third_party/wiredtiger/test/thread/t.c b/src/third_party/wiredtiger/test/thread/t.c index 10fe89b4a75..63d1abab46a 100644 --- a/src/third_party/wiredtiger/test/thread/t.c +++ b/src/third_party/wiredtiger/test/thread/t.c @@ -28,23 +28,22 @@ #include "thread.h" -WT_CONNECTION *conn; /* WiredTiger connection */ -__ftype ftype; /* File type */ -u_int nkeys, max_nops; /* Keys, Operations */ -int vary_nops; /* Vary operations by thread */ -int log_print; /* Log print per operation */ -int multiple_files; /* File per thread */ -int session_per_op; /* New session per operation */ - -static char home[512]; /* Program working dir */ -static FILE *logfp; /* Log file */ - -static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); -static int handle_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); -static void onint(int) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +WT_CONNECTION *conn; /* WiredTiger connection */ +__ftype ftype; /* File type */ +u_int nkeys, max_nops; /* Keys, Operations */ +int vary_nops; /* Vary operations by thread */ +int log_print; /* Log print per operation */ +int multiple_files; /* File per thread */ +int session_per_op; /* New session per operation */ + +static char home[512]; /* Program working dir */ +static FILE *logfp; /* Log file */ + +static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); +static int handle_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); +static void onint(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void shutdown(void); -static int usage(void); +static int usage(void); static void wt_connect(char *); static void wt_shutdown(void); @@ -54,236 +53,225 @@ extern char *__wt_optarg; int main(int argc, char *argv[]) { - u_int readers, writers; - int ch, cnt, runs; - char *config_open, *working_dir; - - (void)testutil_set_progname(argv); - - config_open = NULL; - working_dir = NULL; - ftype = ROW; - log_print = 0; - multiple_files = 0; - nkeys = 1000; - max_nops = 10000; - readers = 10; - runs = 1; - session_per_op = 0; - vary_nops = 0; - writers = 10; - - while ((ch = __wt_getopt( - progname, argc, argv, "C:Fk:h:Ll:n:R:r:St:vW:")) != EOF) - switch (ch) { - case 'C': /* wiredtiger_open config */ - config_open = __wt_optarg; - break; - case 'F': /* multiple files */ - multiple_files = 1; - break; - case 'h': - working_dir = __wt_optarg; - break; - case 'k': /* rows */ - nkeys = (u_int)atoi(__wt_optarg); - break; - case 'L': /* log print per operation */ - log_print = 1; - break; - case 'l': /* log */ - if ((logfp = fopen(__wt_optarg, "w")) == NULL) { - fprintf(stderr, - "%s: %s\n", __wt_optarg, strerror(errno)); - return (EXIT_FAILURE); - } - break; - case 'n': /* operations */ - max_nops = (u_int)atoi(__wt_optarg); - break; - case 'R': - readers = (u_int)atoi(__wt_optarg); - break; - case 'r': /* runs */ - runs = atoi(__wt_optarg); - break; - case 'S': /* new session per operation */ - session_per_op = 1; - break; - case 't': - switch (__wt_optarg[0]) { - case 'f': - ftype = FIX; - break; - case 'r': - ftype = ROW; - break; - case 'v': - ftype = VAR; - break; - default: - return (usage()); - } - break; - case 'v': /* vary operation count */ - vary_nops = 1; - break; - case 'W': - writers = (u_int)atoi(__wt_optarg); - break; - default: - return (usage()); - } - - argc -= __wt_optind; - if (argc != 0) - return (usage()); - - testutil_work_dir_from_path(home, 512, working_dir); - - if (vary_nops && !multiple_files) { - fprintf(stderr, - "Variable op counts only supported with multiple tables\n"); - return (usage()); - } - - /* Clean up on signal. */ - (void)signal(SIGINT, onint); - - printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); - for (cnt = 1; runs == 0 || cnt <= runs; ++cnt) { - printf( - " %d: %u readers, %u writers\n", cnt, readers, writers); - - shutdown(); /* Clean up previous runs */ - - wt_connect(config_open); /* WiredTiger connection */ - - rw_start(readers, writers); /* Loop operations */ - - stats(); /* Statistics */ - - wt_shutdown(); /* WiredTiger shut down */ - } - return (0); + u_int readers, writers; + int ch, cnt, runs; + char *config_open, *working_dir; + + (void)testutil_set_progname(argv); + + config_open = NULL; + working_dir = NULL; + ftype = ROW; + log_print = 0; + multiple_files = 0; + nkeys = 1000; + max_nops = 10000; + readers = 10; + runs = 1; + session_per_op = 0; + vary_nops = 0; + writers = 10; + + while ((ch = __wt_getopt(progname, argc, argv, "C:Fk:h:Ll:n:R:r:St:vW:")) != EOF) + switch (ch) { + case 'C': /* wiredtiger_open config */ + config_open = __wt_optarg; + break; + case 'F': /* multiple files */ + multiple_files = 1; + break; + case 'h': + working_dir = __wt_optarg; + break; + case 'k': /* rows */ + nkeys = (u_int)atoi(__wt_optarg); + break; + case 'L': /* log print per operation */ + log_print = 1; + break; + case 'l': /* log */ + if ((logfp = fopen(__wt_optarg, "w")) == NULL) { + fprintf(stderr, "%s: %s\n", __wt_optarg, strerror(errno)); + return (EXIT_FAILURE); + } + break; + case 'n': /* operations */ + max_nops = (u_int)atoi(__wt_optarg); + break; + case 'R': + readers = (u_int)atoi(__wt_optarg); + break; + case 'r': /* runs */ + runs = atoi(__wt_optarg); + break; + case 'S': /* new session per operation */ + session_per_op = 1; + break; + case 't': + switch (__wt_optarg[0]) { + case 'f': + ftype = FIX; + break; + case 'r': + ftype = ROW; + break; + case 'v': + ftype = VAR; + break; + default: + return (usage()); + } + break; + case 'v': /* vary operation count */ + vary_nops = 1; + break; + case 'W': + writers = (u_int)atoi(__wt_optarg); + break; + default: + return (usage()); + } + + argc -= __wt_optind; + if (argc != 0) + return (usage()); + + testutil_work_dir_from_path(home, 512, working_dir); + + if (vary_nops && !multiple_files) { + fprintf(stderr, "Variable op counts only supported with multiple tables\n"); + return (usage()); + } + + /* Clean up on signal. */ + (void)signal(SIGINT, onint); + + printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); + for (cnt = 1; runs == 0 || cnt <= runs; ++cnt) { + printf(" %d: %u readers, %u writers\n", cnt, readers, writers); + + shutdown(); /* Clean up previous runs */ + + wt_connect(config_open); /* WiredTiger connection */ + + rw_start(readers, writers); /* Loop operations */ + + stats(); /* Statistics */ + + wt_shutdown(); /* WiredTiger shut down */ + } + return (0); } /* * wt_connect -- - * Configure the WiredTiger connection. + * Configure the WiredTiger connection. */ static void wt_connect(char *config_open) { - static WT_EVENT_HANDLER event_handler = { - handle_error, - handle_message, - NULL, - NULL /* Close handler. */ - }; - char config[512]; - - testutil_clean_work_dir(home); - testutil_make_work_dir(home); - - testutil_check(__wt_snprintf(config, sizeof(config), - "create,statistics=(all),error_prefix=\"%s\",%s%s", - progname, - config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open)); - - testutil_check(wiredtiger_open(home, &event_handler, config, &conn)); + static WT_EVENT_HANDLER event_handler = { + handle_error, handle_message, NULL, NULL /* Close handler. */ + }; + char config[512]; + + testutil_clean_work_dir(home); + testutil_make_work_dir(home); + + testutil_check( + __wt_snprintf(config, sizeof(config), "create,statistics=(all),error_prefix=\"%s\",%s%s", + progname, config_open == NULL ? "" : ",", config_open == NULL ? "" : config_open)); + + testutil_check(wiredtiger_open(home, &event_handler, config, &conn)); } /* * wt_shutdown -- - * Flush the file to disk and shut down the WiredTiger connection. + * Flush the file to disk and shut down the WiredTiger connection. */ static void wt_shutdown(void) { - WT_SESSION *session; + WT_SESSION *session; - testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check(session->checkpoint(session, NULL)); + testutil_check(session->checkpoint(session, NULL)); - testutil_check(conn->close(conn, NULL)); + testutil_check(conn->close(conn, NULL)); } /* * shutdown -- - * Clean up from previous runs. + * Clean up from previous runs. */ static void shutdown(void) { - testutil_clean_work_dir(home); + testutil_clean_work_dir(home); } static int -handle_error(WT_EVENT_HANDLER *handler, - WT_SESSION *session, int error, const char *errmsg) +handle_error(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const char *errmsg) { - (void)(handler); - (void)(session); - (void)(error); + (void)(handler); + (void)(session); + (void)(error); - return (fprintf(stderr, "%s\n", errmsg) < 0 ? -1 : 0); + return (fprintf(stderr, "%s\n", errmsg) < 0 ? -1 : 0); } static int -handle_message(WT_EVENT_HANDLER *handler, - WT_SESSION *session, const char *message) +handle_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message) { - (void)(handler); - (void)(session); + (void)(handler); + (void)(session); - if (logfp != NULL) - return (fprintf(logfp, "%s\n", message) < 0 ? -1 : 0); + if (logfp != NULL) + return (fprintf(logfp, "%s\n", message) < 0 ? -1 : 0); - return (printf("%s\n", message) < 0 ? -1 : 0); + return (printf("%s\n", message) < 0 ? -1 : 0); } /* * onint -- - * Interrupt signal handler. + * Interrupt signal handler. */ static void onint(int signo) { - (void)(signo); + (void)(signo); - shutdown(); + shutdown(); - fprintf(stderr, "\n"); - exit(EXIT_FAILURE); + fprintf(stderr, "\n"); + exit(EXIT_FAILURE); } /* * usage -- - * Display usage statement and exit failure. + * Display usage statement and exit failure. */ static int usage(void) { - fprintf(stderr, - "usage: %s " - "[-FLSv] [-C wiredtiger-config] [-k keys] [-l log]\n\t" - "[-n ops] [-R readers] [-r runs] [-t f|r|v] [-W writers]\n", - progname); - fprintf(stderr, "%s", - "\t-C specify wiredtiger_open configuration arguments\n" - "\t-F create a file per thread\n" - "\t-k set number of keys to load\n" - "\t-L log print per operation\n" - "\t-l specify a log file\n" - "\t-n set number of operations each thread does\n" - "\t-R set number of reading threads\n" - "\t-r set number of runs (0 for continuous)\n" - "\t-S open/close a session on every operation\n" - "\t-t set a file type (fix | row | var)\n" - "\t-v do a different number of operations on different tables\n" - "\t-W set number of writing threads\n"); - return (EXIT_FAILURE); + fprintf(stderr, + "usage: %s " + "[-FLSv] [-C wiredtiger-config] [-k keys] [-l log]\n\t" + "[-n ops] [-R readers] [-r runs] [-t f|r|v] [-W writers]\n", + progname); + fprintf(stderr, "%s", + "\t-C specify wiredtiger_open configuration arguments\n" + "\t-F create a file per thread\n" + "\t-k set number of keys to load\n" + "\t-L log print per operation\n" + "\t-l specify a log file\n" + "\t-n set number of operations each thread does\n" + "\t-R set number of reading threads\n" + "\t-r set number of runs (0 for continuous)\n" + "\t-S open/close a session on every operation\n" + "\t-t set a file type (fix | row | var)\n" + "\t-v do a different number of operations on different tables\n" + "\t-W set number of writing threads\n"); + return (EXIT_FAILURE); } diff --git a/src/third_party/wiredtiger/test/thread/thread.h b/src/third_party/wiredtiger/test/thread/thread.h index c485e899eba..ea7965434be 100644 --- a/src/third_party/wiredtiger/test/thread/thread.h +++ b/src/third_party/wiredtiger/test/thread/thread.h @@ -30,20 +30,20 @@ #include <signal.h> -#define FNAME "file:wt.%03d" /* File name */ -#define FNAME_STAT "__stats" /* File name for statistics */ +#define FNAME "file:wt.%03d" /* File name */ +#define FNAME_STAT "__stats" /* File name for statistics */ -extern WT_CONNECTION *conn; /* WiredTiger connection */ +extern WT_CONNECTION *conn; /* WiredTiger connection */ -typedef enum { FIX, ROW, VAR } __ftype; /* File type */ +typedef enum { FIX, ROW, VAR } __ftype; /* File type */ extern __ftype ftype; -extern int log_print; /* Log print per operation */ -extern int multiple_files; /* File per thread */ -extern u_int nkeys; /* Keys to load */ -extern u_int max_nops; /* Operations per thread */ -extern int vary_nops; /* Operations per thread */ -extern int session_per_op; /* New session per operation */ +extern int log_print; /* Log print per operation */ +extern int multiple_files; /* File per thread */ +extern u_int nkeys; /* Keys to load */ +extern u_int max_nops; /* Operations per thread */ +extern int vary_nops; /* Operations per thread */ +extern int session_per_op; /* New session per operation */ void load(const char *); void rw_start(u_int, u_int); diff --git a/src/third_party/wiredtiger/test/utility/misc.c b/src/third_party/wiredtiger/test/utility/misc.c index bebfb5f200a..5f44c080f46 100644 --- a/src/third_party/wiredtiger/test/utility/misc.c +++ b/src/third_party/wiredtiger/test/utility/misc.c @@ -36,327 +36,324 @@ const char *progname = "program name not set"; /* * testutil_die -- - * Report an error and abort. + * Report an error and abort. */ void testutil_die(int e, const char *fmt, ...) { - va_list ap; - - /* Flush output to be sure it doesn't mix with fatal errors. */ - (void)fflush(stdout); - (void)fflush(stderr); - - /* Allow test programs to cleanup on fatal error. */ - if (custom_die != NULL) - (*custom_die)(); - - fprintf(stderr, "%s: FAILED", progname); - if (fmt != NULL) { - fprintf(stderr, ": "); - va_start(ap, fmt); - vfprintf(stderr, fmt, ap); - va_end(ap); - } - if (e != 0) - fprintf(stderr, ": %s", wiredtiger_strerror(e)); - fprintf(stderr, "\n"); - fprintf(stderr, "process aborting\n"); - - abort(); + va_list ap; + + /* Flush output to be sure it doesn't mix with fatal errors. */ + (void)fflush(stdout); + (void)fflush(stderr); + + /* Allow test programs to cleanup on fatal error. */ + if (custom_die != NULL) + (*custom_die)(); + + fprintf(stderr, "%s: FAILED", progname); + if (fmt != NULL) { + fprintf(stderr, ": "); + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + } + if (e != 0) + fprintf(stderr, ": %s", wiredtiger_strerror(e)); + fprintf(stderr, "\n"); + fprintf(stderr, "process aborting\n"); + + abort(); } /* * testutil_set_progname -- - * Set the global program name for error handling. + * Set the global program name for error handling. */ const char * -testutil_set_progname(char * const *argv) +testutil_set_progname(char *const *argv) { - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; - return (progname); + if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) + progname = argv[0]; + else + ++progname; + return (progname); } /* * testutil_work_dir_from_path -- - * Takes a buffer, its size and the intended work directory. - * Creates the full intended work directory in buffer. + * Takes a buffer, its size and the intended work directory. Creates the full intended work + * directory in buffer. */ void testutil_work_dir_from_path(char *buffer, size_t len, const char *dir) { - /* If no directory is provided, use the default. */ - if (dir == NULL) - dir = DEFAULT_DIR; + /* If no directory is provided, use the default. */ + if (dir == NULL) + dir = DEFAULT_DIR; - if (len < strlen(dir) + 1) - testutil_die(ENOMEM, - "Not enough memory in buffer for directory %s", dir); + if (len < strlen(dir) + 1) + testutil_die(ENOMEM, "Not enough memory in buffer for directory %s", dir); - strcpy(buffer, dir); + strcpy(buffer, dir); } /* * testutil_clean_work_dir -- - * Remove the work directory. + * Remove the work directory. */ void testutil_clean_work_dir(const char *dir) { - size_t len; - int ret; - char *buf; + size_t len; + int ret; + char *buf; #ifdef _WIN32 - /* Additional bytes for the Windows rd command. */ - len = 2 * strlen(dir) + strlen(RM_COMMAND) + - strlen(DIR_EXISTS_COMMAND) + 4; - if ((buf = malloc(len)) == NULL) - testutil_die(ENOMEM, "Failed to allocate memory"); - - testutil_check(__wt_snprintf( - buf, len, "%s %s %s %s", DIR_EXISTS_COMMAND, dir, RM_COMMAND, dir)); + /* Additional bytes for the Windows rd command. */ + len = 2 * strlen(dir) + strlen(RM_COMMAND) + strlen(DIR_EXISTS_COMMAND) + 4; + if ((buf = malloc(len)) == NULL) + testutil_die(ENOMEM, "Failed to allocate memory"); + + testutil_check( + __wt_snprintf(buf, len, "%s %s %s %s", DIR_EXISTS_COMMAND, dir, RM_COMMAND, dir)); #else - len = strlen(dir) + strlen(RM_COMMAND) + 1; - if ((buf = malloc(len)) == NULL) - testutil_die(ENOMEM, "Failed to allocate memory"); + len = strlen(dir) + strlen(RM_COMMAND) + 1; + if ((buf = malloc(len)) == NULL) + testutil_die(ENOMEM, "Failed to allocate memory"); - testutil_check(__wt_snprintf(buf, len, "%s%s", RM_COMMAND, dir)); + testutil_check(__wt_snprintf(buf, len, "%s%s", RM_COMMAND, dir)); #endif - if ((ret = system(buf)) != 0 && ret != ENOENT) - testutil_die(ret, "%s", buf); - free(buf); + if ((ret = system(buf)) != 0 && ret != ENOENT) + testutil_die(ret, "%s", buf); + free(buf); } /* * testutil_make_work_dir -- - * Delete the existing work directory, then create a new one. + * Delete the existing work directory, then create a new one. */ void testutil_make_work_dir(const char *dir) { - size_t len; - char *buf; + size_t len; + char *buf; - testutil_clean_work_dir(dir); + testutil_clean_work_dir(dir); - /* Additional bytes for the mkdir command */ - len = strlen(dir) + strlen(MKDIR_COMMAND) + 1; - if ((buf = malloc(len)) == NULL) - testutil_die(ENOMEM, "Failed to allocate memory"); + /* Additional bytes for the mkdir command */ + len = strlen(dir) + strlen(MKDIR_COMMAND) + 1; + if ((buf = malloc(len)) == NULL) + testutil_die(ENOMEM, "Failed to allocate memory"); - /* mkdir shares syntax between Windows and Linux */ - testutil_check(__wt_snprintf(buf, len, "%s%s", MKDIR_COMMAND, dir)); - testutil_check(system(buf)); - free(buf); + /* mkdir shares syntax between Windows and Linux */ + testutil_check(__wt_snprintf(buf, len, "%s%s", MKDIR_COMMAND, dir)); + testutil_check(system(buf)); + free(buf); } /* * testutil_progress -- - * Print a progress message to the progress file. + * Print a progress message to the progress file. */ void testutil_progress(TEST_OPTS *opts, const char *message) { - FILE *fp; - uint64_t now; + FILE *fp; + uint64_t now; - if (opts->progress_fp == NULL) - testutil_checksys((opts->progress_fp = - fopen(opts->progress_file_name, "w")) == NULL); + if (opts->progress_fp == NULL) + testutil_checksys((opts->progress_fp = fopen(opts->progress_file_name, "w")) == NULL); - fp = opts->progress_fp; - __wt_seconds(NULL, &now); - testutil_assert(fprintf(fp, "[%" PRIu64 "] %s\n", now, message) >= 0); - testutil_assert(fflush(fp) == 0); + fp = opts->progress_fp; + __wt_seconds(NULL, &now); + testutil_assert(fprintf(fp, "[%" PRIu64 "] %s\n", now, message) >= 0); + testutil_assert(fflush(fp) == 0); } /* * testutil_cleanup -- - * Delete the existing work directory and free the options structure. + * Delete the existing work directory and free the options structure. */ void testutil_cleanup(TEST_OPTS *opts) { - if (opts->conn != NULL) - testutil_check(opts->conn->close(opts->conn, NULL)); + if (opts->conn != NULL) + testutil_check(opts->conn->close(opts->conn, NULL)); - if (!opts->preserve) - testutil_clean_work_dir(opts->home); + if (!opts->preserve) + testutil_clean_work_dir(opts->home); - if (opts->progress_fp != NULL) - testutil_assert(fclose(opts->progress_fp) == 0); + if (opts->progress_fp != NULL) + testutil_assert(fclose(opts->progress_fp) == 0); - free(opts->uri); - free(opts->progress_file_name); - free(opts->home); + free(opts->uri); + free(opts->progress_file_name); + free(opts->home); } /* * testutil_is_flag_set -- - * Return if an environment variable flag is set. + * Return if an environment variable flag is set. */ bool testutil_is_flag_set(const char *flag) { - const char *res; - bool flag_being_set; + const char *res; + bool flag_being_set; - if (__wt_getenv(NULL, flag, &res) != 0 || res == NULL) - return (false); + if (__wt_getenv(NULL, flag, &res) != 0 || res == NULL) + return (false); - /* - * This is a boolean test. So if the environment variable is set to any - * value other than 0, we return success. - */ - flag_being_set = res[0] != '0'; + /* + * This is a boolean test. So if the environment variable is set to any value other than 0, we + * return success. + */ + flag_being_set = res[0] != '0'; - free((void *)res); + free((void *)res); - return (flag_being_set); + return (flag_being_set); } /* * testutil_print_command_line -- - * Print command line arguments for csuite tests. + * Print command line arguments for csuite tests. */ void -testutil_print_command_line(int argc, char * const *argv) +testutil_print_command_line(int argc, char *const *argv) { - int i; + int i; - printf("Running test command: "); - for (i = 0; i < argc; i++) - printf("%s ", argv[i]); - printf("\n"); + printf("Running test command: "); + for (i = 0; i < argc; i++) + printf("%s ", argv[i]); + printf("\n"); } #ifndef _WIN32 /* * testutil_sleep_wait -- - * Wait for a process up to a number of seconds. + * Wait for a process up to a number of seconds. */ void testutil_sleep_wait(uint32_t seconds, pid_t pid) { - pid_t got; - int status; - - while (seconds > 0) { - if ((got = waitpid(pid, &status, WNOHANG|WUNTRACED)) == pid) { - if (WIFEXITED(status)) - testutil_die(EINVAL, - "Child process %" PRIu64 " exited early" - " with status %d", (uint64_t)pid, - WEXITSTATUS(status)); - if (WIFSIGNALED(status)) - testutil_die(EINVAL, - "Child process %" PRIu64 " terminated " - " with signal %d", (uint64_t)pid, - WTERMSIG(status)); - } else if (got == -1) - testutil_die(errno, "waitpid"); - - --seconds; - sleep(1); - } + pid_t got; + int status; + + while (seconds > 0) { + if ((got = waitpid(pid, &status, WNOHANG | WUNTRACED)) == pid) { + if (WIFEXITED(status)) + testutil_die(EINVAL, "Child process %" PRIu64 + " exited early" + " with status %d", + (uint64_t)pid, WEXITSTATUS(status)); + if (WIFSIGNALED(status)) + testutil_die(EINVAL, "Child process %" PRIu64 + " terminated " + " with signal %d", + (uint64_t)pid, WTERMSIG(status)); + } else if (got == -1) + testutil_die(errno, "waitpid"); + + --seconds; + sleep(1); + } } #endif /* * dcalloc -- - * Call calloc, dying on failure. + * Call calloc, dying on failure. */ void * dcalloc(size_t number, size_t size) { - void *p; + void *p; - if ((p = calloc(number, size)) != NULL) - return (p); - testutil_die(errno, "calloc: %" WT_SIZET_FMT "B", number * size); + if ((p = calloc(number, size)) != NULL) + return (p); + testutil_die(errno, "calloc: %" WT_SIZET_FMT "B", number * size); } /* * dmalloc -- - * Call malloc, dying on failure. + * Call malloc, dying on failure. */ void * dmalloc(size_t len) { - void *p; + void *p; - if ((p = malloc(len)) != NULL) - return (p); - testutil_die(errno, "malloc: %" WT_SIZET_FMT "B", len); + if ((p = malloc(len)) != NULL) + return (p); + testutil_die(errno, "malloc: %" WT_SIZET_FMT "B", len); } /* * drealloc -- - * Call realloc, dying on failure. + * Call realloc, dying on failure. */ void * drealloc(void *p, size_t len) { - void *t; + void *t; - if ((t = realloc(p, len)) != NULL) - return (t); - testutil_die(errno, "realloc: %" WT_SIZET_FMT "B", len); + if ((t = realloc(p, len)) != NULL) + return (t); + testutil_die(errno, "realloc: %" WT_SIZET_FMT "B", len); } /* * dstrdup -- - * Call strdup, dying on failure. + * Call strdup, dying on failure. */ void * dstrdup(const void *str) { - char *p; + char *p; - if ((p = strdup(str)) != NULL) - return (p); - testutil_die(errno, "strdup"); + if ((p = strdup(str)) != NULL) + return (p); + testutil_die(errno, "strdup"); } /* * dstrndup -- - * Call emulating strndup, dying on failure. Don't use actual strndup here - * as it is not supported within MSVC. + * Call emulating strndup, dying on failure. Don't use actual strndup here as it is not + * supported within MSVC. */ void * dstrndup(const char *str, size_t len) { - char *p; + char *p; - p = dcalloc(len + 1, sizeof(char)); - memcpy(p, str, len); - return (p); + p = dcalloc(len + 1, sizeof(char)); + memcpy(p, str, len); + return (p); } /* * example_setup -- - * Set the program name, create a home directory for the example programs. + * Set the program name, create a home directory for the example programs. */ const char * -example_setup(int argc, char * const *argv) +example_setup(int argc, char *const *argv) { - const char *home; + const char *home; - (void)argc; /* Unused variable */ + (void)argc; /* Unused variable */ - (void)testutil_set_progname(argv); + (void)testutil_set_progname(argv); - /* - * Create a clean test directory for this run of the test program if the - * environment variable isn't already set (as is done by make check). - */ - if ((home = getenv("WIREDTIGER_HOME")) == NULL) - home = "WT_HOME"; - testutil_make_work_dir(home); - return (home); + /* + * Create a clean test directory for this run of the test program if the environment variable + * isn't already set (as is done by make check). + */ + if ((home = getenv("WIREDTIGER_HOME")) == NULL) + home = "WT_HOME"; + testutil_make_work_dir(home); + return (home); } diff --git a/src/third_party/wiredtiger/test/utility/parse_opts.c b/src/third_party/wiredtiger/test/utility/parse_opts.c index 06d9ea31538..a1137a90b44 100644 --- a/src/third_party/wiredtiger/test/utility/parse_opts.c +++ b/src/third_party/wiredtiger/test/utility/parse_opts.c @@ -27,119 +27,116 @@ */ #include "test_util.h" -extern char *__wt_optarg; /* argument associated with option */ +extern char *__wt_optarg; /* argument associated with option */ /* * testutil_parse_opts -- - * Parse command line options for a test case. + * Parse command line options for a test case. */ int -testutil_parse_opts(int argc, char * const *argv, TEST_OPTS *opts) +testutil_parse_opts(int argc, char *const *argv, TEST_OPTS *opts) { - size_t len; - int ch; + size_t len; + int ch; - opts->do_data_ops = false; - opts->preserve = false; - opts->running = true; - opts->verbose = false; + opts->do_data_ops = false; + opts->preserve = false; + opts->running = true; + opts->verbose = false; - opts->argv0 = argv[0]; - opts->progname = testutil_set_progname(argv); + opts->argv0 = argv[0]; + opts->progname = testutil_set_progname(argv); - testutil_print_command_line(argc, argv); + testutil_print_command_line(argc, argv); - while ((ch = __wt_getopt(opts->progname, - argc, argv, "A:dh:n:o:pR:T:t:vW:")) != EOF) - switch (ch) { - case 'A': /* Number of append threads */ - opts->n_append_threads = (uint64_t)atoll(__wt_optarg); - break; - case 'd': /* Use data in multi-threaded test programs */ - opts->do_data_ops = true; - break; - case 'h': /* Home directory */ - opts->home = dstrdup(__wt_optarg); - break; - case 'n': /* Number of records */ - opts->nrecords = (uint64_t)atoll(__wt_optarg); - break; - case 'o': /* Number of operations */ - opts->nops = (uint64_t)atoll(__wt_optarg); - break; - case 'p': /* Preserve directory contents */ - opts->preserve = true; - break; - case 'R': /* Number of reader threads */ - opts->n_read_threads = (uint64_t)atoll(__wt_optarg); - break; - case 'T': /* Number of threads */ - opts->nthreads = (uint64_t)atoll(__wt_optarg); - break; - case 't': /* Table type */ - switch (__wt_optarg[0]) { - case 'C': - case 'c': - opts->table_type = TABLE_COL; - break; - case 'F': - case 'f': - opts->table_type = TABLE_FIX; - break; - case 'R': - case 'r': - opts->table_type = TABLE_ROW; - break; - } - break; - case 'v': - opts->verbose = true; - break; - case 'W': /* Number of writer threads */ - opts->n_write_threads = (uint64_t)atoll(__wt_optarg); - break; - case '?': - default: - (void)fprintf(stderr, "usage: %s " - "[-A append thread count] " - "[-d add data] " - "[-h home] " - "[-n record count] " - "[-o op count] " - "[-p] " - "[-R read thread count] " - "[-T thread count] " - "[-t c|f|r table type] " - "[-v] " - "[-W write thread count] ", - opts->progname); - return (1); - } + while ((ch = __wt_getopt(opts->progname, argc, argv, "A:dh:n:o:pR:T:t:vW:")) != EOF) + switch (ch) { + case 'A': /* Number of append threads */ + opts->n_append_threads = (uint64_t)atoll(__wt_optarg); + break; + case 'd': /* Use data in multi-threaded test programs */ + opts->do_data_ops = true; + break; + case 'h': /* Home directory */ + opts->home = dstrdup(__wt_optarg); + break; + case 'n': /* Number of records */ + opts->nrecords = (uint64_t)atoll(__wt_optarg); + break; + case 'o': /* Number of operations */ + opts->nops = (uint64_t)atoll(__wt_optarg); + break; + case 'p': /* Preserve directory contents */ + opts->preserve = true; + break; + case 'R': /* Number of reader threads */ + opts->n_read_threads = (uint64_t)atoll(__wt_optarg); + break; + case 'T': /* Number of threads */ + opts->nthreads = (uint64_t)atoll(__wt_optarg); + break; + case 't': /* Table type */ + switch (__wt_optarg[0]) { + case 'C': + case 'c': + opts->table_type = TABLE_COL; + break; + case 'F': + case 'f': + opts->table_type = TABLE_FIX; + break; + case 'R': + case 'r': + opts->table_type = TABLE_ROW; + break; + } + break; + case 'v': + opts->verbose = true; + break; + case 'W': /* Number of writer threads */ + opts->n_write_threads = (uint64_t)atoll(__wt_optarg); + break; + case '?': + default: + (void)fprintf(stderr, + "usage: %s " + "[-A append thread count] " + "[-d add data] " + "[-h home] " + "[-n record count] " + "[-o op count] " + "[-p] " + "[-R read thread count] " + "[-T thread count] " + "[-t c|f|r table type] " + "[-v] " + "[-W write thread count] ", + opts->progname); + return (1); + } - /* - * Setup the home directory if not explicitly specified. It needs to be - * unique for every test or the auto make parallel tester gets upset. - */ - if (opts->home == NULL) { - len = strlen("WT_TEST.") + strlen(opts->progname) + 10; - opts->home = dmalloc(len); - testutil_check(__wt_snprintf( - opts->home, len, "WT_TEST.%s", opts->progname)); - } + /* + * Setup the home directory if not explicitly specified. It needs to be unique for every test or + * the auto make parallel tester gets upset. + */ + if (opts->home == NULL) { + len = strlen("WT_TEST.") + strlen(opts->progname) + 10; + opts->home = dmalloc(len); + testutil_check(__wt_snprintf(opts->home, len, "WT_TEST.%s", opts->progname)); + } - /* - * Setup the progress file name. - */ - len = strlen(opts->home) + 20; - opts->progress_file_name = dmalloc(len); - testutil_check(__wt_snprintf(opts->progress_file_name, len, - "%s/progress.txt", opts->home)); + /* + * Setup the progress file name. + */ + len = strlen(opts->home) + 20; + opts->progress_file_name = dmalloc(len); + testutil_check(__wt_snprintf(opts->progress_file_name, len, "%s/progress.txt", opts->home)); - /* Setup the default URI string */ - len = strlen("table:") + strlen(opts->progname) + 10; - opts->uri = dmalloc(len); - testutil_check(__wt_snprintf( - opts->uri, len, "table:%s", opts->progname)); + /* Setup the default URI string */ + len = strlen("table:") + strlen(opts->progname) + 10; + opts->uri = dmalloc(len); + testutil_check(__wt_snprintf(opts->uri, len, "table:%s", opts->progname)); - return (0); + return (0); } diff --git a/src/third_party/wiredtiger/test/utility/test_util.h b/src/third_party/wiredtiger/test/utility/test_util.h index 7aa4eaecc0f..398727a6ca8 100644 --- a/src/third_party/wiredtiger/test/utility/test_util.h +++ b/src/third_party/wiredtiger/test/utility/test_util.h @@ -28,19 +28,19 @@ #include "wt_internal.h" #ifdef _WIN32 -#define DIR_DELIM '\\' -#define DIR_DELIM_STR "\\" -#define DIR_EXISTS_COMMAND "IF EXIST " -#define RM_COMMAND "rd /s /q " +#define DIR_DELIM '\\' +#define DIR_DELIM_STR "\\" +#define DIR_EXISTS_COMMAND "IF EXIST " +#define RM_COMMAND "rd /s /q " #else -#define DIR_DELIM '/' -#define DIR_DELIM_STR "/" -#define RM_COMMAND "rm -rf " +#define DIR_DELIM '/' +#define DIR_DELIM_STR "/" +#define RM_COMMAND "rm -rf " #endif -#define DEFAULT_DIR "WT_TEST" -#define DEFAULT_TABLE_SCHEMA "key_format=i,value_format=S" -#define MKDIR_COMMAND "mkdir " +#define DEFAULT_DIR "WT_TEST" +#define DEFAULT_TABLE_SCHEMA "key_format=i,value_format=S" +#define MKDIR_COMMAND "mkdir " #ifdef _WIN32 #include "windows_shim.h" @@ -48,179 +48,178 @@ /* Generic option parsing structure shared by all test cases. */ typedef struct { - char *home; - const char *argv0; /* Exec name */ - const char *progname; /* Truncated program name */ + char *home; + const char *argv0; /* Exec name */ + const char *progname; /* Truncated program name */ - enum { TABLE_COL=1, /* Fixed-length column store */ - TABLE_FIX=2, /* Variable-length column store */ - TABLE_ROW=3 /* Row-store */ - } table_type; + enum { + TABLE_COL = 1, /* Fixed-length column store */ + TABLE_FIX = 2, /* Variable-length column store */ + TABLE_ROW = 3 /* Row-store */ + } table_type; - FILE *progress_fp; /* Progress tracking file */ - char *progress_file_name; + FILE *progress_fp; /* Progress tracking file */ + char *progress_file_name; - bool preserve; /* Don't remove files on exit */ - bool verbose; /* Run in verbose mode */ - bool do_data_ops; /* Have schema ops use data */ - uint64_t nrecords; /* Number of records */ - uint64_t nops; /* Number of operations */ - uint64_t nthreads; /* Number of threads */ - uint64_t n_append_threads; /* Number of append threads */ - uint64_t n_read_threads; /* Number of read threads */ - uint64_t n_write_threads; /* Number of write threads */ + bool preserve; /* Don't remove files on exit */ + bool verbose; /* Run in verbose mode */ + bool do_data_ops; /* Have schema ops use data */ + uint64_t nrecords; /* Number of records */ + uint64_t nops; /* Number of operations */ + uint64_t nthreads; /* Number of threads */ + uint64_t n_append_threads; /* Number of append threads */ + uint64_t n_read_threads; /* Number of read threads */ + uint64_t n_write_threads; /* Number of write threads */ - /* - * Fields commonly shared within a test program. The test cleanup - * function will attempt to automatically free and close non-null - * resources. - */ - WT_CONNECTION *conn; - WT_SESSION *session; - bool running; - char *uri; - volatile uint64_t next_threadid; - uint64_t unique_id; - uint64_t max_inserted_id; + /* + * Fields commonly shared within a test program. The test cleanup function will attempt to + * automatically free and close non-null resources. + */ + WT_CONNECTION *conn; + WT_SESSION *session; + bool running; + char *uri; + volatile uint64_t next_threadid; + uint64_t unique_id; + uint64_t max_inserted_id; } TEST_OPTS; /* - * A structure for the data specific to a single thread of those used by the - * group of threads defined below. + * A structure for the data specific to a single thread of those used by the group of threads + * defined below. */ typedef struct { - TEST_OPTS *testopts; - int threadnum; - int thread_counter; + TEST_OPTS *testopts; + int threadnum; + int thread_counter; } TEST_PER_THREAD_OPTS; /* * testutil_assert -- - * Complain and quit if something isn't true. + * Complain and quit if something isn't true. */ -#define testutil_assert(a) do { \ - if (!(a)) \ - testutil_die(0, "%s/%d: %s", __func__, __LINE__, #a); \ -} while (0) +#define testutil_assert(a) \ + do { \ + if (!(a)) \ + testutil_die(0, "%s/%d: %s", __func__, __LINE__, #a); \ + } while (0) /* * testutil_assertfmt -- - * Complain and quit if something isn't true. + * Complain and quit if something isn't true. */ -#define testutil_assertfmt(a, fmt, ...) do { \ - if (!(a)) \ - testutil_die(0, "%s/%d: %s: " fmt, \ - __func__, __LINE__, #a, __VA_ARGS__); \ -} while (0) +#define testutil_assertfmt(a, fmt, ...) \ + do { \ + if (!(a)) \ + testutil_die(0, "%s/%d: %s: " fmt, __func__, __LINE__, #a, __VA_ARGS__); \ + } while (0) /* * testutil_check -- - * Complain and quit if a function call fails. + * Complain and quit if a function call fails. */ -#define testutil_check(call) do { \ - int __r; \ - if ((__r = (call)) != 0) \ - testutil_die( \ - __r, "%s/%d: %s", __func__, __LINE__, #call); \ -} while (0) +#define testutil_check(call) \ + do { \ + int __r; \ + if ((__r = (call)) != 0) \ + testutil_die(__r, "%s/%d: %s", __func__, __LINE__, #call); \ + } while (0) /* * testutil_checksys -- - * Complain and quit if a function call fails, returning errno. The error - * test must be specified, not just the call, because system calls fail in a - * variety of ways. + * Complain and quit if a function call fails, returning errno. The error test must be + * specified, not just the call, because system calls fail in a variety of ways. */ -#define testutil_checksys(call) do { \ - if (call) \ - testutil_die( \ - errno, "%s/%d: %s", __func__, __LINE__, #call); \ -} while (0) +#define testutil_checksys(call) \ + do { \ + if (call) \ + testutil_die(errno, "%s/%d: %s", __func__, __LINE__, #call); \ + } while (0) /* * testutil_checkfmt -- - * Complain and quit if a function call fails, with additional arguments. + * Complain and quit if a function call fails, with additional arguments. */ -#define testutil_checkfmt(call, fmt, ...) do { \ - int __r; \ - if ((__r = (call)) != 0) \ - testutil_die(__r, "%s/%d: %s: " fmt, \ - __func__, __LINE__, #call, __VA_ARGS__); \ -} while (0) +#define testutil_checkfmt(call, fmt, ...) \ + do { \ + int __r; \ + if ((__r = (call)) != 0) \ + testutil_die(__r, "%s/%d: %s: " fmt, __func__, __LINE__, #call, __VA_ARGS__); \ + } while (0) /* * error_check -- - * Complain and quit if a function call fails. A special name because it - * appears in the documentation. Ignore ENOTSUP to allow library calls which - * might not be included in any particular build. + * Complain and quit if a function call fails. A special name because it appears in the + * documentation. Ignore ENOTSUP to allow library calls which might not be included in any + * particular build. */ -#define error_check(call) do { \ - int __r; \ - if ((__r = (call)) != 0 && __r != ENOTSUP) \ - testutil_die( \ - __r, "%s/%d: %s", __func__, __LINE__, #call); \ -} while (0) +#define error_check(call) \ + do { \ + int __r; \ + if ((__r = (call)) != 0 && __r != ENOTSUP) \ + testutil_die(__r, "%s/%d: %s", __func__, __LINE__, #call); \ + } while (0) /* * scan_end_check -- - * Complain and quit if something isn't true. The same as testutil_assert, - * with a different name because it appears in the documentation. + * Complain and quit if something isn't true. The same as testutil_assert, with a different name + * because it appears in the documentation. */ -#define scan_end_check(a) testutil_assert(a) +#define scan_end_check(a) testutil_assert(a) /* * u64_to_string -- - * Convert a uint64_t to a text string. - * - * Algorithm from Andrei Alexandrescu's talk: "Three Optimization Tips for C++" + * Convert a uint64_t to a text string. Algorithm from Andrei Alexandrescu's talk: "Three + * Optimization Tips for C++" */ static inline void u64_to_string(uint64_t n, char **pp) { - static const char hundred_lookup[201] = - "0001020304050607080910111213141516171819" - "2021222324252627282930313233343536373839" - "4041424344454647484950515253545556575859" - "6061626364656667686970717273747576777879" - "8081828384858687888990919293949596979899"; - u_int i; - char *p; + static const char hundred_lookup[201] = + "0001020304050607080910111213141516171819" + "2021222324252627282930313233343536373839" + "4041424344454647484950515253545556575859" + "6061626364656667686970717273747576777879" + "8081828384858687888990919293949596979899"; + u_int i; + char *p; - /* - * The argument pointer references the last element of a buffer (which - * must be large enough to hold any possible value). - * - * Nul-terminate the buffer. - */ - for (p = *pp, *p-- = '\0'; n >= 100; n /= 100) { - i = (n % 100) * 2; - *p-- = hundred_lookup[i + 1]; - *p-- = hundred_lookup[i]; - } + /* + * The argument pointer references the last element of a buffer (which + * must be large enough to hold any possible value). + * + * Nul-terminate the buffer. + */ + for (p = *pp, *p-- = '\0'; n >= 100; n /= 100) { + i = (n % 100) * 2; + *p-- = hundred_lookup[i + 1]; + *p-- = hundred_lookup[i]; + } - /* Handle the last two digits. */ - i = (u_int)n * 2; - *p = hundred_lookup[i + 1]; - if (n >= 10) - *--p = hundred_lookup[i]; + /* Handle the last two digits. */ + i = (u_int)n * 2; + *p = hundred_lookup[i + 1]; + if (n >= 10) + *--p = hundred_lookup[i]; - /* Return a pointer to the first byte of the text string. */ - *pp = p; + /* Return a pointer to the first byte of the text string. */ + *pp = p; } /* * u64_to_string_zf -- - * Convert a uint64_t to a text string, zero-filling the buffer. + * Convert a uint64_t to a text string, zero-filling the buffer. */ static inline void u64_to_string_zf(uint64_t n, char *buf, size_t len) { - char *p; + char *p; - p = buf + (len - 1); - u64_to_string(n, &p); + p = buf + (len - 1); + u64_to_string(n, &p); - while (p > buf) - *--p = '0'; + while (p > buf) + *--p = '0'; } /* Allow tests to add their own death handling. */ @@ -229,20 +228,18 @@ extern void (*custom_die)(void); #ifdef _WIN32 __declspec(noreturn) #endif -void testutil_die(int, const char *, ...) - WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); + void testutil_die(int, const char *, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); void *dcalloc(size_t, size_t); void *dmalloc(size_t); void *drealloc(void *, size_t); void *dstrdup(const void *); void *dstrndup(const char *, size_t); -const char *example_setup(int, char * const *); +const char *example_setup(int, char *const *); /* - * The functions below can generate errors that we wish to ignore. We have - * handler functions available for them here, to avoid making tests crash - * prematurely. + * The functions below can generate errors that we wish to ignore. We have handler functions + * available for them here, to avoid making tests crash prematurely. */ int handle_op_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); int handle_op_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); @@ -256,8 +253,8 @@ void testutil_clean_work_dir(const char *); void testutil_cleanup(TEST_OPTS *); bool testutil_is_flag_set(const char *); void testutil_make_work_dir(const char *); -int testutil_parse_opts(int, char * const *, TEST_OPTS *); -void testutil_print_command_line(int argc, char * const *argv); +int testutil_parse_opts(int, char *const *, TEST_OPTS *); +void testutil_print_command_line(int argc, char *const *argv); void testutil_progress(TEST_OPTS *, const char *); #ifndef _WIN32 void testutil_sleep_wait(uint32_t, pid_t); @@ -266,4 +263,4 @@ void testutil_work_dir_from_path(char *, size_t, const char *); WT_THREAD_RET thread_append(void *); extern const char *progname; -const char *testutil_set_progname(char * const *); +const char *testutil_set_progname(char *const *); diff --git a/src/third_party/wiredtiger/test/utility/thread.c b/src/third_party/wiredtiger/test/utility/thread.c index 7dd2686fff3..4e6368d19bd 100644 --- a/src/third_party/wiredtiger/test/utility/thread.c +++ b/src/third_party/wiredtiger/test/utility/thread.c @@ -29,102 +29,92 @@ #include "test_util.h" /* - * A thread dedicated to appending records into a table. Works with fixed - * length column stores and variable length column stores. - * One thread (the first thread created by an application) checks for a - * terminating condition after each insert. + * A thread dedicated to appending records into a table. Works with fixed length column stores and + * variable length column stores. One thread (the first thread created by an application) checks for + * a terminating condition after each insert. */ WT_THREAD_RET thread_append(void *arg) { - TEST_OPTS *opts; - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_SESSION *session; - uint64_t id, recno; - char buf[64]; - - opts = (TEST_OPTS *)arg; - conn = opts->conn; - - id = __wt_atomic_fetch_addv64(&opts->next_threadid, 1); - testutil_check(conn->open_session(conn, NULL, NULL, &session)); - testutil_check( - session->open_cursor(session, opts->uri, NULL, "append", &cursor)); - - buf[0] = '\2'; - for (recno = 1; opts->running; ++recno) { - if (opts->table_type == TABLE_FIX) - cursor->set_value(cursor, buf[0]); - else { - testutil_check(__wt_snprintf(buf, sizeof(buf), - "%" PRIu64 " VALUE ------", recno)); - cursor->set_value(cursor, buf); - } - testutil_check(cursor->insert(cursor)); - if (id == 0) { - testutil_check( - cursor->get_key(cursor, &opts->max_inserted_id)); - if (opts->max_inserted_id >= opts->nrecords) - opts->running = false; - } - } - - return (WT_THREAD_RET_VALUE); + TEST_OPTS *opts; + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_SESSION *session; + uint64_t id, recno; + char buf[64]; + + opts = (TEST_OPTS *)arg; + conn = opts->conn; + + id = __wt_atomic_fetch_addv64(&opts->next_threadid, 1); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->open_cursor(session, opts->uri, NULL, "append", &cursor)); + + buf[0] = '\2'; + for (recno = 1; opts->running; ++recno) { + if (opts->table_type == TABLE_FIX) + cursor->set_value(cursor, buf[0]); + else { + testutil_check(__wt_snprintf(buf, sizeof(buf), "%" PRIu64 " VALUE ------", recno)); + cursor->set_value(cursor, buf); + } + testutil_check(cursor->insert(cursor)); + if (id == 0) { + testutil_check(cursor->get_key(cursor, &opts->max_inserted_id)); + if (opts->max_inserted_id >= opts->nrecords) + opts->running = false; + } + } + + return (WT_THREAD_RET_VALUE); } /* - * Below are a series of functions originally designed for test/fops. These - * threads perform a series of simple API access calls, such as opening and - * closing sessions and cursors. These functions require use of the - * TEST_PER_THREAD_OPTS structure in test_util.h. Additionally there are two - * event handler functions that should be used to suppress "expected" errors - * that these functions generate. An example of the use of these functions and - * structures is in the csuite test wt3363_checkpoint_op_races. + * Below are a series of functions originally designed for test/fops. These threads perform a series + * of simple API access calls, such as opening and closing sessions and cursors. These functions + * require use of the TEST_PER_THREAD_OPTS structure in test_util.h. Additionally there are two + * event handler functions that should be used to suppress "expected" errors that these functions + * generate. An example of the use of these functions and structures is in the csuite test + * wt3363_checkpoint_op_races. */ /* - * Handle errors that generated by series of functions below that we can safely - * ignore. + * Handle errors that generated by series of functions below that we can safely ignore. */ int -handle_op_error(WT_EVENT_HANDLER *handler, - WT_SESSION *session, int error, const char *errmsg) +handle_op_error(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const char *errmsg) { - (void)(handler); - (void)(session); - - /* - * Ignore complaints about missing files. It's unlikely but possible - * that checkpoints and cursor open operations can return this due to - * the sequencing of the various ops. - */ - if (error == ENOENT) - return (0); - - /* Ignore complaints about failure to open bulk cursors. */ - if (strstr( - errmsg, "bulk-load is only supported on newly created") != NULL) - return (0); - - return (fprintf(stderr, "%s\n", errmsg) < 0 ? -1 : 0); + (void)(handler); + (void)(session); + + /* + * Ignore complaints about missing files. It's unlikely but possible that checkpoints and cursor + * open operations can return this due to the sequencing of the various ops. + */ + if (error == ENOENT) + return (0); + + /* Ignore complaints about failure to open bulk cursors. */ + if (strstr(errmsg, "bulk-load is only supported on newly created") != NULL) + return (0); + + return (fprintf(stderr, "%s\n", errmsg) < 0 ? -1 : 0); } /* * Handle messages generated by the functions below that we can safely ignore. */ int -handle_op_message(WT_EVENT_HANDLER *handler, - WT_SESSION *session, const char *message) +handle_op_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message) { - (void)(handler); - (void)(session); + (void)(handler); + (void)(session); - /* Ignore messages about failing to create forced checkpoints. */ - if (strstr(message, "forced or named checkpoint") != NULL) - return (0); + /* Ignore messages about failing to create forced checkpoints. */ + if (strstr(message, "forced or named checkpoint") != NULL) + return (0); - return (printf("%s\n", message) < 0 ? -1 : 0); + return (printf("%s\n", message) < 0 ? -1 : 0); } /* @@ -133,34 +123,32 @@ handle_op_message(WT_EVENT_HANDLER *handler, void op_bulk(void *arg) { - TEST_OPTS *opts; - TEST_PER_THREAD_OPTS *args; - WT_CURSOR *c; - WT_SESSION *session; - int ret; - - args = (TEST_PER_THREAD_OPTS *)arg; - opts = args->testopts; - - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - - if ((ret = session->create(session, - opts->uri, DEFAULT_TABLE_SCHEMA)) != 0) - if (ret != EEXIST && ret != EBUSY) - testutil_die(ret, "session.create"); - - if (ret == 0) { - __wt_yield(); - if ((ret = session->open_cursor(session, - opts->uri, NULL, "bulk,checkpoint_wait=false", &c)) == 0) { - testutil_check(c->close(c)); - } else if (ret != ENOENT && ret != EBUSY && ret != EINVAL) - testutil_die(ret, "session.open_cursor bulk"); - } - - testutil_check(session->close(session, NULL)); - args->thread_counter++; + TEST_OPTS *opts; + TEST_PER_THREAD_OPTS *args; + WT_CURSOR *c; + WT_SESSION *session; + int ret; + + args = (TEST_PER_THREAD_OPTS *)arg; + opts = args->testopts; + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + if ((ret = session->create(session, opts->uri, DEFAULT_TABLE_SCHEMA)) != 0) + if (ret != EEXIST && ret != EBUSY) + testutil_die(ret, "session.create"); + + if (ret == 0) { + __wt_yield(); + if ((ret = session->open_cursor( + session, opts->uri, NULL, "bulk,checkpoint_wait=false", &c)) == 0) { + testutil_check(c->close(c)); + } else if (ret != ENOENT && ret != EBUSY && ret != EINVAL) + testutil_die(ret, "session.open_cursor bulk"); + } + + testutil_check(session->close(session, NULL)); + args->thread_counter++; } /* @@ -169,54 +157,51 @@ op_bulk(void *arg) void op_bulk_unique(void *arg) { - TEST_OPTS *opts; - TEST_PER_THREAD_OPTS *args; - WT_CURSOR *c; - WT_RAND_STATE rnd; - WT_SESSION *session; - int ret; - char new_uri[64]; - - args = (TEST_PER_THREAD_OPTS *)arg; - opts = args->testopts; - __wt_random_init_seed(NULL, &rnd); - - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - - /* Generate a unique object name. */ - testutil_check(__wt_snprintf( - new_uri, sizeof(new_uri), "%s.%" PRIu64, - opts->uri, __wt_atomic_add64(&opts->unique_id, 1))); - testutil_check(session->create(session, new_uri, DEFAULT_TABLE_SCHEMA)); - - __wt_yield(); - - /* - * Opening a bulk cursor may have raced with a forced checkpoint - * which created a checkpoint of the empty file, and triggers an EINVAL. - */ - if ((ret = session->open_cursor( - session, new_uri, NULL, "bulk,checkpoint_wait=false", &c)) == 0) { - testutil_check(c->close(c)); - } else if (ret != EINVAL && ret != EBUSY) - testutil_die(ret, - "session.open_cursor bulk unique: %s", new_uri); - - while ((ret = session->drop(session, new_uri, __wt_random(&rnd) & 1 ? - "force,checkpoint_wait=false" : "checkpoint_wait=false")) != 0) - if (ret != EBUSY) - testutil_die(ret, "session.drop: %s", new_uri); - else - /* - * The EBUSY is expected when we run with - * checkpoint_wait set to false, so we increment the - * counter while in this loop to avoid false positives. - */ - args->thread_counter++; - - testutil_check(session->close(session, NULL)); - args->thread_counter++; + TEST_OPTS *opts; + TEST_PER_THREAD_OPTS *args; + WT_CURSOR *c; + WT_RAND_STATE rnd; + WT_SESSION *session; + int ret; + char new_uri[64]; + + args = (TEST_PER_THREAD_OPTS *)arg; + opts = args->testopts; + __wt_random_init_seed(NULL, &rnd); + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + /* Generate a unique object name. */ + testutil_check(__wt_snprintf( + new_uri, sizeof(new_uri), "%s.%" PRIu64, opts->uri, __wt_atomic_add64(&opts->unique_id, 1))); + testutil_check(session->create(session, new_uri, DEFAULT_TABLE_SCHEMA)); + + __wt_yield(); + + /* + * Opening a bulk cursor may have raced with a forced checkpoint which created a checkpoint of + * the empty file, and triggers an EINVAL. + */ + if ((ret = session->open_cursor(session, new_uri, NULL, "bulk,checkpoint_wait=false", &c)) == + 0) { + testutil_check(c->close(c)); + } else if (ret != EINVAL && ret != EBUSY) + testutil_die(ret, "session.open_cursor bulk unique: %s", new_uri); + + while ( + (ret = session->drop(session, new_uri, + __wt_random(&rnd) & 1 ? "force,checkpoint_wait=false" : "checkpoint_wait=false")) != 0) + if (ret != EBUSY) + testutil_die(ret, "session.drop: %s", new_uri); + else + /* + * The EBUSY is expected when we run with checkpoint_wait set to false, so we increment + * the counter while in this loop to avoid false positives. + */ + args->thread_counter++; + + testutil_check(session->close(session, NULL)); + args->thread_counter++; } /* @@ -225,36 +210,34 @@ op_bulk_unique(void *arg) void op_cursor(void *arg) { - TEST_OPTS *opts; - TEST_PER_THREAD_OPTS *args; - WT_CURSOR *cursor; - WT_SESSION *session; - int i, ret; - - args = (TEST_PER_THREAD_OPTS *)arg; - opts = args->testopts; - - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - - if ((ret = session->open_cursor( - session, opts->uri, NULL, NULL, &cursor)) != 0) { - if (ret != ENOENT && ret != EBUSY) - testutil_die(ret, "session.open_cursor"); - } else { - /* Put some data in if asked to. */ - if (args->testopts->do_data_ops) { - for (i = 0; i < 1000; i++) { - cursor->set_key(cursor, i); - cursor->set_value(cursor, "abcdef"); - cursor->insert(cursor); - } - } - testutil_check(cursor->close(cursor)); - } - - testutil_check(session->close(session, NULL)); - args->thread_counter++; + TEST_OPTS *opts; + TEST_PER_THREAD_OPTS *args; + WT_CURSOR *cursor; + WT_SESSION *session; + int i, ret; + + args = (TEST_PER_THREAD_OPTS *)arg; + opts = args->testopts; + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + if ((ret = session->open_cursor(session, opts->uri, NULL, NULL, &cursor)) != 0) { + if (ret != ENOENT && ret != EBUSY) + testutil_die(ret, "session.open_cursor"); + } else { + /* Put some data in if asked to. */ + if (args->testopts->do_data_ops) { + for (i = 0; i < 1000; i++) { + cursor->set_key(cursor, i); + cursor->set_value(cursor, "abcdef"); + cursor->insert(cursor); + } + } + testutil_check(cursor->close(cursor)); + } + + testutil_check(session->close(session, NULL)); + args->thread_counter++; } /* @@ -263,24 +246,22 @@ op_cursor(void *arg) void op_create(void *arg) { - TEST_OPTS *opts; - TEST_PER_THREAD_OPTS *args; - WT_SESSION *session; - int ret; + TEST_OPTS *opts; + TEST_PER_THREAD_OPTS *args; + WT_SESSION *session; + int ret; - args = (TEST_PER_THREAD_OPTS *)arg; - opts = args->testopts; + args = (TEST_PER_THREAD_OPTS *)arg; + opts = args->testopts; - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); - if ((ret = session->create(session, - opts->uri, DEFAULT_TABLE_SCHEMA)) != 0) - if (ret != EEXIST && ret != EBUSY) - testutil_die(ret, "session.create"); + if ((ret = session->create(session, opts->uri, DEFAULT_TABLE_SCHEMA)) != 0) + if (ret != EEXIST && ret != EBUSY) + testutil_die(ret, "session.create"); - testutil_check(session->close(session, NULL)); - args->thread_counter++; + testutil_check(session->close(session, NULL)); + args->thread_counter++; } /* @@ -289,41 +270,39 @@ op_create(void *arg) void op_create_unique(void *arg) { - TEST_OPTS *opts; - TEST_PER_THREAD_OPTS *args; - WT_RAND_STATE rnd; - WT_SESSION *session; - int ret; - char new_uri[64]; - - args = (TEST_PER_THREAD_OPTS *)arg; - opts = args->testopts; - __wt_random_init_seed(NULL, &rnd); - - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - - /* Generate a unique object name. */ - testutil_check(__wt_snprintf( - new_uri, sizeof(new_uri), "%s.%" PRIu64, - opts->uri, __wt_atomic_add64(&opts->unique_id, 1))); - testutil_check(session->create(session, new_uri, DEFAULT_TABLE_SCHEMA)); - - __wt_yield(); - while ((ret = session->drop(session, new_uri, __wt_random(&rnd) & 1 ? - "force,checkpoint_wait=false" : "checkpoint_wait=false")) != 0) - if (ret != EBUSY) - testutil_die(ret, "session.drop: %s", new_uri); - else - /* - * The EBUSY is expected when we run with - * checkpoint_wait set to false, so we increment the - * counter while in this loop to avoid false positives. - */ - args->thread_counter++; - - testutil_check(session->close(session, NULL)); - args->thread_counter++; + TEST_OPTS *opts; + TEST_PER_THREAD_OPTS *args; + WT_RAND_STATE rnd; + WT_SESSION *session; + int ret; + char new_uri[64]; + + args = (TEST_PER_THREAD_OPTS *)arg; + opts = args->testopts; + __wt_random_init_seed(NULL, &rnd); + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + /* Generate a unique object name. */ + testutil_check(__wt_snprintf( + new_uri, sizeof(new_uri), "%s.%" PRIu64, opts->uri, __wt_atomic_add64(&opts->unique_id, 1))); + testutil_check(session->create(session, new_uri, DEFAULT_TABLE_SCHEMA)); + + __wt_yield(); + while ( + (ret = session->drop(session, new_uri, + __wt_random(&rnd) & 1 ? "force,checkpoint_wait=false" : "checkpoint_wait=false")) != 0) + if (ret != EBUSY) + testutil_die(ret, "session.drop: %s", new_uri); + else + /* + * The EBUSY is expected when we run with checkpoint_wait set to false, so we increment + * the counter while in this loop to avoid false positives. + */ + args->thread_counter++; + + testutil_check(session->close(session, NULL)); + args->thread_counter++; } /* @@ -332,24 +311,23 @@ op_create_unique(void *arg) void op_drop(void *arg) { - TEST_OPTS *opts; - TEST_PER_THREAD_OPTS *args; - WT_RAND_STATE rnd; - WT_SESSION *session; - int ret; - - args = (TEST_PER_THREAD_OPTS *)arg; - opts = args->testopts; - __wt_random_init_seed(NULL, &rnd); - - testutil_check( - opts->conn->open_session(opts->conn, NULL, NULL, &session)); - - if ((ret = session->drop(session, opts->uri, __wt_random(&rnd) & 1 ? - "force,checkpoint_wait=false" : "checkpoint_wait=false")) != 0) - if (ret != ENOENT && ret != EBUSY) - testutil_die(ret, "session.drop"); - - testutil_check(session->close(session, NULL)); - args->thread_counter++; + TEST_OPTS *opts; + TEST_PER_THREAD_OPTS *args; + WT_RAND_STATE rnd; + WT_SESSION *session; + int ret; + + args = (TEST_PER_THREAD_OPTS *)arg; + opts = args->testopts; + __wt_random_init_seed(NULL, &rnd); + + testutil_check(opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + if ((ret = session->drop(session, opts->uri, + __wt_random(&rnd) & 1 ? "force,checkpoint_wait=false" : "checkpoint_wait=false")) != 0) + if (ret != ENOENT && ret != EBUSY) + testutil_die(ret, "session.drop"); + + testutil_check(session->close(session, NULL)); + args->thread_counter++; } diff --git a/src/third_party/wiredtiger/test/windows/windows_shim.c b/src/third_party/wiredtiger/test/windows/windows_shim.c index a8bff61a19f..4fa6c8116de 100644 --- a/src/third_party/wiredtiger/test/windows/windows_shim.c +++ b/src/third_party/wiredtiger/test/windows/windows_shim.c @@ -31,102 +31,100 @@ int sleep(int seconds) { - Sleep(seconds * 1000); - return (0); + Sleep(seconds * 1000); + return (0); } int usleep(useconds_t useconds) { - uint32_t milli; - milli = useconds / 1000; + uint32_t milli; + milli = useconds / 1000; - if (milli == 0) - milli++; + if (milli == 0) + milli++; - Sleep(milli); + Sleep(milli); - return (0); + return (0); } int -gettimeofday(struct timeval* tp, void* tzp) +gettimeofday(struct timeval *tp, void *tzp) { - FILETIME time; - uint64_t ns100; + FILETIME time; + uint64_t ns100; - tzp = tzp; + tzp = tzp; - GetSystemTimeAsFileTime(&time); + GetSystemTimeAsFileTime(&time); - ns100 = (((int64_t)time.dwHighDateTime << 32) + time.dwLowDateTime) - - 116444736000000000LL; - tp->tv_sec = ns100 / 10000000; - tp->tv_usec = (long)((ns100 % 10000000) / 10); + ns100 = (((int64_t)time.dwHighDateTime << 32) + time.dwLowDateTime) - 116444736000000000LL; + tp->tv_sec = ns100 / 10000000; + tp->tv_usec = (long)((ns100 % 10000000) / 10); - return (0); + return (0); } int pthread_rwlock_destroy(pthread_rwlock_t *lock) { - lock = lock; /* Unused variable. */ - return (0); + lock = lock; /* Unused variable. */ + return (0); } int -pthread_rwlock_init(pthread_rwlock_t *rwlock, - const pthread_rwlockattr_t *ignored) +pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *ignored) { - ignored = ignored; /* Unused variable. */ - InitializeSRWLock(&rwlock->rwlock); - rwlock->exclusive_locked = 0; + ignored = ignored; /* Unused variable. */ + InitializeSRWLock(&rwlock->rwlock); + rwlock->exclusive_locked = 0; - return (0); + return (0); } int pthread_rwlock_unlock(pthread_rwlock_t *rwlock) { - if (rwlock->exclusive_locked != 0) { - rwlock->exclusive_locked = 0; - ReleaseSRWLockExclusive(&rwlock->rwlock); - } else - ReleaseSRWLockShared(&rwlock->rwlock); + if (rwlock->exclusive_locked != 0) { + rwlock->exclusive_locked = 0; + ReleaseSRWLockExclusive(&rwlock->rwlock); + } else + ReleaseSRWLockShared(&rwlock->rwlock); - return (0); + return (0); } int pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock) { - return (TryAcquireSRWLockShared(&rwlock->rwlock) ? 0 : EBUSY); + return (TryAcquireSRWLockShared(&rwlock->rwlock) ? 0 : EBUSY); } int pthread_rwlock_rdlock(pthread_rwlock_t *rwlock) { - AcquireSRWLockShared(&rwlock->rwlock); - return (0); + AcquireSRWLockShared(&rwlock->rwlock); + return (0); } int pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock) { - if (TryAcquireSRWLockExclusive(&rwlock->rwlock)) { - rwlock->exclusive_locked = GetCurrentThreadId(); - return (0); - } + if (TryAcquireSRWLockExclusive(&rwlock->rwlock)) { + rwlock->exclusive_locked = GetCurrentThreadId(); + return (0); + } - return (EBUSY); + return (EBUSY); } int pthread_rwlock_wrlock(pthread_rwlock_t *rwlock) { - AcquireSRWLockExclusive(&rwlock->rwlock); + AcquireSRWLockExclusive(&rwlock->rwlock); - rwlock->exclusive_locked = GetCurrentThreadId(); + rwlock->exclusive_locked = GetCurrentThreadId(); - return (0); + return (0); } diff --git a/src/third_party/wiredtiger/test/windows/windows_shim.h b/src/third_party/wiredtiger/test/windows/windows_shim.h index b0a2ff7d076..2673c0ce7e8 100644 --- a/src/third_party/wiredtiger/test/windows/windows_shim.h +++ b/src/third_party/wiredtiger/test/windows/windows_shim.h @@ -27,33 +27,33 @@ */ #include "wt_internal.h" -#include <direct.h> /* _mkdir */ +#include <direct.h> /* _mkdir */ /* Windows does not define constants for access() */ -#define R_OK 04 -#define X_OK R_OK +#define R_OK 04 +#define X_OK R_OK /* snprintf does not exist on <= VS 2013 */ #if _MSC_VER < 1900 -#define snprintf __wt_snprintf +#define snprintf __wt_snprintf #endif -#define strcasecmp stricmp +#define strcasecmp stricmp /* * Emulate <sys/stat.h> */ -#define mkdir(path, mode) _mkdir(path) +#define mkdir(path, mode) _mkdir(path) /* * Emulate <sys/time.h> */ struct timeval { - time_t tv_sec; - int64_t tv_usec; + time_t tv_sec; + int64_t tv_usec; }; -int gettimeofday(struct timeval* tp, void* tzp); +int gettimeofday(struct timeval *tp, void *tzp); /* * Emulate <unistd.h> @@ -63,20 +63,18 @@ typedef uint32_t useconds_t; int sleep(int seconds); int usleep(useconds_t useconds); -#define lseek(fd, offset, origin) \ - _lseek(fd, (long)(offset), origin) -#define write(fd, buffer, count) \ - _write(fd, buffer, (unsigned int)(count)) +#define lseek(fd, offset, origin) _lseek(fd, (long)(offset), origin) +#define write(fd, buffer, count) _write(fd, buffer, (unsigned int)(count)) /* * Emulate the <pthread.h> support we need for tests and example code. */ -typedef CRITICAL_SECTION pthread_mutex_t; +typedef CRITICAL_SECTION pthread_mutex_t; typedef CONDITION_VARIABLE pthread_cond_t; struct rwlock_wrapper { - SRWLOCK rwlock; - DWORD exclusive_locked; + SRWLOCK rwlock; + DWORD exclusive_locked; }; struct rwlock_wrapper; @@ -87,8 +85,7 @@ typedef HANDLE pthread_t; typedef int pthread_rwlockattr_t; typedef int pthread_attr_t; -int pthread_create( - pthread_t *, const pthread_attr_t *, void *(*)(void *), void *); +int pthread_create(pthread_t *, const pthread_attr_t *, void *(*)(void *), void *); int pthread_join(pthread_t, void **); int pthread_rwlock_destroy(pthread_rwlock_t *); int pthread_rwlock_init(pthread_rwlock_t *, const pthread_rwlockattr_t *); |