diff options
3 files changed, 196 insertions, 119 deletions
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 025dedd3f32..2292cb1545b 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.1", - "commit": "c7c5bfb98c2cc5d2323df49ffb5d161dea3a333e" + "commit": "0d11c31127fdeefad6a85b9567923e484c28dd0e" } diff --git a/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c b/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c index 8cd8c4b3e77..6e60b1ee229 100644 --- a/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c @@ -45,8 +45,10 @@ * better view on what is happening. */ static const char conn_config[] = "create,cache_size=2GB,statistics=(all)"; -static const char table_config[] = - "allocation_size=4KB,leaf_page_max=4KB,key_format=i,value_format=QQQS"; +static const char table_config_row[] = + "allocation_size=4KB,leaf_page_max=4KB,key_format=Q,value_format=QQQS"; +static const char table_config_col[] = + "allocation_size=4KB,leaf_page_max=4KB,key_format=r,value_format=QQQS"; static char data_str[1024] = ""; static pthread_t thread_compact; @@ -58,42 +60,47 @@ struct thread_data { }; /* Forward declarations. */ -static void run_test(bool stress_test, const char *home, const char *uri); -static void *thread_func_compact(void *arg); -static void *thread_func_checkpoint(void *arg); -static void populate(WT_SESSION *session, const char *uri); -static void remove_records(WT_SESSION *session, const char *uri); -static uint64_t get_file_size(WT_SESSION *session, const char *uri); -static void set_timing_stress_checkpoint(WT_CONNECTION *conn); +static void run_test_clean(bool, bool, bool, const char *, const char *, const char *ri); +static void run_test(bool, bool, const char *, const char *); +static void *thread_func_compact(void *); +static void *thread_func_checkpoint(void *); +static void populate(WT_SESSION *, const char *); +static void remove_records(WT_SESSION *, const char *); +static void get_file_stats(WT_SESSION *, const char *, uint64_t *, uint64_t *); +static void set_timing_stress_checkpoint(WT_CONNECTION *); +static bool check_db_size(WT_SESSION *, const char *); /* Methods implementation. */ int main(int argc, char *argv[]) { TEST_OPTS *opts, _opts; - char home_cv[512]; opts = &_opts; memset(opts, 0, sizeof(*opts)); testutil_check(testutil_parse_opts(argc, argv, opts)); /* - * First run test with WT_TIMING_STRESS_CHECKPOINT_SLOW. + * First, run test with WT_TIMING_STRESS_CHECKPOINT_SLOW. Row store case. */ - printf("Running stress test...\n"); - run_test(true, opts->home, opts->uri); + run_test_clean(true, false, opts->preserve, opts->home, "SR", opts->uri); /* - * Now run test where compact and checkpoint threads are synchronized using condition variable. + * Now, run test where compact and checkpoint threads are synchronized using condition variable. + * Row store case. */ - printf("Running normal test...\n"); - testutil_assert(sizeof(home_cv) > strlen(opts->home) + 3); - sprintf(home_cv, "%s.CV", opts->home); - run_test(false, home_cv, opts->uri); + run_test_clean(false, false, opts->preserve, opts->home, "NR", opts->uri); - /* Cleanup */ - if (!opts->preserve) - testutil_clean_work_dir(home_cv); + /* + * Next, run test with WT_TIMING_STRESS_CHECKPOINT_SLOW. Column store case. + */ + run_test_clean(true, true, opts->preserve, opts->home, "SC", opts->uri); + + /* + * Finally, run test where compact and checkpoint threads are synchronized using condition + * variable. Column store case. + */ + run_test_clean(false, true, opts->preserve, opts->home, "NC", opts->uri); testutil_cleanup(opts); @@ -101,13 +108,31 @@ main(int argc, char *argv[]) } static void -run_test(bool stress_test, const char *home, const char *uri) +run_test_clean(bool stress_test, bool column_store, bool preserve, const char *home, + const char *suffix, const char *uri) +{ + char home_full[512]; + + printf("\n"); + printf("Running %s test with %s store...\n", stress_test ? "stress" : "normal", + column_store ? "column" : "row"); + testutil_assert(sizeof(home_full) > strlen(home) + strlen(suffix) + 2); + sprintf(home_full, "%s.%s", home, suffix); + run_test(stress_test, column_store, home_full, uri); + + /* Cleanup */ + if (!preserve) + testutil_clean_work_dir(home_full); +} + +static void +run_test(bool stress_test, bool column_store, const char *home, const char *uri) { struct thread_data td; WT_CONNECTION *conn; WT_SESSION *session; pthread_t thread_checkpoint; - uint64_t file_sz_after, file_sz_before; + bool size_check_res; testutil_make_work_dir(home); testutil_check(wiredtiger_open(home, NULL, conn_config, &conn)); @@ -123,7 +148,8 @@ run_test(bool stress_test, const char *home, const char *uri) testutil_check(conn->open_session(conn, NULL, NULL, &session)); /* Create and populate table. Checkpoint the data after that. */ - testutil_check(session->create(session, uri, table_config)); + testutil_check( + session->create(session, uri, column_store ? table_config_col : table_config_row)); populate(session, uri); testutil_check(session->checkpoint(session, NULL)); @@ -134,8 +160,6 @@ run_test(bool stress_test, const char *home, const char *uri) */ remove_records(session, uri); - file_sz_before = get_file_size(session, uri); - td.conn = conn; td.uri = uri; td.cond = NULL; @@ -156,7 +180,7 @@ run_test(bool stress_test, const char *home, const char *uri) (void)pthread_join(thread_checkpoint, NULL); (void)pthread_join(thread_compact, NULL); - file_sz_after = get_file_size(session, uri); + size_check_res = check_db_size(session, uri); /* Cleanup */ if (!stress_test) { @@ -170,12 +194,11 @@ run_test(bool stress_test, const char *home, const char *uri) testutil_check(conn->close(conn, NULL)); conn = NULL; - /* Check if there's at least 10% compaction. */ - printf(" - Compressed file size MB: %f\n - Original file size MB: %f\n", - file_sz_after / (1024.0 * 1024), file_sz_before / (1024.0 * 1024)); - - /* Make sure the compact operation has reduced the file size by at least 20%. */ - testutil_assert((file_sz_before / 100) * 80 > file_sz_after); + /* + * Check if there's more than 10% available space in the file. Checking result here to allow + * connection to close properly. + */ + testutil_assert(size_check_res); } static void * @@ -274,20 +297,19 @@ populate(WT_SESSION *session, const char *uri) { WT_CURSOR *cursor; WT_RAND_STATE rnd; - uint64_t val; - int i, str_len; + uint64_t i, str_len, val; __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); str_len = sizeof(data_str) / sizeof(data_str[0]); for (i = 0; i < str_len - 1; i++) - data_str[i] = 'a' + __wt_random(&rnd) % 26; + data_str[i] = 'a' + (uint32_t)__wt_random(&rnd) % 26; data_str[str_len - 1] = '\0'; testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); for (i = 0; i < NUM_RECORDS; i++) { - cursor->set_key(cursor, i); + cursor->set_key(cursor, i + 1); val = (uint64_t)__wt_random(&rnd); cursor->set_value(cursor, val, val, val, data_str); testutil_check(cursor->insert(cursor)); @@ -301,13 +323,13 @@ static void remove_records(WT_SESSION *session, const char *uri) { WT_CURSOR *cursor; - int i; + uint64_t i; testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); /* Remove 1/3 of the records from the middle of the key range. */ for (i = NUM_RECORDS / 3; i < (NUM_RECORDS * 2) / 3; i++) { - cursor->set_key(cursor, i); + cursor->set_key(cursor, i + 1); testutil_check(cursor->remove(cursor)); } @@ -315,22 +337,27 @@ remove_records(WT_SESSION *session, const char *uri) cursor = NULL; } -static uint64_t -get_file_size(WT_SESSION *session, const char *uri) +static void +get_file_stats(WT_SESSION *session, const char *uri, uint64_t *file_sz, uint64_t *avail_bytes) { WT_CURSOR *cur_stat; - uint64_t val; char *descr, *str_val, stat_uri[128]; sprintf(stat_uri, "statistics:%s", uri); testutil_check(session->open_cursor(session, stat_uri, NULL, "statistics=(all)", &cur_stat)); + + /* Get file size. */ cur_stat->set_key(cur_stat, WT_STAT_DSRC_BLOCK_SIZE); testutil_check(cur_stat->search(cur_stat)); - testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, &val)); + testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, file_sz)); + + /* Get bytes available for reuse. */ + cur_stat->set_key(cur_stat, WT_STAT_DSRC_BLOCK_REUSE_BYTES); + testutil_check(cur_stat->search(cur_stat)); + testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, avail_bytes)); + testutil_check(cur_stat->close(cur_stat)); cur_stat = NULL; - - return (val); } static void @@ -341,3 +368,19 @@ set_timing_stress_checkpoint(WT_CONNECTION *conn) conn_impl = (WT_CONNECTION_IMPL *)conn; conn_impl->timing_stress_flags |= WT_TIMING_STRESS_CHECKPOINT_SLOW; } + +static bool +check_db_size(WT_SESSION *session, const char *uri) +{ + uint64_t file_sz, avail_bytes, available_pct; + + get_file_stats(session, uri, &file_sz, &avail_bytes); + + /* Check if there's maximum of 10% space available after compaction. */ + available_pct = (avail_bytes * 100) / file_sz; + printf(" - Compacted file size: %" PRIu64 "MB (%" PRIu64 "B)\n - Available for reuse: %" PRIu64 + "MB (%" PRIu64 "B)\n - %" PRIu64 "%% space available in the file.\n", + file_sz / WT_MEGABYTE, file_sz, avail_bytes / WT_MEGABYTE, avail_bytes, available_pct); + + return (available_pct <= 10); +} diff --git a/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c b/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c index 1a495ccf06f..22a96c50f78 100644 --- a/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c @@ -46,12 +46,15 @@ * better view on what is happening. */ static const char conn_config[] = "create,cache_size=2GB,statistics=(all)"; -static const char table_config[] = - "allocation_size=4KB,leaf_page_max=4KB,key_format=i,value_format=S"; +static const char table_config_row[] = + "allocation_size=4KB,leaf_page_max=4KB,key_format=Q,value_format=" WT_UNCHECKED_STRING(QS); +static const char table_config_col[] = + "allocation_size=4KB,leaf_page_max=4KB,key_format=r,value_format=" WT_UNCHECKED_STRING(QS); static char data_str[1024] = ""; -static const char ckpt_file[] = "WT_TEST.compact-stress/checkpoint_done"; -static const char working_dir[] = "WT_TEST.compact-stress"; +static const char ckpt_file_fmt[] = "%s/checkpoint_done"; +static const char working_dir_row[] = "WT_TEST.compact-stress-row"; +static const char working_dir_col[] = "WT_TEST.compact-stress-col"; static const char uri1[] = "table:compact1"; static const char uri2[] = "table:compact2"; @@ -79,12 +82,14 @@ static WT_EVENT_HANDLER event_handler = { static void sig_handler(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); /* Forward declarations. */ -static void run_test(const char *home); -static void populate(WT_SESSION *session, int start, int end); -static void remove_records(WT_SESSION *session, const char *uri, int start, int end); -static void verify_tables(WT_SESSION *session); -static int verify_tables_helper(WT_SESSION *session, const char *table1, const char *table2); -static uint64_t get_file_size(WT_SESSION *session, const char *uri); +static int run_test(bool, bool); +static void workload_compact(const char *, const char *); +static void populate(WT_SESSION *, uint64_t, uint64_t); +static void remove_records(WT_SESSION *, const char *, uint64_t, uint64_t); +static void verify_tables(WT_SESSION *); +static int verify_tables_helper(WT_SESSION *, const char *, const char *); +static void get_file_stats(WT_SESSION *, const char *, uint64_t *, uint64_t *); +static void log_db_size(WT_SESSION *, const char *); /* * Signal handler to catch if the child died unexpectedly. @@ -107,21 +112,36 @@ int main(int argc, char *argv[]) { TEST_OPTS *opts, _opts; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + + testutil_assert(run_test(false, opts->preserve) == EXIT_SUCCESS); + + testutil_assert(run_test(true, opts->preserve) == EXIT_SUCCESS); + + testutil_cleanup(opts); + + return (EXIT_SUCCESS); +} + +static int +run_test(bool column_store, bool preserve) +{ WT_CONNECTION *conn; WT_SESSION *session; - char home[1024]; + char ckpt_file[2048], home[1024]; int status; pid_t pid; struct sigaction sa; struct stat sb; - opts = &_opts; - memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); - - testutil_work_dir_from_path(home, sizeof(home), working_dir); + testutil_work_dir_from_path( + home, sizeof(home), column_store ? working_dir_col : working_dir_row); + printf("\n"); printf("Work directory: %s\n", home); testutil_make_work_dir(home); @@ -133,7 +153,7 @@ main(int argc, char *argv[]) if (pid == 0) { /* child */ - run_test(home); + workload_compact(home, column_store ? table_config_col : table_config_row); /* * We do not expect test to reach here. The child process should have been killed by the * parent process. @@ -148,6 +168,7 @@ main(int argc, char *argv[]) * time we notice that child process has written a checkpoint. That allows the test to run * correctly on really slow machines. */ + sprintf(ckpt_file, ckpt_file_fmt, home); while (stat(ckpt_file, &sb) != 0) testutil_sleep_wait(1, pid); @@ -168,34 +189,37 @@ main(int argc, char *argv[]) /* Clean-up. */ testutil_check(session->close(session, NULL)); + session = NULL; + testutil_check(conn->close(conn, NULL)); - if (!opts->preserve) + conn = NULL; + + if (!preserve) testutil_clean_work_dir(home); - testutil_cleanup(opts); return (EXIT_SUCCESS); } static void -run_test(const char *home) +workload_compact(const char *home, const char *table_config) { - + FILE *fp; WT_CONNECTION *conn; + WT_RAND_STATE rnd; WT_SESSION *session; - FILE *fp; bool first_ckpt; - int i; - int key_range_start; - time_t t; - uint64_t file_sz_after, file_sz_before; + char ckpt_file[2048]; + uint32_t i; + uint64_t key_range_start; first_ckpt = false; - srand((u_int)time(&t)); testutil_check(wiredtiger_open(home, &event_handler, conn_config, &conn)); testutil_check(conn->open_session(conn, NULL, NULL, &session)); + __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); + /* Create and populate table. Checkpoint the data after that. */ testutil_check(session->create(session, uri1, table_config)); testutil_check(session->create(session, uri2, table_config)); @@ -208,13 +232,14 @@ run_test(const char *home) */ for (i = 0; i < 40; i++) { - printf("Running Loop: %i\n", i + 1); + printf("Running Loop: %" PRIu32 "\n", i + 1); testutil_check(session->checkpoint(session, NULL)); /* * Create the checkpoint file so that the parent process knows at least one checkpoint has * finished and can start its timer. */ if (!first_ckpt) { + sprintf(ckpt_file, ckpt_file_fmt, home); testutil_checksys((fp = fopen(ckpt_file, "w")) == NULL); testutil_checksys(fclose(fp) != 0); first_ckpt = true; @@ -224,20 +249,14 @@ run_test(const char *home) * Remove 1/3 of data from the middle of the key range to let compact relocate blocks from * the end of the file. */ - key_range_start = rand() % (((NUM_RECORDS * 2) / 3) - 1); + key_range_start = (uint64_t)__wt_random(&rnd) % (((NUM_RECORDS * 2) / 3) - 1); remove_records(session, uri1, key_range_start, key_range_start + NUM_RECORDS / 3); remove_records(session, uri2, key_range_start, key_range_start + NUM_RECORDS / 3); /* Only perform compaction on the first table. */ - file_sz_before = get_file_size(session, uri1); - testutil_check(session->compact(session, uri1, NULL)); - file_sz_after = get_file_size(session, uri1); - - /* Check if there's at least 10% compaction. */ - printf(" - Compressed file size MB: %f\n - Original file size MB: %f\n", - file_sz_after / (1024.0 * 1024), file_sz_before / (1024.0 * 1024)); + log_db_size(session, uri1); /* Put the deleted records back. */ populate(session, key_range_start, key_range_start + NUM_RECORDS / 3); @@ -251,30 +270,32 @@ run_test(const char *home) } static void -populate(WT_SESSION *session, int start, int end) +populate(WT_SESSION *session, uint64_t start, uint64_t end) { WT_CURSOR *cursor_1; WT_CURSOR *cursor_2; - time_t t; - int i, str_len; + WT_RAND_STATE rnd; - srand((u_int)time(&t)); + uint64_t i, str_len, val; + + __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); str_len = sizeof(data_str) / sizeof(data_str[0]); for (i = 0; i < str_len - 1; i++) - data_str[i] = 'a' + rand() % 26; + data_str[i] = 'a' + (uint32_t)__wt_random(&rnd) % 26; data_str[str_len - 1] = '\0'; testutil_check(session->open_cursor(session, uri1, NULL, NULL, &cursor_1)); testutil_check(session->open_cursor(session, uri2, NULL, NULL, &cursor_2)); for (i = start; i < end; i++) { - cursor_1->set_key(cursor_1, i); - cursor_1->set_value(cursor_1, data_str); + val = (uint64_t)__wt_random(&rnd); + cursor_1->set_key(cursor_1, i + 1); + cursor_1->set_value(cursor_1, val, data_str); testutil_check(cursor_1->insert(cursor_1)); - cursor_2->set_key(cursor_2, i); - cursor_2->set_value(cursor_2, data_str); + cursor_2->set_key(cursor_2, i + 1); + cursor_2->set_value(cursor_2, val, data_str); testutil_check(cursor_2->insert(cursor_2)); } @@ -283,15 +304,15 @@ populate(WT_SESSION *session, int start, int end) } static void -remove_records(WT_SESSION *session, const char *uri, int start, int end) +remove_records(WT_SESSION *session, const char *uri, uint64_t start, uint64_t end) { WT_CURSOR *cursor; - int i; + uint64_t i; testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor)); for (i = start; i < end; i++) { - cursor->set_key(cursor, i); + cursor->set_key(cursor, i + 1); testutil_check(cursor->remove(cursor)); } @@ -301,18 +322,12 @@ remove_records(WT_SESSION *session, const char *uri, int start, int end) static int verify_tables_helper(WT_SESSION *session, const char *table1, const char *table2) { - WT_CURSOR *cursor_1; - WT_CURSOR *cursor_2; - - char *val_1; - char *val_2; - - int key; - int ret; - int total_keys; + WT_CURSOR *cursor_1, *cursor_2; - size_t val_1_size; - size_t val_2_size; + char *str_val_1, *str_val_2; + int ret, total_keys; + size_t val_1_size, val_2_size; + uint64_t key, val_1, val_2; testutil_check(session->open_cursor(session, table1, NULL, NULL, &cursor_1)); testutil_check(session->open_cursor(session, table2, NULL, NULL, &cursor_2)); @@ -323,14 +338,15 @@ verify_tables_helper(WT_SESSION *session, const char *table1, const char *table2 cursor_1->get_key(cursor_1, &key); cursor_2->set_key(cursor_2, key); testutil_check(cursor_2->search(cursor_2)); - testutil_check(cursor_1->get_value(cursor_1, &val_1)); - testutil_check(cursor_2->get_value(cursor_2, &val_2)); + testutil_check(cursor_1->get_value(cursor_1, &val_1, &str_val_1)); + testutil_check(cursor_2->get_value(cursor_2, &val_2, &str_val_2)); - val_1_size = strlen(val_1); - val_2_size = strlen(val_2); + val_1_size = strlen(str_val_1); + val_2_size = strlen(str_val_2); + testutil_assert(val_1 == val_2); testutil_assert(val_1_size == val_2_size); - testutil_assert(memcmp(val_1, val_2, val_1_size) == 0); + testutil_assert(memcmp(str_val_1, str_val_2, val_1_size) == 0); ++total_keys; } @@ -346,8 +362,7 @@ verify_tables_helper(WT_SESSION *session, const char *table1, const char *table2 static void verify_tables(WT_SESSION *session) { - int total_keys_1; - int total_keys_2; + int total_keys_1, total_keys_2; /* * Run over all keys in first table and verify they are present in the second table. Repeat with @@ -361,20 +376,39 @@ verify_tables(WT_SESSION *session) printf("%i Keys verified from the two tables. \n", total_keys_1); } -static uint64_t -get_file_size(WT_SESSION *session, const char *uri) +static void +get_file_stats(WT_SESSION *session, const char *uri, uint64_t *file_sz, uint64_t *avail_bytes) { WT_CURSOR *cur_stat; - uint64_t val; - char *descr, *str_val; - char stat_uri[128]; + char *descr, stat_uri[128], *str_val; sprintf(stat_uri, "statistics:%s", uri); testutil_check(session->open_cursor(session, stat_uri, NULL, "statistics=(all)", &cur_stat)); + + /* Get file size. */ cur_stat->set_key(cur_stat, WT_STAT_DSRC_BLOCK_SIZE); testutil_check(cur_stat->search(cur_stat)); - testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, &val)); + testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, file_sz)); + + /* Get bytes available for reuse. */ + cur_stat->set_key(cur_stat, WT_STAT_DSRC_BLOCK_REUSE_BYTES); + testutil_check(cur_stat->search(cur_stat)); + testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, avail_bytes)); + testutil_check(cur_stat->close(cur_stat)); + cur_stat = NULL; +} + +static void +log_db_size(WT_SESSION *session, const char *uri) +{ + uint64_t avail_bytes, available_pct, file_sz; + + get_file_stats(session, uri, &file_sz, &avail_bytes); - return (val); + /* Check if there's maximum of 10% space available after compaction. */ + available_pct = (avail_bytes * 100) / file_sz; + printf(" - Compacted file size: %" PRIu64 "MB (%" PRIu64 "B)\n - Available for reuse: %" PRIu64 + "MB (%" PRIu64 "B)\n - %" PRIu64 "%% space available in the file.\n", + file_sz / WT_MEGABYTE, file_sz, avail_bytes / WT_MEGABYTE, avail_bytes, available_pct); } |