diff options
author | Chenhao Qu <chenhao.qu@mongodb.com> | 2022-06-29 13:09:45 +1000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-06-29 03:38:45 +0000 |
commit | 35fbd67eaa07bb08619b48a8cd47d89b7427e04d (patch) | |
tree | bacd242a5ed2d3df6413cd5a9573a0f4baac7118 /src/third_party | |
parent | e9021b9422d761cf20c4361cc311b2dbbcba2ea4 (diff) | |
download | mongo-35fbd67eaa07bb08619b48a8cd47d89b7427e04d.tar.gz |
Import wiredtiger: f7aa95cb1b523c76e570c6457d2c70e0c32c6dce from branch mongodb-master
ref: 1f7c41168f..f7aa95cb1b
for: 6.1.0-rc0
WT-9370 Add tiered support to test/checkpoint (#8034)
Diffstat (limited to 'src/third_party')
10 files changed, 186 insertions, 85 deletions
diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index ae591538174..c78bfd4a62e 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -37,8 +37,10 @@ BBBBBBBBBB BBBBBBBBBBBBB BBBBBBBBBBBBBBBB BBBBBBBBBBBBBBBBBB +BCmvxz BDB BDB's +BDmpvXx BIGENDIAN BLKCACHE BOOL @@ -91,7 +93,6 @@ Checksum Checksums CityHash CloseHandle -Cmvxz Cmvz Collet Comparator @@ -125,7 +126,6 @@ Decrypt DeleteFileW Destructor Deterministically -DmpvXx EACCES EAGAIN EB diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 7e491cc300c..745db55be16 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "1f7c41168f83c2aab15f5a8e2e565718f5672e02" + "commit": "f7aa95cb1b523c76e570c6457d2c70e0c32c6dce" } diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index 4aa541b28ab..98e048bd07d 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -1017,12 +1017,18 @@ int __wt_btree_switch_object(WT_SESSION_IMPL *session, uint32_t objectid) { WT_BM *bm; + WT_BTREE *btree; + + btree = S2BT(session); + /* If the btree is readonly, there is nothing to do. */ + if (F_ISSET(btree, WT_BTREE_READONLY)) + return (0); /* * When initially opening a tiered Btree, a tier switch is done internally without the btree * being fully opened. That's okay, the btree will be told later about the current object * number. */ - bm = S2BT(session)->bm; + bm = btree->bm; return (bm == NULL ? 0 : bm->switch_object(bm, session, objectid)); } diff --git a/src/third_party/wiredtiger/src/schema/schema_create.c b/src/third_party/wiredtiger/src/schema/schema_create.c index cbc70ef6f07..44f012fd762 100644 --- a/src/third_party/wiredtiger/src/schema/schema_create.c +++ b/src/third_party/wiredtiger/src/schema/schema_create.c @@ -1226,8 +1226,9 @@ __schema_create_config_check( */ if (is_tiered && __wt_config_getones(session, config, "type", &cval) == 0 && !WT_STRING_MATCH("file", cval.str, cval.len)) - WT_RET_MSG( - session, ENOTSUP, "unsupported type configuration: type must be file for tiered storage"); + WT_RET_MSG(session, ENOTSUP, + "unsupported type configuration: %.*s: type must be file for tiered storage", + (int)cval.len, cval.str); return (0); } diff --git a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c index 59db5a98e69..b7f73f20071 100644 --- a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c +++ b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c @@ -30,6 +30,7 @@ static WT_THREAD_RET checkpointer(void *); static WT_THREAD_RET clock_thread(void *); +static WT_THREAD_RET flush_thread(void *); static int compare_cursors(WT_CURSOR *, table_type, WT_CURSOR *, table_type); static int diagnose_key_error(WT_CURSOR *, table_type, int, WT_CURSOR *, table_type, int); static int real_checkpointer(void); @@ -52,14 +53,18 @@ set_stable(void) } /* - * start_checkpoints -- - * Responsible for creating the checkpoint thread. + * start_threads -- + * Responsible for creating the service threads. */ void -start_checkpoints(void) +start_threads(void) { set_stable(); testutil_check(__wt_thread_create(NULL, &g.checkpoint_thread, checkpointer, NULL)); + if (g.tiered) { + testutil_check(__wt_rwlock_init(NULL, &g.flush_lock)); + testutil_check(__wt_thread_create(NULL, &g.flush_thread, flush_thread, NULL)); + } if (g.use_timestamps) { testutil_check(__wt_rwlock_init(NULL, &g.clock_lock)); testutil_check(__wt_thread_create(NULL, &g.clock_thread, clock_thread, NULL)); @@ -67,17 +72,22 @@ start_checkpoints(void) } /* - * end_checkpoints -- - * Responsible for cleanly shutting down the checkpoint thread. + * end_threads -- + * Responsible for cleanly shutting down the service threads. */ void -end_checkpoints(void) +end_threads(void) { - testutil_check(__wt_thread_join(NULL, &g.checkpoint_thread)); + if (g.tiered) { + testutil_check(__wt_thread_join(NULL, &g.flush_thread)); + __wt_rwlock_destroy(NULL, &g.flush_lock); + } if (g.use_timestamps) { testutil_check(__wt_thread_join(NULL, &g.clock_thread)); __wt_rwlock_destroy(NULL, &g.clock_lock); } + /* Shutdown checkpoint after flush thread completes because flush depends on checkpoint. */ + testutil_check(__wt_thread_join(NULL, &g.checkpoint_thread)); } /* @@ -130,6 +140,52 @@ clock_thread(void *arg) } /* + * flush_thread -- + * Flush thread to call flush_tier. + */ +static WT_THREAD_RET +flush_thread(void *arg) +{ + WT_RAND_STATE rnd; + WT_SESSION *wt_session; + WT_SESSION_IMPL *session; + uint64_t delay; + char tid[128]; + + WT_UNUSED(arg); + + __wt_random_init(&rnd); + testutil_check(g.conn->open_session(g.conn, NULL, NULL, &wt_session)); + session = (WT_SESSION_IMPL *)wt_session; + + testutil_check(__wt_thread_str(tid, sizeof(tid))); + printf("flush thread starting: tid: %s\n", tid); + fflush(stdout); + + while (g.running) { + /* FIXME-WT-7833 Remove this lock when that ticket merges. */ + __wt_writelock(session, &g.flush_lock); + testutil_check(wt_session->flush_tier(wt_session, NULL)); + __wt_writeunlock(session, &g.flush_lock); + printf("Finished a flush_tier\n"); + fflush(stdout); + + if (!g.running) + goto done; + /* + * Random value between 5000 and 10000. + */ + delay = __wt_random(&rnd) % 5001; + __wt_sleep(0, delay + 5000); + } + +done: + testutil_check(wt_session->close(wt_session, NULL)); + + return (WT_THREAD_RET_VALUE); +} + +/* * checkpointer -- * Checkpoint thread start function. */ @@ -171,7 +227,7 @@ real_checkpointer(void) return (log_print_err("Checkpoint thread started stopped\n", EINVAL, 1)); __wt_random_init(&rnd); - while (g.ntables > g.ntables_created) + while (g.ntables > g.ntables_created && g.running) __wt_yield(); if ((ret = g.conn->open_session(g.conn, NULL, NULL, &session)) != 0) diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c index 3ef36e622e4..ff5d7532d4c 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c @@ -76,8 +76,11 @@ main(int argc, char *argv[]) runs = 1; verify_only = false; - while ((ch = __wt_getopt(progname, argc, argv, "C:c:Dh:k:l:mn:pr:s:T:t:vW:xX")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "BC:c:Dh:k:l:mn:pr:s:T:t:vW:xX")) != EOF) switch (ch) { + case 'B': + g.tiered = true; + break; case 'c': g.checkpoint_name = __wt_optarg; break; @@ -202,9 +205,12 @@ main(int argc, char *argv[]) for (i = 0; i < g.ntables; ++i) { g.cookies[i].id = i; - if (ttype == MIX) + if (ttype == MIX) { g.cookies[i].type = (table_type)((i % MAX_TABLE_TYPE) + 1); - else + /* LSM is not supported with tiered storage. Just use ROW. */ + if (g.tiered && g.cookies[i].type == LSM) + g.cookies[i].type = ROW; + } else g.cookies[i].type = ttype; testutil_check(__wt_snprintf( g.cookies[i].uri, sizeof(g.cookies[i].uri), "%s%04d", URI_BASE, g.cookies[i].id)); @@ -229,15 +235,15 @@ main(int argc, char *argv[]) goto run_complete; } - start_checkpoints(); - if ((ret = start_workers()) != 0) { + start_threads(); + ret = start_workers(); + g.running = 0; + end_threads(); + if (ret != 0) { (void)log_print_err("Start workers failed", ret, 1); break; } - g.running = 0; - end_checkpoints(); - run_complete: free(g.cookies); g.cookies = NULL; @@ -256,6 +262,11 @@ run_complete: } #define DEBUG_MODE_CFG ",debug_mode=(eviction=true,table_logging=true),verbose=(recovery)" +#define SWEEP_CFG ",file_manager=(close_handle_minimum=1,close_idle_time=1,close_scan_interval=1)" +#define TIER_CFG \ + ",extensions=(../../ext/storage_sources/dir_store/" \ + "libwiredtiger_dir_store.so=(early_load=true)),tiered_storage=" \ + "(bucket=bucket,bucket_prefix=ckpt-,local_retention=2,name=dir_store)" /* * wt_connect -- * Configure the WiredTiger connection. @@ -268,53 +279,53 @@ wt_connect(const char *config_open) }; WT_RAND_STATE rnd; int ret; - char config[512]; - char timing_stress_config[512]; - bool fast_eviction, timing_stress; + char buf[512], config[1024]; + bool fast_eviction; fast_eviction = false; - timing_stress = false; + + /* + * Randomly decide on the eviction rate (fast or default). + */ + __wt_random_init_seed(NULL, &rnd); + if ((__wt_random(&rnd) % 15) % 2 == 0) + fast_eviction = true; + + /* Set up the basic configuration string first. */ + testutil_check(__wt_snprintf(config, sizeof(config), + "create,cache_cursors=false,statistics=(fast),statistics_log=(json,wait=1),log=(enabled)," + "error_prefix=\"%s\",cache_size=1G, eviction_dirty_trigger=%i, " + "eviction_dirty_target=%i,%s%s%s", + progname, fast_eviction ? 5 : 20, fast_eviction ? 1 : 5, g.debug_mode ? DEBUG_MODE_CFG : "", + config_open == NULL ? "" : ",", config_open == NULL ? "" : config_open)); + if (g.evict_reposition_timing_stress || g.sweep_stress || g.failpoint_hs_delete_key_from_ts || g.hs_checkpoint_timing_stress || g.reserved_txnid_timing_stress || g.checkpoint_slow_timing_stress) { - timing_stress = true; - testutil_check(__wt_snprintf(timing_stress_config, sizeof(timing_stress_config), - ",timing_stress_for_test=[%s%s%s%s%s%s]", + testutil_check(__wt_snprintf(buf, sizeof(buf), ",timing_stress_for_test=[%s%s%s%s%s%s]", + g.checkpoint_slow_timing_stress ? "checkpoint_slow" : "", g.evict_reposition_timing_stress ? "evict_reposition" : "", - g.sweep_stress ? "aggressive_sweep" : "", g.failpoint_hs_delete_key_from_ts ? "failpoint_history_store_delete_key_from_ts" : "", g.hs_checkpoint_timing_stress ? "history_store_checkpoint_delay" : "", g.reserved_txnid_timing_stress ? "checkpoint_reserved_txnid_delay" : "", - g.checkpoint_slow_timing_stress ? "checkpoint_slow" : "")); + g.sweep_stress ? "aggressive_sweep" : "")); + strcat(config, buf); } /* - * Randomly decide on the eviction rate (fast or default). + * If we want to stress sweep, we have a lot of additional configuration settings to set. */ - __wt_random_init_seed(NULL, &rnd); - if ((__wt_random(&rnd) % 15) % 2 == 0) - fast_eviction = true; + if (g.sweep_stress) + strcat(config, SWEEP_CFG); /* - * If we want to stress sweep, we have a lot of additional configuration settings to set. + * If we are using tiered add in the extension and tiered storage configuration. */ - if (g.sweep_stress) - testutil_check(__wt_snprintf(config, sizeof(config), - "create,cache_cursors=false,statistics=(fast),statistics_log=(json,wait=1),error_prefix=" - "\"%s\",file_manager=(close_handle_minimum=1,close_idle_time=1,close_scan_interval=1)," - "log=(enabled),cache_size=1GB, eviction_dirty_trigger=%i, " - "eviction_dirty_target=%i,%s%s%s%s", - progname, fast_eviction ? 5 : 20, fast_eviction ? 1 : 5, timing_stress_config, - g.debug_mode ? DEBUG_MODE_CFG : "", config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open)); - else - testutil_check(__wt_snprintf(config, sizeof(config), - "create,cache_cursors=false,statistics=(fast),statistics_log=(json,wait=1),log=(enabled)," - "error_prefix=\"%s\",cache_size=1G, eviction_dirty_trigger=%i, " - "eviction_dirty_target=%i,%s%s%s%s", - progname, fast_eviction ? 5 : 20, fast_eviction ? 1 : 5, - g.debug_mode ? DEBUG_MODE_CFG : "", config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open, timing_stress ? timing_stress_config : "")); + if (g.tiered) { + testutil_check(__wt_snprintf(buf, sizeof(buf), "%s/bucket", g.home)); + testutil_make_work_dir(buf); + strcat(config, TIER_CFG); + } printf("WT open config: %s\n", config); fflush(stdout); @@ -588,10 +599,11 @@ usage(void) { fprintf(stderr, "usage: %s\n" - " [-DmpvXx] [-C wiredtiger-config] [-c checkpoint] [-h home] [-k keys] [-l log]\n" + " [-BDmpvXx] [-C wiredtiger-config] [-c checkpoint] [-h home] [-k keys] [-l log]\n" " [-n ops] [-r runs] [-s 1|2|3|4|5] [-T table-config] [-t f|r|v] [-W workers]\n", progname); fprintf(stderr, "%s", + "\t-B use tiered storage\n" "\t-C specify wiredtiger_open configuration arguments\n" "\t-c checkpoint name to used named checkpoints\n" "\t-D debug mode\n" diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h index 395af7105da..321f4321961 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h @@ -55,44 +55,51 @@ typedef struct { } COOKIE; typedef struct { - char *home; /* Home directory */ - const char *checkpoint_name; /* Checkpoint name */ - WT_CONNECTION *conn; /* WiredTiger connection */ - bool debug_mode; /* History store stress test */ - u_int nkeys; /* Keys to load */ - u_int nops; /* Operations per thread */ - FILE *logfp; /* Message log file. */ - int nworkers; /* Number workers configured */ - int ntables; /* Number tables configured */ - int ntables_created; /* Number tables opened */ - volatile int running; /* Whether to stop */ - int status; /* Exit status */ - bool evict_reposition_timing_stress; /* Reposition the cursor for read operations */ - bool sweep_stress; /* Sweep stress test */ - bool failpoint_hs_delete_key_from_ts; /* Failpoint for hs key deletion. */ - bool hs_checkpoint_timing_stress; /* History store checkpoint timing stress */ - bool reserved_txnid_timing_stress; /* Reserved transaction id timing stress */ - bool checkpoint_slow_timing_stress; /* Checkpoint slow timing stress */ + char *home; /* Home directory */ + const char *checkpoint_name; /* Checkpoint name */ + WT_CONNECTION *conn; /* WiredTiger connection */ + bool debug_mode; /* History store stress test */ + u_int nkeys; /* Keys to load */ + u_int nops; /* Operations per thread */ + FILE *logfp; /* Message log file. */ + int ntables; /* Number tables configured */ + int ntables_created; /* Number tables opened */ + int nworkers; /* Number workers configured */ + volatile int running; /* Whether to stop */ + int status; /* Exit status */ + + bool checkpoint_slow_timing_stress; /* Checkpoint slow timing stress */ + bool evict_reposition_timing_stress; /* Reposition the cursor for read operations */ + bool hs_checkpoint_timing_stress; /* History store checkpoint timing stress */ + bool reserved_txnid_timing_stress; /* Reserved transaction id timing stress */ + bool sweep_stress; /* Sweep stress test */ + uint64_t ts_oldest; /* Current oldest timestamp */ uint64_t ts_stable; /* Current stable timestamp */ + bool failpoint_hs_delete_key_from_ts; /* Failpoint for hs key deletion. */ bool no_ts_deletes; /* Run with deletes without timestamps */ - bool use_timestamps; /* Use txn timestamps */ - bool race_timestamps; /* Async update to oldest timestamp */ bool prepare; /* Use prepare transactions */ - COOKIE *cookies; /* Per-thread info */ - WT_RWLOCK clock_lock; /* Clock synchronization */ - wt_thread_t checkpoint_thread; /* Checkpoint thread */ - wt_thread_t clock_thread; /* Clock thread */ + bool race_timestamps; /* Async update to oldest timestamp */ + + bool tiered; /* Use tiered storage. Start flush thread */ + bool use_timestamps; /* Use txn timestamps. Start clock thread */ + + COOKIE *cookies; /* Per-thread info */ + WT_RWLOCK clock_lock; /* Clock synchronization */ + WT_RWLOCK flush_lock; /* Flush synchronization */ + wt_thread_t checkpoint_thread; /* Checkpoint thread */ + wt_thread_t clock_thread; /* Clock thread */ + wt_thread_t flush_thread; /* Flush thread */ } GLOBAL; extern GLOBAL g; #define log_print_err(m, e, fatal) log_print_err_worker(__func__, __LINE__, m, e, fatal) -void end_checkpoints(void); +void end_threads(void); uint8_t flcs_encode(const char *); uint8_t flcs_modify(WT_MODIFY *, int, uint8_t); int log_print_err_worker(const char *, int, const char *, int, int); -void start_checkpoints(void); +void start_threads(void); int start_workers(void); const char *type_to_string(table_type); int verify_consistency(WT_SESSION *, wt_timestamp_t, bool); diff --git a/src/third_party/wiredtiger/test/checkpoint/workers.c b/src/third_party/wiredtiger/test/checkpoint/workers.c index cc419e09b7f..cfc594a483d 100644 --- a/src/third_party/wiredtiger/test/checkpoint/workers.c +++ b/src/third_party/wiredtiger/test/checkpoint/workers.c @@ -392,6 +392,8 @@ real_worker(void) } for (i = 0; i < g.nops && g.running; ++i, __wt_yield()) { + if (i > 0 && i % 5000 == 0) + printf("Worker %u of %u ops\n", i, g.nops); if (start_txn) { if ((ret = session->begin_transaction(session, begin_cfg)) != 0) { (void)log_print_err("real_worker:begin_transaction", ret, 1); @@ -399,6 +401,8 @@ real_worker(void) } new_txn = true; start_txn = false; + if (g.tiered) + __wt_readlock((WT_SESSION_IMPL *)session, &g.flush_lock); } keyno = __wt_random(&rnd) % g.nkeys + 1; /* If we have specified to run with mix mode deletes we need to do it in it's own txn. */ @@ -424,6 +428,8 @@ real_worker(void) } } start_txn = true; + if (g.tiered) + __wt_readunlock((WT_SESSION_IMPL *)session, &g.flush_lock); continue; } else new_txn = false; @@ -488,6 +494,8 @@ real_worker(void) } } start_txn = true; + if (g.tiered) + __wt_readunlock((WT_SESSION_IMPL *)session, &g.flush_lock); } } else if (next_rnd % 15 == 0) /* Occasionally reopen cursors during a running transaction. */ @@ -498,6 +506,8 @@ real_worker(void) goto err; } start_txn = true; + if (g.tiered) + __wt_readunlock((WT_SESSION_IMPL *)session, &g.flush_lock); } if (reopen_cursors) { for (j = 0; j < g.ntables; j++) { diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c index 456087b3e04..88ea6e3a31c 100644 --- a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c +++ b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c @@ -149,7 +149,7 @@ static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void usage(void) { - fprintf(stderr, "usage: %s [-h dir] [-T threads] [-t time] [-Cmvxz]\n", progname); + fprintf(stderr, "usage: %s [-h dir] [-T threads] [-t time] [-BCmvxz]\n", progname); exit(EXIT_FAILURE); } diff --git a/src/third_party/wiredtiger/test/suite/test_tiered17.py b/src/third_party/wiredtiger/test/suite/test_tiered17.py index a9b3dae59f3..3abaf4fa61f 100644 --- a/src/third_party/wiredtiger/test/suite/test_tiered17.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered17.py @@ -50,6 +50,18 @@ class test_tiered17(TieredConfigMixin, wttest.WiredTigerTestCase): scenarios = make_scenarios(tiered_storage_sources, shutdown) + def get_object_files(self): + object_files = fnmatch.filter(os.listdir('.'), "*.wtobj") + fnmatch.filter(os.listdir('.'), '*.wt') + return object_files + + def verify_checkpoint(self): + obj_files_orig = self.get_object_files() + ckpt_cursor = self.session.open_cursor(self.uri, None, 'checkpoint=WiredTigerCheckpoint') + ckpt_cursor.close() + obj_files = self.get_object_files() + # Check that no additional object files have been created after opening the checkpoint. + self.assertTrue(len(obj_files_orig) == len(obj_files)) + def populate(self): # Create and populate a table. self.session.create(self.uri, "key_format=S,value_format=S") @@ -67,12 +79,9 @@ class test_tiered17(TieredConfigMixin, wttest.WiredTigerTestCase): c["d"] = "d" c.close() - def get_object_files(self): - object_files = fnmatch.filter(os.listdir('.'), "*.wtobj") + fnmatch.filter(os.listdir('.'), '*.wt') - return object_files - def test_open_readonly_conn(self): self.populate() + self.verify_checkpoint() obj_files_orig = self.get_object_files() # Re-open the connection but in readonly mode. |