diff options
author | Alex Gorrod <alexander.gorrod@mongodb.com> | 2017-08-31 16:44:19 +1000 |
---|---|---|
committer | Alex Gorrod <alexander.gorrod@mongodb.com> | 2017-08-31 16:46:48 +1000 |
commit | 2730bd049022954bc7cd43392be20e6d54cf330d (patch) | |
tree | cf451c093d9b999e4da33b78d9d912b816864a93 /src/third_party | |
parent | ac84a0d028eff2b8ca42817d998cb8b34563816c (diff) | |
download | mongo-2730bd049022954bc7cd43392be20e6d54cf330d.tar.gz |
Import wiredtiger: bacc9ada53a46128642343c162a993d15bc80b57 from branch mongodb-3.6
ref: b055251678..bacc9ada53
for: 3.5.13
WT-3386 Fix test/checkpoint with timestamps
WT-3454 Enhance Python test_timestamp03 to test with logged and non-logged tables
WT-3476 Make tables first class data handles
WT-3495 Don't allow log truncation if log cursors are open
WT-3516 Some potential cursor open performance improvements
WT-3522 Coverity 1379736, full build Friday lint
WT-3523 Fix incorrect use of configure API in test format
WT-3525 test_timestamp07 failing automated testing on develop
WT-3527 Use power of two hash table size
WT-3528 Don't allow setting a commit timestamp on a non-running transaction
WT-3529 Add undocumented debug API
WT-3535 Fix ASAN build on Jenkins
WT-3536 Enable more strict compiler options for clang
WT-3538 Misaligned reads when using the lookaside table and timestamps
WT-3539 Enhance documentation around APIs that need exclusive handle access
WT-3540 Test failure running recovery test case
WT-3541 Python test_reconfig02 timeout on the PPC
WT-3542 Python test_stats_log_on_json_with_tables timeout on the PPC
WT-3544 WT rollback_to_stable hits "memory stomping bug"
WT-3545 Avoid array-bounds warnings with gcc 7
WT-3546 Coverity 1380021 dereference before NULL check, lint
WT-3547 Hang in test/fops on LSM table read lock
WT-3549 Deadlock in test_schema03
WT-3550 Minor output changes to debug_info
Diffstat (limited to 'src/third_party')
104 files changed, 1676 insertions, 1405 deletions
diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct index 501dae2ad9f..96012456b34 100644 --- a/src/third_party/wiredtiger/SConstruct +++ b/src/third_party/wiredtiger/SConstruct @@ -469,39 +469,6 @@ t = env.Program("t_manydbs", LIBS=[wtlib, shim, testutil] + wtlibs) Default(t) -# t_readonly doesn't currently build/run. -#t = env.Program("t_readonly", -# "test/readonly/readonly.c", -# LIBS=[wtlib, shim, testutil] + wtlibs) -#Default(t) - -# t_random-abort doesn't currently build/run. -#t = env.Program("t_random-abort", -# "test/recovery/random-abort.c", -# LIBS=[wtlib, shim, testutil] + wtlibs) -#Default(t) - -# t_truncated-log doesn't currently build/run. -#t = env.Program("t_truncated-log", -# "test/recovery/truncated-log.c", -# LIBS=[wtlib, shim, testutil] + wtlibs) -#Default(t) - -# t_salvage-log doesn't currently build/run. -#t = env.Program("t_salvage", -# "test/salvage/salvage.c", -# LIBS=[wtlib, shim, testutil] + wtlibs) -#Default(t) - -# t_thread doesn't currently build/run. -#t = env.Program("t_thread", -# ["test/thread/file.c", -# "test/thread/rw.c", -# "test/thread/stats.c", -# "test/thread/t.c"], -# LIBS=[wtlib, shim, testutil] + wtlibs) -#Default(t) - t = env.Program("wtperf", [ "bench/wtperf/config.c", "bench/wtperf/idle_table_cycle.c", diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 37f9baedc70..6936ab6c96f 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -1213,6 +1213,20 @@ methods = { don't free memory during close''', type='boolean'), ]), +'WT_CONNECTION.debug_info' : Method([ + Config('cache', 'false', r''' + print cache information''', type='boolean'), + Config('cursors', 'false', r''' + print all open cursor information''', type='boolean'), + Config('handles', 'false', r''' + print open handles information''', type='boolean'), + Config('log', 'false', r''' + print log information''', type='boolean'), + Config('sessions', 'false', r''' + print open session information''', type='boolean'), + Config('txn', 'false', r''' + print global txn information''', type='boolean'), +]), 'WT_CONNECTION.reconfigure' : Method( connection_reconfigure_log_configuration +\ connection_reconfigure_statistics_log_configuration +\ diff --git a/src/third_party/wiredtiger/dist/s_define.list b/src/third_party/wiredtiger/dist/s_define.list index 9f94132f584..dcaf975434f 100644 --- a/src/third_party/wiredtiger/dist/s_define.list +++ b/src/third_party/wiredtiger/dist/s_define.list @@ -65,4 +65,5 @@ __F __WIREDTIGER_EXT_H_ __WIREDTIGER_H_ __WT_INTERNAL_H +__func__ __wt_bswap16 diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 58b8137cad9..e6597734999 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -55,6 +55,7 @@ CET CFLAGS CHECKKEY CKPT +CLOEXEC CMP CONCAT CONFIG @@ -176,6 +177,7 @@ INMEM INSN INTL INULL +INUSE ISA ITEMs ITER @@ -923,6 +925,7 @@ mem memalign membar memcpy +memdup memget memmove memset @@ -1066,6 +1069,7 @@ qrrSS qsort quartile queueable +quiesce qup rN rS diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index c0f667140d0..b2e75ab4218 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "b055251678e6b4fcc74a1f651432aadbfeecc0e4", + "commit": "bacc9ada53a46128642343c162a993d15bc80b57", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-3.6" diff --git a/src/third_party/wiredtiger/lang/java/java_doc.i b/src/third_party/wiredtiger/lang/java/java_doc.i index c1062f313f3..28990234e59 100644 --- a/src/third_party/wiredtiger/lang/java/java_doc.i +++ b/src/third_party/wiredtiger/lang/java/java_doc.i @@ -29,10 +29,10 @@ COPYDOC(__wt_async_op, WT_ASYNC_OP, remove) COPYDOC(__wt_async_op, WT_ASYNC_OP, compact) COPYDOC(__wt_async_op, WT_ASYNC_OP, get_id) COPYDOC(__wt_async_op, WT_ASYNC_OP, get_type) -COPYDOC(__wt_session, WT_SESSION, alter) COPYDOC(__wt_session, WT_SESSION, close) COPYDOC(__wt_session, WT_SESSION, reconfigure) COPYDOC(__wt_session, WT_SESSION, open_cursor) +COPYDOC(__wt_session, WT_SESSION, alter) COPYDOC(__wt_session, WT_SESSION, create) COPYDOC(__wt_session, WT_SESSION, compact) COPYDOC(__wt_session, WT_SESSION, drop) @@ -57,6 +57,7 @@ COPYDOC(__wt_session, WT_SESSION, transaction_sync) COPYDOC(__wt_connection, WT_CONNECTION, async_flush) COPYDOC(__wt_connection, WT_CONNECTION, async_new_op) COPYDOC(__wt_connection, WT_CONNECTION, close) +COPYDOC(__wt_connection, WT_CONNECTION, debug_info) COPYDOC(__wt_connection, WT_CONNECTION, reconfigure) COPYDOC(__wt_connection, WT_CONNECTION, configure_method) COPYDOC(__wt_connection, WT_CONNECTION, is_new) diff --git a/src/third_party/wiredtiger/src/btree/bt_ovfl.c b/src/third_party/wiredtiger/src/btree/bt_ovfl.c index fab38f3cc8d..ebd0eb0cb71 100644 --- a/src/third_party/wiredtiger/src/btree/bt_ovfl.c +++ b/src/third_party/wiredtiger/src/btree/bt_ovfl.c @@ -132,7 +132,7 @@ __ovfl_cache(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL_UNPACK *unpack) WT_ERR(__wt_realloc_def(session, &track->remove_allocated, track->remove_next + 1, &track->remove)); track->remove[track->remove_next].cell = unpack->cell; - WT_ERR(__wt_strndup(session, + WT_ERR(__wt_memdup(session, tmp->data, tmp->size, &track->remove[track->remove_next].data)); track->remove[track->remove_next].size = tmp->size; ++track->remove_next; diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index 0c3cb026421..26ddde8d6ad 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -136,8 +136,9 @@ __las_page_instantiate(WT_SESSION_IMPL *session, WT_CURSOR *cursor; WT_CURSOR_BTREE cbt; WT_DECL_ITEM(current_key); - WT_ITEM las_addr, las_key, las_timestamp, las_value; WT_DECL_RET; + WT_DECL_TIMESTAMP(timestamp) + WT_ITEM las_addr, las_key, las_timestamp, las_value; WT_PAGE *page; WT_UPDATE *first_upd, *last_upd, *upd; size_t incr, total_incr; @@ -154,7 +155,6 @@ __las_page_instantiate(WT_SESSION_IMPL *session, current_recno = recno = WT_RECNO_OOB; session_flags = 0; /* [-Werror=maybe-uninitialized] */ WT_CLEAR(las_key); - WT_CLEAR(las_timestamp); __wt_btcur_init(session, &cbt); __wt_btcur_open(&cbt); @@ -176,6 +176,7 @@ __las_page_instantiate(WT_SESSION_IMPL *session, */ las_addr.data = addr; las_addr.size = addr_size; + las_timestamp.size = 0; cursor->set_key(cursor, read_id, &las_addr, (uint64_t)0, (uint32_t)0, &las_timestamp, &las_key); if ((ret = cursor->search_near(cursor, &exact)) == 0 && exact < 0) @@ -195,13 +196,16 @@ __las_page_instantiate(WT_SESSION_IMPL *session, /* * If the on-page value has become globally visible, this record - * is no longer needed. We clear the las_timestamp structure - * above to avoid reading uninitialized memory here when - * timestamps are disabled (even though it is unused in that - * case). + * is no longer needed. + * + * Copy the timestamp from the cursor to avoid unaligned reads. */ +#ifdef HAVE_TIMESTAMPS + WT_ASSERT(session, las_timestamp.size == WT_TIMESTAMP_SIZE); + memcpy(×tamp, las_timestamp.data, las_timestamp.size); +#endif if (__wt_txn_visible_all( - session, las_txnid, las_timestamp.data)) + session, las_txnid, WT_TIMESTAMP_NULL(×tamp))) continue; /* Allocate the WT_UPDATE structure. */ @@ -213,7 +217,7 @@ __las_page_instantiate(WT_SESSION_IMPL *session, upd->txnid = upd_txnid; #ifdef HAVE_TIMESTAMPS WT_ASSERT(session, las_timestamp.size == WT_TIMESTAMP_SIZE); - __wt_timestamp_set(&upd->timestamp, las_timestamp.data); + memcpy(&upd->timestamp, las_timestamp.data, las_timestamp.size); #endif switch (page->type) { diff --git a/src/third_party/wiredtiger/src/btree/bt_rebalance.c b/src/third_party/wiredtiger/src/btree/bt_rebalance.c index 47c7888af35..a616ab7fc42 100644 --- a/src/third_party/wiredtiger/src/btree/bt_rebalance.c +++ b/src/third_party/wiredtiger/src/btree/bt_rebalance.c @@ -83,7 +83,7 @@ __rebalance_leaf_append(WT_SESSION_IMPL *session, WT_RET(__wt_calloc_one(session, ©_addr)); copy->addr = copy_addr; - WT_RET(__wt_strndup(session, addr, addr_len, ©_addr->addr)); + WT_RET(__wt_memdup(session, addr, addr_len, ©_addr->addr)); copy_addr->size = (uint8_t)addr_len; copy_addr->type = (uint8_t)addr_type; @@ -110,7 +110,7 @@ __rebalance_fl_append(WT_SESSION_IMPL *session, session, &rs->fl_allocated, rs->fl_next + 1, &rs->fl)); copy = &rs->fl[rs->fl_next++]; - WT_RET(__wt_strndup(session, addr, addr_len, ©->addr)); + WT_RET(__wt_memdup(session, addr, addr_len, ©->addr)); copy->size = (uint8_t)addr_len; copy->type = 0; diff --git a/src/third_party/wiredtiger/src/btree/bt_ret.c b/src/third_party/wiredtiger/src/btree/bt_ret.c index 120daed3935..49a2481062e 100644 --- a/src/third_party/wiredtiger/src/btree/bt_ret.c +++ b/src/third_party/wiredtiger/src/btree/bt_ret.c @@ -129,9 +129,17 @@ __value_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) } /* + * When threads race modifying a record, we can end up with more than the usual + * maximum number of modifications in an update list. We'd prefer not to + * allocate memory in a return path, so add a few additional slots to the array + * we use to build up a list of modify records to apply. + */ +#define WT_MODIFY_ARRAY_SIZE (WT_MAX_MODIFY_UPDATE + 10) + +/* * __value_return_upd -- * Change the cursor to reference an internal update structure return - * value. + * value. */ static inline int __value_return_upd( @@ -139,7 +147,7 @@ __value_return_upd( { WT_CURSOR *cursor; WT_DECL_RET; - WT_UPDATE **listp, *list[WT_MAX_MODIFY_UPDATE]; + WT_UPDATE **listp, *list[WT_MODIFY_ARRAY_SIZE]; u_int i; size_t allocated_bytes; @@ -178,12 +186,12 @@ __value_return_upd( * avoid memory allocation in normal cases, but we have * to handle the edge cases too. */ - if (i >= WT_MAX_MODIFY_UPDATE) { - if (i == WT_MAX_MODIFY_UPDATE) + if (i >= WT_MODIFY_ARRAY_SIZE) { + if (i == WT_MODIFY_ARRAY_SIZE) listp = NULL; WT_ERR(__wt_realloc_def( session, &allocated_bytes, i + 1, &listp)); - if (i == WT_MAX_MODIFY_UPDATE) + if (i == WT_MODIFY_ARRAY_SIZE) memcpy(listp, list, sizeof(list)); } listp[i++] = upd; diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index 1f686d904ec..2b5fb8b2db0 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -496,7 +496,7 @@ __slvg_trk_init(WT_SESSION_IMPL *session, trk->shared->ref = 1; trk->ss = ss; - WT_ERR(__wt_strndup(session, addr, addr_size, &trk->trk_addr)); + WT_ERR(__wt_memdup(session, addr, addr_size, &trk->trk_addr)); trk->trk_addr_size = (uint8_t)addr_size; trk->trk_size = size; trk->trk_gen = gen; @@ -683,7 +683,7 @@ __slvg_trk_leaf_ovfl( WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { __wt_cell_unpack(cell, unpack); if (unpack->ovfl) { - WT_RET(__wt_strndup(session, unpack->data, + WT_RET(__wt_memdup(session, unpack->data, unpack->size, &trk->trk_ovfl_addr[ovfl_cnt].addr)); trk->trk_ovfl_addr[ovfl_cnt].size = (uint8_t)unpack->size; @@ -1171,7 +1171,7 @@ __slvg_col_build_internal( ref->page = NULL; WT_ERR(__wt_calloc_one(session, &addr)); - WT_ERR(__wt_strndup( + WT_ERR(__wt_memdup( session, trk->trk_addr, trk->trk_addr_size, &addr->addr)); addr->size = trk->trk_addr_size; addr->type = @@ -1824,7 +1824,7 @@ __slvg_row_build_internal( ref->page = NULL; WT_ERR(__wt_calloc_one(session, &addr)); - WT_ERR(__wt_strndup( + WT_ERR(__wt_memdup( session, trk->trk_addr, trk->trk_addr_size, &addr->addr)); addr->size = trk->trk_addr_size; addr->type = diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index ac90d6693d3..42fd1bd1bd2 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -290,7 +290,7 @@ __split_ref_move(WT_SESSION_IMPL *session, WT_PAGE *from_home, if (ref_addr != NULL && !__wt_off_page(from_home, ref_addr)) { __wt_cell_unpack((WT_CELL *)ref_addr, &unpack); WT_RET(__wt_calloc_one(session, &addr)); - if ((ret = __wt_strndup( + if ((ret = __wt_memdup( session, unpack.data, unpack.size, &addr->addr)) != 0) { __wt_free(session, addr); return (ret); @@ -1612,7 +1612,7 @@ __wt_multi_to_ref(WT_SESSION_IMPL *session, ref->addr = addr; addr->size = multi->addr.size; addr->type = multi->addr.type; - WT_RET(__wt_strndup(session, + WT_RET(__wt_memdup(session, multi->addr.addr, addr->size, &addr->addr)); ref->state = WT_REF_DISK; } diff --git a/src/third_party/wiredtiger/src/cache/cache_las.c b/src/third_party/wiredtiger/src/cache/cache_las.c index a118e5d5e41..e1e47b9eecb 100644 --- a/src/third_party/wiredtiger/src/cache/cache_las.c +++ b/src/third_party/wiredtiger/src/cache/cache_las.c @@ -288,9 +288,10 @@ __wt_las_sweep(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; WT_CURSOR *cursor; - WT_ITEM las_addr, las_key, las_timestamp; WT_DECL_RET; + WT_DECL_TIMESTAMP(timestamp) WT_ITEM *key; + WT_ITEM las_addr, las_key, las_timestamp; uint64_t cnt, las_counter, las_txnid, remove_cnt; uint32_t las_id, session_flags; int notused; @@ -300,7 +301,6 @@ __wt_las_sweep(WT_SESSION_IMPL *session) key = &conn->las_sweep_key; remove_cnt = 0; session_flags = 0; /* [-Werror=maybe-uninitialized] */ - WT_CLEAR(las_timestamp); __wt_las_cursor(session, &cursor, &session_flags); @@ -359,6 +359,11 @@ __wt_las_sweep(WT_SESSION_IMPL *session) session, key, key->data, key->size)); } + /* + * Cursor opened overwrite=true: won't return WT_NOTFOUND should + * another thread remove the record before we do, and the cursor + * remains positioned in that case. + */ WT_ERR(cursor->get_key(cursor, &las_id, &las_addr, &las_counter, &las_txnid, &las_timestamp, &las_key)); @@ -366,16 +371,14 @@ __wt_las_sweep(WT_SESSION_IMPL *session) * If the on-page record transaction ID associated with the * record is globally visible, the record can be discarded. * - * Cursor opened overwrite=true: won't return WT_NOTFOUND should - * another thread remove the record before we do, and the cursor - * remains positioned in that case. - * - * We clear the las_timestamp structure above to avoid reading - * uninitialized memory here when timestamps are disabled (even - * though it is unused in that case). + * Copy the timestamp from the cursor to avoid unaligned reads. */ +#ifdef HAVE_TIMESTAMPS + WT_ASSERT(session, las_timestamp.size == WT_TIMESTAMP_SIZE); + memcpy(×tamp, las_timestamp.data, las_timestamp.size); +#endif if (__wt_txn_visible_all( - session, las_txnid, las_timestamp.data)) { + session, las_txnid, WT_TIMESTAMP_NULL(×tamp))) { WT_ERR(cursor->remove(cursor)); ++remove_cnt; } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 764006b024d..9f08872dc61 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -15,6 +15,16 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_close[] = { { NULL, NULL, NULL, NULL, NULL, 0 } }; +static const WT_CONFIG_CHECK confchk_WT_CONNECTION_debug_info[] = { + { "cache", "boolean", NULL, NULL, NULL, 0 }, + { "cursors", "boolean", NULL, NULL, NULL, 0 }, + { "handles", "boolean", NULL, NULL, NULL, 0 }, + { "log", "boolean", NULL, NULL, NULL, 0 }, + { "sessions", "boolean", NULL, NULL, NULL, 0 }, + { "txn", "boolean", NULL, NULL, NULL, 0 }, + { NULL, NULL, NULL, NULL, NULL, 0 } +}; + static const WT_CONFIG_CHECK confchk_WT_CONNECTION_load_extension[] = { { "config", "string", NULL, NULL, NULL, 0 }, { "early_load", "boolean", NULL, NULL, NULL, 0 }, @@ -1114,6 +1124,11 @@ static const WT_CONFIG_ENTRY config_entries[] = { "leak_memory=false", confchk_WT_CONNECTION_close, 1 }, + { "WT_CONNECTION.debug_info", + "cache=false,cursors=false,handles=false,log=false,sessions=false" + ",txn=false", + confchk_WT_CONNECTION_debug_info, 6 + }, { "WT_CONNECTION.load_extension", "config=,early_load=false,entry=wiredtiger_extension_init," "terminate=wiredtiger_extension_terminate", diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index b29b6184ce3..bacfcc6d5e6 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1093,6 +1093,49 @@ err: /* } /* + * __conn_debug_info -- + * WT_CONNECTION->debug_info method. + */ +static int +__conn_debug_info(WT_CONNECTION *wt_conn, const char *config) +{ + WT_CONFIG_ITEM cval; + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + WT_SESSION_IMPL *session; + + conn = (WT_CONNECTION_IMPL *)wt_conn; + + CONNECTION_API_CALL(conn, session, debug_info, config, cfg); + + WT_ERR(__wt_config_gets(session, cfg, "cache", &cval)); + if (cval.val != 0) + WT_ERR(__wt_verbose_dump_cache(session)); + + WT_ERR(__wt_config_gets(session, cfg, "cursors", &cval)); + if (cval.val != 0) + WT_ERR(__wt_verbose_dump_sessions(session, true)); + + WT_ERR(__wt_config_gets(session, cfg, "handles", &cval)); + if (cval.val != 0) + WT_ERR(__wt_verbose_dump_handles(session)); + + WT_ERR(__wt_config_gets(session, cfg, "log", &cval)); + if (cval.val != 0) + WT_ERR(__wt_verbose_dump_log(session)); + + WT_ERR(__wt_config_gets(session, cfg, "sessions", &cval)); + if (cval.val != 0) + WT_ERR(__wt_verbose_dump_sessions(session, false)); + + WT_ERR(__wt_config_gets(session, cfg, "txn", &cval)); + if (cval.val != 0) + WT_ERR(__wt_verbose_dump_txn(session)); +err: + API_END_RET(session, ret); +} + +/* * __conn_reconfigure -- * WT_CONNECTION->reconfigure method. */ @@ -1780,6 +1823,120 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) } /* + * __wt_verbose_dump_sessions -- + * Print out debugging information about sessions. + */ +int +__wt_verbose_dump_sessions(WT_SESSION_IMPL *session, bool show_cursors) +{ + WT_CONNECTION_IMPL *conn; + WT_CURSOR *cursor; + WT_DECL_ITEM(buf); + WT_DECL_RET; + WT_SESSION_IMPL *s; + uint32_t i, internal; + + conn = S2C(session); + WT_RET(__wt_msg(session, "%s", WT_DIVIDER)); + WT_RET(__wt_msg(session, "Active sessions: %" PRIu32 " Max: %" PRIu32, + conn->session_cnt, conn->session_size)); + WT_RET(__wt_scr_alloc(session, 0, &buf)); + internal = 0; + for (s = conn->sessions, i = 0; i < conn->session_cnt; ++s, ++i) { + /* + * If it is not active or it is an internal session + * it is not interesting. + */ + if (!s->active) + continue; + if (F_ISSET(s, WT_SESSION_INTERNAL)) { + ++internal; + continue; + } + WT_ASSERT(session, i == s->id); + WT_ERR(__wt_msg(session, + "Session: ID: %" PRIu32 " @: 0x%p", i, (void *)s)); + WT_ERR(__wt_msg(session, " Name: %s", + s->name == NULL ? "EMPTY" : s->name)); + if (!show_cursors) { + WT_ERR(__wt_msg(session, " Last operation: %s", + s->lastop == NULL ? "NONE" : s->lastop)); + WT_ERR(__wt_msg(session, " Current dhandle: %s", + s->dhandle == NULL ? "NONE" : s->dhandle->name)); + WT_ERR(__wt_msg(session, " Backup in progress: %s", + s->bkp_cursor == NULL ? "no" : "yes")); + WT_ERR(__wt_msg(session, " Compact state: %s", + s->compact_state == WT_COMPACT_NONE ? "none" : + (s->compact_state == WT_COMPACT_RUNNING ? + "running" : "success"))); + WT_ERR(__wt_msg(session, + " Flags: 0x%" PRIx32, s->flags)); + WT_ERR(__wt_msg(session, " Isolation level: %s", + s->isolation == WT_ISO_READ_COMMITTED ? + "read-committed" : + (s->isolation == WT_ISO_READ_UNCOMMITTED ? + "read-uncommitted" : "snapshot"))); + WT_ERR(__wt_msg(session, " Transaction:")); + WT_ERR(__wt_verbose_dump_txn_one(session, &s->txn)); + } else { + WT_ERR(__wt_msg(session, + " Number of positioned cursors: %u", s->ncursors)); + TAILQ_FOREACH(cursor, &s->cursors, q) { + WT_ERR(__wt_msg(session, + "Cursor @ %p:", (void *)cursor)); + WT_ERR(__wt_msg(session, + " URI: %s, Internal URI: %s", + cursor->uri == NULL ? "EMPTY" : cursor->uri, + cursor->internal_uri == NULL ? "EMPTY" : + cursor->internal_uri)); + if (F_ISSET(cursor, WT_CURSTD_OPEN)) { + WT_ERR(__wt_buf_fmt( + session, buf, "OPEN")); + if (F_ISSET(cursor, + WT_CURSTD_KEY_SET) || + F_ISSET(cursor, + WT_CURSTD_VALUE_SET)) + WT_ERR(__wt_buf_catfmt(session, + buf, ", POSITIONED")); + else + WT_ERR(__wt_buf_catfmt(session, + buf, ", RESET")); + if (F_ISSET(cursor, WT_CURSTD_APPEND)) + WT_ERR(__wt_buf_catfmt(session, + buf, ", APPEND")); + if (F_ISSET(cursor, WT_CURSTD_BULK)) + WT_ERR(__wt_buf_catfmt(session, + buf, ", BULK")); + if (F_ISSET(cursor, + WT_CURSTD_META_INUSE)) + WT_ERR(__wt_buf_catfmt(session, + buf, ", META_INUSE")); + if (F_ISSET(cursor, + WT_CURSTD_OVERWRITE)) + WT_ERR(__wt_buf_catfmt(session, + buf, ", OVERWRITE")); + WT_ERR(__wt_msg(session, + " %s", (const char *)buf->data)); + } + WT_ERR(__wt_msg(session, + " Flags: 0x%" PRIx32, cursor->flags)); + WT_ERR(__wt_msg(session, + " Key_format: %s, Value_format: %s", + cursor->key_format == NULL ? "EMPTY" : + cursor->key_format, + cursor->value_format == NULL ? "EMPTY" : + cursor->value_format)); + } + } + } + if (!show_cursors) + WT_ERR(__wt_msg(session, + "Internal sessions: %" PRIu32, internal)); +err: __wt_scr_free(session, &buf); + return (ret); +} + +/* * __wt_timing_stress_config -- * Set timing stress for test delay configuration. */ @@ -2045,6 +2202,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, __conn_async_flush, __conn_async_new_op, __conn_close, + __conn_debug_info, __conn_reconfigure, __conn_get_home, __conn_configure_method, diff --git a/src/third_party/wiredtiger/src/conn/conn_cache_pool.c b/src/third_party/wiredtiger/src/conn/conn_cache_pool.c index 55393a86ba1..2791277b0d8 100644 --- a/src/third_party/wiredtiger/src/conn/conn_cache_pool.c +++ b/src/third_party/wiredtiger/src/conn/conn_cache_pool.c @@ -47,7 +47,7 @@ int __wt_cache_pool_config(WT_SESSION_IMPL *session, const char **cfg) { WT_CACHE_POOL *cp; - WT_CONFIG_ITEM cval; + WT_CONFIG_ITEM cval, cval_cache_size; WT_CONNECTION_IMPL *conn, *entry; WT_DECL_RET; char *pool_name; @@ -78,7 +78,7 @@ __wt_cache_pool_config(WT_SESSION_IMPL *session, const char **cfg) } if (__wt_config_gets(session, - &cfg[1], "cache_size", &cval) != WT_NOTFOUND) + &cfg[1], "cache_size", &cval_cache_size) != WT_NOTFOUND) WT_RET_MSG(session, EINVAL, "Only one of cache_size and shared_cache can be " "in the configuration"); @@ -221,8 +221,7 @@ __wt_cache_pool_config(WT_SESSION_IMPL *session, const char **cfg) F_SET(conn, WT_CONN_CACHE_POOL); err: __wt_spin_unlock(session, &__wt_process.spinlock); - if (!updating) - __wt_free(session, pool_name); + __wt_free(session, pool_name); if (ret != 0 && created) { __wt_free(session, cp->name); __wt_cond_destroy(session, &cp->cache_pool_cond); diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index 45f04b66247..aa12718cd69 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -17,7 +17,15 @@ __conn_dhandle_destroy(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle) { WT_DECL_RET; - WT_WITH_DHANDLE(session, dhandle, ret = __wt_btree_discard(session)); + switch (dhandle->type) { + case WT_DHANDLE_TYPE_BTREE: + WT_WITH_DHANDLE(session, dhandle, + ret = __wt_btree_discard(session)); + break; + case WT_DHANDLE_TYPE_TABLE: + ret = __wt_schema_close_table(session, (WT_TABLE *)dhandle); + break; + } __wt_rwlock_destroy(session, &dhandle->rwlock); __wt_free(session, dhandle->name); @@ -40,6 +48,7 @@ __wt_conn_dhandle_alloc( WT_BTREE *btree; WT_DATA_HANDLE *dhandle; WT_DECL_RET; + WT_TABLE *table; uint64_t bucket; /* @@ -50,24 +59,34 @@ __wt_conn_dhandle_alloc( __wt_conn_dhandle_find(session, uri, checkpoint)) != WT_NOTFOUND) return (ret); - WT_RET(__wt_calloc_one(session, &dhandle)); + if (WT_PREFIX_MATCH(uri, "file:")) { + WT_RET(__wt_calloc_one(session, &dhandle)); + dhandle->type = WT_DHANDLE_TYPE_BTREE; + } else if (WT_PREFIX_MATCH(uri, "table:")) { + WT_RET(__wt_calloc_one(session, &table)); + dhandle = &table->iface; + dhandle->type = WT_DHANDLE_TYPE_TABLE; + } else + return (__wt_illegal_value(session, NULL)); + + /* Btree handles keep their data separate from the interface. */ + if (dhandle->type == WT_DHANDLE_TYPE_BTREE) { + WT_ERR(__wt_calloc_one(session, &btree)); + dhandle->handle = btree; + btree->dhandle = dhandle; + } + + if (strcmp(uri, WT_METAFILE_URI) == 0) + F_SET(dhandle, WT_DHANDLE_IS_METADATA); WT_ERR(__wt_rwlock_init(session, &dhandle->rwlock)); dhandle->name_hash = __wt_hash_city64(uri, strlen(uri)); WT_ERR(__wt_strdup(session, uri, &dhandle->name)); WT_ERR(__wt_strdup(session, checkpoint, &dhandle->checkpoint)); - /* TODO: abstract this out for other data handle types */ - WT_ERR(__wt_calloc_one(session, &btree)); - dhandle->handle = btree; - btree->dhandle = dhandle; - WT_ERR(__wt_spin_init( session, &dhandle->close_lock, "data handle close")); - if (strcmp(uri, WT_METAFILE_URI) == 0) - F_SET(dhandle, WT_DHANDLE_IS_METADATA); - /* * We are holding the data handle list lock, which protects most * threads from seeing the new handle until that lock is released. @@ -136,11 +155,11 @@ __wt_conn_dhandle_find( } /* - * __wt_conn_btree_sync_and_close -- + * __wt_conn_dhandle_close -- * Sync and close the underlying btree handle. */ int -__wt_conn_btree_sync_and_close( +__wt_conn_dhandle_close( WT_SESSION_IMPL *session, bool final, bool mark_dead) { WT_BM *bm; @@ -148,18 +167,28 @@ __wt_conn_btree_sync_and_close( WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle; WT_DECL_RET; - bool discard, marked_dead, no_schema_lock; + bool discard, is_btree, marked_dead, no_schema_lock; conn = S2C(session); - btree = S2BT(session); - bm = btree->bm; dhandle = session->dhandle; if (!F_ISSET(dhandle, WT_DHANDLE_OPEN)) return (0); - /* Turn off eviction. */ - WT_RET(__wt_evict_file_exclusive_on(session)); + /* + * The only data handle type that uses the "handle" field is btree. + * For other data handle types, it should be NULL. + */ + is_btree = dhandle->type == WT_DHANDLE_TYPE_BTREE; + btree = is_btree ? dhandle->handle : NULL; + + if (is_btree) { + /* Turn off eviction. */ + WT_RET(__wt_evict_file_exclusive_on(session)); + + /* Reset the tree's eviction priority (if any). */ + __wt_evict_priority_clear(session); + } /* * If we don't already have the schema lock, make it an error to try to @@ -182,11 +211,8 @@ __wt_conn_btree_sync_and_close( */ __wt_spin_lock(session, &dhandle->close_lock); - /* Reset the tree's eviction priority (if any). */ - __wt_evict_priority_clear(session); - discard = marked_dead = false; - if (!F_ISSET(btree, + if (is_btree && !F_ISSET(btree, WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY)) { /* * If the handle is already marked dead, we're just here to @@ -204,6 +230,7 @@ __wt_conn_btree_sync_and_close( * memory-mapped trees contain pointers into memory that become * invalid if the mapping is closed.) */ + bm = btree->bm; if (!discard && mark_dead && !final && (bm == NULL || !bm->is_mapped(bm, session))) marked_dead = true; @@ -231,9 +258,16 @@ __wt_conn_btree_sync_and_close( } } - /* Discard the underlying btree handle. */ - WT_TRET(__wt_btree_close(session)); - F_CLR(btree, WT_BTREE_SPECIAL_FLAGS); + /* Close the underlying handle. */ + switch (dhandle->type) { + case WT_DHANDLE_TYPE_BTREE: + WT_TRET(__wt_btree_close(session)); + F_CLR(btree, WT_BTREE_SPECIAL_FLAGS); + break; + case WT_DHANDLE_TYPE_TABLE: + WT_TRET(__wt_schema_close_table(session, (WT_TABLE *)dhandle)); + break; + } /* * If marking the handle dead, do so after closing the underlying btree. @@ -276,17 +310,18 @@ err: __wt_spin_unlock(session, &dhandle->close_lock); if (no_schema_lock) F_CLR(session, WT_SESSION_NO_SCHEMA_LOCK); - __wt_evict_file_exclusive_off(session); + if (is_btree) + __wt_evict_file_exclusive_off(session); return (ret); } /* - * __conn_btree_config_clear -- + * __conn_dhandle_config_clear -- * Clear the underlying object's configuration information. */ static void -__conn_btree_config_clear(WT_SESSION_IMPL *session) +__conn_dhandle_config_clear(WT_SESSION_IMPL *session) { WT_DATA_HANDLE *dhandle; const char **a; @@ -301,11 +336,11 @@ __conn_btree_config_clear(WT_SESSION_IMPL *session) } /* - * __conn_btree_config_set -- + * __conn_dhandle_config_set -- * Set up a btree handle's configuration information. */ static int -__conn_btree_config_set(WT_SESSION_IMPL *session) +__conn_dhandle_config_set(WT_SESSION_IMPL *session) { WT_DATA_HANDLE *dhandle; WT_DECL_RET; @@ -325,11 +360,12 @@ __conn_btree_config_set(WT_SESSION_IMPL *session) } /* - * The defaults are included because underlying objects have persistent - * configuration information stored in the metadata file. If defaults - * are included in the configuration, we can add new configuration - * strings without upgrading the metadata file or writing special code - * in case a configuration string isn't initialized, as long as the new + * The defaults are included because persistent configuration + * information is stored in the metadata file and it may be from an + * earlier version of WiredTiger. If defaults are included in the + * configuration, we can add new configuration strings without + * upgrading the metadata file or writing special code in case a + * configuration string isn't initialized, as long as the new * configuration string has an appropriate default value. * * The error handling is a little odd, but be careful: we're holding a @@ -338,8 +374,16 @@ __conn_btree_config_set(WT_SESSION_IMPL *session) * it, after the copy, we don't want to free it. */ WT_ERR(__wt_calloc_def(session, 3, &dhandle->cfg)); - WT_ERR(__wt_strdup( - session, WT_CONFIG_BASE(session, file_meta), &dhandle->cfg[0])); + switch (dhandle->type) { + case WT_DHANDLE_TYPE_BTREE: + WT_ERR(__wt_strdup(session, + WT_CONFIG_BASE(session, file_meta), &dhandle->cfg[0])); + break; + case WT_DHANDLE_TYPE_TABLE: + WT_ERR(__wt_strdup(session, + WT_CONFIG_BASE(session, table_meta), &dhandle->cfg[0])); + break; + } dhandle->cfg[1] = metaconf; return (0); @@ -348,11 +392,11 @@ err: __wt_free(session, metaconf); } /* - * __wt_conn_btree_open -- - * Open the current btree handle. + * __wt_conn_dhandle_open -- + * Open the current data handle. */ int -__wt_conn_btree_open( +__wt_conn_dhandle_open( WT_SESSION_IMPL *session, const char *cfg[], uint32_t flags) { WT_BTREE *btree; @@ -360,7 +404,7 @@ __wt_conn_btree_open( WT_DECL_RET; dhandle = session->dhandle; - btree = S2BT(session); + btree = dhandle->handle; WT_ASSERT(session, F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE) && @@ -382,26 +426,34 @@ __wt_conn_btree_open( * in the tree that can block the close. */ if (F_ISSET(dhandle, WT_DHANDLE_OPEN)) - WT_RET(__wt_conn_btree_sync_and_close(session, false, false)); + WT_RET(__wt_conn_dhandle_close(session, false, false)); /* Discard any previous configuration, set up the new configuration. */ - __conn_btree_config_clear(session); - WT_RET(__conn_btree_config_set(session)); + __conn_dhandle_config_clear(session); + WT_RET(__conn_dhandle_config_set(session)); - /* Set any special flags on the handle. */ - F_SET(btree, LF_MASK(WT_BTREE_SPECIAL_FLAGS)); + switch (dhandle->type) { + case WT_DHANDLE_TYPE_BTREE: + /* Set any special flags on the btree handle. */ + F_SET(btree, LF_MASK(WT_BTREE_SPECIAL_FLAGS)); - /* - * Allocate data-source statistics memory. We don't allocate that memory - * when allocating the data-handle because not all data handles need - * statistics (for example, handles used for checkpoint locking). If we - * are reopening the handle, then it may already have statistics memory, - * check to avoid the leak. - */ - if (dhandle->stat_array == NULL) - WT_ERR(__wt_stat_dsrc_init(session, dhandle)); - - WT_ERR(__wt_btree_open(session, cfg)); + /* + * Allocate data-source statistics memory. We don't allocate + * that memory when allocating the data handle because not all + * data handles need statistics (for example, handles used for + * checkpoint locking). If we are reopening the handle, then + * it may already have statistics memory, check to avoid the + * leak. + */ + if (dhandle->stat_array == NULL) + WT_ERR(__wt_stat_dsrc_init(session, dhandle)); + + WT_ERR(__wt_btree_open(session, cfg)); + break; + case WT_DHANDLE_TYPE_TABLE: + WT_ERR(__wt_schema_open_table(session, cfg)); + break; + } /* * Bulk handles require true exclusive access, otherwise, handles @@ -423,7 +475,8 @@ __wt_conn_btree_open( ++S2C(session)->open_btree_count; if (0) { -err: F_CLR(btree, WT_BTREE_SPECIAL_FLAGS); +err: if (btree != NULL) + F_CLR(btree, WT_BTREE_SPECIAL_FLAGS); } return (ret); @@ -456,12 +509,12 @@ __conn_btree_apply_internal(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle, * sure it's referenced to stop other internal code dropping the handle * (e.g in LSM when cleaning up obsolete chunks). */ - if ((ret = __wt_session_get_btree(session, + if ((ret = __wt_session_get_dhandle(session, dhandle->name, dhandle->checkpoint, NULL, 0)) != 0) return (ret == EBUSY ? 0 : ret); WT_SAVE_DHANDLE(session, ret = file_func(session, cfg)); - WT_TRET(__wt_session_release_btree(session)); + WT_TRET(__wt_session_release_dhandle(session)); return (ret); } @@ -514,8 +567,8 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, if (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || F_ISSET(dhandle, WT_DHANDLE_DEAD) || + dhandle->type != WT_DHANDLE_TYPE_BTREE || dhandle->checkpoint != NULL || - !WT_PREFIX_MATCH(dhandle->name, "file:") || WT_IS_METADATA(dhandle)) continue; WT_ERR(__conn_btree_apply_internal(session, @@ -542,7 +595,7 @@ __conn_dhandle_close_one(WT_SESSION_IMPL *session, * operation (indicated by metadata tracking being enabled), hold the * lock for the duration of the operation. */ - WT_RET(__wt_session_get_btree(session, uri, checkpoint, + WT_RET(__wt_session_get_dhandle(session, uri, checkpoint, NULL, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_LOCK_ONLY)); if (WT_META_TRACKING(session)) WT_RET(__wt_meta_track_handle_lock(session, false)); @@ -553,7 +606,7 @@ __conn_dhandle_close_one(WT_SESSION_IMPL *session, */ if (F_ISSET(session->dhandle, WT_DHANDLE_OPEN)) { __wt_meta_track_sub_on(session); - ret = __wt_conn_btree_sync_and_close(session, false, mark_dead); + ret = __wt_conn_dhandle_close(session, false, mark_dead); /* * If the close succeeded, drop any locks it acquired. If @@ -565,7 +618,7 @@ __conn_dhandle_close_one(WT_SESSION_IMPL *session, } if (!WT_META_TRACKING(session)) - WT_TRET(__wt_session_release_btree(session)); + WT_TRET(__wt_session_release_dhandle(session)); return (ret); } @@ -657,8 +710,7 @@ __wt_conn_dhandle_discard_single( dhandle = session->dhandle; if (F_ISSET(dhandle, WT_DHANDLE_OPEN)) { - tret = - __wt_conn_btree_sync_and_close(session, final, mark_dead); + tret = __wt_conn_dhandle_close(session, final, mark_dead); if (final && tret != 0) { __wt_err(session, tret, "Final close of %s failed", dhandle->name); @@ -688,7 +740,7 @@ __wt_conn_dhandle_discard_single( * After successfully removing the handle, clean it up. */ if (ret == 0 || final) { - __conn_btree_config_clear(session); + __conn_dhandle_config_clear(session); WT_TRET(__conn_dhandle_destroy(session, dhandle)); session->dhandle = NULL; } @@ -758,3 +810,42 @@ restart: return (ret); } + +/* + * __wt_verbose_dump_handles -- + * Dump information about all data handles. + */ +int +__wt_verbose_dump_handles(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_DATA_HANDLE *dhandle; + + conn = S2C(session); + + WT_RET(__wt_msg(session, "%s", WT_DIVIDER)); + WT_RET(__wt_msg(session, "Data handle dump:")); + for (dhandle = NULL;;) { + WT_WITH_HANDLE_LIST_READ_LOCK(session, + WT_DHANDLE_NEXT(session, dhandle, &conn->dhqh, q)); + if (dhandle == NULL) + break; + WT_RET(__wt_msg(session, "Name: %s", dhandle->name)); + if (dhandle->checkpoint != NULL) + WT_RET(__wt_msg(session, + "Checkpoint: %s", dhandle->checkpoint)); + WT_RET(__wt_msg(session, " Sessions referencing handle: %" + PRIu32, dhandle->session_ref)); + WT_RET(__wt_msg(session, " Sessions using handle: %" + PRId32, dhandle->session_inuse)); + WT_RET(__wt_msg(session, " Exclusive references to handle: %" + PRIu32, dhandle->excl_ref)); + if (dhandle->excl_ref != 0) + WT_RET(__wt_msg(session, + " Session with exclusive use: %p", + (void *)dhandle->excl_session)); + WT_RET(__wt_msg(session, + " Flags: 0x%08" PRIx32, dhandle->flags)); + } + return (0); +} diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index fdf9bc8627e..b89efd884f5 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -543,7 +543,8 @@ __log_file_server(void *arg) if (!conn->hot_backup) { __wt_readlock( session, &conn->hot_backup_lock); - if (!conn->hot_backup) + if (!conn->hot_backup && + conn->log_cursors == 0) WT_ERR_ERROR_OK( __wt_ftruncate(session, close_fh, diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index d1d7d264d53..af40108602c 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -175,7 +175,6 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) if ((s = conn->sessions) != NULL) for (i = 0; i < conn->session_size; ++s, ++i) { __wt_free(session, s->dhhash); - __wt_free(session, s->tablehash); __wt_stash_discard_all(session, s); __wt_free(session, s->hazard); } diff --git a/src/third_party/wiredtiger/src/conn/conn_reconfig.c b/src/third_party/wiredtiger/src/conn/conn_reconfig.c index e67f2c9a18d..ae240443f0b 100644 --- a/src/third_party/wiredtiger/src/conn/conn_reconfig.c +++ b/src/third_party/wiredtiger/src/conn/conn_reconfig.c @@ -21,8 +21,7 @@ __wt_conn_compat_config(WT_SESSION_IMPL *session, const char **cfg) bool txn_active; conn = S2C(session); - WT_RET(__wt_config_gets(session, cfg, - "compatibility.release", &cval)); + WT_RET(__wt_config_gets(session, cfg, "compatibility.release", &cval)); if (cval.len == 0) { conn->compat_major = WIREDTIGER_VERSION_MAJOR; conn->compat_minor = WIREDTIGER_VERSION_MINOR; @@ -30,9 +29,9 @@ __wt_conn_compat_config(WT_SESSION_IMPL *session, const char **cfg) } /* - * Accept either a major.minor release string or a - * major.minor.patch release string. We ignore the patch - * value, but allow it in the string. + * Accept either a major.minor release string or a major.minor.patch + * release string. We ignore the patch value, but allow it in the + * string. */ if (sscanf(cval.str, "%" SCNu16 ".%" SCNu16, &conn->compat_major, &conn->compat_minor) != 2 && @@ -40,10 +39,10 @@ __wt_conn_compat_config(WT_SESSION_IMPL *session, const char **cfg) &conn->compat_major, &conn->compat_minor, &patch) != 3) WT_RET_MSG(session, EINVAL, "illegal compatibility release"); if (conn->compat_major > WIREDTIGER_VERSION_MAJOR) - WT_RET_MSG(session, EINVAL, "unknown major version"); + WT_RET_MSG(session, ENOTSUP, "unsupported major version"); if (conn->compat_major == WIREDTIGER_VERSION_MAJOR && conn->compat_minor > WIREDTIGER_VERSION_MINOR) - WT_RET_MSG(session, EINVAL, "illegal compatibility version"); + WT_RET_MSG(session, ENOTSUP, "unsupported minor version"); /* * We're doing an upgrade or downgrade, check whether transactions are @@ -52,7 +51,7 @@ __wt_conn_compat_config(WT_SESSION_IMPL *session, const char **cfg) WT_RET(__wt_txn_activity_check(session, &txn_active)); if (txn_active) WT_RET_MSG(session, ENOTSUP, - "upgrade / downgrade must run single-threaded"); + "system must be quiescent for upgrade or downgrade"); return (0); } @@ -188,6 +187,7 @@ __wt_conn_reconfig(WT_SESSION_IMPL *session, const char **cfg) */ WT_WITH_CHECKPOINT_LOCK(session, ret = __wt_conn_compat_config(session, cfg)); + WT_ERR(ret); WT_ERR(__wt_conn_statistics_config(session, cfg)); WT_ERR(__wt_async_reconfig(session, cfg)); WT_ERR(__wt_cache_config(session, true, cfg)); diff --git a/src/third_party/wiredtiger/src/conn/conn_sweep.c b/src/third_party/wiredtiger/src/conn/conn_sweep.c index 7236735715f..008aa6c08d8 100644 --- a/src/third_party/wiredtiger/src/conn/conn_sweep.c +++ b/src/third_party/wiredtiger/src/conn/conn_sweep.c @@ -63,8 +63,8 @@ __sweep_expire_one(WT_SESSION_IMPL *session) WT_DATA_HANDLE *dhandle; WT_DECL_RET; - btree = S2BT(session); dhandle = session->dhandle; + btree = dhandle->type == WT_DHANDLE_TYPE_BTREE ? dhandle->handle : NULL; /* * Acquire an exclusive lock on the handle and mark it dead. @@ -84,16 +84,17 @@ __sweep_expire_one(WT_SESSION_IMPL *session) WT_RET(__wt_try_writelock(session, &dhandle->rwlock)); /* Only sweep clean trees where all updates are visible. */ - if (btree->modified || !__wt_txn_visible_all(session, - btree->rec_max_txn, WT_TIMESTAMP_NULL(&btree->rec_max_timestamp))) + if (btree != NULL && (btree->modified || !__wt_txn_visible_all(session, + btree->rec_max_txn, WT_TIMESTAMP_NULL(&btree->rec_max_timestamp)))) goto err; /* - * Mark the handle dead and close the underlying file handle. - * Closing the handle decrements the open file count, meaning the close - * loop won't overrun the configured minimum. + * Mark the handle dead and close the underlying handle. + * + * For btree handles, closing the handle decrements the open file + * count, meaning the close loop won't overrun the configured minimum. */ - ret = __wt_conn_btree_sync_and_close(session, false, true); + ret = __wt_conn_dhandle_close(session, false, true); err: __wt_writeunlock(session, &dhandle->rwlock); @@ -130,8 +131,17 @@ __sweep_expire(WT_SESSION_IMPL *session, time_t now) conn->sweep_idle_time) continue; - WT_WITH_DHANDLE(session, dhandle, - ret = __sweep_expire_one(session)); + /* + * For tables, we need to hold the table lock to avoid racing + * with cursor opens. + */ + if (dhandle->type == WT_DHANDLE_TYPE_TABLE) + WT_WITH_TABLE_WRITE_LOCK(session, + WT_WITH_DHANDLE(session, dhandle, + ret = __sweep_expire_one(session))); + else + WT_WITH_DHANDLE(session, dhandle, + ret = __sweep_expire_one(session)); WT_RET_BUSY_OK(ret); } @@ -163,7 +173,7 @@ __sweep_discard_trees(WT_SESSION_IMPL *session, u_int *dead_handlesp) /* If the handle is marked dead, flush it from cache. */ WT_WITH_DHANDLE(session, dhandle, ret = - __wt_conn_btree_sync_and_close(session, false, false)); + __wt_conn_dhandle_close(session, false, false)); /* We closed the btree handle. */ if (ret == 0) { @@ -230,8 +240,13 @@ __sweep_remove_handles(WT_SESSION_IMPL *session) if (!WT_DHANDLE_CAN_DISCARD(dhandle)) continue; - WT_WITH_HANDLE_LIST_WRITE_LOCK(session, - ret = __sweep_remove_one(session, dhandle)); + if (dhandle->type == WT_DHANDLE_TYPE_TABLE) + WT_WITH_TABLE_WRITE_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, + ret = __sweep_remove_one(session, dhandle))); + else + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, + ret = __sweep_remove_one(session, dhandle)); if (ret == 0) WT_STAT_CONN_INCR(session, dh_sweep_remove); else diff --git a/src/third_party/wiredtiger/src/cursor/cur_file.c b/src/third_party/wiredtiger/src/cursor/cur_file.c index b5a8e1353ca..9aa93ade372 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_file.c +++ b/src/third_party/wiredtiger/src/cursor/cur_file.c @@ -441,7 +441,7 @@ __curfile_close(WT_CURSOR *cursor) if (session->dhandle != NULL) { /* Decrement the data-source's in-use counter. */ __wt_cursor_dhandle_decr_use(session); - WT_TRET(__wt_session_release_btree(session)); + WT_TRET(__wt_session_release_dhandle(session)); } err: API_END_RET(session, ret); @@ -629,29 +629,28 @@ __wt_curfile_open(WT_SESSION_IMPL *session, const char *uri, if (bulk) LF_SET(WT_BTREE_BULK | WT_DHANDLE_EXCLUSIVE); + WT_ASSERT(session, WT_PREFIX_MATCH(uri, "file:")); + /* Get the handle and lock it while the cursor is using it. */ - if (WT_PREFIX_MATCH(uri, "file:")) { - /* - * If we are opening exclusive and don't want a bulk cursor - * open to fail with EBUSY due to a database-wide checkpoint, - * get the handle while holding the checkpoint lock. - */ - if (LF_ISSET(WT_DHANDLE_EXCLUSIVE) && checkpoint_wait) - WT_WITH_CHECKPOINT_LOCK(session, - ret = __wt_session_get_btree_ckpt( - session, uri, cfg, flags)); - else - ret = __wt_session_get_btree_ckpt( - session, uri, cfg, flags); - WT_RET(ret); - } else - WT_RET(__wt_bad_object_type(session, uri)); + /* + * If we are opening exclusive and don't want a bulk cursor + * open to fail with EBUSY due to a database-wide checkpoint, + * get the handle while holding the checkpoint lock. + */ + if (LF_ISSET(WT_DHANDLE_EXCLUSIVE) && checkpoint_wait) + WT_WITH_CHECKPOINT_LOCK(session, + ret = __wt_session_get_btree_ckpt( + session, uri, cfg, flags)); + else + ret = __wt_session_get_btree_ckpt( + session, uri, cfg, flags); + WT_RET(ret); WT_ERR(__curfile_create(session, owner, cfg, bulk, bitmap, cursorp)); return (0); err: /* If the cursor could not be opened, release the handle. */ - WT_TRET(__wt_session_release_btree(session)); + WT_TRET(__wt_session_release_dhandle(session)); return (ret); } diff --git a/src/third_party/wiredtiger/src/cursor/cur_index.c b/src/third_party/wiredtiger/src/cursor/cur_index.c index e8fcb1b2702..c523ec620a2 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_index.c +++ b/src/third_party/wiredtiger/src/cursor/cur_index.c @@ -382,7 +382,7 @@ __curindex_close(WT_CURSOR *cursor) if (cindex->child != NULL) WT_TRET(cindex->child->close(cindex->child)); - __wt_schema_release_table(session, cindex->table); + WT_TRET(__wt_schema_release_table(session, cindex->table)); /* The URI is owned by the index. */ cursor->internal_uri = NULL; WT_TRET(__wt_cursor_close(cursor)); @@ -472,7 +472,7 @@ __wt_curindex_open(WT_SESSION_IMPL *session, ++idxname; if ((ret = __wt_schema_get_table(session, - tablename, namesize, false, &table)) != 0) { + tablename, namesize, false, 0, &table)) != 0) { if (ret == WT_NOTFOUND) WT_RET_MSG(session, EINVAL, "Cannot open cursor '%s' on unknown table", uri); @@ -487,7 +487,7 @@ __wt_curindex_open(WT_SESSION_IMPL *session, if ((ret = __wt_schema_open_index( session, table, idxname, namesize, &idx)) != 0) { - __wt_schema_release_table(session, table); + WT_TRET(__wt_schema_release_table(session, table)); return (ret); } WT_RET(__wt_calloc_one(session, &cindex)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_join.c b/src/third_party/wiredtiger/src/cursor/cur_join.c index e3ae9dbd9f6..06829e77809 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_join.c +++ b/src/third_party/wiredtiger/src/cursor/cur_join.c @@ -326,7 +326,8 @@ __curjoin_close(WT_CURSOR *cursor) JOINABLE_CURSOR_API_CALL(cursor, session, close, NULL); - __wt_schema_release_table(session, cjoin->table); + WT_TRET(__wt_schema_release_table(session, cjoin->table)); + /* This is owned by the table */ cursor->key_format = NULL; if (cjoin->projection != NULL) { @@ -457,7 +458,6 @@ __curjoin_entry_in_range(WT_SESSION_IMPL *session, WT_CURSOR_JOIN_ENTRY *entry, collator = (entry->index != NULL) ? entry->index->collator : NULL; endmax = &entry->ends[entry->ends_next]; disjunction = F_ISSET(entry, WT_CURJOIN_ENTRY_DISJUNCTION); - passed = false; /* * The iterator may have already satisfied some endpoint conditions. @@ -776,10 +776,10 @@ __curjoin_init_bloom(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, * For joins on the main table, we just need the primary * key for comparison, we don't need any values. */ - size = strlen(cjoin->table->name) + 3; + size = strlen(cjoin->table->iface.name) + 3; WT_ERR(__wt_scr_alloc(session, size, &uribuf)); WT_ERR(__wt_buf_fmt(session, uribuf, "%s()", - cjoin->table->name)); + cjoin->table->iface.name)); uri = uribuf->data; } WT_ERR(__wt_open_cursor(session, uri, &cjoin->iface, raw_cfg, &c)); @@ -926,7 +926,7 @@ __curjoin_init_next(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, config = &raw_cfg[0]; else config = &def_cfg[0]; - urimain = cjoin->table->name; + urimain = cjoin->table->iface.name; if ((proj = cjoin->projection) != NULL) { size = strlen(urimain) + strlen(proj) + 1; WT_ERR(__wt_calloc(session, size, 1, &mainbuf)); @@ -1147,10 +1147,10 @@ __curjoin_open_main(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, main_uri = newformat = NULL; idx = entry->index; - newsize = strlen(cjoin->table->name) + idx->colconf.len + 1; + newsize = strlen(cjoin->table->iface.name) + idx->colconf.len + 1; WT_ERR(__wt_calloc(session, 1, newsize, &main_uri)); WT_ERR(__wt_snprintf(main_uri, newsize, "%s%.*s", - cjoin->table->name, (int)idx->colconf.len, idx->colconf.str)); + cjoin->table->iface.name, (int)idx->colconf.len, idx->colconf.str)); WT_ERR(__wt_open_cursor(session, main_uri, (WT_CURSOR *)cjoin, raw_cfg, &entry->main)); if (idx->extractor == NULL) { @@ -1324,7 +1324,8 @@ __wt_curjoin_open(WT_SESSION_IMPL *session, size = strlen(tablename); else size = WT_PTRDIFF(columns, tablename); - WT_RET(__wt_schema_get_table(session, tablename, size, 0, &table)); + WT_RET(__wt_schema_get_table( + session, tablename, size, false, 0, &table)); WT_RET(__wt_calloc_one(session, &cjoin)); cursor = &cjoin->iface; diff --git a/src/third_party/wiredtiger/src/cursor/cur_log.c b/src/third_party/wiredtiger/src/cursor/cur_log.c index 0ba3ce83e59..bcfac1bfe3e 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_log.c +++ b/src/third_party/wiredtiger/src/cursor/cur_log.c @@ -317,8 +317,10 @@ __curlog_close(WT_CURSOR *cursor) cl = (WT_CURSOR_LOG *)cursor; conn = S2C(session); - if (F_ISSET(cl, WT_CURLOG_ARCHIVE_LOCK)) + if (F_ISSET(cl, WT_CURLOG_ARCHIVE_LOCK)) { + (void)__wt_atomic_sub32(&conn->log_cursors, 1); __wt_readunlock(session, &conn->log->log_archive_lock); + } __wt_free(session, cl->cur_lsn); __wt_free(session, cl->next_lsn); @@ -398,6 +400,8 @@ __wt_curlog_open(WT_SESSION_IMPL *session, /* Log cursors block archiving. */ __wt_readlock(session, &log->log_archive_lock); F_SET(cl, WT_CURLOG_ARCHIVE_LOCK); + (void)__wt_atomic_add32(&conn->log_cursors, 1); + } if (0) { diff --git a/src/third_party/wiredtiger/src/cursor/cur_stat.c b/src/third_party/wiredtiger/src/cursor/cur_stat.c index a1ec1d75918..bf13114a47e 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_stat.c +++ b/src/third_party/wiredtiger/src/cursor/cur_stat.c @@ -407,7 +407,7 @@ __curstat_file_init(WT_SESSION_IMPL *session, } /* Release the handle, we're done with it. */ - WT_TRET(__wt_session_release_btree(session)); + WT_TRET(__wt_session_release_dhandle(session)); return (ret); } diff --git a/src/third_party/wiredtiger/src/cursor/cur_table.c b/src/third_party/wiredtiger/src/cursor/cur_table.c index 000fcae99f2..9010e761ee7 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_table.c +++ b/src/third_party/wiredtiger/src/cursor/cur_table.c @@ -829,7 +829,8 @@ __curtable_close(WT_CURSOR *cursor) __wt_free(session, ctable->cg_cursors); __wt_free(session, ctable->cg_valcopy); __wt_free(session, ctable->idx_cursors); - __wt_schema_release_table(session, ctable->table); + + WT_TRET(__wt_schema_release_table(session, ctable->table)); /* The URI is owned by the table. */ cursor->internal_uri = NULL; WT_TRET(__wt_cursor_close(cursor)); @@ -854,7 +855,7 @@ __curtable_complete(WT_SESSION_IMPL *session, WT_TABLE *table) if (!complete) WT_RET_MSG(session, EINVAL, "'%s' not available until all column groups are created", - table->name); + table->iface.name); return (0); } @@ -971,14 +972,16 @@ __wt_curtable_open(WT_SESSION_IMPL *session, ctable = NULL; tablename = uri; - if (!WT_PREFIX_SKIP(tablename, "table:")) - return (__wt_unexpected_object_type(session, uri, "table:")); + WT_PREFIX_SKIP_REQUIRED(session, tablename, "table:"); columns = strchr(tablename, '('); if (columns == NULL) - size = strlen(tablename); - else + WT_RET(__wt_schema_get_table_uri( + session, uri, false, 0, &table)); + else { size = WT_PTRDIFF(columns, tablename); - WT_RET(__wt_schema_get_table(session, tablename, size, false, &table)); + WT_RET(__wt_schema_get_table( + session, tablename, size, false, 0, &table)); + } WT_RET(__curtable_complete(session, table)); /* completeness check */ @@ -987,7 +990,7 @@ __wt_curtable_open(WT_SESSION_IMPL *session, ret = __wt_open_cursor(session, table->cgroups[0]->source, NULL, cfg, cursorp); - __wt_schema_release_table(session, table); + WT_TRET(__wt_schema_release_table(session, table)); if (ret == 0) { /* Fix up the public URI to match what was passed in. */ cursor = *cursorp; @@ -1002,7 +1005,7 @@ __wt_curtable_open(WT_SESSION_IMPL *session, cursor = &ctable->iface; *cursor = iface; cursor->session = &session->iface; - cursor->internal_uri = table->name; + cursor->internal_uri = table->iface.name; cursor->key_format = table->key_format; cursor->value_format = table->value_format; diff --git a/src/third_party/wiredtiger/src/docs/Doxyfile b/src/third_party/wiredtiger/src/docs/Doxyfile index e95d8babe48..178655bf6ed 100644 --- a/src/third_party/wiredtiger/src/docs/Doxyfile +++ b/src/third_party/wiredtiger/src/docs/Doxyfile @@ -205,12 +205,15 @@ TAB_SIZE = 8 # will result in a user-defined paragraph with heading "Side Effects:". # You can put \n's in the value part of an alias to insert newlines. -ALIASES = "notyet{1}=Note: <b>"\1"</b> not yet supported in WiredTiger.\n@todo fix when \1 supported\n\n" \ +ALIASES = \ + "config{3}= @row{<tt>\1</tt>,\2,\3}" \ + "configempty{2}=@param config\n Configuration string, see @ref config_strings. No values currently permitted." \ + "configend= </table>" \ + "configstart{2}=@param config\n Configuration string, see @ref config_strings. Permitted values:\n <table>@hrow{Name,Effect,Values}" \ +"ebusy_errors=@returns zero on success, EBUSY if the object is not available for exclusive access, and a non-zero error code on failure. See @ref error_handling \"Error handling\" for details." \ "errors=@returns zero on success and a non-zero error code on failure. See @ref error_handling \"Error handling\" for details." \ - "ebusy_errors=@returns zero on success, EBUSY if the object is not available for exclusive access, and a non-zero error code on failure. See @ref error_handling \"Error handling\" for details." \ + "exclusive=This method requires exclusive access to the specified data source(s). If any cursors are open with the specified name(s) or a data source is otherwise in use, the call will fail and return \c EBUSY.\n\n" \ "ex_ref{1}=@ref \1 \"\1\"" \ - "ref_single=@ref" \ - "subpage_single=@subpage" \ "hrow{1}=<tr><th>\1</th></tr>" \ "hrow{2}=<tr><th>\1</th><th>\2</th></tr>" \ "hrow{3}=<tr><th>\1</th><th>\2</th><th>\3</th></tr>" \ @@ -220,6 +223,12 @@ ALIASES = "notyet{1}=Note: <b>"\1"</b> not yet supported in Wired "hrow{7}=<tr><th>\1</th><th>\2</th><th>\3</th><th>\4</th><th>\5</th><th>\6</th><th>\7</th></tr>" \ "hrow{8}=<tr><th>\1</th><th>\2</th><th>\3</th><th>\4</th><th>\5</th><th>\6</th><th>\7</th><th>\8</th></tr>" \ "hrow{9}=<tr><th>\1</th><th>\2</th><th>\3</th><th>\4</th><th>\5</th><th>\6</th><th>\7</th><th>\8</th><th>\9</th></tr>" \ + "notyet{1}=Note: <b>"\1"</b> not yet supported in WiredTiger.\n@todo fix when \1 supported\n\n" \ + "plantuml_end=PlantUML template end -->" \ + "plantuml_start{1}=\image html \1\n\image latex \1\n<!-- PlantUML template begins" \ + "requires_notransaction=This method must not be called on a session with an active transaction.\n\n" \ + "requires_transaction=This method must be called on a session with an active transaction.\n\n" \ + "ref_single=@ref" \ "row{1}=<tr><td>\1</td></tr>" \ "row{2}=<tr><td>\1</td><td>\2</td></tr>" \ "row{3}=<tr><td>\1</td><td>\2</td><td>\3</td></tr>" \ @@ -229,12 +238,7 @@ ALIASES = "notyet{1}=Note: <b>"\1"</b> not yet supported in Wired "row{7}=<tr><td>\1</td><td>\2</td><td>\3</td><td>\4</td><td>\5</td><td>\6</td><td>\7</td></tr>" \ "row{8}=<tr><td>\1</td><td>\2</td><td>\3</td><td>\4</td><td>\5</td><td>\6</td><td>\7</td><td>\8</td></tr>" \ "row{9}=<tr><td>\1</td><td>\2</td><td>\3</td><td>\4</td><td>\5</td><td>\6</td><td>\7</td><td>\8</td><td>\9</td></tr>" \ - "configstart{2}=@param config\n Configuration string, see @ref config_strings. Permitted values:\n <table>@hrow{Name,Effect,Values}" \ - "config{3}= @row{<tt>\1</tt>,\2,\3}" \ - "configend= </table>" \ - "configempty{2}=@param config\n Configuration string, see @ref config_strings. No values currently permitted." \ - "plantuml_start{1}=\image html \1\n\image latex \1\n<!-- PlantUML template begins" \ - "plantuml_end=PlantUML template end -->" + "subpage_single=@subpage" \ # This tag can be used to specify a number of word-keyword mappings (TCL only). # A mapping has the form "name=value". For example adding diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 6b341a85df8..11e3758f910 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -809,7 +809,7 @@ __evict_clear_all_walks(WT_SESSION_IMPL *session) conn = S2C(session); TAILQ_FOREACH(dhandle, &conn->dhqh, q) - if (WT_PREFIX_MATCH(dhandle->name, "file:")) + if (dhandle->type == WT_DHANDLE_TYPE_BTREE) WT_WITH_DHANDLE(session, dhandle, WT_TRET(__evict_clear_walk(session))); return (ret); @@ -1399,8 +1399,8 @@ retry: while (slot < max_entries) { if (dhandle == NULL) break; - /* Ignore non-file handles, or handles that aren't open. */ - if (!WT_PREFIX_MATCH(dhandle->name, "file:") || + /* Ignore non-btree handles, or handles that aren't open. */ + if (dhandle->type != WT_DHANDLE_TYPE_BTREE || !F_ISSET(dhandle, WT_DHANDLE_OPEN)) continue; @@ -1730,7 +1730,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, * eviction fairly visits all pages in trees with a lot of in-cache * content. */ - switch ((WT_EVICT_WALK_START)btree->evict_start_type) { + switch (btree->evict_start_type) { case WT_EVICT_WALK_NEXT: break; case WT_EVICT_WALK_PREV: @@ -1788,9 +1788,23 @@ __evict_walk_file(WT_SESSION_IMPL *session, * Try a different walk start point next time if a * walk gave up. */ - btree->evict_start_type = - (btree->evict_start_type + 1) % - WT_EVICT_WALK_START_NUM; + switch (btree->evict_start_type) { + case WT_EVICT_WALK_NEXT: + btree->evict_start_type = WT_EVICT_WALK_PREV; + break; + case WT_EVICT_WALK_PREV: + btree->evict_start_type = + WT_EVICT_WALK_RAND_PREV; + break; + case WT_EVICT_WALK_RAND_PREV: + btree->evict_start_type = + WT_EVICT_WALK_RAND_NEXT; + break; + case WT_EVICT_WALK_RAND_NEXT: + btree->evict_start_type = WT_EVICT_WALK_NEXT; + break; + } + /* * We differentiate the reasons we gave up on this walk * and increment the stats accordingly. @@ -2404,7 +2418,6 @@ __wt_evict_priority_clear(WT_SESSION_IMPL *session) S2BT(session)->evict_priority = 0; } -#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) /* * __verbose_dump_cache_single -- * Output diagnostic information about a single file in the cache. @@ -2527,7 +2540,7 @@ __wt_verbose_dump_cache(WT_SESSION_IMPL *session) WT_DHANDLE_NEXT(session, dhandle, &conn->dhqh, q)); if (dhandle == NULL) break; - if (!WT_PREFIX_MATCH(dhandle->name, "file:") || + if (dhandle->type != WT_DHANDLE_TYPE_BTREE || !F_ISSET(dhandle, WT_DHANDLE_OPEN)) continue; @@ -2553,8 +2566,6 @@ __wt_verbose_dump_cache(WT_SESSION_IMPL *session) WT_RET(__wt_msg(session, "total dirty bytes: %" PRIu64 "MB", total_dirty_bytes / WT_MEGABYTE)); - WT_RET(__wt_msg(session, "%s", WT_DIVIDER)); return (0); } -#endif diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h index 4d2a4f0c4d9..f69fc355383 100644 --- a/src/third_party/wiredtiger/src/include/btree.h +++ b/src/third_party/wiredtiger/src/include/btree.h @@ -154,8 +154,12 @@ struct __wt_btree { u_int evict_walk_skips; /* Number of walks skipped */ int32_t evict_disabled; /* Eviction disabled count */ volatile uint32_t evict_busy; /* Count of threads in eviction */ - int evict_start_type; /* Start position for eviction walk - (see WT_EVICT_WALK_START). */ + enum { /* Start position for eviction walk */ + WT_EVICT_WALK_NEXT, + WT_EVICT_WALK_PREV, + WT_EVICT_WALK_RAND_NEXT, + WT_EVICT_WALK_RAND_PREV + } evict_start_type; /* * We flush pages from the tree (in order to make checkpoint faster), diff --git a/src/third_party/wiredtiger/src/include/cache.h b/src/third_party/wiredtiger/src/include/cache.h index 9b5057d0b61..9d7489732aa 100644 --- a/src/third_party/wiredtiger/src/include/cache.h +++ b/src/third_party/wiredtiger/src/include/cache.h @@ -14,15 +14,6 @@ #define WT_EVICT_WALK_BASE 300 /* Pages tracked across file visits */ #define WT_EVICT_WALK_INCR 100 /* Pages added each walk */ -/* Ways to position when starting an eviction walk. */ -typedef enum { - WT_EVICT_WALK_NEXT, - WT_EVICT_WALK_PREV, - WT_EVICT_WALK_RAND_NEXT, - WT_EVICT_WALK_RAND_PREV -} WT_EVICT_WALK_START; -#define WT_EVICT_WALK_START_NUM (WT_EVICT_WALK_RAND_PREV + 1) - /* * WT_EVICT_ENTRY -- * Encapsulation of an eviction candidate. diff --git a/src/third_party/wiredtiger/src/include/cache.i b/src/third_party/wiredtiger/src/include/cache.i index 1e058a3ec1b..6e0d2541c3c 100644 --- a/src/third_party/wiredtiger/src/include/cache.i +++ b/src/third_party/wiredtiger/src/include/cache.i @@ -216,7 +216,7 @@ __wt_session_can_wait(WT_SESSION_IMPL *session) /* * __wt_eviction_clean_needed -- * Return if an application thread should do eviction due to the total - * volume of dirty data in cache. + * volume of data in cache. */ static inline bool __wt_eviction_clean_needed(WT_SESSION_IMPL *session, u_int *pct_fullp) diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h index d02ec21023b..3698290f21b 100644 --- a/src/third_party/wiredtiger/src/include/config.h +++ b/src/third_party/wiredtiger/src/include/config.h @@ -56,50 +56,51 @@ struct __wt_config_parser_impl { #define WT_CONFIG_ENTRY_WT_CONNECTION_add_extractor 4 #define WT_CONFIG_ENTRY_WT_CONNECTION_async_new_op 5 #define WT_CONFIG_ENTRY_WT_CONNECTION_close 6 -#define WT_CONFIG_ENTRY_WT_CONNECTION_load_extension 7 -#define WT_CONFIG_ENTRY_WT_CONNECTION_open_session 8 -#define WT_CONFIG_ENTRY_WT_CONNECTION_query_timestamp 9 -#define WT_CONFIG_ENTRY_WT_CONNECTION_reconfigure 10 -#define WT_CONFIG_ENTRY_WT_CONNECTION_rollback_to_stable 11 -#define WT_CONFIG_ENTRY_WT_CONNECTION_set_file_system 12 -#define WT_CONFIG_ENTRY_WT_CONNECTION_set_timestamp 13 -#define WT_CONFIG_ENTRY_WT_CURSOR_close 14 -#define WT_CONFIG_ENTRY_WT_CURSOR_reconfigure 15 -#define WT_CONFIG_ENTRY_WT_SESSION_alter 16 -#define WT_CONFIG_ENTRY_WT_SESSION_begin_transaction 17 -#define WT_CONFIG_ENTRY_WT_SESSION_checkpoint 18 -#define WT_CONFIG_ENTRY_WT_SESSION_close 19 -#define WT_CONFIG_ENTRY_WT_SESSION_commit_transaction 20 -#define WT_CONFIG_ENTRY_WT_SESSION_compact 21 -#define WT_CONFIG_ENTRY_WT_SESSION_create 22 -#define WT_CONFIG_ENTRY_WT_SESSION_drop 23 -#define WT_CONFIG_ENTRY_WT_SESSION_join 24 -#define WT_CONFIG_ENTRY_WT_SESSION_log_flush 25 -#define WT_CONFIG_ENTRY_WT_SESSION_log_printf 26 -#define WT_CONFIG_ENTRY_WT_SESSION_open_cursor 27 -#define WT_CONFIG_ENTRY_WT_SESSION_rebalance 28 -#define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 29 -#define WT_CONFIG_ENTRY_WT_SESSION_rename 30 -#define WT_CONFIG_ENTRY_WT_SESSION_reset 31 -#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 32 -#define WT_CONFIG_ENTRY_WT_SESSION_salvage 33 -#define WT_CONFIG_ENTRY_WT_SESSION_snapshot 34 -#define WT_CONFIG_ENTRY_WT_SESSION_strerror 35 -#define WT_CONFIG_ENTRY_WT_SESSION_timestamp_transaction 36 -#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 37 -#define WT_CONFIG_ENTRY_WT_SESSION_truncate 38 -#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 39 -#define WT_CONFIG_ENTRY_WT_SESSION_verify 40 -#define WT_CONFIG_ENTRY_colgroup_meta 41 -#define WT_CONFIG_ENTRY_file_config 42 -#define WT_CONFIG_ENTRY_file_meta 43 -#define WT_CONFIG_ENTRY_index_meta 44 -#define WT_CONFIG_ENTRY_lsm_meta 45 -#define WT_CONFIG_ENTRY_table_meta 46 -#define WT_CONFIG_ENTRY_wiredtiger_open 47 -#define WT_CONFIG_ENTRY_wiredtiger_open_all 48 -#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 49 -#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 50 +#define WT_CONFIG_ENTRY_WT_CONNECTION_debug_info 7 +#define WT_CONFIG_ENTRY_WT_CONNECTION_load_extension 8 +#define WT_CONFIG_ENTRY_WT_CONNECTION_open_session 9 +#define WT_CONFIG_ENTRY_WT_CONNECTION_query_timestamp 10 +#define WT_CONFIG_ENTRY_WT_CONNECTION_reconfigure 11 +#define WT_CONFIG_ENTRY_WT_CONNECTION_rollback_to_stable 12 +#define WT_CONFIG_ENTRY_WT_CONNECTION_set_file_system 13 +#define WT_CONFIG_ENTRY_WT_CONNECTION_set_timestamp 14 +#define WT_CONFIG_ENTRY_WT_CURSOR_close 15 +#define WT_CONFIG_ENTRY_WT_CURSOR_reconfigure 16 +#define WT_CONFIG_ENTRY_WT_SESSION_alter 17 +#define WT_CONFIG_ENTRY_WT_SESSION_begin_transaction 18 +#define WT_CONFIG_ENTRY_WT_SESSION_checkpoint 19 +#define WT_CONFIG_ENTRY_WT_SESSION_close 20 +#define WT_CONFIG_ENTRY_WT_SESSION_commit_transaction 21 +#define WT_CONFIG_ENTRY_WT_SESSION_compact 22 +#define WT_CONFIG_ENTRY_WT_SESSION_create 23 +#define WT_CONFIG_ENTRY_WT_SESSION_drop 24 +#define WT_CONFIG_ENTRY_WT_SESSION_join 25 +#define WT_CONFIG_ENTRY_WT_SESSION_log_flush 26 +#define WT_CONFIG_ENTRY_WT_SESSION_log_printf 27 +#define WT_CONFIG_ENTRY_WT_SESSION_open_cursor 28 +#define WT_CONFIG_ENTRY_WT_SESSION_rebalance 29 +#define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 30 +#define WT_CONFIG_ENTRY_WT_SESSION_rename 31 +#define WT_CONFIG_ENTRY_WT_SESSION_reset 32 +#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 33 +#define WT_CONFIG_ENTRY_WT_SESSION_salvage 34 +#define WT_CONFIG_ENTRY_WT_SESSION_snapshot 35 +#define WT_CONFIG_ENTRY_WT_SESSION_strerror 36 +#define WT_CONFIG_ENTRY_WT_SESSION_timestamp_transaction 37 +#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 38 +#define WT_CONFIG_ENTRY_WT_SESSION_truncate 39 +#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 40 +#define WT_CONFIG_ENTRY_WT_SESSION_verify 41 +#define WT_CONFIG_ENTRY_colgroup_meta 42 +#define WT_CONFIG_ENTRY_file_config 43 +#define WT_CONFIG_ENTRY_file_meta 44 +#define WT_CONFIG_ENTRY_index_meta 45 +#define WT_CONFIG_ENTRY_lsm_meta 46 +#define WT_CONFIG_ENTRY_table_meta 47 +#define WT_CONFIG_ENTRY_wiredtiger_open 48 +#define WT_CONFIG_ENTRY_wiredtiger_open_all 49 +#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 50 +#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 51 /* * configuration section: END * DO NOT EDIT: automatically built by dist/flags.py. diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 2370444b681..e5593357347 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -7,10 +7,10 @@ */ /* - * Default hash table size; use a prime number of buckets rather than assuming - * a good hash (Reference Sedgewick, Algorithms in C, "Hash Functions"). + * Default hash table size; we don't need a prime number of buckets + * because we always use a good hash function. */ -#define WT_HASH_ARRAY_SIZE 509 +#define WT_HASH_ARRAY_SIZE 512 /******************************************* * Global per-process structure. @@ -336,6 +336,7 @@ struct __wt_connection_impl { bool log_wrlsn_tid_set;/* Log write lsn thread set */ WT_LOG *log; /* Logging structure */ WT_COMPRESSOR *log_compressor;/* Logging compressor */ + uint32_t log_cursors; /* Log cursor count */ wt_off_t log_file_max; /* Log file max size */ const char *log_path; /* Logging path format */ uint32_t log_prealloc; /* Log file pre-allocation */ @@ -379,6 +380,7 @@ struct __wt_connection_impl { /* Set of btree IDs not being rolled back */ uint8_t *stable_rollback_bitstring; + uint32_t stable_rollback_maxfile; /* Locked: collator list */ TAILQ_HEAD(__wt_coll_qh, __wt_named_collator) collqh; diff --git a/src/third_party/wiredtiger/src/include/dhandle.h b/src/third_party/wiredtiger/src/include/dhandle.h index 32574f05fa1..2844c6e3e73 100644 --- a/src/third_party/wiredtiger/src/include/dhandle.h +++ b/src/third_party/wiredtiger/src/include/dhandle.h @@ -64,6 +64,11 @@ struct __wt_data_handle { TAILQ_ENTRY(__wt_data_handle) q; TAILQ_ENTRY(__wt_data_handle) hashq; + const char *name; /* Object name as a URI */ + uint64_t name_hash; /* Hash of name */ + const char *checkpoint; /* Checkpoint name (or NULL) */ + const char **cfg; /* Configuration information */ + /* * Sessions caching a connection's data handle will have a non-zero * reference count; sessions using a connection's data handle will @@ -75,16 +80,16 @@ struct __wt_data_handle { time_t timeofdeath; /* Use count went to 0 */ WT_SESSION_IMPL *excl_session; /* Session with exclusive use, if any */ - uint64_t name_hash; /* Hash of name */ - const char *name; /* Object name as a URI */ - const char *checkpoint; /* Checkpoint name (or NULL) */ - const char **cfg; /* Configuration information */ - - bool compact_skip; /* If the handle failed to compact */ - WT_DATA_SOURCE *dsrc; /* Data source for this handle */ void *handle; /* Generic handle */ + enum { + WT_DHANDLE_TYPE_BTREE, + WT_DHANDLE_TYPE_TABLE + } type; + + bool compact_skip; /* If the handle failed to compact */ + /* * Data handles can be closed without holding the schema lock; threads * walk the list of open handles, operating on them (checkpoint is the diff --git a/src/third_party/wiredtiger/src/include/error.h b/src/third_party/wiredtiger/src/include/error.h index 2d302fb6879..3b3c0769c90 100644 --- a/src/third_party/wiredtiger/src/include/error.h +++ b/src/third_party/wiredtiger/src/include/error.h @@ -118,7 +118,7 @@ #ifdef HAVE_DIAGNOSTIC #define WT_ASSERT(session, exp) do { \ if (!(exp)) \ - __wt_assert(session, 0, __FILE__, __LINE__, "%s", #exp);\ + __wt_assert(session, 0, __func__, __LINE__, "%s", #exp);\ } while (0) #else #define WT_ASSERT(session, exp) \ diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 9a86dbc1a26..f4846029026 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -248,6 +248,7 @@ extern int __wt_conn_remove_encryptor(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL extern int __wt_extractor_config(WT_SESSION_IMPL *session, const char *uri, const char *config, WT_EXTRACTOR **extractorp, int *ownp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_remove_extractor(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_verbose_dump_sessions(WT_SESSION_IMPL *session, bool show_cursors) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_timing_stress_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cache_config(WT_SESSION_IMPL *session, bool reconfigure, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cache_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -262,12 +263,13 @@ extern int __wt_checkpoint_server_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_ extern void __wt_checkpoint_signal(WT_SESSION_IMPL *session, wt_off_t logsize); extern int __wt_conn_dhandle_alloc( WT_SESSION_IMPL *session, const char *uri, const char *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_dhandle_find( WT_SESSION_IMPL *session, const char *uri, const char *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_conn_btree_sync_and_close( WT_SESSION_IMPL *session, bool final, bool mark_dead) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_conn_btree_open( WT_SESSION_IMPL *session, const char *cfg[], uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_dhandle_close( WT_SESSION_IMPL *session, bool final, bool mark_dead) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_conn_dhandle_open( WT_SESSION_IMPL *session, const char *cfg[], uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_dhandle_close_all( WT_SESSION_IMPL *session, const char *uri, bool mark_dead) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_dhandle_discard_single( WT_SESSION_IMPL *session, bool final, bool mark_dead) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_dhandle_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_verbose_dump_handles(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_connection_destroy(WT_CONNECTION_IMPL *conn); extern int __wt_logmgr_reconfig(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -436,6 +438,7 @@ extern int64_t __wt_log_slot_release(WT_MYSLOT *myslot, int64_t size); extern void __wt_log_slot_free(WT_SESSION_IMPL *session, WT_LOGSLOT *slot); extern int __wt_log_system_record( WT_SESSION_IMPL *session, WT_FH *log_fh, WT_LSN *lsn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_recover_system(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, WT_LSN *lsnp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_verbose_dump_log(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_clsm_request_switch(WT_CURSOR_LSM *clsm) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_clsm_await_switch(WT_CURSOR_LSM *clsm) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_clsm_init_merge( WT_CURSOR *cursor, u_int start_chunk, uint32_t start_id, u_int nchunks) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -531,6 +534,7 @@ extern int __wt_malloc(WT_SESSION_IMPL *session, size_t bytes_to_allocate, void extern int __wt_realloc(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_realloc_noclear(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_realloc_aligned(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_memdup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_strndup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_free_int(WT_SESSION_IMPL *session, const void *p_arg) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_errno(void) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -587,21 +591,19 @@ extern int __wt_schema_colgroup_source(WT_SESSION_IMPL *session, WT_TABLE *table extern int __wt_schema_index_source(WT_SESSION_IMPL *session, WT_TABLE *table, const char *idxname, const char *config, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_create( WT_SESSION_IMPL *session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_schema_get_table(WT_SESSION_IMPL *session, const char *name, size_t namelen, bool ok_incomplete, WT_TABLE **tablep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_schema_release_table(WT_SESSION_IMPL *session, WT_TABLE *table); +extern int __wt_schema_get_table_uri(WT_SESSION_IMPL *session, const char *uri, bool ok_incomplete, uint32_t flags, WT_TABLE **tablep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_get_table(WT_SESSION_IMPL *session, const char *name, size_t namelen, bool ok_incomplete, uint32_t flags, WT_TABLE **tablep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_release_table(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_schema_destroy_colgroup(WT_SESSION_IMPL *session, WT_COLGROUP **colgroupp); extern int __wt_schema_destroy_index(WT_SESSION_IMPL *session, WT_INDEX **idxp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_schema_destroy_table(WT_SESSION_IMPL *session, WT_TABLE **tablep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_schema_remove_table(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_schema_close_tables(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_schema_sweep_tables(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_close_table(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_colgroup_name(WT_SESSION_IMPL *session, WT_TABLE *table, const char *cgname, size_t len, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_open_colgroups(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_open_index(WT_SESSION_IMPL *session, WT_TABLE *table, const char *idxname, size_t len, WT_INDEX **indexp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_open_indices(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_get_colgroup(WT_SESSION_IMPL *session, const char *uri, bool quiet, WT_TABLE **tablep, WT_COLGROUP **colgroupp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_schema_get_index(WT_SESSION_IMPL *session, const char *uri, bool quiet, WT_TABLE **tablep, WT_INDEX **indexp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_schema_open_table(WT_SESSION_IMPL *session, const char *name, size_t namelen, bool ok_incomplete, WT_TABLE **tablep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_get_index(WT_SESSION_IMPL *session, const char *uri, bool invalidate, bool quiet, WT_INDEX **indexp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_open_table(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_colcheck(WT_SESSION_IMPL *session, const char *key_format, const char *value_format, WT_CONFIG_ITEM *colconf, u_int *kcolsp, u_int *vcolsp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_table_check(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_struct_plan(WT_SESSION_IMPL *session, WT_TABLE *table, const char *columns, size_t len, bool value_only, WT_ITEM *plan) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -637,10 +639,10 @@ extern int __wt_session_compact_check_timeout(WT_SESSION_IMPL *session) WT_GCC_F extern int __wt_session_compact( WT_SESSION *wt_session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_compact_readonly( WT_SESSION *wt_session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_lock_dhandle( WT_SESSION_IMPL *session, uint32_t flags, bool *is_deadp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_session_release_btree(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_release_dhandle(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_get_btree_ckpt(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_session_close_cache(WT_SESSION_IMPL *session); -extern int __wt_session_get_btree(WT_SESSION_IMPL *session, const char *uri, const char *checkpoint, const char *cfg[], uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_get_dhandle(WT_SESSION_IMPL *session, const char *uri, const char *checkpoint, const char *cfg[], uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_lock_checkpoint(WT_SESSION_IMPL *session, const char *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_salvage(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cond_auto_alloc(WT_SESSION_IMPL *session, const char *name, uint64_t min, uint64_t max, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -787,6 +789,7 @@ extern void __wt_txn_destroy(WT_SESSION_IMPL *session); extern int __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_txn_global_destroy(WT_SESSION_IMPL *session); extern int __wt_txn_global_shutdown(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_verbose_dump_txn_one(WT_SESSION_IMPL *session, WT_TXN *txn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_verbose_dump_txn(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[], bool waiting) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/log.h b/src/third_party/wiredtiger/src/include/log.h index e6acdd1e834..3ec874ad5b6 100644 --- a/src/third_party/wiredtiger/src/include/log.h +++ b/src/third_party/wiredtiger/src/include/log.h @@ -58,6 +58,13 @@ union __wt_lsn { #define WT_IS_ZERO_LSN(l) ((l)->file_offset == 0) /* + * Macro to print an LSN. + */ +#define WT_LSN_MSG(lsn, msg) \ + __wt_msg(session, "%s LSN: [%" PRIu32 "][%" PRIu32 "]", \ + (msg), (lsn)->l.file, (lsn)->l.offset) + +/* * Both of the macros below need to change if the content of __wt_lsn * ever changes. The value is the following: * txnid, record type, operation type, file id, operation key, operation value diff --git a/src/third_party/wiredtiger/src/include/misc.h b/src/third_party/wiredtiger/src/include/misc.h index a6cb56dd852..e7b3e1931ac 100644 --- a/src/third_party/wiredtiger/src/include/misc.h +++ b/src/third_party/wiredtiger/src/include/misc.h @@ -214,6 +214,12 @@ #define WT_PREFIX_SKIP(str, pfx) \ (WT_PREFIX_MATCH(str, pfx) ? ((str) += strlen(pfx), 1) : 0) +/* Assert that a string matches a prefix, and move past it. */ +#define WT_PREFIX_SKIP_REQUIRED(session, str, pfx) do { \ + WT_ASSERT(session, WT_PREFIX_MATCH(str, pfx)); \ + (str) += strlen(pfx); \ +} while (0) + /* * Check if a variable string equals a constant string. Inline the common * case for WiredTiger of a single byte string. This is required because not @@ -279,11 +285,11 @@ typedef void wt_timestamp_t; */ #ifdef HAVE_DIAGNOSTIC #define __wt_scr_alloc(session, size, scratchp) \ - __wt_scr_alloc_func(session, size, scratchp, __FILE__, __LINE__) + __wt_scr_alloc_func(session, size, scratchp, __func__, __LINE__) #define __wt_page_in(session, ref, flags) \ - __wt_page_in_func(session, ref, flags, __FILE__, __LINE__) + __wt_page_in_func(session, ref, flags, __func__, __LINE__) #define __wt_page_swap(session, held, want, flags) \ - __wt_page_swap_func(session, held, want, flags, __FILE__, __LINE__) + __wt_page_swap_func(session, held, want, flags, __func__, __LINE__) #else #define __wt_scr_alloc(session, size, scratchp) \ __wt_scr_alloc_func(session, size, scratchp) @@ -295,7 +301,7 @@ typedef void wt_timestamp_t; /* Called on unexpected code path: locate the failure. */ #define __wt_illegal_value(session, msg) \ - __wt_illegal_value_func(session, msg, __FILE__, __LINE__) + __wt_illegal_value_func(session, msg, __func__, __LINE__) /* Random number generator state. */ union __wt_rand_state { diff --git a/src/third_party/wiredtiger/src/include/msvc.h b/src/third_party/wiredtiger/src/include/msvc.h index f1fab2add9e..d9bbd2d505b 100644 --- a/src/third_party/wiredtiger/src/include/msvc.h +++ b/src/third_party/wiredtiger/src/include/msvc.h @@ -13,6 +13,11 @@ #define inline __inline +/* MSVC Doesn't provide __func__, it has __FUNCTION__ */ +#ifdef _MSC_VER +#define __func__ __FUNCTION__ +#endif + #define WT_PTRDIFFT_FMT "Id" /* ptrdiff_t format string */ #define WT_SIZET_FMT "Iu" /* size_t format string */ diff --git a/src/third_party/wiredtiger/src/include/schema.h b/src/third_party/wiredtiger/src/include/schema.h index 8b8ee5616d1..2bdbc6268fb 100644 --- a/src/third_party/wiredtiger/src/include/schema.h +++ b/src/third_party/wiredtiger/src/include/schema.h @@ -52,9 +52,10 @@ struct __wt_index { * in an index key that can be used to reconstruct the primary key. */ struct __wt_table { - const char *name, *config, *plan; + WT_DATA_HANDLE iface; + + const char *plan; const char *key_format, *value_format; - uint64_t name_hash; /* Hash of name */ WT_CONFIG_ITEM cgconf, colconf; @@ -62,14 +63,8 @@ struct __wt_table { WT_INDEX **indices; size_t idx_alloc; - TAILQ_ENTRY(__wt_table) q; - TAILQ_ENTRY(__wt_table) hashq; - bool cg_complete, idx_complete, is_simple; u_int ncolgroups, nindices, nkey_columns; - - uint32_t refcnt; /* Number of open cursors */ - uint64_t schema_gen; /* Cached schema generation number */ }; /* diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index f3a11d1dba4..bd69cc36405 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -57,10 +57,10 @@ struct __wt_session_impl { WT_DATA_HANDLE *dhandle; /* Current data handle */ /* - * Each session keeps a cache of data handles. The set of handles - * can grow quite large so we maintain both a simple list and a hash - * table of lists. The hash table key is based on a hash of the table - * URI. The hash table list is kept in allocated memory that lives + * Each session keeps a cache of data handles. The set of handles can + * grow quite large so we maintain both a simple list and a hash table + * of lists. The hash table key is based on a hash of the data handle's + * URI. The hash table list is kept in allocated memory that lives * across session close - so it is declared further down. */ /* Session handle reference list */ @@ -90,20 +90,6 @@ struct __wt_session_impl { int meta_track_nest; /* Nesting level of meta transaction */ #define WT_META_TRACKING(session) ((session)->meta_track_next != NULL) - /* - * Each session keeps a cache of table handles. The set of handles - * can grow quite large so we maintain both a simple list and a hash - * table of lists. The hash table list is kept in allocated memory - * that lives across session close - so it is declared further down. - */ - TAILQ_HEAD(__tables, __wt_table) tables; - - /* - * Updated when the table cache is swept of all tables older than the - * current schema generation. - */ - uint64_t table_sweep_gen; - /* Current rwlock for callback. */ WT_RWLOCK *current_rwlock; uint8_t current_rwticket; @@ -176,16 +162,13 @@ struct __wt_session_impl { /* Hashed handle reference list array */ TAILQ_HEAD(__dhandles_hash, __wt_data_handle_cache) *dhhash; - /* Hashed table reference list array */ - TAILQ_HEAD(__tables_hash, __wt_table) *tablehash; /* Generations manager */ #define WT_GEN_CHECKPOINT 0 /* Checkpoint generation */ #define WT_GEN_EVICT 1 /* Eviction generation */ #define WT_GEN_HAZARD 2 /* Hazard pointer */ -#define WT_GEN_SCHEMA 3 /* Schema version */ -#define WT_GEN_SPLIT 4 /* Page splits */ -#define WT_GENERATIONS 5 /* Total generation manager entries */ +#define WT_GEN_SPLIT 3 /* Page splits */ +#define WT_GENERATIONS 4 /* Total generation manager entries */ volatile uint64_t generations[WT_GENERATIONS]; /* diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 7825962d89f..7eee17e285c 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -911,40 +911,6 @@ struct __wt_session { #endif /*! - * Alter a table. - * - * This will allow modification of some table settings after - * creation. - * - * @snippet ex_all.c Alter a table - * - * @param session the session handle - * @param name the URI of the object to alter, such as \c "table:stock" - * @configstart{WT_SESSION.alter, see dist/api_data.py} - * @config{access_pattern_hint, It is recommended that workloads that - * consist primarily of updates and/or point queries specify \c random. - * Workloads that do many cursor scans through large ranges of data - * specify \c sequential and other workloads specify \c none. The - * option leads to an advisory call to an appropriate operating system - * API where available., a string\, chosen from the following options: - * \c "none"\, \c "random"\, \c "sequential"; default \c none.} - * @config{cache_resident, do not ever evict the object's pages from - * cache. Not compatible with LSM tables; see @ref - * tuning_cache_resident for more information., a boolean flag; default - * \c false.} - * @config{log = (, the transaction log configuration for this object. - * Only valid if log is enabled in ::wiredtiger_open., a set of related - * configuration options defined below.} - * @config{ enabled, if false\, this object has - * checkpoint-level durability., a boolean flag; default \c true.} - * @config{ ),,} - * @configend - * @errors - */ - int __F(alter)(WT_HANDLE_CLOSED(WT_SESSION) *session, - const char *name, const char *config); - - /*! * Close the session handle. * * This will release the resources associated with the session handle, @@ -1131,6 +1097,42 @@ struct __wt_session { * @{ */ /*! + * Alter a table. + * + * This will allow modification of some table settings after + * creation. + * + * @exclusive + * + * @snippet ex_all.c Alter a table + * + * @param session the session handle + * @param name the URI of the object to alter, such as \c "table:stock" + * @configstart{WT_SESSION.alter, see dist/api_data.py} + * @config{access_pattern_hint, It is recommended that workloads that + * consist primarily of updates and/or point queries specify \c random. + * Workloads that do many cursor scans through large ranges of data + * specify \c sequential and other workloads specify \c none. The + * option leads to an advisory call to an appropriate operating system + * API where available., a string\, chosen from the following options: + * \c "none"\, \c "random"\, \c "sequential"; default \c none.} + * @config{cache_resident, do not ever evict the object's pages from + * cache. Not compatible with LSM tables; see @ref + * tuning_cache_resident for more information., a boolean flag; default + * \c false.} + * @config{log = (, the transaction log configuration for this object. + * Only valid if log is enabled in ::wiredtiger_open., a set of related + * configuration options defined below.} + * @config{ enabled, if false\, this object has + * checkpoint-level durability., a boolean flag; default \c true.} + * @config{ ),,} + * @configend + * @errors + */ + int __F(alter)(WT_SESSION *session, + const char *name, const char *config); + + /*! * Create a table, column group, index or file. * * @snippet ex_all.c Create a table @@ -1391,6 +1393,8 @@ struct __wt_session { /*! * Drop (delete) an object. * + * @exclusive + * * @snippet ex_all.c Drop a table * * @param session the session handle @@ -1504,7 +1508,9 @@ struct __wt_session { int __F(log_printf)(WT_SESSION *session, const char *fmt, ...); /*! - * Rebalance a table, see @ref rebalance. + * Rebalance a table or file, see @ref rebalance. + * + * @exclusive * * @snippet ex_all.c Rebalance a table * @@ -1521,6 +1527,8 @@ struct __wt_session { * * @snippet ex_all.c Rename a table * + * @exclusive + * * @param session the session handle * @param uri the current URI of the object, such as \c "table:old" * @param newuri the new URI of the object, such as \c "table:new" @@ -1533,12 +1541,10 @@ struct __wt_session { /*! * Reset the session handle. * - * This will reset all cursors associated with this session and clear - * any buffers that each cursor held. After this the buffers associated - * with the session will be discarded. The session can be re-used - * immediately after this call returns. If a transaction is running on - * this session, then this call will take no action and return a busy - * error. + * This method resets all cursors associated with this session and + * discards cached resources. The session can be re-used immediately + * after this call returns. If a transaction is running on this + * session, then this call takes no action and return an error. * * @snippet ex_all.c Reset the session * @@ -1548,7 +1554,7 @@ struct __wt_session { int __F(reset)(WT_SESSION *session); /*! - * Salvage a file or table + * Salvage a table or file. * * Salvage rebuilds the file, or files of which a table is comprised, * discarding any corrupted file blocks. @@ -1556,16 +1562,18 @@ struct __wt_session { * Previously deleted records may re-appear, and inserted records may * disappear, when salvage is done, so salvage should not be run * unless it is known to be necessary. Normally, salvage should be - * called after a file or table has been corrupted, as reported by the + * called after a table or file has been corrupted, as reported by the * WT_SESSION::verify method. * * Files are rebuilt in place, the salvage method overwrites the * existing files. * + * @exclusive + * * @snippet ex_all.c Salvage a table * * @param session the session handle - * @param name the URI of the file or table to salvage + * @param name the URI of the table or file to salvage * @configstart{WT_SESSION.salvage, see dist/api_data.py} * @config{force, force salvage even of files that do not appear to be * WiredTiger files., a boolean flag; default \c false.} @@ -1578,7 +1586,7 @@ struct __wt_session { /*! * Truncate a file, table or cursor range. * - * Truncate a file or table. + * Truncate a table or file. * @snippet ex_all.c Truncate a table * * Truncate a cursor range. When truncating based on a cursor position, @@ -1593,7 +1601,7 @@ struct __wt_session { * beginning (end) of the table. * * @param session the session handle - * @param name the URI of the file or table to truncate + * @param name the URI of the table or file to truncate * @param start optional cursor marking the first record discarded; * if <code>NULL</code>, the truncate starts from the beginning of * the object @@ -1610,14 +1618,16 @@ struct __wt_session { const char *config); /*! - * Upgrade a file or table. + * Upgrade a table or file. + * + * Upgrade upgrades a table or file, if upgrade is required. * - * Upgrade upgrades a file or table, if upgrade is required. + * @exclusive * * @snippet ex_all.c Upgrade a table * * @param session the session handle - * @param name the URI of the file or table to upgrade + * @param name the URI of the table or file to upgrade * @configempty{WT_SESSION.upgrade, see dist/api_data.py} * @ebusy_errors */ @@ -1625,7 +1635,7 @@ struct __wt_session { const char *name, const char *config); /*! - * Verify a file or table. + * Verify a table or file. * * Verify reports if a file, or the files of which a table is * comprised, have been corrupted. The WT_SESSION::salvage method @@ -1633,8 +1643,10 @@ struct __wt_session { * * @snippet ex_all.c Verify a table * + * @exclusive + * * @param session the session handle - * @param name the URI of the file or table to verify + * @param name the URI of the table or file to verify * @configstart{WT_SESSION.verify, see dist/api_data.py} * @config{dump_address, Display addresses and page types as pages are * verified\, using the application's message handler\, intended for @@ -1677,8 +1689,7 @@ struct __wt_session { * before the transaction ends, will operate in the context of the * transaction. * - * WT_SESSION::begin_transaction will fail if a transaction is already - * in progress in the session. + * @requires_notransaction * * @snippet ex_all.c transaction commit/rollback * @@ -1713,6 +1724,8 @@ struct __wt_session { * If WT_SESSION::commit_transaction returns an error, the transaction * was rolled back, not committed. * + * @requires_transaction + * * @snippet ex_all.c transaction commit/rollback * * @param session the session handle @@ -1741,6 +1754,8 @@ struct __wt_session { * * All cursors are reset. * + * @requires_transaction + * * @snippet ex_all.c transaction commit/rollback * * @param session the session handle @@ -1754,6 +1769,8 @@ struct __wt_session { * * @snippet ex_all.c transaction timestamp * + * @requires_transaction + * * @param session the session handle * @configstart{WT_SESSION.timestamp_transaction, see dist/api_data.py} * @config{commit_timestamp, set the commit timestamp for the current @@ -1781,6 +1798,8 @@ struct __wt_session { * Additionally, existing named checkpoints may optionally be * discarded. * + * @requires_notransaction + * * @snippet ex_all.c Checkpoint examples * * @param session the session handle @@ -1863,8 +1882,9 @@ struct __wt_session { /*! * Wait for a transaction to become synchronized. This method is * only useful when ::wiredtiger_open is configured with the - * \c transaction_sync setting disabled. This method must be called - * when no transactions are active in the session. + * \c transaction_sync setting disabled. + * + * @requires_notransaction * * @snippet ex_all.c Transaction sync * @@ -1960,6 +1980,33 @@ struct __wt_connection { int __F(close)(WT_HANDLE_CLOSED(WT_CONNECTION) *connection, const char *config); +#ifndef DOXYGEN + /*! + * Output debug information for various subsystems. The output format + * may change over time, gathering the debug information may be + * invasive, and the information reported may not provide a point in + * time view of the system. + * + * @param connection the connection handle + * @configstart{WT_CONNECTION.debug_info, see dist/api_data.py} + * @config{cache, print cache information., a boolean flag; default \c + * false.} + * @config{cursors, print all open cursor information., a boolean flag; + * default \c false.} + * @config{handles, print open handles information., a boolean flag; + * default \c false.} + * @config{log, print log information., a boolean flag; default \c + * false.} + * @config{sessions, print open session information., a boolean flag; + * default \c false.} + * @config{txn, print global txn information., a boolean flag; default + * \c false.} + * @configend + * @errors + */ + int __F(debug_info)(WT_CONNECTION *connection, const char *config); +#endif + /*! * Reconfigure a connection handle. * diff --git a/src/third_party/wiredtiger/src/log/log_sys.c b/src/third_party/wiredtiger/src/log/log_sys.c index 4eb2a8e23d2..6956aa99fe7 100644 --- a/src/third_party/wiredtiger/src/log/log_sys.c +++ b/src/third_party/wiredtiger/src/log/log_sys.c @@ -85,3 +85,64 @@ __wt_log_recover_system(WT_SESSION_IMPL *session, return (0); } + +/* + * __wt_verbose_dump_log -- + * Dump information about the logging subsystem. + */ +int +__wt_verbose_dump_log(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_LOG *log; + + conn = S2C(session); + log = conn->log; + + WT_RET(__wt_msg(session, "%s", WT_DIVIDER)); + WT_RET(__wt_msg(session, "Logging subsystem: Enabled: %s", + FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED) ? "yes" : "no")); + if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) + return (0); + /* + * Logging is enabled, print out the other information. + */ + WT_RET(__wt_msg(session, "Archiving: %s", + FLD_ISSET(conn->log_flags, WT_CONN_LOG_ARCHIVE) ? "yes" : "no")); + WT_RET(__wt_msg(session, "Running downgraded: %s", + FLD_ISSET(conn->log_flags, WT_CONN_LOG_DOWNGRADED) ? "yes" : "no")); + WT_RET(__wt_msg(session, "Zero fill files: %s", + FLD_ISSET(conn->log_flags, WT_CONN_LOG_ZERO_FILL) ? "yes" : "no")); + WT_RET(__wt_msg(session, "Pre-allocate files: %s", + conn->log_prealloc > 0 ? "yes" : "no")); + WT_RET(__wt_msg(session, "Logging directory: %s", conn->log_path)); + WT_RET(__wt_msg(session, "Logging maximum file size: %" PRId64, + (int64_t)conn->log_file_max)); + WT_RET(__wt_msg(session, "Log sync setting: %s", + !FLD_ISSET(conn->txn_logsync, WT_LOG_SYNC_ENABLED) ? "none" : + FLD_ISSET(conn->txn_logsync, WT_LOG_DSYNC) ? "dsync" : + FLD_ISSET(conn->txn_logsync, WT_LOG_FLUSH) ? "write to OS" : + FLD_ISSET(conn->txn_logsync, WT_LOG_FSYNC) ? + "fsync to disk": "unknown sync setting")); + WT_RET(__wt_msg(session, "Log record allocation alignment: %" PRIu32, + log->allocsize)); + WT_RET(__wt_msg(session, "Current log file number: %" PRIu32, + log->fileid)); + WT_RET(__wt_msg(session, "Current log version number: %" PRIu16, + log->log_version)); + WT_RET(WT_LSN_MSG(&log->alloc_lsn, "Next allocation")); + WT_RET(WT_LSN_MSG(&log->bg_sync_lsn, "Last background sync")); + WT_RET(WT_LSN_MSG(&log->ckpt_lsn, "Last checkpoint")); + WT_RET(WT_LSN_MSG(&log->sync_dir_lsn, "Last directory sync")); + WT_RET(WT_LSN_MSG(&log->sync_lsn, "Last sync")); + WT_RET(WT_LSN_MSG(&log->trunc_lsn, "Recovery truncate")); + WT_RET(WT_LSN_MSG(&log->write_lsn, "Last written")); + WT_RET(WT_LSN_MSG(&log->write_start_lsn, "Start of last written")); + /* + * If we wanted a dump of the slots, it would go here. Walking + * the slot pool may not require a lock since they're statically + * allocated, but output could be inconsistent without it. + */ + + return (0); +} diff --git a/src/third_party/wiredtiger/src/lsm/lsm_meta.c b/src/third_party/wiredtiger/src/lsm/lsm_meta.c index 66ad24dee5b..3a4960aaec1 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_meta.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_meta.c @@ -309,12 +309,10 @@ __lsm_meta_read_v1( continue; } WT_ERR(__wt_realloc_def(session, - &lsm_tree->old_alloc, nchunks + 1, - &lsm_tree->old_chunks)); + &lsm_tree->old_alloc, nchunks + 1, &lsm_tree->old_chunks)); WT_ERR(__wt_calloc_one(session, &chunk)); lsm_tree->old_chunks[nchunks++] = chunk; - WT_ERR(__wt_strndup(session, - lk.str, lk.len, &chunk->uri)); + WT_ERR(__wt_strndup(session, lk.str, lk.len, &chunk->uri)); F_SET(chunk, WT_LSM_CHUNK_ONDISK); } WT_ERR_NOTFOUND_OK(ret); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_tree.c b/src/third_party/wiredtiger/src/lsm/lsm_tree.c index e6eccf96467..3a1165ec239 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_tree.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_tree.c @@ -282,8 +282,8 @@ __wt_lsm_tree_setup_chunk( WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_SCHEMA)); __wt_epoch(session, &chunk->create_time); - __wt_spin_init(session, - &chunk->timestamp_spinlock, "LSM chunk timestamp"); + WT_RET(__wt_spin_init(session, + &chunk->timestamp_spinlock, "LSM chunk timestamp")); WT_RET(__wt_lsm_tree_chunk_name( session, lsm_tree, chunk->id, &chunk->uri)); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c index 816eafebe99..8c01ecf0e42 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c @@ -373,7 +373,7 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, * We can wait here for checkpoints and fsyncs to complete, which can * take a long time. */ - WT_ERR(__wt_session_get_btree(session, chunk->uri, NULL, NULL, 0)); + WT_ERR(__wt_session_get_dhandle(session, chunk->uri, NULL, NULL, 0)); release_btree = true; /* @@ -408,7 +408,7 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, WT_ERR_MSG(session, ret, "LSM checkpoint"); release_btree = false; - WT_ERR(__wt_session_release_btree(session)); + WT_ERR(__wt_session_release_dhandle(session)); /* Now the file is written, get the chunk size. */ WT_ERR(__wt_lsm_tree_set_chunk_size(session, chunk)); @@ -449,7 +449,7 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, err: if (flush_set) WT_PUBLISH(chunk->flushing, 0); if (release_btree) - WT_TRET(__wt_session_release_btree(session)); + WT_TRET(__wt_session_release_dhandle(session)); return (ret); } @@ -539,11 +539,11 @@ __lsm_discard_handle( WT_SESSION_IMPL *session, const char *uri, const char *checkpoint) { /* This will fail with EBUSY if the file is still in use. */ - WT_RET(__wt_session_get_btree(session, uri, checkpoint, NULL, + WT_RET(__wt_session_get_dhandle(session, uri, checkpoint, NULL, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_LOCK_ONLY)); F_SET(session->dhandle, WT_DHANDLE_DISCARD_KILL); - return (__wt_session_release_btree(session)); + return (__wt_session_release_dhandle(session)); } /* diff --git a/src/third_party/wiredtiger/src/meta/meta_apply.c b/src/third_party/wiredtiger/src/meta/meta_apply.c index 9fb70dac081..dbd3293dc2e 100644 --- a/src/third_party/wiredtiger/src/meta/meta_apply.c +++ b/src/third_party/wiredtiger/src/meta/meta_apply.c @@ -41,11 +41,11 @@ __meta_btree_apply(WT_SESSION_IMPL *session, WT_CURSOR *cursor, * dropping the handle (e.g in LSM when cleaning up obsolete * chunks). Holding the metadata lock isn't enough. */ - if ((ret = __wt_session_get_btree( + if ((ret = __wt_session_get_dhandle( session, uri, NULL, NULL, 0)) != 0) return (ret == EBUSY ? 0 : ret); WT_SAVE_DHANDLE(session, ret = file_func(session, cfg)); - WT_TRET(__wt_session_release_btree(session)); + WT_TRET(__wt_session_release_dhandle(session)); WT_RET(ret); } WT_RET_NOTFOUND_OK(ret); diff --git a/src/third_party/wiredtiger/src/meta/meta_track.c b/src/third_party/wiredtiger/src/meta/meta_track.c index fe7b467c199..dbe39012591 100644 --- a/src/third_party/wiredtiger/src/meta/meta_track.c +++ b/src/third_party/wiredtiger/src/meta/meta_track.c @@ -148,7 +148,7 @@ __meta_track_apply(WT_SESSION_IMPL *session, WT_META_TRACK *trk) break; case WT_ST_LOCK: WT_WITH_DHANDLE(session, trk->dhandle, - ret = __wt_session_release_btree(session)); + ret = __wt_session_release_dhandle(session)); break; case WT_ST_FILEOP: case WT_ST_REMOVE: @@ -180,7 +180,7 @@ __meta_track_unroll(WT_SESSION_IMPL *session, WT_META_TRACK *trk) if (trk->created) F_SET(trk->dhandle, WT_DHANDLE_DISCARD); WT_WITH_DHANDLE(session, trk->dhandle, - ret = __wt_session_release_btree(session)); + ret = __wt_session_release_dhandle(session)); break; case WT_ST_FILEOP: /* File operation */ /* diff --git a/src/third_party/wiredtiger/src/os_common/os_alloc.c b/src/third_party/wiredtiger/src/os_common/os_alloc.c index 388c9c8c18b..fe31403c3f5 100644 --- a/src/third_party/wiredtiger/src/os_common/os_alloc.c +++ b/src/third_party/wiredtiger/src/os_common/os_alloc.c @@ -251,29 +251,45 @@ __wt_realloc_aligned(WT_SESSION_IMPL *session, } /* + * __wt_memdup -- + * Duplicate a byte string of a given length. + */ +int +__wt_memdup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp) +{ + void *p; + + WT_RET(__wt_malloc(session, len, &p)); + + WT_ASSERT(session, p != NULL); /* quiet clang scan-build */ + + memcpy(p, str, len); + + *(void **)retp = p; + return (0); +} + +/* * __wt_strndup -- - * Duplicate a byte string of a given length (and NUL-terminate). + * ANSI strndup function. */ int __wt_strndup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp) { - void *p; + uint8_t *p; if (str == NULL) { *(void **)retp = NULL; return (0); } + /* Copy and nul-terminate. */ WT_RET(__wt_malloc(session, len + 1, &p)); WT_ASSERT(session, p != NULL); /* quiet clang scan-build */ - /* - * Don't change this to strncpy, we rely on this function to duplicate - * "strings" that contain nul bytes. - */ memcpy(p, str, len); - ((uint8_t *)p)[len] = '\0'; + p[len] = '\0'; *(void **)retp = p; return (0); diff --git a/src/third_party/wiredtiger/src/os_posix/os_fs.c b/src/third_party/wiredtiger/src/os_posix/os_fs.c index d0391537543..c72c116015f 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_fs.c +++ b/src/third_party/wiredtiger/src/os_posix/os_fs.c @@ -550,7 +550,7 @@ __posix_open_file_cloexec(WT_SESSION_IMPL *session, int fd, const char *name) if ((f = fcntl(fd, F_GETFD)) == -1 || fcntl(fd, F_SETFD, f | FD_CLOEXEC) == -1) WT_RET_MSG(session, __wt_errno(), - "%s: handle-open: fcntl", name); + "%s: handle-open: fcntl(FD_CLOEXEC)", name); return (0); #else WT_UNUSED(session); @@ -602,7 +602,8 @@ __posix_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, WT_SYSCALL_RETRY(( (pfh->fd = open(name, f, 0444)) == -1 ? -1 : 0), ret); if (ret != 0) - WT_ERR_MSG(session, ret, "%s: handle-open: open", name); + WT_ERR_MSG(session, ret, + "%s: handle-open: open-directory", name); WT_ERR(__posix_open_file_cloexec(session, pfh->fd, name)); goto directory_open; } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 10c2c0dc937..5d5b909eee5 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -3131,7 +3131,7 @@ no_slots: */ if (F_ISSET(r, WT_EVICT_SCRUB) || (F_ISSET(r, WT_EVICT_UPDATE_RESTORE) && r->supd_next > 0)) { - WT_RET(__wt_strndup(session, dsk, + WT_RET(__wt_memdup(session, dsk, dsk_dst->mem_size, &last->disk_image)); disk_image = last->disk_image; disk_image->recno = last->max_bnd_recno; @@ -3674,7 +3674,7 @@ supd_check_complete: #ifdef HAVE_DIAGNOSTIC verify_image = false; #endif - WT_ERR(__wt_strndup(session, addr, addr_size, &bnd->addr.addr)); + WT_ERR(__wt_memdup(session, addr, addr_size, &bnd->addr.addr)); bnd->addr.size = (uint8_t)addr_size; /* @@ -3711,7 +3711,7 @@ copy_image: __wt_verify_dsk_image( session, "[reconcile-image]", buf->data, 0, true) == 0); #endif - WT_ERR(__wt_strndup( + WT_ERR(__wt_memdup( session, buf->data, buf->size, &bnd->disk_image)); } if (!need_image) diff --git a/src/third_party/wiredtiger/src/schema/schema_alter.c b/src/third_party/wiredtiger/src/schema/schema_alter.c index 7af4b86bc85..3b52b4bd222 100644 --- a/src/third_party/wiredtiger/src/schema/schema_alter.c +++ b/src/third_party/wiredtiger/src/schema/schema_alter.c @@ -63,9 +63,6 @@ err: __wt_free(session, config); if (ret == WT_NOTFOUND) ret = ENOENT; - /* Bump the schema generation so that stale data is ignored. */ - (void)__wt_gen_next(session, WT_GEN_SCHEMA); - WT_TRET(__wt_meta_track_off(session, true, ret != 0)); return (ret); diff --git a/src/third_party/wiredtiger/src/schema/schema_create.c b/src/third_party/wiredtiger/src/schema/schema_create.c index 1ba0961cced..641c99793c3 100644 --- a/src/third_party/wiredtiger/src/schema/schema_create.c +++ b/src/third_party/wiredtiger/src/schema/schema_create.c @@ -69,8 +69,7 @@ __create_file(WT_SESSION_IMPL *session, is_metadata = strcmp(uri, WT_METAFILE_URI) == 0; filename = uri; - if (!WT_PREFIX_SKIP(filename, "file:")) - return (__wt_unexpected_object_type(session, uri, "file:")); + WT_PREFIX_SKIP_REQUIRED(session, filename, "file:"); /* Check if the file already exists. */ if (!is_metadata && (ret = @@ -116,12 +115,12 @@ __create_file(WT_SESSION_IMPL *session, * Keep the handle exclusive until it is released at the end of the * call, otherwise we could race with a drop. */ - WT_ERR(__wt_session_get_btree( + WT_ERR(__wt_session_get_dhandle( session, uri, NULL, NULL, WT_DHANDLE_EXCLUSIVE)); if (WT_META_TRACKING(session)) WT_ERR(__wt_meta_track_handle_lock(session, true)); else - WT_ERR(__wt_session_release_btree(session)); + WT_ERR(__wt_session_release_dhandle(session)); err: __wt_scr_free(session, &val); __wt_free(session, fileconf); @@ -141,7 +140,7 @@ __wt_schema_colgroup_source(WT_SESSION_IMPL *session, size_t len; const char *prefix, *suffix, *tablename; - tablename = table->name + strlen("table:"); + tablename = table->iface.name + strlen("table:"); if ((ret = __wt_config_getones(session, config, "type", &cval)) == 0 && !WT_STRING_MATCH("file", cval.str, cval.len)) { prefix = cval.str; @@ -182,19 +181,17 @@ __create_colgroup(WT_SESSION_IMPL *session, const char *sourcecfg[] = { config, NULL, NULL }; const char *cgname, *source, *sourceconf, *tablename; char *cgconf, *origconf; - bool exists; + bool exists, tracked; sourceconf = NULL; cgconf = origconf = NULL; WT_CLEAR(fmt); WT_CLEAR(confbuf); WT_CLEAR(namebuf); - exists = false; + exists = tracked = false; tablename = name; - if (!WT_PREFIX_SKIP(tablename, "colgroup:")) - return ( - __wt_unexpected_object_type(session, name, "colgroup:")); + WT_PREFIX_SKIP_REQUIRED(session, tablename, "colgroup:"); cgname = strchr(tablename, ':'); if (cgname != NULL) { tlen = (size_t)(cgname - tablename); @@ -202,12 +199,19 @@ __create_colgroup(WT_SESSION_IMPL *session, } else tlen = strlen(tablename); - if ((ret = - __wt_schema_get_table(session, tablename, tlen, true, &table)) != 0) + if ((ret = __wt_schema_get_table( + session, tablename, tlen, true, WT_DHANDLE_EXCLUSIVE, &table)) != 0) WT_RET_MSG(session, (ret == WT_NOTFOUND) ? ENOENT : ret, "Can't create '%s' for non-existent table '%.*s'", name, (int)tlen, tablename); + if (WT_META_TRACKING(session)) { + WT_WITH_DHANDLE(session, &table->iface, + ret = __wt_meta_track_handle_lock(session, false)); + WT_ERR(ret); + tracked = true; + } + /* Make sure the column group is referenced from the table. */ if (cgname != NULL && (ret = __wt_config_subgets(session, &table->cgconf, cgname, &cval)) != 0) @@ -273,7 +277,8 @@ err: __wt_free(session, cgconf); __wt_buf_free(session, &fmt); __wt_buf_free(session, &namebuf); - __wt_schema_release_table(session, table); + if (!tracked) + WT_TRET(__wt_schema_release_table(session, table)); return (ret); } @@ -290,7 +295,7 @@ __wt_schema_index_source(WT_SESSION_IMPL *session, size_t len; const char *prefix, *suffix, *tablename; - tablename = table->name + strlen("table:"); + tablename = table->iface.name + strlen("table:"); if ((ret = __wt_config_getones(session, config, "type", &cval)) == 0 && !WT_STRING_MATCH("file", cval.str, cval.len)) { prefix = cval.str; @@ -335,7 +340,7 @@ __fill_index(WT_SESSION_IMPL *session, WT_TABLE *table, WT_INDEX *idx) WT_ERR(wt_session->open_cursor(wt_session, idx->source, NULL, "bulk=unordered", &icur)); WT_ERR(wt_session->open_cursor(wt_session, - table->name, NULL, "readonly", &tcur)); + table->iface.name, NULL, "readonly", &tcur)); while ((ret = tcur->next(tcur)) == 0) WT_ERR(__wt_apply_single_idx(session, idx, @@ -384,16 +389,26 @@ __create_index(WT_SESSION_IMPL *session, exists = have_extractor = false; tablename = name; - if (!WT_PREFIX_SKIP(tablename, "index:")) - return (__wt_unexpected_object_type(session, name, "index:")); + WT_PREFIX_SKIP_REQUIRED(session, tablename, "index:"); idxname = strchr(tablename, ':'); if (idxname == NULL) WT_RET_MSG(session, EINVAL, "Invalid index name, " "should be <table name>:<index name>: %s", name); + /* + * Note: it would be better to get the table exclusive here, while + * changing its indexes. We don't because some operation we perform + * below reacquire the table handle (such as opening a cursor on the + * table in order to fill the index). If we get the handle exclusive + * here, those operations wanting ordinary access will conflict, + * leading to errors. + * + * Instead, we rely on the global table lock to protect the set of + * available indexes. + */ tlen = (size_t)(idxname++ - tablename); - if ((ret = - __wt_schema_get_table(session, tablename, tlen, true, &table)) != 0) + if ((ret = __wt_schema_get_table( + session, tablename, tlen, true, 0, &table)) != 0) WT_RET_MSG(session, ret, "Can't create an index for a non-existent table: %.*s", (int)tlen, tablename); @@ -540,7 +555,7 @@ err: __wt_free(session, idxconf); __wt_buf_free(session, &fmt); __wt_buf_free(session, &namebuf); - __wt_schema_release_table(session, table); + WT_TRET(__wt_schema_release_table(session, table)); return (ret); } @@ -550,7 +565,7 @@ err: __wt_free(session, idxconf); */ static int __create_table(WT_SESSION_IMPL *session, - const char *name, bool exclusive, const char *config) + const char *uri, bool exclusive, const char *config) { WT_CONFIG conf; WT_CONFIG_ITEM cgkey, cgval, cval; @@ -562,24 +577,23 @@ __create_table(WT_SESSION_IMPL *session, char *tableconf, *cgname; size_t cgsize; int ncolgroups; - bool exists; cgname = NULL; table = NULL; tableconf = NULL; - exists = false; - tablename = name; - if (!WT_PREFIX_SKIP(tablename, "table:")) - return (__wt_unexpected_object_type(session, name, "table:")); + WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TABLE_WRITE)); - if ((ret = __wt_schema_get_table(session, - tablename, strlen(tablename), false, &table)) == 0) { + tablename = uri; + WT_PREFIX_SKIP_REQUIRED(session, tablename, "table:"); + + /* Check if the table already exists. */ + if ((ret = __wt_metadata_search( + session, uri, &tableconf)) != WT_NOTFOUND) { if (exclusive) - WT_ERR(EEXIST); - exists = true; + WT_TRET(EEXIST); + goto err; } - WT_ERR_NOTFOUND_OK(ret); WT_ERR(__wt_config_gets(session, cfg, "colgroups", &cval)); __wt_config_subinit(session, &conf, &cval); @@ -590,32 +604,30 @@ __create_table(WT_SESSION_IMPL *session, WT_ERR_NOTFOUND_OK(ret); WT_ERR(__wt_config_collapse(session, cfg, &tableconf)); + WT_ERR(__wt_metadata_insert(session, uri, tableconf)); - if (!exists) { - WT_ERR(__wt_metadata_insert(session, name, tableconf)); - - /* Attempt to open the table now to catch any errors. */ - WT_ERR(__wt_schema_get_table( - session, tablename, strlen(tablename), true, &table)); - - if (ncolgroups == 0) { - cgsize = strlen("colgroup:") + strlen(tablename) + 1; - WT_ERR(__wt_calloc_def(session, cgsize, &cgname)); - WT_ERR(__wt_snprintf( - cgname, cgsize, "colgroup:%s", tablename)); - WT_ERR(__create_colgroup( - session, cgname, exclusive, config)); - } + if (ncolgroups == 0) { + cgsize = strlen("colgroup:") + strlen(tablename) + 1; + WT_ERR(__wt_calloc_def(session, cgsize, &cgname)); + WT_ERR(__wt_snprintf(cgname, cgsize, "colgroup:%s", tablename)); + WT_ERR(__create_colgroup(session, cgname, exclusive, config)); } - if (0) { -err: if (table != NULL) { - WT_TRET(__wt_schema_remove_table(session, table)); - table = NULL; - } + /* + * Open the table to check that it was setup correctly. Keep the + * handle exclusive until it is released at the end of the call. + */ + WT_ERR(__wt_schema_get_table_uri( + session, uri, true, WT_DHANDLE_EXCLUSIVE, &table)); + if (WT_META_TRACKING(session)) { + WT_WITH_DHANDLE(session, &table->iface, + ret = __wt_meta_track_handle_lock(session, true)); + WT_ERR(ret); + table = NULL; } - if (table != NULL) - __wt_schema_release_table(session, table); + +err: if (table != NULL) + WT_TRET(__wt_schema_release_table(session, table)); __wt_free(session, cgname); __wt_free(session, tableconf); return (ret); diff --git a/src/third_party/wiredtiger/src/schema/schema_drop.c b/src/third_party/wiredtiger/src/schema/schema_drop.c index ec12ec3752f..d058dae69e1 100644 --- a/src/third_party/wiredtiger/src/schema/schema_drop.c +++ b/src/third_party/wiredtiger/src/schema/schema_drop.c @@ -25,8 +25,7 @@ __drop_file( remove_files = cval.val != 0; filename = uri; - if (!WT_PREFIX_SKIP(filename, "file:")) - return (__wt_unexpected_object_type(session, uri, "file:")); + WT_PREFIX_SKIP_REQUIRED(session, filename, "file:"); WT_RET(__wt_schema_backup_check(session, filename)); /* Close all btree handles associated with this file. */ @@ -60,13 +59,14 @@ __drop_colgroup( WT_DECL_RET; WT_TABLE *table; - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TABLE)); + WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TABLE_WRITE)); /* If we can get the colgroup, detach it from the table. */ if ((ret = __wt_schema_get_colgroup( session, uri, force, &table, &colgroup)) == 0) { - table->cg_complete = false; WT_TRET(__wt_schema_drop(session, colgroup->source, cfg)); + if (ret == 0) + table->cg_complete = false; } WT_TRET(__wt_metadata_remove(session, uri)); @@ -83,14 +83,10 @@ __drop_index( { WT_INDEX *idx; WT_DECL_RET; - WT_TABLE *table; /* If we can get the index, detach it from the table. */ - if ((ret = __wt_schema_get_index( - session, uri, force, &table, &idx)) == 0) { - table->idx_complete = false; + if ((ret = __wt_schema_get_index(session, uri, true, force, &idx)) == 0) WT_TRET(__wt_schema_drop(session, idx->source, cfg)); - } WT_TRET(__wt_metadata_remove(session, uri)); return (ret); @@ -101,7 +97,8 @@ __drop_index( * WT_SESSION::drop for a table. */ static int -__drop_table(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +__drop_table( + WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) { WT_COLGROUP *colgroup; WT_DECL_RET; @@ -109,13 +106,26 @@ __drop_table(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_TABLE *table; const char *name; u_int i; + bool tracked; + + WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TABLE_WRITE)); name = uri; - (void)WT_PREFIX_SKIP(name, "table:"); + WT_PREFIX_SKIP_REQUIRED(session, name, "table:"); table = NULL; - WT_ERR(__wt_schema_get_table( - session, name, strlen(name), true, &table)); + tracked = false; + + /* + * Open the table so we can drop its column groups and indexes. + * + * Ideally we would keep the table locked exclusive across the drop, + * but for now we rely on the global table lock to prevent the table + * being reopened while it is being dropped. One issue is that the + * WT_WITHOUT_LOCKS macro can drop and reacquire the global table lock, + * avoiding deadlocks while waiting for LSM operation to quiesce. + */ + WT_ERR(__wt_schema_get_table_uri(session, uri, true, 0, &table)); /* Drop the column groups. */ for (i = 0; i < WT_COLGROUPS(table); i++) { @@ -144,14 +154,23 @@ __drop_table(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_ERR(__wt_metadata_remove(session, idx->name)); } - WT_ERR(__wt_schema_remove_table(session, table)); - table = NULL; + /* Make sure the table data handle is closed. */ + WT_TRET(__wt_schema_release_table(session, table)); + WT_ERR(__wt_schema_get_table_uri( + session, uri, true, WT_DHANDLE_EXCLUSIVE, &table)); + F_SET(&table->iface, WT_DHANDLE_DISCARD); + if (WT_META_TRACKING(session)) { + WT_WITH_DHANDLE(session, &table->iface, + ret = __wt_meta_track_handle_lock(session, false)); + WT_ERR(ret); + tracked = true; + } /* Remove the metadata entry (ignore missing items). */ WT_ERR(__wt_metadata_remove(session, uri)); -err: if (table != NULL) - __wt_schema_release_table(session, table); +err: if (table != NULL && !tracked) + WT_TRET(__wt_schema_release_table(session, table)); return (ret); } @@ -200,9 +219,6 @@ __wt_schema_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) if (ret == WT_NOTFOUND || ret == ENOENT) ret = force ? 0 : ENOENT; - /* Bump the schema generation so that stale data is ignored. */ - (void)__wt_gen_next(session, WT_GEN_SCHEMA); - WT_TRET(__wt_meta_track_off(session, true, ret != 0)); return (ret); diff --git a/src/third_party/wiredtiger/src/schema/schema_list.c b/src/third_party/wiredtiger/src/schema/schema_list.c index fe86a37850d..6907f0922d7 100644 --- a/src/third_party/wiredtiger/src/schema/schema_list.c +++ b/src/third_party/wiredtiger/src/schema/schema_list.c @@ -9,78 +9,35 @@ #include "wt_internal.h" /* - * __schema_add_table -- - * Add a table handle to the session's cache. + * __wt_schema_get_table_uri -- + * Get the table handle for the named table. */ -static int -__schema_add_table(WT_SESSION_IMPL *session, - const char *name, size_t namelen, bool ok_incomplete, WT_TABLE **tablep) +int +__wt_schema_get_table_uri(WT_SESSION_IMPL *session, + const char *uri, bool ok_incomplete, uint32_t flags, + WT_TABLE **tablep) { + WT_DATA_HANDLE *saved_dhandle; WT_DECL_RET; WT_TABLE *table; - uint64_t bucket; - - table = NULL; /* -Wconditional-uninitialized */ - - /* Make sure the metadata is open before getting other locks. */ - WT_RET(__wt_metadata_cursor(session, NULL)); - - WT_WITH_TABLE_READ_LOCK(session, - ret = __wt_schema_open_table( - session, name, namelen, ok_incomplete, &table)); - WT_RET(ret); - - bucket = table->name_hash % WT_HASH_ARRAY_SIZE; - TAILQ_INSERT_HEAD(&session->tables, table, q); - TAILQ_INSERT_HEAD(&session->tablehash[bucket], table, hashq); - *tablep = table; - return (0); -} + saved_dhandle = session->dhandle; -/* - * __schema_find_table -- - * Find the table handle for the named table in the session cache. - */ -static int -__schema_find_table(WT_SESSION_IMPL *session, - const char *name, size_t namelen, WT_TABLE **tablep) -{ - WT_TABLE *table; - const char *tablename; - uint64_t bucket; - - bucket = __wt_hash_city64(name, namelen) % WT_HASH_ARRAY_SIZE; + *tablep = NULL; -restart: - TAILQ_FOREACH(table, &session->tablehash[bucket], hashq) { - tablename = table->name; - (void)WT_PREFIX_SKIP(tablename, "table:"); - if (WT_STRING_MATCH(tablename, name, namelen)) { - /* - * Ignore stale tables. - * - * XXX: should be managed the same as btree handles, - * with a local cache in each session and a shared list - * in the connection. There is still a race here - * between checking the generation and opening the - * first column group. - */ - if (table->schema_gen != - __wt_gen(session, WT_GEN_SCHEMA)) { - if (table->refcnt == 0) { - WT_RET(__wt_schema_remove_table( - session, table)); - goto restart; - } - continue; - } - *tablep = table; - return (0); - } + WT_ERR(__wt_session_get_dhandle(session, uri, NULL, NULL, flags)); + table = (WT_TABLE *)session->dhandle; + if (!ok_incomplete && !table->cg_complete) { + ret = EINVAL; + WT_TRET(__wt_session_release_dhandle(session)); + WT_ERR_MSG(session, ret, "'%s' cannot be used " + "until all column groups are created", + table->iface.name); } + *tablep = table; - return (WT_NOTFOUND); +err: session->dhandle = saved_dhandle; + return (ret); } /* @@ -89,23 +46,20 @@ restart: */ int __wt_schema_get_table(WT_SESSION_IMPL *session, - const char *name, size_t namelen, bool ok_incomplete, WT_TABLE **tablep) + const char *name, size_t namelen, bool ok_incomplete, uint32_t flags, + WT_TABLE **tablep) { WT_DECL_RET; - WT_TABLE *table; + WT_DECL_ITEM(namebuf); - *tablep = table = NULL; - ret = __schema_find_table(session, name, namelen, &table); + WT_RET(__wt_scr_alloc(session, namelen + 1, &namebuf)); + WT_ERR(__wt_buf_fmt( + session, namebuf, "table:%.*s", (int)namelen, name)); - if (ret == WT_NOTFOUND) - ret = __schema_add_table( - session, name, namelen, ok_incomplete, &table); - - if (ret == 0) { - ++table->refcnt; - *tablep = table; - } + WT_ERR(__wt_schema_get_table_uri( + session, namebuf->data, ok_incomplete, flags, tablep)); +err: __wt_scr_free(session, &namebuf); return (ret); } @@ -113,11 +67,15 @@ __wt_schema_get_table(WT_SESSION_IMPL *session, * __wt_schema_release_table -- * Release a table handle. */ -void +int __wt_schema_release_table(WT_SESSION_IMPL *session, WT_TABLE *table) { - WT_ASSERT(session, table->refcnt > 0); - --table->refcnt; + WT_DECL_RET; + + WT_WITH_DHANDLE(session, &table->iface, + ret = __wt_session_release_dhandle(session)); + + return (ret); } /* @@ -185,22 +143,15 @@ __wt_schema_destroy_index(WT_SESSION_IMPL *session, WT_INDEX **idxp) } /* - * __wt_schema_destroy_table -- - * Free a table handle. + * __wt_schema_close_table -- + * Close a table handle. */ int -__wt_schema_destroy_table(WT_SESSION_IMPL *session, WT_TABLE **tablep) +__wt_schema_close_table(WT_SESSION_IMPL *session, WT_TABLE *table) { WT_DECL_RET; - WT_TABLE *table; u_int i; - if ((table = *tablep) == NULL) - return (0); - *tablep = NULL; - - __wt_free(session, table->name); - __wt_free(session, table->config); __wt_free(session, table->plan); __wt_free(session, table->key_format); __wt_free(session, table->value_format); @@ -216,70 +167,11 @@ __wt_schema_destroy_table(WT_SESSION_IMPL *session, WT_TABLE **tablep) session, &table->indices[i])); __wt_free(session, table->indices); } - __wt_free(session, table); - return (ret); -} - -/* - * __wt_schema_remove_table -- - * Remove the table handle from the session, closing if necessary. - */ -int -__wt_schema_remove_table(WT_SESSION_IMPL *session, WT_TABLE *table) -{ - uint64_t bucket; - WT_ASSERT(session, table->refcnt <= 1); + table->idx_alloc = 0; - bucket = table->name_hash % WT_HASH_ARRAY_SIZE; - TAILQ_REMOVE(&session->tables, table, q); - TAILQ_REMOVE(&session->tablehash[bucket], table, hashq); - return (__wt_schema_destroy_table(session, &table)); -} - -/* - * __wt_schema_close_tables -- - * Close all of the tables in a session. - */ -int -__wt_schema_close_tables(WT_SESSION_IMPL *session) -{ - WT_DECL_RET; - WT_TABLE *table, *table_tmp; - - WT_TAILQ_SAFE_REMOVE_BEGIN(table, &session->tables, q, table_tmp) { - WT_TRET(__wt_schema_remove_table(session, table)); - } WT_TAILQ_SAFE_REMOVE_END + WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TABLE_WRITE) || + F_ISSET(S2C(session), WT_CONN_CLOSING)); + table->cg_complete = table->idx_complete = false; return (ret); } - -/* - * __wt_schema_sweep_tables -- - * Close all idle, obsolete tables in a session. - */ -int -__wt_schema_sweep_tables(WT_SESSION_IMPL *session) -{ - WT_TABLE *table, *next; - uint64_t schema_gen; - bool old_table_busy; - - schema_gen = __wt_gen(session, WT_GEN_SCHEMA); - if (schema_gen == session->table_sweep_gen) - return (0); - - old_table_busy = false; - TAILQ_FOREACH_SAFE(table, &session->tables, q, next) - if (table->schema_gen != schema_gen) { - if (table->refcnt == 0) - WT_RET(__wt_schema_remove_table( - session, table)); - else - old_table_busy = true; - } - - if (!old_table_busy) - session->table_sweep_gen = schema_gen; - - return (0); -} diff --git a/src/third_party/wiredtiger/src/schema/schema_open.c b/src/third_party/wiredtiger/src/schema/schema_open.c index df89607e0ef..a2ccbbe83b1 100644 --- a/src/third_party/wiredtiger/src/schema/schema_open.c +++ b/src/third_party/wiredtiger/src/schema/schema_open.c @@ -20,8 +20,8 @@ __wt_schema_colgroup_name(WT_SESSION_IMPL *session, { const char *tablename; - tablename = table->name; - (void)WT_PREFIX_SKIP(tablename, "table:"); + tablename = table->iface.name; + WT_PREFIX_SKIP_REQUIRED(session, tablename, "table:"); return ((table->ncolgroups == 0) ? __wt_buf_fmt(session, buf, "colgroup:%s", tablename) : @@ -284,8 +284,8 @@ __schema_open_index(WT_SESSION_IMPL *session, match = false; /* Build a search key. */ - tablename = table->name; - (void)WT_PREFIX_SKIP(tablename, "table:"); + tablename = table->iface.name; + WT_PREFIX_SKIP_REQUIRED(session, tablename, "table:"); WT_ERR(__wt_scr_alloc(session, 512, &tmp)); WT_ERR(__wt_buf_fmt(session, tmp, "index:%s:", tablename)); @@ -412,57 +412,34 @@ __wt_schema_open_indices(WT_SESSION_IMPL *session, WT_TABLE *table) /* * __schema_open_table -- - * Open a named table (internal version). + * Open the data handle for a table (internal version). */ static int -__schema_open_table(WT_SESSION_IMPL *session, - const char *name, size_t namelen, bool ok_incomplete, WT_TABLE **tablep) +__schema_open_table(WT_SESSION_IMPL *session, const char *cfg[]) { WT_CONFIG cparser; WT_CONFIG_ITEM ckey, cval; - WT_CURSOR *cursor; - WT_DECL_ITEM(buf); WT_DECL_RET; WT_TABLE *table; - const char *tconfig; - - *tablep = NULL; + const char *tablename; + const char **table_cfg; - cursor = NULL; - table = NULL; + table = (WT_TABLE *)session->dhandle; + table_cfg = table->iface.cfg; + tablename = table->iface.name; WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TABLE)); + WT_UNUSED(cfg); - WT_ERR(__wt_calloc_one(session, &table)); - table->name_hash = __wt_hash_city64(name, namelen); - - WT_ERR(__wt_scr_alloc(session, 0, &buf)); - WT_ERR(__wt_buf_fmt(session, buf, "table:%.*s", (int)namelen, name)); - WT_ERR(__wt_strndup(session, buf->data, buf->size, &table->name)); - - /* - * Don't hold the metadata cursor pinned, we call functions that use it - * to retrieve column group information. - */ - WT_ERR(__wt_metadata_cursor(session, &cursor)); - cursor->set_key(cursor, table->name); - if ((ret = cursor->search(cursor)) == 0 && - (ret = cursor->get_value(cursor, &tconfig)) == 0) - ret = __wt_strdup(session, tconfig, &table->config); - WT_TRET(__wt_metadata_cursor_release(session, &cursor)); - WT_ERR(ret); - - WT_ERR(__wt_config_getones(session, table->config, "columns", &cval)); - WT_ERR(__wt_config_getones( - session, table->config, "key_format", &cval)); - WT_ERR(__wt_strndup(session, cval.str, cval.len, &table->key_format)); - WT_ERR(__wt_config_getones( - session, table->config, "value_format", &cval)); - WT_ERR(__wt_strndup(session, cval.str, cval.len, &table->value_format)); + WT_RET(__wt_config_gets(session, table_cfg, "columns", &cval)); + WT_RET(__wt_config_gets(session, table_cfg, "key_format", &cval)); + WT_RET(__wt_strndup(session, cval.str, cval.len, &table->key_format)); + WT_RET(__wt_config_gets(session, table_cfg, "value_format", &cval)); + WT_RET(__wt_strndup(session, cval.str, cval.len, &table->value_format)); /* Point to some items in the copy to save re-parsing. */ - WT_ERR(__wt_config_getones(session, table->config, - "columns", &table->colconf)); + WT_RET(__wt_config_gets( + session, table_cfg, "columns", &table->colconf)); /* * Count the number of columns: tables are "simple" if the columns @@ -472,49 +449,32 @@ __schema_open_table(WT_SESSION_IMPL *session, table->is_simple = true; while ((ret = __wt_config_next(&cparser, &ckey, &cval)) == 0) table->is_simple = false; - if (ret != WT_NOTFOUND) - goto err; + WT_RET_NOTFOUND_OK(ret); /* Check that the columns match the key and value formats. */ if (!table->is_simple) - WT_ERR(__wt_schema_colcheck(session, + WT_RET(__wt_schema_colcheck(session, table->key_format, table->value_format, &table->colconf, &table->nkey_columns, NULL)); - WT_ERR(__wt_config_getones(session, table->config, - "colgroups", &table->cgconf)); + WT_RET(__wt_config_gets( + session, table_cfg, "colgroups", &table->cgconf)); /* Count the number of column groups. */ __wt_config_subinit(session, &cparser, &table->cgconf); table->ncolgroups = 0; while ((ret = __wt_config_next(&cparser, &ckey, &cval)) == 0) ++table->ncolgroups; - if (ret != WT_NOTFOUND) - goto err; + WT_RET_NOTFOUND_OK(ret); if (table->ncolgroups > 0 && table->is_simple) - WT_ERR_MSG(session, EINVAL, - "%s requires a table with named columns", table->name); + WT_RET_MSG(session, EINVAL, + "%s requires a table with named columns", tablename); - WT_ERR(__wt_calloc_def(session, WT_COLGROUPS(table), &table->cgroups)); - WT_ERR(__wt_schema_open_colgroups(session, table)); + WT_RET(__wt_calloc_def(session, WT_COLGROUPS(table), &table->cgroups)); + WT_RET(__wt_schema_open_colgroups(session, table)); - if (!ok_incomplete && !table->cg_complete) - WT_ERR_MSG(session, EINVAL, "'%s' cannot be used " - "until all column groups are created", - table->name); - - /* Copy the schema generation into the new table. */ - table->schema_gen = __wt_gen(session, WT_GEN_SCHEMA); - - *tablep = table; - - if (0) { -err: WT_TRET(__wt_schema_destroy_table(session, &table)); - } - - __wt_scr_free(session, &buf); - return (ret); + return (0); } /* @@ -542,7 +502,7 @@ __wt_schema_get_colgroup(WT_SESSION_IMPL *session, tend = tablename + strlen(tablename); WT_RET(__wt_schema_get_table(session, - tablename, WT_PTRDIFF(tend, tablename), false, &table)); + tablename, WT_PTRDIFF(tend, tablename), false, 0, &table)); for (i = 0; i < WT_COLGROUPS(table); i++) { colgroup = table->cgroups[i]; @@ -551,12 +511,13 @@ __wt_schema_get_colgroup(WT_SESSION_IMPL *session, if (tablep != NULL) *tablep = table; else - __wt_schema_release_table(session, table); + WT_RET( + __wt_schema_release_table(session, table)); return (0); } } - __wt_schema_release_table(session, table); + WT_RET(__wt_schema_release_table(session, table)); if (quiet) WT_RET(ENOENT); WT_RET_MSG(session, ENOENT, "%s not found in table", uri); @@ -568,7 +529,7 @@ __wt_schema_get_colgroup(WT_SESSION_IMPL *session, */ int __wt_schema_get_index(WT_SESSION_IMPL *session, - const char *uri, bool quiet, WT_TABLE **tablep, WT_INDEX **indexp) + const char *uri, bool invalidate, bool quiet, WT_INDEX **indexp) { WT_DECL_RET; WT_INDEX *idx; @@ -576,8 +537,6 @@ __wt_schema_get_index(WT_SESSION_IMPL *session, const char *tablename, *tend; u_int i; - if (tablep != NULL) - *tablep = NULL; *indexp = NULL; tablename = uri; @@ -586,28 +545,25 @@ __wt_schema_get_index(WT_SESSION_IMPL *session, return (__wt_bad_object_type(session, uri)); WT_RET(__wt_schema_get_table(session, - tablename, WT_PTRDIFF(tend, tablename), false, &table)); + tablename, WT_PTRDIFF(tend, tablename), false, 0, &table)); /* Try to find the index in the table. */ for (i = 0; i < table->nindices; i++) { idx = table->indices[i]; if (idx != NULL && strcmp(idx->name, uri) == 0) { - if (tablep != NULL) - *tablep = table; - else - __wt_schema_release_table(session, table); *indexp = idx; - return (0); + goto done; } } /* Otherwise, open it. */ WT_ERR(__wt_schema_open_index( session, table, tend + 1, strlen(tend + 1), indexp)); - if (tablep != NULL) - *tablep = table; -err: __wt_schema_release_table(session, table); +done: if (invalidate) + table->idx_complete = false; + +err: WT_TRET(__wt_schema_release_table(session, table)); WT_RET(ret); if (*indexp != NULL) @@ -623,13 +579,13 @@ err: __wt_schema_release_table(session, table); * Open a named table. */ int -__wt_schema_open_table(WT_SESSION_IMPL *session, - const char *name, size_t namelen, bool ok_incomplete, WT_TABLE **tablep) +__wt_schema_open_table(WT_SESSION_IMPL *session, const char *cfg[]) { WT_DECL_RET; - WT_WITH_TXN_ISOLATION(session, WT_ISO_READ_UNCOMMITTED, - ret = __schema_open_table( - session, name, namelen, ok_incomplete, tablep)); + WT_WITH_TABLE_WRITE_LOCK(session, + WT_WITH_TXN_ISOLATION(session, WT_ISO_READ_UNCOMMITTED, + ret = __schema_open_table(session, cfg))); + return (ret); } diff --git a/src/third_party/wiredtiger/src/schema/schema_plan.c b/src/third_party/wiredtiger/src/schema/schema_plan.c index cef8260d265..c45ad86ccd5 100644 --- a/src/third_party/wiredtiger/src/schema/schema_plan.c +++ b/src/third_party/wiredtiger/src/schema/schema_plan.c @@ -153,7 +153,7 @@ __wt_table_check(WT_SESSION_IMPL *session, WT_TABLE *table) WT_RET_MSG(session, EINVAL, "Column '%.*s' in '%s' does not appear in a " "column group", - (int)k.len, k.str, table->name); + (int)k.len, k.str, table->iface.name); /* * Column groups can't store key columns in their value: * __wt_struct_reformat should have already detected this case. diff --git a/src/third_party/wiredtiger/src/schema/schema_rename.c b/src/third_party/wiredtiger/src/schema/schema_rename.c index 1868d907d00..06133814291 100644 --- a/src/third_party/wiredtiger/src/schema/schema_rename.c +++ b/src/third_party/wiredtiger/src/schema/schema_rename.c @@ -24,11 +24,9 @@ __rename_file( newvalue = oldvalue = NULL; filename = uri; - if (!WT_PREFIX_SKIP(filename, "file:")) - return (__wt_unexpected_object_type(session, uri, "file:")); + WT_PREFIX_SKIP_REQUIRED(session, filename, "file:"); newfile = newuri; - if (!WT_PREFIX_SKIP(newfile, "file:")) - return (__wt_unexpected_object_type(session, newuri, "file:")); + WT_PREFIX_SKIP_REQUIRED(session, newfile, "file:"); WT_RET(__wt_schema_backup_check(session, filename)); WT_RET(__wt_schema_backup_check(session, newfile)); @@ -94,11 +92,11 @@ __rename_tree(WT_SESSION_IMPL *session, const char *newname, *olduri, *suffix; char *value; - olduri = table->name; + olduri = table->iface.name; value = NULL; newname = newuri; - (void)WT_PREFIX_SKIP(newname, "table:"); + WT_PREFIX_SKIP_REQUIRED(session, newname, "table:"); /* * Create the new data source URI and update the schema value. @@ -134,7 +132,7 @@ __rename_tree(WT_SESSION_IMPL *session, * and substitute the new name temporarily. */ WT_ERR(__wt_scr_alloc(session, 0, &ns)); - table->name = newuri; + table->iface.name = newuri; if (is_colgroup) WT_ERR(__wt_schema_colgroup_source( session, table, suffix, value, ns)); @@ -174,7 +172,7 @@ err: __wt_scr_free(session, &nn); __wt_scr_free(session, &nv); __wt_scr_free(session, &os); __wt_free(session, value); - table->name = olduri; + table->iface.name = olduri; return (ret); } @@ -208,12 +206,23 @@ __rename_table(WT_SESSION_IMPL *session, WT_TABLE *table; u_int i; const char *oldname; + bool tracked; oldname = uri; (void)WT_PREFIX_SKIP(oldname, "table:"); + tracked = false; + /* + * Open the table so we can rename its column groups and indexes. + * + * Ideally we would keep the table locked exclusive across the rename, + * but for now we rely on the global table lock to prevent the table + * being reopened while it is being renamed. One issue is that the + * WT_WITHOUT_LOCKS macro can drop and reacquire the global table lock, + * avoiding deadlocks while waiting for LSM operation to quiesce. + */ WT_RET(__wt_schema_get_table( - session, oldname, strlen(oldname), false, &table)); + session, oldname, strlen(oldname), false, 0, &table)); /* Rename the column groups. */ for (i = 0; i < WT_COLGROUPS(table); i++) @@ -226,14 +235,23 @@ __rename_table(WT_SESSION_IMPL *session, WT_ERR(__rename_tree(session, table, newuri, table->indices[i]->name, cfg)); - WT_ERR(__wt_schema_remove_table(session, table)); - table = NULL; + /* Make sure the table data handle is closed. */ + WT_TRET(__wt_schema_release_table(session, table)); + WT_ERR(__wt_schema_get_table_uri( + session, uri, true, WT_DHANDLE_EXCLUSIVE, &table)); + F_SET(&table->iface, WT_DHANDLE_DISCARD); + if (WT_META_TRACKING(session)) { + WT_WITH_DHANDLE(session, &table->iface, + ret = __wt_meta_track_handle_lock(session, false)); + WT_ERR(ret); + tracked = true; + } /* Rename the table. */ - WT_ERR(__metadata_rename(session, uri, newuri)); + ret = __metadata_rename(session, uri, newuri); -err: if (table != NULL) - __wt_schema_release_table(session, table); +err: if (!tracked) + WT_TRET(__wt_schema_release_table(session, table)); return (ret); } @@ -276,9 +294,6 @@ __wt_schema_rename(WT_SESSION_IMPL *session, else ret = __wt_bad_object_type(session, uri); - /* Bump the schema generation so that stale data is ignored. */ - (void)__wt_gen_next(session, WT_GEN_SCHEMA); - WT_TRET(__wt_meta_track_off(session, true, ret != 0)); /* If we didn't find a metadata entry, map that error to ENOENT. */ diff --git a/src/third_party/wiredtiger/src/schema/schema_stat.c b/src/third_party/wiredtiger/src/schema/schema_stat.c index d2d61febc39..06518ffd50a 100644 --- a/src/third_party/wiredtiger/src/schema/schema_stat.c +++ b/src/third_party/wiredtiger/src/schema/schema_stat.c @@ -42,7 +42,7 @@ __wt_curstat_index_init(WT_SESSION_IMPL *session, WT_DECL_RET; WT_INDEX *idx; - WT_RET(__wt_schema_get_index(session, uri, false, NULL, &idx)); + WT_RET(__wt_schema_get_index(session, uri, false, false, &idx)); WT_RET(__wt_scr_alloc(session, 0, &buf)); WT_ERR(__wt_buf_fmt(session, buf, "statistics:%s", idx->source)); @@ -145,7 +145,7 @@ __wt_curstat_table_init(WT_SESSION_IMPL *session, name = uri + strlen("table:"); WT_RET(__wt_schema_get_table( - session, name, strlen(name), false, &table)); + session, name, strlen(name), false, 0, &table)); WT_ERR(__wt_scr_alloc(session, 0, &buf)); @@ -184,7 +184,7 @@ __wt_curstat_table_init(WT_SESSION_IMPL *session, __wt_curstat_dsrc_final(cst); -err: __wt_schema_release_table(session, table); +err: WT_TRET(__wt_schema_release_table(session, table)); __wt_scr_free(session, &buf); return (ret); diff --git a/src/third_party/wiredtiger/src/schema/schema_truncate.c b/src/third_party/wiredtiger/src/schema/schema_truncate.c index b3a69dd5abd..7d5168fe72f 100644 --- a/src/third_party/wiredtiger/src/schema/schema_truncate.c +++ b/src/third_party/wiredtiger/src/schema/schema_truncate.c @@ -19,7 +19,8 @@ __truncate_table(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_TABLE *table; u_int i; - WT_RET(__wt_schema_get_table(session, uri, strlen(uri), false, &table)); + WT_RET(__wt_schema_get_table( + session, uri, strlen(uri), false, 0, &table)); WT_STAT_DATA_INCR(session, cursor_truncate); /* Truncate the column groups. */ @@ -33,7 +34,7 @@ __truncate_table(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_ERR(__wt_schema_truncate( session, table->indices[i]->source, cfg)); -err: __wt_schema_release_table(session, table); +err: WT_TRET(__wt_schema_release_table(session, table)); return (ret); } diff --git a/src/third_party/wiredtiger/src/schema/schema_worker.c b/src/third_party/wiredtiger/src/schema/schema_worker.c index 7655456b243..ce930c20818 100644 --- a/src/third_party/wiredtiger/src/schema/schema_worker.c +++ b/src/third_party/wiredtiger/src/schema/schema_worker.c @@ -26,12 +26,10 @@ __wt_schema_worker(WT_SESSION_IMPL *session, WT_INDEX *idx; WT_SESSION *wt_session; WT_TABLE *table; - const char *tablename; u_int i; bool skip; table = NULL; - tablename = uri; skip = false; if (name_func != NULL) @@ -59,7 +57,7 @@ __wt_schema_worker(WT_SESSION_IMPL *session, session, uri, cfg, open_flags)); WT_SAVE_DHANDLE(session, ret = file_func(session, cfg)); - WT_TRET(__wt_session_release_btree(session)); + WT_TRET(__wt_session_release_dhandle(session)); WT_ERR(ret); } } else if (WT_PREFIX_MATCH(uri, "colgroup:")) { @@ -67,18 +65,23 @@ __wt_schema_worker(WT_SESSION_IMPL *session, session, uri, false, NULL, &colgroup)); WT_ERR(__wt_schema_worker(session, colgroup->source, file_func, name_func, cfg, open_flags)); - } else if (WT_PREFIX_SKIP(tablename, "index:")) { + } else if (WT_PREFIX_MATCH(uri, "index:")) { idx = NULL; - WT_ERR(__wt_schema_get_index(session, uri, false, NULL, &idx)); + WT_ERR(__wt_schema_get_index(session, uri, false, false, &idx)); WT_ERR(__wt_schema_worker(session, idx->source, file_func, name_func, cfg, open_flags)); } else if (WT_PREFIX_MATCH(uri, "lsm:")) { WT_ERR(__wt_lsm_tree_worker(session, uri, file_func, name_func, cfg, open_flags)); - } else if (WT_PREFIX_SKIP(tablename, "table:")) { - WT_ERR(__wt_schema_get_table(session, - tablename, strlen(tablename), false, &table)); - WT_ASSERT(session, session->dhandle == NULL); + } else if (WT_PREFIX_MATCH(uri, "table:")) { + /* + * Note: we would like to use open_flags here (e.g., to lock + * the table exclusive during schema-changing operations), but + * that is currently problematic because we get the table again + * in order to discover column groups and indexes. + */ + WT_ERR(__wt_schema_get_table_uri( + session, uri, false, 0, &table)); /* * We could make a recursive call for each colgroup or index @@ -128,6 +131,6 @@ __wt_schema_worker(WT_SESSION_IMPL *session, WT_ERR(__wt_bad_object_type(session, uri)); err: if (table != NULL) - __wt_schema_release_table(session, table); + WT_TRET(__wt_schema_release_table(session, table)); return (ret); } diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 1a63ed675b5..73dea95aa2d 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -136,68 +136,6 @@ __session_clear(WT_SESSION_IMPL *session) } /* - * __session_alter -- - * Alter a table setting. - */ -static int -__session_alter(WT_SESSION *wt_session, const char *uri, const char *config) -{ - WT_DECL_RET; - WT_SESSION_IMPL *session; - - session = (WT_SESSION_IMPL *)wt_session; - - SESSION_API_CALL(session, alter, config, cfg); - - /* In-memory ignores alter operations. */ - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) - goto err; - - /* Disallow objects in the WiredTiger name space. */ - WT_ERR(__wt_str_name_check(session, uri)); - - /* - * We replace the default configuration listing with the current - * configuration. Otherwise the defaults for values that can be - * altered would override settings used by the user in create. - */ - cfg[0] = cfg[1]; - cfg[1] = NULL; - WT_WITH_CHECKPOINT_LOCK(session, - WT_WITH_SCHEMA_LOCK(session, - ret = __wt_schema_worker(session, uri, __wt_alter, NULL, cfg, - WT_BTREE_ALTER | WT_DHANDLE_EXCLUSIVE))); - -err: if (ret != 0) - WT_STAT_CONN_INCR(session, session_table_alter_fail); - else - WT_STAT_CONN_INCR(session, session_table_alter_success); - API_END_RET_NOTFOUND_MAP(session, ret); -} - -/* - * __session_alter_readonly -- - * WT_SESSION->alter method; readonly version. - */ -static int -__session_alter_readonly( - WT_SESSION *wt_session, const char *uri, const char *config) -{ - WT_DECL_RET; - WT_SESSION_IMPL *session; - - WT_UNUSED(uri); - WT_UNUSED(config); - - session = (WT_SESSION_IMPL *)wt_session; - SESSION_API_CALL_NOCONF(session, alter); - - WT_STAT_CONN_INCR(session, session_table_alter_fail); - ret = __wt_session_notsup(session); -err: API_END_RET(session, ret); -} - -/* * __session_close -- * WT_SESSION->close method. */ @@ -244,9 +182,6 @@ __session_close(WT_SESSION *wt_session, const char *config) /* Discard cached handles. */ __wt_session_close_cache(session); - /* Close all tables. */ - WT_TRET(__wt_schema_close_tables(session)); - /* Confirm we're not holding any hazard pointers. */ __wt_hazard_close(session); @@ -524,6 +459,68 @@ err: if (cursor != NULL) } /* + * __session_alter -- + * Alter a table setting. + */ +static int +__session_alter(WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_DECL_RET; + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)wt_session; + + SESSION_API_CALL(session, alter, config, cfg); + + /* In-memory ignores alter operations. */ + if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) + goto err; + + /* Disallow objects in the WiredTiger name space. */ + WT_ERR(__wt_str_name_check(session, uri)); + + /* + * We replace the default configuration listing with the current + * configuration. Otherwise the defaults for values that can be + * altered would override settings used by the user in create. + */ + cfg[0] = cfg[1]; + cfg[1] = NULL; + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, + ret = __wt_schema_worker(session, uri, __wt_alter, NULL, cfg, + WT_BTREE_ALTER | WT_DHANDLE_EXCLUSIVE))); + +err: if (ret != 0) + WT_STAT_CONN_INCR(session, session_table_alter_fail); + else + WT_STAT_CONN_INCR(session, session_table_alter_success); + API_END_RET_NOTFOUND_MAP(session, ret); +} + +/* + * __session_alter_readonly -- + * WT_SESSION->alter method; readonly version. + */ +static int +__session_alter_readonly( + WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_DECL_RET; + WT_SESSION_IMPL *session; + + WT_UNUSED(uri); + WT_UNUSED(config); + + session = (WT_SESSION_IMPL *)wt_session; + SESSION_API_CALL_NOCONF(session, alter); + + WT_STAT_CONN_INCR(session, session_table_alter_fail); + ret = __wt_session_notsup(session); +err: API_END_RET(session, ret); +} + +/* * __wt_session_create -- * Internal version of WT_SESSION::create. */ @@ -836,8 +833,6 @@ __session_reset(WT_SESSION *wt_session) WT_TRET(__wt_session_reset_cursors(session, true)); - WT_TRET(__wt_schema_sweep_tables(session)); - /* Release common session resources. */ WT_TRET(__wt_session_release_resources(session)); @@ -1768,11 +1763,11 @@ __open_session(WT_CONNECTION_IMPL *conn, static const WT_SESSION stds = { NULL, NULL, - __session_alter, __session_close, __session_reconfigure, __wt_session_strerror, __session_open_cursor, + __session_alter, __session_create, __wt_session_compact, __session_drop, @@ -1797,11 +1792,11 @@ __open_session(WT_CONNECTION_IMPL *conn, }, stds_readonly = { NULL, NULL, - __session_alter_readonly, __session_close, __session_reconfigure, __wt_session_strerror, __session_open_cursor, + __session_alter_readonly, __session_create_readonly, __wt_session_compact_readonly, __session_drop_readonly, @@ -1883,13 +1878,8 @@ __open_session(WT_CONNECTION_IMPL *conn, if (session_ret->dhhash == NULL) WT_ERR(__wt_calloc(session, WT_HASH_ARRAY_SIZE, sizeof(struct __dhandles_hash), &session_ret->dhhash)); - if (session_ret->tablehash == NULL) - WT_ERR(__wt_calloc(session, WT_HASH_ARRAY_SIZE, - sizeof(struct __tables_hash), &session_ret->tablehash)); - for (i = 0; i < WT_HASH_ARRAY_SIZE; i++) { + for (i = 0; i < WT_HASH_ARRAY_SIZE; i++) TAILQ_INIT(&session_ret->dhhash[i]); - TAILQ_INIT(&session_ret->tablehash[i]); - } /* Initialize transaction support: default to read-committed. */ session_ret->isolation = WT_ISO_READ_COMMITTED; diff --git a/src/third_party/wiredtiger/src/session/session_compact.c b/src/third_party/wiredtiger/src/session/session_compact.c index 10328e0db5b..65f2fd345d6 100644 --- a/src/third_party/wiredtiger/src/session/session_compact.c +++ b/src/third_party/wiredtiger/src/session/session_compact.c @@ -160,12 +160,12 @@ __compact_handle_append(WT_SESSION_IMPL *session, const char *cfg[]) WT_UNUSED(cfg); - WT_RET(__wt_session_get_btree( + WT_RET(__wt_session_get_dhandle( session, session->dhandle->name, NULL, NULL, 0)); /* Set compact active on the handle. */ if ((ret = __compact_start(session)) != 0) { - WT_TRET(__wt_session_release_btree(session)); + WT_TRET(__wt_session_release_dhandle(session)); return (ret); } @@ -381,7 +381,7 @@ err: session->compact = NULL; WT_WITH_DHANDLE(session, session->op_handle[i], WT_TRET(__compact_end(session))); WT_WITH_DHANDLE(session, session->op_handle[i], - WT_TRET(__wt_session_release_btree(session))); + WT_TRET(__wt_session_release_dhandle(session))); } __wt_free(session, session->op_handle); diff --git a/src/third_party/wiredtiger/src/session/session_dhandle.c b/src/third_party/wiredtiger/src/session/session_dhandle.c index 54fbceb65d7..ca85215debf 100644 --- a/src/third_party/wiredtiger/src/session/session_dhandle.c +++ b/src/third_party/wiredtiger/src/session/session_dhandle.c @@ -139,7 +139,7 @@ __wt_session_lock_dhandle( if (dhandle->excl_session == session) { if (!LF_ISSET(WT_DHANDLE_LOCK_ONLY) && (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || - F_ISSET(btree, WT_BTREE_SPECIAL_FLAGS))) + (btree != NULL && F_ISSET(btree, WT_BTREE_SPECIAL_FLAGS)))) return (EBUSY); ++dhandle->excl_ref; return (0); @@ -166,7 +166,7 @@ __wt_session_lock_dhandle( * If the handle is already open for a special operation, * give up. */ - if (F_ISSET(btree, WT_BTREE_SPECIAL_FLAGS)) + if (btree != NULL && F_ISSET(btree, WT_BTREE_SPECIAL_FLAGS)) return (EBUSY); /* @@ -241,11 +241,11 @@ __wt_session_lock_dhandle( } /* - * __wt_session_release_btree -- - * Unlock a btree handle. + * __wt_session_release_dhandle -- + * Unlock a data handle. */ int -__wt_session_release_btree(WT_SESSION_IMPL *session) +__wt_session_release_dhandle(WT_SESSION_IMPL *session) { WT_BTREE *btree; WT_DATA_HANDLE *dhandle; @@ -253,8 +253,8 @@ __wt_session_release_btree(WT_SESSION_IMPL *session) WT_DECL_RET; bool locked, write_locked; - btree = S2BT(session); dhandle = session->dhandle; + btree = dhandle->handle; write_locked = F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE); locked = true; @@ -269,7 +269,11 @@ __wt_session_release_btree(WT_SESSION_IMPL *session) __session_discard_dhandle(session, dhandle_cache); } - if (F_ISSET(btree, WT_BTREE_BULK)) { + /* + * Close the handle if we are finishing a bulk load or if the handle is + * set to discard on release. + */ + if (btree != NULL && F_ISSET(btree, WT_BTREE_BULK)) { WT_ASSERT(session, F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE) && !F_ISSET(dhandle, WT_DHANDLE_DISCARD)); /* @@ -278,11 +282,12 @@ __wt_session_release_btree(WT_SESSION_IMPL *session) * of handles. */ WT_WITH_SCHEMA_LOCK(session, ret = - __wt_conn_btree_sync_and_close(session, false, false)); - } else if (F_ISSET(btree, WT_BTREE_SPECIAL_FLAGS) || + __wt_conn_dhandle_close(session, false, false)); + } else if ((btree != NULL && F_ISSET(btree, WT_BTREE_SPECIAL_FLAGS)) || F_ISSET(dhandle, WT_DHANDLE_DISCARD | WT_DHANDLE_DISCARD_KILL)) { WT_ASSERT(session, F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE)); - ret = __wt_conn_btree_sync_and_close(session, false, + + ret = __wt_conn_dhandle_close(session, false, F_ISSET(dhandle, WT_DHANDLE_DISCARD_KILL)); F_CLR(dhandle, WT_DHANDLE_DISCARD | WT_DHANDLE_DISCARD_KILL); } @@ -342,7 +347,7 @@ retry: WT_RET(__wt_meta_checkpoint_last_name( session, cval.str, cval.len, &checkpoint)); } - ret = __wt_session_get_btree(session, uri, checkpoint, cfg, flags); + ret = __wt_session_get_dhandle(session, uri, checkpoint, cfg, flags); __wt_free(session, checkpoint); /* @@ -484,11 +489,11 @@ __session_get_dhandle( } /* - * __wt_session_get_btree -- - * Get a btree handle for the given name, set session->dhandle. + * __wt_session_get_dhandle -- + * Get a data handle for the given name, set session->dhandle. */ int -__wt_session_get_btree(WT_SESSION_IMPL *session, +__wt_session_get_dhandle(WT_SESSION_IMPL *session, const char *uri, const char *checkpoint, const char *cfg[], uint32_t flags) { WT_DATA_HANDLE *dhandle; @@ -531,14 +536,14 @@ __wt_session_get_btree(WT_SESSION_IMPL *session, __wt_writeunlock(session, &dhandle->rwlock); WT_WITH_SCHEMA_LOCK(session, - ret = __wt_session_get_btree( + ret = __wt_session_get_dhandle( session, uri, checkpoint, cfg, flags)); return (ret); } /* Open the handle. */ - if ((ret = __wt_conn_btree_open(session, cfg, flags)) == 0 && + if ((ret = __wt_conn_dhandle_open(session, cfg, flags)) == 0 && LF_ISSET(WT_DHANDLE_EXCLUSIVE)) break; @@ -581,10 +586,10 @@ __wt_session_lock_checkpoint(WT_SESSION_IMPL *session, const char *checkpoint) * while we are creating the new checkpoint. Hold the lock until the * checkpoint completes. */ - WT_ERR(__wt_session_get_btree(session, saved_dhandle->name, + WT_ERR(__wt_session_get_dhandle(session, saved_dhandle->name, checkpoint, NULL, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_LOCK_ONLY)); if ((ret = __wt_meta_track_handle_lock(session, false)) != 0) { - WT_TRET(__wt_session_release_btree(session)); + WT_TRET(__wt_session_release_dhandle(session)); goto err; } diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 09efb2924bf..68271fa97cf 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -1028,7 +1028,70 @@ __wt_txn_global_shutdown(WT_SESSION_IMPL *session) return (0); } -#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) +/* + * __wt_verbose_dump_txn_one -- + * Output diagnostic information about a transaction structure. + */ +int +__wt_verbose_dump_txn_one(WT_SESSION_IMPL *session, WT_TXN *txn) +{ +#ifdef HAVE_TIMESTAMPS + char hex_timestamp[3][2 * WT_TIMESTAMP_SIZE + 1]; +#endif + const char *iso_tag; + + iso_tag = "INVALID"; + switch (txn->isolation) { + case WT_ISO_READ_COMMITTED: + iso_tag = "WT_ISO_READ_COMMITTED"; + break; + case WT_ISO_READ_UNCOMMITTED: + iso_tag = "WT_ISO_READ_UNCOMMITTED"; + break; + case WT_ISO_SNAPSHOT: + iso_tag = "WT_ISO_SNAPSHOT"; + break; + } +#ifdef HAVE_TIMESTAMPS + WT_RET(__wt_timestamp_to_hex_string( + session, hex_timestamp[0], &txn->commit_timestamp)); + WT_RET(__wt_timestamp_to_hex_string( + session, hex_timestamp[1], &txn->first_commit_timestamp)); + WT_RET(__wt_timestamp_to_hex_string( + session, hex_timestamp[2], &txn->read_timestamp)); + WT_RET(__wt_msg(session, + "mod count: %u" + ", snap min: %" PRIu64 + ", snap max: %" PRIu64 + ", commit_timestamp: %s" + ", first_commit_timestamp: %s" + ", read_timestamp: %s" + ", flags: 0x%08" PRIx32 + ", isolation: %s", + txn->mod_count, + txn->snap_min, + txn->snap_max, + hex_timestamp[0], + hex_timestamp[1], + hex_timestamp[2], + txn->flags, + iso_tag)); +#else + WT_RET(__wt_msg(session, + "mod count: %u" + ", snap min: %" PRIu64 + ", snap max: %" PRIu64 + ", flags: 0x%08" PRIx32 + ", isolation: %s", + txn->mod_count, + txn->snap_min, + txn->snap_max, + txn->flags, + iso_tag)); +#endif + return (0); +} + /* * __wt_verbose_dump_txn -- * Output diagnostic information about the global transaction state. @@ -1037,15 +1100,15 @@ int __wt_verbose_dump_txn(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; + WT_SESSION_IMPL *sess; WT_TXN_GLOBAL *txn_global; - WT_TXN *txn; WT_TXN_STATE *s; - const char *iso_tag; uint64_t id; uint32_t i, session_cnt; #ifdef HAVE_TIMESTAMPS char hex_timestamp[3][2 * WT_TIMESTAMP_SIZE + 1]; #endif + conn = S2C(session); txn_global = &conn->txn_global; @@ -1110,78 +1173,17 @@ __wt_verbose_dump_txn(WT_SESSION_IMPL *session) /* Skip sessions with no active transaction */ if ((id = s->id) == WT_TXN_NONE && s->pinned_id == WT_TXN_NONE) continue; - - txn = &conn->sessions[i].txn; - iso_tag = "INVALID"; - switch (txn->isolation) { - case WT_ISO_READ_COMMITTED: - iso_tag = "WT_ISO_READ_COMMITTED"; - break; - case WT_ISO_READ_UNCOMMITTED: - iso_tag = "WT_ISO_READ_UNCOMMITTED"; - break; - case WT_ISO_SNAPSHOT: - iso_tag = "WT_ISO_SNAPSHOT"; - break; - } -#ifdef HAVE_TIMESTAMPS - WT_RET(__wt_timestamp_to_hex_string( - session, hex_timestamp[0], &txn->commit_timestamp)); - WT_RET(__wt_timestamp_to_hex_string( - session, hex_timestamp[1], &txn->first_commit_timestamp)); - WT_RET(__wt_timestamp_to_hex_string( - session, hex_timestamp[2], &txn->read_timestamp)); + sess = &conn->sessions[i]; WT_RET(__wt_msg(session, - "ID: %8" PRIu64 - ", mod count: %u" - ", pinned ID: %8" PRIu64 - ", snap min: %" PRIu64 - ", snap max: %" PRIu64 - ", commit_timestamp: %s" - ", first_commit_timestamp: %s" - ", read_timestamp: %s" - ", metadata pinned ID: %" PRIu64 - ", flags: 0x%08" PRIx32 - ", name: %s" - ", isolation: %s", - id, - txn->mod_count, - s->pinned_id, - txn->snap_min, - txn->snap_max, - hex_timestamp[0], - hex_timestamp[1], - hex_timestamp[2], - s->metadata_pinned, - txn->flags, - conn->sessions[i].name == NULL ? - "EMPTY" : conn->sessions[i].name, - iso_tag)); -#else - WT_RET(__wt_msg(session, - "ID: %6" PRIu64 - ", mod count: %u" + "ID: %" PRIu64 ", pinned ID: %" PRIu64 - ", snap min: %" PRIu64 - ", snap max: %" PRIu64 ", metadata pinned ID: %" PRIu64 - ", flags: 0x%08" PRIx32 - ", name: %s" - ", isolation: %s", - id, - txn->mod_count, - s->pinned_id, - txn->snap_min, - txn->snap_max, - s->metadata_pinned, - txn->flags, - conn->sessions[i].name == NULL ? - "EMPTY" : conn->sessions[i].name, - iso_tag)); -#endif + ", name: %s", + id, s->pinned_id, s->metadata_pinned, + sess->name == NULL ? + "EMPTY" : sess->name)); + WT_RET(__wt_verbose_dump_txn_one(sess, &sess->txn)); } - WT_RET(__wt_msg(session, "%s", WT_DIVIDER)); return (0); } -#endif diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index b9da630cca3..e54ec99b590 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -271,8 +271,6 @@ __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) const char *name; bool force; - btree = S2BT(session); - /* Find out if we have to force a checkpoint. */ WT_RET(__wt_config_gets_def(session, cfg, "force", 0, &cval)); force = cval.val != 0; @@ -281,9 +279,11 @@ __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) force = cval.len != 0; } - /* Should not be called with anything other than a file object. */ - WT_ASSERT(session, session->dhandle->checkpoint == NULL); - WT_ASSERT(session, WT_PREFIX_MATCH(session->dhandle->name, "file:")); + /* Should not be called with anything other than a live btree handle. */ + WT_ASSERT(session, session->dhandle->type == WT_DHANDLE_TYPE_BTREE && + session->dhandle->checkpoint == NULL); + + btree = S2BT(session); /* Skip files that are never involved in a checkpoint. */ if (F_ISSET(btree, WT_BTREE_NO_CHECKPOINT)) @@ -309,11 +309,7 @@ __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) ret = __wt_curfile_insert_check(meta_cursor); if (ret == WT_ROLLBACK) { metadata_race = true; - // Temporarily returning WT_ROLLBACK again from this function, - // to allow a mongodb unit test to pass. -#if 0 ret = 0; -#endif } else metadata_race = false; WT_TRET(__wt_metadata_cursor_release(session, &meta_cursor)); @@ -349,7 +345,7 @@ __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) name = session->dhandle->name; session->dhandle = NULL; - if ((ret = __wt_session_get_btree(session, name, NULL, NULL, 0)) != 0) + if ((ret = __wt_session_get_dhandle(session, name, NULL, NULL, 0)) != 0) return (ret == EBUSY ? 0 : ret); /* @@ -959,7 +955,7 @@ err: /* WT_WITH_DHANDLE(session, session->ckpt_handle[i], __checkpoint_fail_reset(session)); WT_WITH_DHANDLE(session, session->ckpt_handle[i], - WT_TRET(__wt_session_release_btree(session))); + WT_TRET(__wt_session_release_dhandle(session))); } __wt_free(session, session->ckpt_handle); diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index e19bbc73bb3..54634c03dfb 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -53,6 +53,10 @@ __txn_rollback_to_stable_lookaside_fixup(WT_SESSION_IMPL *session) &las_txnid, &las_timestamp, &las_key)); /* Check the file ID so we can skip durable tables */ + if (las_id >= conn->stable_rollback_maxfile) + WT_PANIC_RET(session, EINVAL, "file ID %" PRIu32 + " in lookaside table larger than max %" PRIu32, + las_id, conn->stable_rollback_maxfile); if (__bit_test(conn->stable_rollback_bitstring, las_id)) continue; @@ -303,18 +307,19 @@ __txn_rollback_to_stable_btree_walk( * Called for each open handle - choose to either skip or wipe the commits */ static int -__txn_rollback_to_stable_btree( - WT_SESSION_IMPL *session, const char *cfg[]) +__txn_rollback_to_stable_btree(WT_SESSION_IMPL *session, const char *cfg[]) { + WT_BTREE *btree; + WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_DECL_TIMESTAMP(rollback_timestamp) - WT_BTREE *btree; WT_TXN_GLOBAL *txn_global; WT_UNUSED(cfg); btree = S2BT(session); - txn_global = &S2C(session)->txn_global; + conn = S2C(session); + txn_global = &conn->txn_global; /* * Immediately durable files don't get their commits wiped. This case @@ -329,7 +334,11 @@ __txn_rollback_to_stable_btree( * Add the btree ID to the bitstring, so we can exclude any * lookaside entries for this btree. */ - __bit_set(S2C(session)->stable_rollback_bitstring, btree->id); + if (btree->id >= conn->stable_rollback_maxfile) + WT_PANIC_RET(session, EINVAL, "btree file ID %" PRIu32 + " larger than max %" PRIu32, + btree->id, conn->stable_rollback_maxfile); + __bit_set(conn->stable_rollback_bitstring, btree->id); return (0); } @@ -415,9 +424,14 @@ __wt_txn_rollback_to_stable(WT_SESSION_IMPL *session, const char *cfg[]) conn = S2C(session); WT_RET(__txn_rollback_to_stable_check(session)); - /* Allocate a non-durable btree bitstring */ + /* + * Allocate a non-durable btree bitstring. We increment the global + * value before using it, so the current value is already in use, and + * hence we need to add one here. + */ + conn->stable_rollback_maxfile = conn->next_file_id + 1; WT_RET(__bit_alloc(session, - conn->next_file_id, &conn->stable_rollback_bitstring)); + conn->stable_rollback_maxfile, &conn->stable_rollback_bitstring)); WT_ERR(__wt_conn_btree_apply(session, NULL, __txn_rollback_to_stable_btree, NULL, cfg)); diff --git a/src/third_party/wiredtiger/src/txn/txn_timestamp.c b/src/third_party/wiredtiger/src/txn/txn_timestamp.c index 275ef941490..0e04de7fdd5 100644 --- a/src/third_party/wiredtiger/src/txn/txn_timestamp.c +++ b/src/third_party/wiredtiger/src/txn/txn_timestamp.c @@ -262,8 +262,7 @@ __wt_txn_global_query_timestamp( WT_UNUSED(cfg); WT_RET_MSG(session, ENOTSUP, - "WT_CONNECTION.query_timestamp requires a version of WiredTiger " - "built with timestamp support"); + "requires a version of WiredTiger built with timestamp support"); #endif } @@ -408,7 +407,7 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) __wt_timestamp_cmp(&oldest_ts, &stable_ts) > 0) { __wt_writeunlock(session, &txn_global->rwlock); WT_RET_MSG(session, EINVAL, - "set_timestamp: oldest timestamp must be later than " + "set_timestamp: oldest timestamp must not be later than " "stable timestamp"); } @@ -454,7 +453,7 @@ __wt_txn_global_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_txn_update_pinned_timestamp(session)); } #else - WT_RET_MSG(session, EINVAL, "set_timestamp requires a " + WT_RET_MSG(session, ENOTSUP, "set_timestamp requires a " "version of WiredTiger built with timestamp support"); #endif return (0); @@ -478,11 +477,15 @@ __wt_txn_set_timestamp(WT_SESSION_IMPL *session, const char *cfg[]) #ifdef HAVE_TIMESTAMPS WT_TXN *txn = &session->txn; + if (!F_ISSET(txn, WT_TXN_RUNNING)) + WT_RET_MSG(session, EINVAL, + "Transaction must be running " + "to set a commit_timestamp"); WT_RET(__wt_txn_parse_timestamp( session, "commit", &txn->commit_timestamp, &cval)); __wt_txn_set_commit_timestamp(session); #else - WT_RET_MSG(session, EINVAL, "commit_timestamp requires a " + WT_RET_MSG(session, ENOTSUP, "commit_timestamp requires a " "version of WiredTiger built with timestamp support"); #endif } diff --git a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c index c735c0360c2..3f962d53ae0 100644 --- a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c +++ b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c @@ -83,7 +83,7 @@ static int real_checkpointer(void) { WT_SESSION *session; - char *checkpoint_config, _buf[128]; + char *checkpoint_config, buf[128]; int ret; if (g.running == 0) @@ -96,19 +96,15 @@ real_checkpointer(void) if ((ret = g.conn->open_session(g.conn, NULL, NULL, &session)) != 0) return (log_print_err("conn.open_session", ret, 1)); + if (WT_PREFIX_MATCH(g.checkpoint_name, "WiredTigerCheckpoint")) + checkpoint_config = NULL; + else { + testutil_check(__wt_snprintf( + buf, sizeof(buf), "name=%s", g.checkpoint_name)); + checkpoint_config = buf; + } + while (g.running) { - if (WT_PREFIX_MATCH(g.checkpoint_name, "WiredTigerCheckpoint")) - strcpy(_buf, ""); - else - testutil_check(__wt_snprintf( - _buf, sizeof(_buf), "name=%s", g.checkpoint_name)); - - if (g.use_timestamps && g.timestamp > 0) - testutil_check(__wt_snprintf( - _buf + strlen(_buf), sizeof(_buf) - strlen(_buf), - ",read_timestamp=%" PRIx64, g.timestamp)); - - checkpoint_config = strlen(_buf) > 0 ? _buf : NULL; /* Execute a checkpoint */ if ((ret = session->checkpoint( session, checkpoint_config)) != 0) diff --git a/src/third_party/wiredtiger/test/checkpoint/smoke.sh b/src/third_party/wiredtiger/test/checkpoint/smoke.sh index 18ae42af49b..39b1f428c2c 100755 --- a/src/third_party/wiredtiger/test/checkpoint/smoke.sh +++ b/src/third_party/wiredtiger/test/checkpoint/smoke.sh @@ -6,11 +6,6 @@ set -e echo "checkpoint: 3 mixed tables" $TEST_WRAPPER ./t -T 3 -t m -# Smoke-test timestamps -# Timestamp testing is commented as part of WT-3446 -# echo "checkpoint: 3 mixed tables with timestamps" -# $TEST_WRAPPER ./t -T 3 -t m -s - # We are done unless long tests are enabled. test "$TESTUTIL_ENABLE_LONG_TESTS" = "1" || exit 0 @@ -23,14 +18,8 @@ $TEST_WRAPPER ./t -T 6 -t l echo "checkpoint: 6 mixed tables" $TEST_WRAPPER ./t -T 6 -t m -# echo "checkpoint: 6 mixed tables with timestamps" -# $TEST_WRAPPER ./t -T 6 -t m -s - echo "checkpoint: 6 row-store tables" $TEST_WRAPPER ./t -T 6 -t r echo "checkpoint: 6 row-store tables, named checkpoint" $TEST_WRAPPER ./t -c 'TeSt' -T 6 -t r - -# echo "checkpoint: 6 row-store tables, named checkpoint" -# $TEST_WRAPPER ./t -c 'TeSt' -T 6 -t r -s diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c index bf5e645bb51..a646482b392 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c @@ -65,7 +65,7 @@ main(int argc, char *argv[]) runs = 1; while ((ch = __wt_getopt( - progname, argc, argv, "C:c:h:k:l:n:r:sT:t:W:")) != EOF) + progname, argc, argv, "C:c:h:k:l:n:r:T:t:W:")) != EOF) switch (ch) { case 'c': g.checkpoint_name = __wt_optarg; @@ -92,25 +92,6 @@ main(int argc, char *argv[]) case 'r': /* runs */ runs = atoi(__wt_optarg); break; - case 's': - /* - * disabled below block temporarily to avoid spurious - * test failures as per ticket WT-3446 and - * to be reverted when WT-3386 is merged. - */ -#if 0 -#ifdef HAVE_TIMESTAMPS - g.use_timestamps = true; -#endif - break; -#endif - /* - * The below code segment to be deleted as part of - * reverting the above block i.e. WT-3386 - */ - fprintf(stderr, - "Checkpoint Timestamp testing is not supported\n"); - return (EXIT_FAILURE); case 't': switch (__wt_optarg[0]) { case 'c': @@ -339,12 +320,10 @@ type_to_string(table_type type) static int usage(void) { - // progname, argc, argv, "c:C:h:k:l:n:r:t:T:W:")) != EOF) - fprintf(stderr, "usage: %s " "[-C wiredtiger-config] [-c checkpoint] [-h home] [-k keys]\n\t" - "[-l log] [-n ops] [-r runs] [-s] [-t f|r|v] [-T table-config]\n\t" + "[-l log] [-n ops] [-r runs] [-T table-config] [-t f|r|v]\n\t" "[-W workers]\n", progname); fprintf(stderr, "%s", @@ -355,9 +334,8 @@ usage(void) "\t-l specify a log file\n" "\t-n set number of operations each thread does\n" "\t-r set number of runs (0 for continuous)\n" - "\t-s enable transaction timestamps\n" - "\t-t set a file type ( col | mix | row | lsm )\n" "\t-T specify a table configuration\n" + "\t-t set a file type ( col | mix | row | lsm )\n" "\t-W set number of worker threads\n"); return (EXIT_FAILURE); } diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h index 3737131eb01..36551211b7e 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h @@ -57,8 +57,6 @@ typedef struct { WT_CONNECTION *conn; /* WiredTiger connection */ u_int nkeys; /* Keys to load */ u_int nops; /* Operations per thread */ - uint64_t timestamp; /* Current timestamp */ - bool use_timestamps; /* Test with timestamps */ FILE *logfp; /* Message log file. */ int nworkers; /* Number workers configured */ int ntables; /* Number tables configured */ diff --git a/src/third_party/wiredtiger/test/checkpoint/workers.c b/src/third_party/wiredtiger/test/checkpoint/workers.c index 8d6d2c53188..7ac35b1bb7d 100644 --- a/src/third_party/wiredtiger/test/checkpoint/workers.c +++ b/src/third_party/wiredtiger/test/checkpoint/workers.c @@ -166,10 +166,8 @@ real_worker(void) WT_CURSOR **cursors; WT_SESSION *session; WT_RAND_STATE rnd; - uint64_t ts; u_int i, keyno; int j, ret, t_ret; - char config_buf[64]; ret = t_ret = 0; @@ -204,27 +202,12 @@ real_worker(void) break; } if (ret == 0) { - if (g.use_timestamps) { - ts = __wt_atomic_add64(&g.timestamp, 1); - if (ts > 100 && ts % 100 == 0) { - testutil_check(__wt_snprintf( - config_buf, sizeof(config_buf), - "oldest_timestamp=%" PRIx64, - ts - 100)); - testutil_check(g.conn->set_timestamp( - g.conn, config_buf)); - } - testutil_check(__wt_snprintf( - config_buf, sizeof(config_buf), - "commit_timestamp=%" PRIx64, ts)); - } - - if ((ret = session->commit_transaction(session, - g.use_timestamps ? config_buf : NULL)) != 0) { + if ((ret = session->commit_transaction( + session, NULL)) != 0) { (void)log_print_err( "real_worker:commit_transaction", ret, 1); goto err; - } + } } else if (ret == WT_ROLLBACK) { if ((ret = session->rollback_transaction( session, NULL)) != 0) { diff --git a/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c b/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c index cef95490c7e..10824aec744 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c @@ -61,6 +61,9 @@ static const char * const list[] = { ",checkpoint=(wait=2)", ",checkpoint=(wait=0)", + ",compatibility=(release=2.6)", + ",compatibility=(release=3.0)", + ",error_prefix=\"prefix\"", ",eviction=(threads_min=7,threads_max=10)", diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 8d39e99c050..ef52da66a6c 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -30,7 +30,6 @@ #include "config.h" static void config_checksum(void); -static void config_compatibility(void); static void config_compression(const char *); static void config_encryption(void); static const char *config_file_type(u_int); @@ -41,7 +40,6 @@ static int config_is_perm(const char *); static void config_isolation(void); static void config_lrt(void); static void config_map_checksum(const char *, u_int *); -static void config_map_compatibility(const char *, u_int *); static void config_map_compression(const char *, u_int *); static void config_map_encryption(const char *, u_int *); static void config_map_file_type(const char *, u_int *); @@ -163,7 +161,6 @@ config_setup(void) g.c_threads = 1; config_checksum(); - config_compatibility(); config_compression("compression"); config_compression("logging_compression"); config_encryption(); @@ -314,36 +311,6 @@ config_compression(const char *conf_name) } /* - * config_compatibility -- - * Compatibility configuration. - */ -static void -config_compatibility(void) -{ - /* - * Compatibility is only relevant if logging is enabled. - * Skip it no matter what if we're not logging. - */ - if (g.c_logging == 0) { - config_single("compatibility=none", 0); - return; - } - /* Choose a compatibility mode if nothing was specified. */ - if (!config_is_perm("compatibility")) - switch (mmrand(NULL, 1, 10)) { - case 1: /* 10% */ - config_single("compatibility=v1", 0); - break; - case 2: /* 10% */ - config_single("compatibility=v2", 0); - break; - default: /* 80% */ - config_single("compatibility=none", 0); - break; - } -} - -/* * config_encryption -- * Encryption configuration. */ @@ -423,8 +390,6 @@ config_in_memory_check(void) config_single("backups=off", 0); if (!config_is_perm("checkpoints")) config_single("checkpoints=off", 0); - if (!config_is_perm("compatibility")) - config_single("compatibility=none", 0); if (!config_is_perm("compression")) config_single("compression=none", 0); if (!config_is_perm("logging")) @@ -789,10 +754,6 @@ config_single(const char *s, int perm) config_map_checksum(ep, &g.c_checksum_flag); *cp->vstr = dstrdup(ep); } else if (strncmp( - s, "compatibility", strlen("compatibility")) == 0) { - config_map_compatibility(ep, &g.c_compat_flag); - *cp->vstr = dstrdup(ep); - } else if (strncmp( s, "compression", strlen("compression")) == 0) { config_map_compression(ep, &g.c_compression_flag); *cp->vstr = dstrdup(ep); @@ -888,24 +849,6 @@ config_map_checksum(const char *s, u_int *vp) } /* - * config_map_compatibility -- - * Map a compatibility configuration to a flag. - */ -static void -config_map_compatibility(const char *s, u_int *vp) -{ - if (strcmp(s, "none") == 0) - *vp = COMPAT_NONE; - else if (strcmp(s, "v1") == 0) - *vp = COMPAT_V1; - else if (strcmp(s, "v2") == 0) - *vp = COMPAT_V2; - else - testutil_die(EINVAL, - "illegal compatibility configuration: %s", s); -} - -/* * config_map_compression -- * Map a compression configuration to a flag. */ diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index ef3fcf8c028..ea30986f453 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -117,10 +117,6 @@ static CONFIG c[] = { "if compaction is running", /* 10% */ C_BOOL, 10, 0, 0, &g.c_compact, NULL }, - { "compatibility", - "log file compatibility (none | v1 | v2 )", - C_IGNORE|C_STRING, 0, 0, 0, NULL, &g.c_compat }, - { "compression", "type of compression " COMPRESSION_LIST, C_IGNORE|C_STRING, 0, 0, 0, NULL, &g.c_compression }, diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 7ba36a700c6..72f9713f58b 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -155,7 +155,6 @@ typedef struct { char *c_checksum; uint32_t c_chunk_size; uint32_t c_compact; - char *c_compat; char *c_compression; char *c_config_open; uint32_t c_data_extend; @@ -221,11 +220,6 @@ typedef struct { #define CHECKSUM_UNCOMPRESSED 3 u_int c_checksum_flag; /* Checksum flag value */ -#define COMPAT_NONE 1 -#define COMPAT_V1 2 -#define COMPAT_V2 3 - u_int c_compat_flag; /* Compatibility flag value */ - #define COMPRESS_NONE 1 #define COMPRESS_LZ4 2 #define COMPRESS_LZ4_NO_RAW 3 @@ -291,7 +285,6 @@ void bdb_update(const void *, size_t, const void *, size_t); WT_THREAD_RET alter(void *); WT_THREAD_RET backup(void *); WT_THREAD_RET compact(void *); -WT_THREAD_RET compat(void *); void config_clear(void); void config_error(void); void config_file(const char *); diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 8725243ba05..8e118772500 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -76,7 +76,7 @@ wts_ops(int lastrun) TINFO **tinfo_list, *tinfo, total; WT_CONNECTION *conn; WT_SESSION *session; - wt_thread_t alter_tid, backup_tid, compact_tid, compat_tid, lrt_tid; + wt_thread_t alter_tid, backup_tid, compact_tid, lrt_tid; int64_t fourths, thread_ops; uint32_t i; int running; @@ -87,7 +87,6 @@ wts_ops(int lastrun) memset(&alter_tid, 0, sizeof(alter_tid)); memset(&backup_tid, 0, sizeof(backup_tid)); memset(&compact_tid, 0, sizeof(compact_tid)); - memset(&compat_tid, 0, sizeof(compat_tid)); memset(&lrt_tid, 0, sizeof(lrt_tid)); modify_repl_init(); @@ -155,9 +154,6 @@ wts_ops(int lastrun) if (g.c_compact) testutil_check( __wt_thread_create(NULL, &compact_tid, compact, NULL)); - if (g.c_compat_flag != COMPAT_NONE) - testutil_check( - __wt_thread_create(NULL, &compat_tid, compat, NULL)); if (!SINGLETHREADED && g.c_long_running_txn) testutil_check(__wt_thread_create(NULL, &lrt_tid, lrt, NULL)); @@ -225,8 +221,6 @@ wts_ops(int lastrun) testutil_check(__wt_thread_join(NULL, backup_tid)); if (g.c_compact) testutil_check(__wt_thread_join(NULL, compact_tid)); - if (g.c_compat_flag != COMPAT_NONE) - testutil_check(__wt_thread_join(NULL, compat_tid)); if (!SINGLETHREADED && g.c_long_running_txn) testutil_check(__wt_thread_join(NULL, lrt_tid)); g.workers_finished = 0; diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index 29c80183673..37a4ad074a5 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -550,48 +550,6 @@ alter(void *arg) return (WT_THREAD_RET_VALUE); } -#define COMPATSTR_V1 "compatibility=(release=2.6)" -#define COMPATSTR_V2 "compatibility=(release=3.0)" - -/* - * compat -- - * Periodically reconfigure the compatibility option. - */ -WT_THREAD_RET -compat(void *arg) -{ - WT_CONNECTION *conn; - WT_DECL_RET; - u_int count, period; - const char *str; - - (void)(arg); - - conn = g.wts_conn; - str = NULL; - /* - * Perform compatibility swaps at somewhere under 10 seconds (so we - * get at least one done), and then at 7 second intervals. - */ - for (period = mmrand(NULL, 1, 10), count = 0;; ++count, period = 7) { - if (count % 2 == 0) - str = COMPATSTR_V1; - else - str = COMPATSTR_V2; - if ((ret = conn->reconfigure(conn, str)) != 0) - testutil_die(ret, "conn.reconfigure"); - - /* Sleep for short periods so we don't make the run wait. */ - while (period > 0 && !g.workers_finished) { - --period; - sleep(1); - } - if (g.workers_finished) - break; - } - return (WT_THREAD_RET_VALUE); -} - /* * print_item_data -- * Display a single data/size pair, with a tag. diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index 031e3bb25af..37b59e81c65 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -72,34 +72,6 @@ compressor(uint32_t compress_flag) } /* - * compatibility -- - * Configure compatibility. - */ -static const char * -compatibility(uint32_t compat_flag) -{ - const char *p; - - p = "unrecognized compatibility flag"; - switch (compat_flag) { - case COMPAT_NONE: - p = ""; - break; - case COMPAT_V1: - p = "2.6"; - break; - case COMPAT_V2: - p = "3.0"; - break; - default: - testutil_die(EINVAL, - "illegal compatibility flag: %#" PRIx32, compat_flag); - /* NOTREACHED */ - } - return (p); -} - -/* * encryptor -- * Configure encryption. */ @@ -219,17 +191,13 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) ",eviction=(threads_max=%" PRIu32 ")", g.c_evict_max); /* Logging configuration. */ - if (g.c_logging) { + if (g.c_logging) CONFIG_APPEND(p, ",log=(enabled=true,archive=%d,prealloc=%d" ",compressor=\"%s\")", g.c_logging_archive ? 1 : 0, g.c_logging_prealloc ? 1 : 0, compressor(g.c_logging_compression_flag)); - CONFIG_APPEND(p, - ",compatibility=(release=%s)", - compatibility(g.c_compat_flag)); - } if (g.c_encryption) CONFIG_APPEND(p, diff --git a/src/third_party/wiredtiger/test/packing/intpack-test.c b/src/third_party/wiredtiger/test/packing/intpack-test.c index 172575c3d3d..2b482d289dc 100644 --- a/src/third_party/wiredtiger/test/packing/intpack-test.c +++ b/src/third_party/wiredtiger/test/packing/intpack-test.c @@ -32,8 +32,10 @@ int main(void) { uint64_t ncalls, r, r2, s; - uint8_t buf[WT_INTPACK64_MAXSIZE], *p; + uint8_t buf[WT_INTPACK64_MAXSIZE + 8]; /* -Werror=array-bounds */ + uint8_t *p; const uint8_t *cp; + size_t used_len; int i; memset(buf, 0xff, sizeof(buf)); /* -Werror=maybe-uninitialized */ @@ -46,6 +48,8 @@ main(void) #if 1 p = buf; testutil_check(__wt_vpack_uint(&p, sizeof(buf), r)); + used_len = (size_t)(p - buf); + testutil_assert(used_len <= WT_INTPACK64_MAXSIZE); cp = buf; testutil_check( __wt_vunpack_uint(&cp, sizeof(buf), &r2)); diff --git a/src/third_party/wiredtiger/test/packing/intpack-test2.c b/src/third_party/wiredtiger/test/packing/intpack-test2.c index 13f9ab0d2d7..82a4e1d36d2 100644 --- a/src/third_party/wiredtiger/test/packing/intpack-test2.c +++ b/src/third_party/wiredtiger/test/packing/intpack-test2.c @@ -31,13 +31,19 @@ int main(void) { - uint8_t buf[WT_INTPACK64_MAXSIZE], *p, *end; + uint8_t buf[WT_INTPACK64_MAXSIZE + 8]; /* -Werror=array-bounds */ + uint8_t *p, *end; int64_t i; + size_t used_len; + + memset(buf, 0xff, sizeof(buf)); /* -Werror=maybe-uninitialized */ for (i = 1; i < 1LL << 60; i <<= 1) { end = buf; testutil_check( __wt_vpack_uint(&end, sizeof(buf), (uint64_t)i)); + used_len = (size_t)(end - buf); + testutil_assert(used_len <= WT_INTPACK64_MAXSIZE); printf("%" PRId64 " ", i); for (p = buf; p < end; p++) printf("%02x", *p); @@ -45,6 +51,8 @@ main(void) end = buf; testutil_check(__wt_vpack_int(&end, sizeof(buf), -i)); + used_len = (size_t)(end - buf); + testutil_assert(used_len <= WT_INTPACK64_MAXSIZE); printf("%" PRId64 " ", -i); for (p = buf; p < end; p++) printf("%02x", *p); diff --git a/src/third_party/wiredtiger/test/packing/intpack-test3.c b/src/third_party/wiredtiger/test/packing/intpack-test3.c index dad31fd7f7d..2f74c84c6d8 100644 --- a/src/third_party/wiredtiger/test/packing/intpack-test3.c +++ b/src/third_party/wiredtiger/test/packing/intpack-test3.c @@ -35,17 +35,20 @@ void test_value(int64_t val) { const uint8_t *cp; - uint8_t buf[WT_INTPACK64_MAXSIZE], *p; + uint8_t buf[WT_INTPACK64_MAXSIZE + 8]; /* -Werror=array-bounds */ + uint8_t *p; int64_t sinput, soutput; uint64_t uinput, uoutput; size_t used_len; - soutput = 0; /* -Werror=maybe-uninitialized */ + memset(buf, 0xff, sizeof(buf)); /* -Werror=maybe-uninitialized */ sinput = val; - soutput = 0; /* Make GCC happy. */ + soutput = 0; /* -Werror=maybe-uninitialized */ + p = buf; testutil_check(__wt_vpack_int(&p, sizeof(buf), sinput)); used_len = (size_t)(p - buf); + testutil_assert(used_len <= WT_INTPACK64_MAXSIZE); cp = buf; testutil_check(__wt_vunpack_int(&cp, used_len, &soutput)); /* Ensure we got the correct value back */ @@ -70,6 +73,8 @@ test_value(int64_t val) p = buf; testutil_check(__wt_vpack_uint(&p, sizeof(buf), uinput)); + used_len = (size_t)(p - buf); + testutil_assert(used_len <= WT_INTPACK64_MAXSIZE); cp = buf; testutil_check(__wt_vunpack_uint(&cp, sizeof(buf), &uoutput)); /* Ensure we got the correct value back */ diff --git a/src/third_party/wiredtiger/test/recovery/random-abort.c b/src/third_party/wiredtiger/test/recovery/random-abort.c index 5588da00bd9..c9c6773800f 100644 --- a/src/third_party/wiredtiger/test/recovery/random-abort.c +++ b/src/third_party/wiredtiger/test/recovery/random-abort.c @@ -233,7 +233,7 @@ main(int argc, char *argv[]) uint32_t i, nth, timeout; int ch, status, ret; const char *working_dir; - char fname[64], kname[64], statname[1024]; + char buf[1024], fname[64], kname[64]; bool fatal, rand_th, rand_time, verify_only; (void)testutil_set_progname(argv); @@ -326,8 +326,8 @@ main(int argc, char *argv[]) * don't stay in this loop forever. */ testutil_check(__wt_snprintf( - statname, sizeof(statname), "%s/%s", home, fs_main)); - while (stat(statname, &sb) != 0 && kill(pid, 0) == 0) + buf, sizeof(buf), "%s/%s", home, fs_main)); + while (stat(buf, &sb) != 0 && kill(pid, 0) == 0) sleep(1); sleep(timeout); @@ -348,6 +348,14 @@ main(int argc, char *argv[]) */ if (chdir(home) != 0) testutil_die(errno, "parent chdir: %s", home); + + testutil_check(__wt_snprintf(buf, sizeof(buf), + "rm -rf ../%s.SAVE; mkdir ../%s.SAVE; " + "cp -p WiredTigerLog.* ../%s.SAVE;", + home, home, home)); + if ((status = system(buf)) < 0) + testutil_die(status, "system: %s", buf); + printf("Open database, run recovery and verify content\n"); if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); diff --git a/src/third_party/wiredtiger/test/recovery/timestamp-abort.c b/src/third_party/wiredtiger/test/recovery/timestamp-abort.c index 7e912b1fe26..b0b128c2dd9 100644 --- a/src/third_party/wiredtiger/test/recovery/timestamp-abort.c +++ b/src/third_party/wiredtiger/test/recovery/timestamp-abort.c @@ -403,7 +403,7 @@ main(int argc, char *argv[]) uint32_t i, nth, timeout; int ch, status, ret; const char *working_dir; - char buf[128], fname[64], kname[64], statname[1024]; + char buf[512], fname[64], kname[64], statname[1024]; bool fatal, rand_th, rand_time, verify_only; (void)testutil_set_progname(argv); @@ -523,7 +523,8 @@ main(int argc, char *argv[]) if (chdir(home) != 0) testutil_die(errno, "parent chdir: %s", home); testutil_check(__wt_snprintf(buf, sizeof(buf), - "rm -rf ../%s.SAVE && mkdir ../%s.SAVE && cp -rp * ../%s.SAVE", + "rm -rf ../%s.SAVE && mkdir ../%s.SAVE && " + "cp -p WiredTigerLog.* ../%s.SAVE", home, home, home)); (void)system(buf); printf("Open database, run recovery and verify content\n"); diff --git a/src/third_party/wiredtiger/test/suite/test_alter02.py b/src/third_party/wiredtiger/test/suite/test_alter02.py index 3c5b67aeba5..3a288d7edf1 100644 --- a/src/third_party/wiredtiger/test/suite/test_alter02.py +++ b/src/third_party/wiredtiger/test/suite/test_alter02.py @@ -82,7 +82,7 @@ class test_alter02(wttest.WiredTigerTestCase): self.session = self.conn.open_session() # Verify the metadata string for this URI and that its setting in the - # metdata file is correct. + # metadata file is correct. def verify_metadata(self, metastr): if metastr == '': return diff --git a/src/third_party/wiredtiger/test/suite/test_bug010.py b/src/third_party/wiredtiger/test/suite/test_bug010.py index dfe317bf94e..1584c173053 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug010.py +++ b/src/third_party/wiredtiger/test/suite/test_bug010.py @@ -40,7 +40,7 @@ class test_bug010(wttest.WiredTigerTestCase): num_tables = 1000 # Disable checkpoint sync, to make checkpoints faster and - # increase the likelyhood of triggering the symptom + # increase the likelihood of triggering the symptom conn_config = 'checkpoint_sync=false' def test_checkpoint_dirty(self): diff --git a/src/third_party/wiredtiger/test/suite/test_bulk01.py b/src/third_party/wiredtiger/test/suite/test_bulk01.py index da399faba2d..5cbb722d7a5 100644 --- a/src/third_party/wiredtiger/test/suite/test_bulk01.py +++ b/src/third_party/wiredtiger/test/suite/test_bulk01.py @@ -187,7 +187,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): if not wiredtiger.diagnostic_build(): self.skipTest('requires a diagnostic build') - # Close explicitly, there's going to be a fallure. + # Close explicitly, there's going to be a failure. msg = '/are incorrectly sorted/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.conn.close(), msg) diff --git a/src/third_party/wiredtiger/test/suite/test_colgap.py b/src/third_party/wiredtiger/test/suite/test_colgap.py index 01e52ea1da5..73feaf0dd8e 100644 --- a/src/third_party/wiredtiger/test/suite/test_colgap.py +++ b/src/third_party/wiredtiger/test/suite/test_colgap.py @@ -169,7 +169,7 @@ class test_colmax(wttest.WiredTigerTestCase): bulk_config = "bulk" cursor = self.session.open_cursor(uri, None, bulk_config) - # Optionaly make the big record the only record in the table. + # Optionally make the big record the only record in the table. if not self.single: for i in range(1, 723): cursor[simple_key(cursor, i)] = simple_value(cursor, i) diff --git a/src/third_party/wiredtiger/test/suite/test_compat01.py b/src/third_party/wiredtiger/test/suite/test_compat01.py index 1c9f07c02f2..ce87e072fc9 100644 --- a/src/third_party/wiredtiger/test/suite/test_compat01.py +++ b/src/third_party/wiredtiger/test/suite/test_compat01.py @@ -29,10 +29,11 @@ # test_compat01.py # Check compatibility API -import fnmatch, os, shutil, sys, time +import fnmatch, os +import wiredtiger, wttest from suite_subprocess import suite_subprocess +from wtdataset import SimpleDataSet, simple_key from wtscenario import make_scenarios -import wttest class test_compat01(wttest.WiredTigerTestCase, suite_subprocess): # Add enough entries and use a small log size to generate more than @@ -178,5 +179,24 @@ class test_compat01(wttest.WiredTigerTestCase, suite_subprocess): def test_restart(self): self.run_test(False) +class test_reconfig_fail(wttest.WiredTigerTestCase): + + # Test an active transaction blocks upgrade/downgrade configuration. + def test_reconfig_fail(self): + uri = 'table:reconfig_fail' + ds = SimpleDataSet(self, uri, 100, key_format='S') + ds.populate() + + self.session.begin_transaction("isolation=snapshot") + c = self.session.open_cursor(uri, None) + c.set_key(ds.key(20)) + c.set_value("abcde") + self.assertEquals(c.update(), 0) + + compat_str = 'compatibility=(release="3.0.0")' + msg = '/system must be quiescent/' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda:self.conn.reconfigure(compat_str), msg) + if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_debug_info.py b/src/third_party/wiredtiger/test/suite/test_debug_info.py new file mode 100644 index 00000000000..97fa8efbc86 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_debug_info.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest + +# test_debug_info.py +# Test WT_CONNECTION::debug_info undocumented feature +class test_debug_info(wttest.WiredTigerTestCase): + conn_config = 'create,log=(enabled),statistics=(fast)' + uri = 'file:test_conndump' + def conn_cursors(self): + + self.session.create(self.uri, 'key_format=i,value_format=i') + c = self.session.open_cursor(self.uri, None) + keys = range(1, 101) + for k in keys: + c[k] = 1 + c.close() + c = self.session.open_cursor(self.uri, None) + val = c[50] + self.conn.debug_info('cursors') + c.close() + + def conn_cursors_special(self, special_uri): + c = self.session.open_cursor(special_uri, None, None) + self.conn.debug_info('cursors') + c.close() + + def test_debug(self): + with self.expectedStdoutPattern('Data handle dump'): + self.conn.debug_info('handles') + + with self.expectedStdoutPattern('Active'): + self.conn.debug_info('sessions') + + with self.expectedStdoutPattern('POSITIONED'): + self.conn_cursors() + + special = ['backup:', 'log:', 'metadata:', 'statistics:'] + for s in special: + pat = 'URI: ' + s + with self.expectedStdoutPattern(pat): + self.conn_cursors_special(s) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_empty.py b/src/third_party/wiredtiger/test/suite/test_empty.py index 82a3bb406ee..ad57bc78036 100644 --- a/src/third_party/wiredtiger/test/suite/test_empty.py +++ b/src/third_party/wiredtiger/test/suite/test_empty.py @@ -53,7 +53,7 @@ class test_empty(wttest.WiredTigerTestCase): name = name + '.wt' self.assertEquals(os.stat(name).st_size, 4*1024) - # Open a new sesssion, add a few rows to an object and then remove them, + # Open a new session, add a few rows to an object and then remove them, # then close the object. We open/close the object so it's flushed from # the underlying cache each time. def empty(self): diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt04.py b/src/third_party/wiredtiger/test/suite/test_encrypt04.py index 7bbc4c617f1..3f9058404a8 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt04.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt04.py @@ -42,7 +42,7 @@ class test_encrypt04(wttest.WiredTigerTestCase, suite_subprocess): # For tests that are mismatching, we use a secretkey. The 'rotn' # encryptor without a secretkey is too simple, and may leave - # substantional portions of its input unchanged - a root page decoded + # substantial portions of its input unchanged - a root page decoded # with simply the wrong keyid may appear valid when initially verified, # but may result in error on first use. The odds that a real encryptor # would leave a lot of its input unchanged is infinitesimally small. @@ -175,7 +175,7 @@ class test_encrypt04(wttest.WiredTigerTestCase, suite_subprocess): self.create_records(cursor, r, 0, self.nrecords) cursor.close() - # Now intentially expose the test to mismatched configuration + # Now intentionally expose the test to mismatched configuration self.part = 2 self.name = self.name2 self.keyid = self.keyid2 diff --git a/src/third_party/wiredtiger/test/suite/test_reconfig02.py b/src/third_party/wiredtiger/test/suite/test_reconfig02.py index bba224d1bb5..0694f3621d8 100644 --- a/src/third_party/wiredtiger/test/suite/test_reconfig02.py +++ b/src/third_party/wiredtiger/test/suite/test_reconfig02.py @@ -81,7 +81,7 @@ class test_reconfig02(wttest.WiredTigerTestCase): # # Potentially loop a few times in case it is a very slow system. self.conn.reconfigure("log=(prealloc=true)") - for x in xrange(0, 20): + for x in xrange(0, 100): time.sleep(1) prep_logs = fnmatch.filter(os.listdir('.'), "*Prep*") if len(prep_logs) != 0: diff --git a/src/third_party/wiredtiger/test/suite/test_schema03.py b/src/third_party/wiredtiger/test/suite/test_schema03.py index e5471a4de73..ce6e3684980 100644 --- a/src/third_party/wiredtiger/test/suite/test_schema03.py +++ b/src/third_party/wiredtiger/test/suite/test_schema03.py @@ -133,7 +133,7 @@ class tabconfig: rand = suite_random.suite_random(ncolumns, totalindices) # Initially, all indices get one column from the collist. - # Overlaps are allowed. Then probalistically, add some + # Overlaps are allowed. Then probabalistically, add some # more columns. for idx in self.idxlist: prob = 1.0 diff --git a/src/third_party/wiredtiger/test/suite/test_stat02.py b/src/third_party/wiredtiger/test/suite/test_stat02.py index d3bc18cb3e6..d7f66424c5d 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat02.py +++ b/src/third_party/wiredtiger/test/suite/test_stat02.py @@ -243,7 +243,7 @@ class test_stat_cursor_dsrc_cache_walk(wttest.WiredTigerTestCase): self.assertGreater(c[stat.dsrc.btree_entries][2], 0) c.close() - # Verify that cache and tree walk can operate independantly + # Verify that cache and tree walk can operate independently self.conn.reconfigure('statistics=(all,clear)') c = self.session.open_cursor( 'statistics:' + self.uri, None, 'statistics=(cache_walk,fast)') diff --git a/src/third_party/wiredtiger/test/suite/test_stat_log02.py b/src/third_party/wiredtiger/test/suite/test_stat_log02.py index e85b64721df..96f4ccf3da8 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat_log02.py +++ b/src/third_party/wiredtiger/test/suite/test_stat_log02.py @@ -26,64 +26,20 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import glob -import json -import os.path -import time +import glob, json import helper, wiredtiger, wttest from wiredtiger import stat # test_stat_log02.py -# Statistics log JSON testing +# Statistics log sources argument and JSON testing. class test_stat_log02(wttest.WiredTigerTestCase): - """ - Test statistics log JSON outputs - """ # Tests need to setup the connection in their own way. def setUpConnectionOpen(self, dir): return None - def setUpSessionOpen(self, conn): return None - def test_stats_log_json(self): - self.conn = self.wiredtiger_open( - None, "create,statistics=(fast),statistics_log=(wait=1,json)") - - self.wait_for_stats_file(".") - self.check_stats_file(".") - - def test_stats_log_on_json_with_tables(self): - self.conn = self.wiredtiger_open(None, - "create,statistics=(fast),statistics_log=(wait=1,json,sources=[file:])") - - # Create a session and table to give us some stats - session = self.conn.open_session() - session.create("table:foo") - c = session.open_cursor("table:foo") - c["foo"] = "foo" - c.close() - session.close() - - self.wait_for_stats_file(".") - self.close_conn() - self.check_stats_file(".") - self.check_file_contains_tables(".") - - def wait_for_stats_file(self, dir): - # We wait for 30 sleeps then fail - number_sleeps = 0 - while True: - time.sleep(1) - files = glob.glob(dir + '/' + 'WiredTigerStat.[0-9]*') - for f in files: - if os.stat(f).st_size != 0: - return - - number_sleeps += 1 - self.assertLess(number_sleeps, 30) - def check_stats_file(self, dir): files = glob.glob(dir + '/' + 'WiredTigerStat.[0-9]*') self.assertTrue(files) @@ -96,21 +52,40 @@ class test_stat_log02(wttest.WiredTigerTestCase): json.loads(line) def check_file_contains_tables(self, dir): - # We wait for another 30 sleeps here to avoid erroring in the case where - # the stat log has only made the first pass and not yet printed the - # individual table stats. - number_sleeps = 0 - while True: - files = glob.glob(dir + '/' + 'WiredTigerStat.[0-9]*') - f = open(files[0], 'r') - for line in f: - data = json.loads(line) - if "wiredTigerTables" in data: - if "file:foo.wt" in data["wiredTigerTables"]: - return + files = glob.glob(dir + '/' + 'WiredTigerStat.[0-9]*') + f = open(files[0], 'r') + for line in f: + data = json.loads(line) + if "wiredTigerTables" in data: + if "file:foo.wt" in data["wiredTigerTables"]: + return + self.fail('Did not find expected sources in statistics log output') + + def test_stats_log_json(self): + self.conn = self.wiredtiger_open(None, + "create,statistics=(fast),statistics_log=(wait=1,json,on_close=1)") + + # Closing the connection forces statistics to be written. + self.close_conn() + + self.check_stats_file(".") - number_sleeps += 1 - self.assertLess(number_sleeps, 30) + def test_stats_log_on_json_with_tables(self): + self.conn = self.wiredtiger_open(None, + "create,statistics=(fast)," +\ + "statistics_log=(wait=1,json,on_close=1,sources=[file:])") + self.session = self.conn.open_session(None) + + # Create a session and table to give us some stats. + self.session.create("table:foo", "key_format=S,value_format=S") + c = self.session.open_cursor("table:foo") + c["foo"] = "foo" + + # Closing the connection forces statistics to be written. + self.close_conn() + + self.check_stats_file(".") + self.check_file_contains_tables(".") if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp01.py b/src/third_party/wiredtiger/test/suite/test_timestamp01.py index c8938296908..c7a5df66ae0 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp01.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp01.py @@ -41,6 +41,12 @@ class test_timestamp01(wttest.WiredTigerTestCase, suite_subprocess): if not wiredtiger.timestamp_build(): self.skipTest('requires a timestamp build') + # Cannot set a timestamp on a non-running transaction + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.timestamp_transaction( + 'commit_timestamp=' + timestamp_str(1 << 5000)), + '/must be running/') + # Zero is not permitted self.session.begin_transaction() self.assertRaisesWithMessage(wiredtiger.WiredTigerError, diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp03.py b/src/third_party/wiredtiger/test/suite/test_timestamp03.py index 728200e528a..d2d5cba6073 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp03.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp03.py @@ -46,9 +46,10 @@ def timestamp_ret_str(t): return s class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): - tablename = 'ts03_ts_nologged' - tablename2 = 'ts03_nots_logged' - tablename3 = 'ts03_ts_logged' + table_ts_log = 'ts03_ts_logged' + table_ts_nolog = 'ts03_ts_nologged' + table_nots_log = 'ts03_nots_logged' + table_nots_nolog = 'ts03_nots_nologged' types = [ ('file', dict(uri='file:', use_cg=False, use_index=False)), @@ -73,101 +74,114 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): scenarios = make_scenarios(types, ckpt, conncfg) # Binary values. - value = u'\u0001\u0002abcd\u0003\u0004' + value = u'\u0001\u0002abcd\u0003\u0004' value2 = u'\u0001\u0002dcba\u0003\u0004' value3 = u'\u0001\u0002cdef\u0003\u0004' # Check that a cursor (optionally started in a new transaction), sees the # expected values. - def check(self, session, txn_config, expected): + def check(self, session, txn_config, tablename, expected): + if txn_config: session.begin_transaction(txn_config) - c = session.open_cursor(self.uri + self.tablename, None) - actual = dict((k, v) for k, v in c if v != 0) + + cur = session.open_cursor(self.uri + tablename, None) + actual = dict((k, v) for k, v in cur if v != 0) self.assertEqual(actual, expected) # Search for the expected items as well as iterating for k, v in expected.iteritems(): - self.assertEqual(c[k], v, "for key " + str(k)) - c.close() + self.assertEqual(cur[k], v, "for key " + str(k)) + cur.close() if txn_config: session.commit_transaction() # # Take a backup of the database and verify that the value we want to # check exists in the tables the expected number of times. # - def backup_check(self, check_value, valcnt, valcnt2, valcnt3): + def backup_check( + self, check_value, expected_ts_log, expected_ts_nolog, expected_nots_log, + expected_nots_nolog): + newdir = "BACKUP" copy_wiredtiger_home('.', newdir, True) conn = self.setUpConnectionOpen(newdir) session = self.setUpSessionOpen(conn) - c = session.open_cursor(self.uri + self.tablename, None) - c2 = session.open_cursor(self.uri + self.tablename2, None) - c3 = session.open_cursor(self.uri + self.tablename3, None) - # Count how many times the second value is present - count = 0 - for k, v in c: + cur_ts_log = session.open_cursor(self.uri + self.table_ts_log, None) + cur_ts_nolog = session.open_cursor(self.uri + self.table_ts_nolog, None) + cur_nots_log = session.open_cursor(self.uri + self.table_nots_log, None) + cur_nots_nolog = session.open_cursor(self.uri + self.table_nots_nolog, None) + # Count how many times the check_value is present in the + # logged timestamp table. + actual_ts_log = 0 + for k, v in cur_ts_log: if check_value in str(v): - # print "check_value found in key " + str(k) - count += 1 - c.close() - # Count how many times the second value is present in the - # non-timestamp table. - count2 = 0 - for k, v in c2: + actual_ts_log += 1 + cur_ts_log.close() + # Count how many times the check_value is present in the + # not logged timestamp table + actual_ts_nolog = 0 + for k, v in cur_ts_nolog: if check_value in str(v): - # print "check_value found in key " + str(k) - count2 += 1 - c2.close() - # Count how many times the second value is present in the - # logged timestamp table. - count3 = 0 - for k, v in c3: + actual_ts_nolog += 1 + cur_ts_nolog.close() + # Count how many times the check_value is present in the + # logged non-timestamp table. + actual_nots_log = 0 + for k, v in cur_nots_log: + if check_value in str(v): + actual_nots_log += 1 + cur_nots_log.close() + # Count how many times the check_value is present in the + # not logged non-timestamp table. + actual_nots_nolog = 0 + for k, v in cur_nots_nolog: if check_value in str(v): - count3 += 1 - c3.close() + actual_nots_nolog += 1 + cur_nots_nolog.close() conn.close() - # print "CHECK BACKUP: Count " + str(count) + " Count2 " + str(count2) + " Count3 " + str(count3) - # print "CHECK BACKUP: Expect value2 count " + str(valcnt) - # print "CHECK BACKUP: 2nd table Expect value2 count " + str(valcnt2) - # print "CHECK BACKUP: 3rd table Expect value2 count " + str(valcnt3) - # print "CHECK BACKUP: config " + str(self.ckptcfg) - self.assertEqual(count, valcnt) - self.assertEqual(count2, valcnt2) - self.assertEqual(count3, valcnt3) + self.assertEqual(actual_ts_log, expected_ts_log) + self.assertEqual(actual_ts_nolog, expected_ts_nolog) + self.assertEqual(actual_nots_log, expected_nots_log) + self.assertEqual(actual_nots_nolog, expected_nots_nolog) # Check that a cursor sees the expected values after a checkpoint. - def ckpt_backup(self, check_value, valcnt, valcnt2, valcnt3): + def ckpt_backup( + self, check_value, valcnt_ts_log, valcnt_ts_nolog, valcnt_nots_log, + valcnt_nots_nolog): # Take a checkpoint. Make a copy of the database. Open the # copy and verify whether or not the expected data is in there. self.pr("CKPT: " + self.ckptcfg) ckptcfg = self.ckptcfg - if ckptcfg == 'read_timestamp': - ckptcfg = self.ckptcfg + '=' + self.oldts - # print "CKPT: " + ckptcfg + self.session.checkpoint(ckptcfg) - self.backup_check(check_value, valcnt, valcnt2, valcnt3) + self.backup_check(check_value, valcnt_ts_log, valcnt_ts_nolog, + valcnt_nots_log, valcnt_nots_nolog) def test_timestamp03(self): if not wiredtiger.timestamp_build(): self.skipTest('requires a timestamp build') - uri = self.uri + self.tablename - uri2 = self.uri + self.tablename2 - uri3 = self.uri + self.tablename3 + uri_ts_log = self.uri + self.table_ts_log + uri_ts_nolog = self.uri + self.table_ts_nolog + uri_nots_log = self.uri + self.table_nots_log + uri_nots_nolog = self.uri + self.table_nots_nolog # - # Open three tables: - # 1. Table is not logged and uses timestamps. - # 2. Table is logged and does not use timestamps. - # 3. Table is logged and uses timestamps. + # Open four tables: + # 1. Table is logged and uses timestamps. + # 2. Table is not logged and uses timestamps. + # 3. Table is logged and does not use timestamps. + # 4. Table is not logged and does not use timestamps. # - self.session.create(uri, 'key_format=i,value_format=S,log=(enabled=false)') - c = self.session.open_cursor(uri) - self.session.create(uri2, 'key_format=i,value_format=S') - c2 = self.session.open_cursor(uri2) - self.session.create(uri3, 'key_format=i,value_format=S') - c3 = self.session.open_cursor(uri3) + self.session.create(uri_ts_log, 'key_format=i,value_format=S') + cur_ts_log = self.session.open_cursor(uri_ts_log) + self.session.create(uri_ts_nolog, 'key_format=i,value_format=S,log=(enabled=false)') + cur_ts_nolog = self.session.open_cursor(uri_ts_nolog) + self.session.create(uri_nots_log, 'key_format=i,value_format=S') + cur_nots_log = self.session.open_cursor(uri_nots_log) + self.session.create(uri_nots_nolog, 'key_format=i,value_format=S, log=(enabled=false)') + cur_nots_nolog = self.session.open_cursor(uri_nots_nolog) # Insert keys 1..100 each with timestamp=key, in some order nkeys = 100 @@ -176,26 +190,61 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): random.shuffle(keys) for k in keys: - c2[k] = self.value + cur_nots_log[k] = self.value + cur_nots_nolog[k] = self.value self.session.begin_transaction() - c[k] = self.value - c3[k] = self.value + cur_ts_log[k] = self.value + cur_ts_nolog[k] = self.value self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) - # Now check that we see the expected state when reading at each - # timestamp + # Scenario: 1 + # Check that we see all the inserted values as per transaction + # visibility when reading with out the read_timestamp. + # All tables should see all the values. + self.check(self.session, "", self.table_ts_log, + dict((k, self.value) for k in orig_keys)) + self.check(self.session, "", self.table_ts_nolog, + dict((k, self.value) for k in orig_keys)) + self.check(self.session, "", self.table_nots_log, + dict((k, self.value) for k in orig_keys)) + self.check(self.session, "", self.table_nots_nolog, + dict((k, self.value) for k in orig_keys)) + + # Scenario: 2 + # Check that we see the inserted values as per the timestamp. for i, t in enumerate(orig_keys): + # Tables using the timestamps should see the values as per the + # given read_timestamp + self.check(self.session, 'read_timestamp=' + timestamp_str(t), + self.table_ts_log, dict((k, self.value) for k in orig_keys[:i+1])) self.check(self.session, 'read_timestamp=' + timestamp_str(t), - dict((k, self.value) for k in orig_keys[:i+1])) + self.table_ts_nolog, dict((k, self.value) for k in orig_keys[:i+1])) + # Tables not using the timestamps should see all the values. + self.check(self.session, 'read_timestamp=' + timestamp_str(t), + self.table_nots_log, dict((k, self.value) for k in orig_keys)) + self.check(self.session, 'read_timestamp=' + timestamp_str(t), + self.table_nots_nolog, dict((k, self.value) for k in orig_keys)) - # Bump the oldest timestamp, we're not going back... + # Bump the oldest_timestamp, we're not going back... self.assertEqual(self.conn.query_timestamp(), timestamp_ret_str(100)) self.oldts = timestamp_str(100) self.conn.set_timestamp('oldest_timestamp=' + self.oldts) self.conn.set_timestamp('stable_timestamp=' + self.oldts) - # print "Oldest " + self.oldts - # Update them and retry. + # Scenario: 3 + # Check that we see all the data values after moving the oldest_timestamp + # to the current timestamp + # All tables should see all the values. + self.check(self.session, 'read_timestamp=' + self.oldts, + self.table_ts_log, dict((k, self.value) for k in orig_keys)) + self.check(self.session, 'read_timestamp=' + self.oldts, + self.table_ts_nolog, dict((k, self.value) for k in orig_keys)) + self.check(self.session, 'read_timestamp=' + self.oldts, + self.table_nots_log, dict((k, self.value) for k in orig_keys)) + self.check(self.session, 'read_timestamp=' + self.oldts, + self.table_nots_nolog, dict((k, self.value) for k in orig_keys)) + + # Update the keys and checkpoint using the stable_timestamp. random.shuffle(keys) count = 0 for k in keys: @@ -203,66 +252,101 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): # tests the scenario for a bug we found where recovery replayed # the last record written into the log. # - # print "Key " + str(k) + " to value2" - c2[k] = self.value2 + cur_nots_log[k] = self.value2 + cur_nots_nolog[k] = self.value2 self.session.begin_transaction() - c[k] = self.value2 - c3[k] = self.value2 + cur_ts_log[k] = self.value2 + cur_ts_nolog[k] = self.value2 ts = timestamp_str(k + 100) self.session.commit_transaction('commit_timestamp=' + ts) - # print "Commit key " + str(k) + " ts " + ts count += 1 - # print "Updated " + str(count) + " keys to value2" + + # Scenario: 4 + # Check that we don't see the updated data of timestamp tables + # with the read_timestamp as oldest_timestamp + # Tables using the timestamps should see old values (i.e. value) only + self.check(self.session, 'read_timestamp=' + self.oldts, + self.table_ts_log, dict((k, self.value) for k in orig_keys)) + self.check(self.session, 'read_timestamp=' + self.oldts, + self.table_ts_nolog, dict((k, self.value) for k in orig_keys)) + # Tables not using the timestamps should see updated values (i.e. value2). + self.check(self.session, 'read_timestamp=' + self.oldts, + self.table_nots_log, dict((k, self.value2) for k in orig_keys)) + self.check(self.session, 'read_timestamp=' + self.oldts, + self.table_nots_nolog, dict((k, self.value2) for k in orig_keys)) + + # Scenario: 5 + # Check that we see the updated values as per the timestamp. + # Construct expected values. + expected_dict = dict((k, self.value) for k in orig_keys) + for i, t in enumerate(orig_keys): + # update expected value + expected_dict[i+1] = self.value2 + # Tables using the timestamps should see the updated values as per + # the given read_timestamp + self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.table_ts_log, expected_dict) + self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.table_ts_nolog, expected_dict) + # Tables not using the timestamps should see all the data values as + # updated values (i.e. value2). + self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.table_nots_log, dict((k, self.value2) for k in orig_keys)) + self.check(self.session, 'read_timestamp=' + timestamp_str(t + 100), + self.table_nots_nolog, dict((k, self.value2) for k in orig_keys)) # Take a checkpoint using the given configuration. Then verify # whether value2 appears in a copy of that data or not. - valcnt2 = valcnt3 = nkeys + valcnt_ts_log = valcnt_nots_log = valcnt_nots_nolog = nkeys if self.val == 'all': - valcnt = nkeys + # if use_timestamp is false, then all updates will be checkpointed. + valcnt_ts_nolog = nkeys else: - valcnt = 0 - self.ckpt_backup(self.value2, valcnt, valcnt2, valcnt3) - if self.ckptcfg != 'read_timestamp': - # Update the stable timestamp to the latest, but not the oldest - # timestamp and make sure we can see the data. Once the stable - # timestamp is moved we should see all keys with value2. - self.conn.set_timestamp('stable_timestamp=' + \ - timestamp_str(100+nkeys)) - self.ckpt_backup(self.value2, nkeys, nkeys, nkeys) + # checkpoint will happen with stable_timestamp=100, hence only + # table_ts_nolog will still have the old values (i.e. value) + self.ckpt_backup(self.value, 0, nkeys, 0, 0) + # table_ts_nolog will not have any new values (i.e. value2) + valcnt_ts_nolog = 0 + self.ckpt_backup(self.value2, valcnt_ts_log, valcnt_ts_nolog, + valcnt_nots_log, valcnt_nots_nolog) + + # Update the stable_timestamp to the latest, but not the + # oldest_timestamp and make sure we can see the data. Once the + # stable_timestamp is moved we should see all keys with value2. + self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100+nkeys)) + self.ckpt_backup(self.value2, nkeys, nkeys, nkeys, nkeys) # If we're not using the log we're done. if not self.using_log: return - # Update them and retry. This time take a backup and recover. + # Scenario: 7 + # Update the keys and log_flush with out checkpoint. random.shuffle(keys) count = 0 for k in keys: - # Make sure a timestamp cursor is the last one to update. This - # tests the scenario for a bug we found where recovery replayed - # the last record written into the log. + # Make sure a timestamp cursor is the last one to update. # - # print "Key " + str(k) + " to value3" - c2[k] = self.value3 + cur_nots_log[k] = self.value3 + cur_nots_nolog[k] = self.value3 self.session.begin_transaction() - c[k] = self.value3 - c3[k] = self.value3 + cur_ts_log[k] = self.value3 + cur_ts_nolog[k] = self.value3 ts = timestamp_str(k + 200) self.session.commit_transaction('commit_timestamp=' + ts) - # print "Commit key " + str(k) + " ts " + ts count += 1 - # print "Updated " + str(count) + " keys to value3" # Flush the log but don't checkpoint self.session.log_flush('sync=on') # Take a backup and then verify whether value3 appears in a copy # of that data or not. Both tables that are logged should see - # all the data regardless of timestamps. The table that is not + # all the data regardless of timestamps. Both tables that are not # logged should not see any of it. - valcnt = 0 - valcnt2 = valcnt3 = nkeys - self.backup_check(self.value3, valcnt, valcnt2, valcnt3) + valcnt_ts_nolog = valcnt_nots_nolog = 0 + valcnt_ts_log = valcnt_nots_log = nkeys + self.backup_check(self.value3, valcnt_ts_log, valcnt_ts_nolog, + valcnt_nots_log, valcnt_nots_nolog) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp04.py b/src/third_party/wiredtiger/test/suite/test_timestamp04.py index 3af0feed31b..aa5b4fd9716 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp04.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp04.py @@ -88,7 +88,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): self.skipTest('requires a timestamp build') # Configure small page sizes to ensure eviction comes through and we have a - # somewhat complex tree + # somewhat complex tree self.session.create(self.uri, 'key_format=i,value_format=i,memory_page_max=32k,leaf_page_max=8k,internal_page_max=8k' + self.extra_config) @@ -130,7 +130,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): self.check(self.session, 'read_timestamp=' + timestamp_str(2 * key_range), dict((k, 2) for k in keys[:])) - # Rollback a quater of the new commits + # Rollback a quarter of the new commits self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1 + key_range + key_range / 4)) self.conn.rollback_to_stable() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp07.py b/src/third_party/wiredtiger/test/suite/test_timestamp07.py index c1f70e0cb1a..eb3e2e1c3ec 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp07.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp07.py @@ -27,14 +27,13 @@ # OTHER DEALINGS IN THE SOFTWARE. # # test_timestamp07.py -# Timestamps: checkpoints and eviction +# Timestamps: checkpoints. # from helper import copy_wiredtiger_home import random from suite_subprocess import suite_subprocess import wiredtiger, wttest -from wiredtiger import stat from wtscenario import make_scenarios def timestamp_str(t): @@ -51,20 +50,18 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): ] conncfg = [ - ('nolog', dict(conn_config='create,cache_size=1M,statistics=(fast)', using_log=False)), - ('log', dict(conn_config='create,log=(enabled),cache_size=1M,statistics=(fast)', using_log=True)), + ('nolog', dict(conn_config='create,cache_size=1M', using_log=False)), + ('log', dict(conn_config='create,log=(enabled),cache_size=1M', using_log=True)), ] nkeys = [ - ('100keys', dict(nkeys=100,evicts=False)), - ('500keys', dict(nkeys=500,evicts=True)), -# ('1000keys', dict(nkeys=1000,evicts=True)), + ('100keys', dict(nkeys=100)), +# ('500keys', dict(nkeys=500)), +# ('1000keys', dict(nkeys=1000)), ] scenarios = make_scenarios(types, conncfg, nkeys) - modified_evicted = 0 - # Binary values. value = u'\u0001\u0002abcd\u0007\u0004' value2 = u'\u0001\u0002dcba\u0007\u0004' @@ -74,16 +71,9 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): # expected values. def check(self, session, txn_config, expected): if txn_config: - #print "Check: txn_config:" - #print txn_config session.begin_transaction(txn_config) c = session.open_cursor(self.uri + self.tablename, None) actual = dict((k, v) for k, v in c if v != 0) - self.maxDiff = None - #print "Expected:" - #print expected - #print "Actual:" - #print actual self.assertEqual(actual, expected) # Search for the expected items as well as iterating for k, v in expected.iteritems(): @@ -135,26 +125,6 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): self.assertEqual(count2, valcnt2) self.assertEqual(count3, valcnt3) - # Return whether or not eviction happened since the last call. - def check_eviction(self): - # Get a statistics cursor and look at the number of dirty pages - # evicted. Keep track of the last read value so we can determine - # if the value changed since the last call to this function. - stat_cursor = self.session.open_cursor('statistics:', None, None) - evict_dirty = stat_cursor[stat.conn.cache_eviction_dirty][2] - - # Return True if the new value is more, False otherwise. - #print "Old: " + str(self.modified_evicted) - # print "New: " + str(evict_dirty) - did_eviction = self.modified_evicted < evict_dirty - stat_cursor.close() - self.modified_evicted = evict_dirty - # print "Evict ret: " + str(ret) - - # XXX we can't guarantee that eviction will always happen, but make - # sure it doesn't happen if not expected. - self.assertTrue(not did_eviction or self.evicts) - # Check that a cursor sees the expected values after a checkpoint. def ckpt_backup(self, check_value, valcnt, valcnt2, valcnt3): @@ -196,7 +166,6 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): c3[k] = self.value self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) - self.check_eviction() # Now check that we see the expected state when reading at each # timestamp. for i, t in enumerate(orig_keys): @@ -228,8 +197,6 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): # print "Commit key " + str(k) + " ts " + ts count += 1 - self.check_eviction() - # print "Updated " + str(count) + " keys to value2" # Take a checkpoint using the given configuration. Then verify @@ -266,7 +233,6 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): # print "Commit key " + str(k) + " ts " + ts count += 1 - self.check_eviction() # print "Updated " + str(count) + " keys to value3" # Flush the log but don't checkpoint diff --git a/src/third_party/wiredtiger/test/suite/test_util02.py b/src/third_party/wiredtiger/test/suite/test_util02.py index 59c34e6ef0e..b9bc3bd26c1 100644 --- a/src/third_party/wiredtiger/test/suite/test_util02.py +++ b/src/third_party/wiredtiger/test/suite/test_util02.py @@ -178,7 +178,7 @@ class test_load_commandline(wttest.WiredTigerTestCase, suite_subprocess): else: self.check_empty_file(errfile) - # Empty arguments should suceed. + # Empty arguments should succeed. def test_load_commandline_1(self): self.load_commandline([], False) @@ -199,7 +199,7 @@ class test_load_commandline(wttest.WiredTigerTestCase, suite_subprocess): self.load_commandline([self.uri, "block_allocation=first"], False) self.load_commandline(["table:bar", "block_allocation=first"], True) - # You can specify multipleconfiguration arguments for the same object. + # You can specify multiple configuration arguments for the same object. def test_load_commandline_5(self): self.load_commandline([ self.uri, "block_allocation=first", diff --git a/src/third_party/wiredtiger/test/windows/windows_shim.h b/src/third_party/wiredtiger/test/windows/windows_shim.h index ad9de676deb..5a065ed8a79 100644 --- a/src/third_party/wiredtiger/test/windows/windows_shim.h +++ b/src/third_party/wiredtiger/test/windows/windows_shim.h @@ -33,11 +33,6 @@ #define R_OK 04 #define X_OK R_OK -/* MSVC Doesn't provide __func__, it has __FUNCTION__ */ -#ifdef _MSC_VER -#define __func__ __FUNCTION__ -#endif - /* snprintf does not exist on <= VS 2013 */ #if _MSC_VER < 1900 #define snprintf __wt_snprintf |