diff options
author | Luke Chen <luke.chen@mongodb.com> | 2019-08-01 10:34:36 +1000 |
---|---|---|
committer | Luke Chen <luke.chen@mongodb.com> | 2019-08-01 10:34:36 +1000 |
commit | 8a4b8b9fb7121417d33e8b1dd99f696c2f4fd765 (patch) | |
tree | c081094a03f66fa9d8fa29e31175e2e7f48c42f4 | |
parent | 943da6c9cbf1b55b2bc7248d20f483bd5ff66bdf (diff) | |
download | mongo-8a4b8b9fb7121417d33e8b1dd99f696c2f4fd765.tar.gz |
Import wiredtiger: c29f4c6030e37794b8c2c1195829ba2d14954677 from branch mongodb-4.4
ref: 1055c64267..c29f4c6030
for: 4.3.1
WT-4858 Add compatibility support for gcc-9 and clang-8
WT-4917 Fix test/format where truncate operations not correctly compared with previous updates
WT-4946 Tighten up failure checks for cursor join
WT-4957 Revert part of a change about when pages are queued for urgent eviction
WT-4963 Add debugging for rare test hang
WT-5036 Fix test/format where tracking buffer wrap causes repeatable operation failure
-rw-r--r-- | src/third_party/wiredtiger/build_posix/aclocal/strict.m4 | 4 | ||||
-rw-r--r-- | src/third_party/wiredtiger/examples/c/ex_async.c | 2 | ||||
-rw-r--r-- | src/third_party/wiredtiger/import.data | 2 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/async/async_api.c | 3 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/conn/conn_capacity.c | 3 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/cursor/cur_join.c | 24 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/include/btree.i | 40 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/include/gcc.h | 43 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/include/lint.h | 9 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/include/msvc.h | 5 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/include/mutex.i | 6 | ||||
-rw-r--r-- | src/third_party/wiredtiger/test/format/format.h | 5 | ||||
-rw-r--r-- | src/third_party/wiredtiger/test/format/ops.c | 21 | ||||
-rw-r--r-- | src/third_party/wiredtiger/test/format/snap.c | 148 | ||||
-rw-r--r-- | src/third_party/wiredtiger/test/suite/test_compat02.py | 8 |
15 files changed, 161 insertions, 162 deletions
diff --git a/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 b/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 index 3600a39fe43..b912335fd16 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 @@ -134,6 +134,10 @@ AC_DEFUN([AM_CLANG_WARNINGS], [ w="$w -Wno-unused-command-line-argument";; esac + # We occasionally use an extra semicolon to indicate an empty loop or + # conditional body. + w="$w -Wno-extra-semi-stmt" + # Ignore unrecognized options. w="$w -Wno-unknown-warning-option" diff --git a/src/third_party/wiredtiger/examples/c/ex_async.c b/src/third_party/wiredtiger/examples/c/ex_async.c index e9ffad4807c..85f783092fa 100644 --- a/src/third_party/wiredtiger/examples/c/ex_async.c +++ b/src/third_party/wiredtiger/examples/c/ex_async.c @@ -37,7 +37,7 @@ static const char *home; #elif defined(_WIN32) #define ATOMIC_ADD(v, val) (_InterlockedExchangeAdd(&(v), val) + val) #else -#define ATOMIC_ADD(v, val) __sync_add_and_fetch(&(v), val) +#define ATOMIC_ADD(v, val) __atomic_add_fetch(&(v), val, __ATOMIC_SEQ_CST) #endif static int global_error = 0; diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 49e3237bd7b..882ae91c078 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "1055c64267ae9ee9a1ef11a9826dd409259b13d2", + "commit": "c29f4c6030e37794b8c2c1195829ba2d14954677", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-4.4" diff --git a/src/third_party/wiredtiger/src/async/async_api.c b/src/third_party/wiredtiger/src/async/async_api.c index 9f9aa979139..0ef85b8cd28 100644 --- a/src/third_party/wiredtiger/src/async/async_api.c +++ b/src/third_party/wiredtiger/src/async/async_api.c @@ -160,8 +160,7 @@ retry: WT_RET(__async_get_format(conn, uri, config, op)); op->unique_id = __wt_atomic_add64(&async->op_id, 1); op->optype = WT_AOP_NONE; - (void)__wt_atomic_store32( - &async->ops_index, (i + 1) % conn->async_size); + async->ops_index = (i + 1) % conn->async_size; *opp = op; return (0); } diff --git a/src/third_party/wiredtiger/src/conn/conn_capacity.c b/src/third_party/wiredtiger/src/conn/conn_capacity.c index a75bdd259c4..38052a8e412 100644 --- a/src/third_party/wiredtiger/src/conn/conn_capacity.c +++ b/src/third_party/wiredtiger/src/conn/conn_capacity.c @@ -270,8 +270,7 @@ __capacity_reserve(uint64_t *reservation, uint64_t bytes, uint64_t capacity, * If the reservation clock is out of date, bring it * to within a second of a current time. */ - (void)__wt_atomic_store64(reservation, - (now_ns - WT_BILLION) + res_len); + *reservation = (now_ns - WT_BILLION) + res_len; } else res_value = now_ns; diff --git a/src/third_party/wiredtiger/src/cursor/cur_join.c b/src/third_party/wiredtiger/src/cursor/cur_join.c index 07bfe02a142..12be6929022 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_join.c +++ b/src/third_party/wiredtiger/src/cursor/cur_join.c @@ -508,10 +508,8 @@ __curjoin_entry_in_range(WT_SESSION_IMPL *session, WT_CURSOR_JOIN_ENTRY *entry, if (!passed) { if (iter != NULL && (iter->is_equal || - F_ISSET(end, WT_CURJOIN_END_LT))) { - WT_RET(__curjoin_iter_bump(iter)); + F_ISSET(end, WT_CURJOIN_END_LT))) return (WT_NOTFOUND); - } if (!disjunction) return (WT_NOTFOUND); iter = NULL; @@ -606,6 +604,9 @@ __curjoin_entry_member(WT_SESSION_IMPL *session, WT_CURSOR_JOIN_ENTRY *entry, WT_ITEM v; bool bloom_found; + /* We cannot have a bloom filter on a join entry with subordinates. */ + WT_ASSERT(session, entry->bloom == NULL || entry->subjoin == NULL); + if (entry->subjoin == NULL && iter != NULL && (iter->end_pos + iter->end_skip >= entry->ends_next || (iter->end_skip > 0 && @@ -633,16 +634,19 @@ __curjoin_entry_member(WT_SESSION_IMPL *session, WT_CURSOR_JOIN_ENTRY *entry, bloom_found = true; } if (entry->subjoin != NULL) { + /* + * If we have a subordinate join, the membership + * check is delegated to it. + */ WT_ASSERT(session, iter == NULL || entry->subjoin == iter->child->cjoin); - ret = __curjoin_entries_in_range(session, entry->subjoin, - key, iter == NULL ? NULL : iter->child); + WT_ERR(__curjoin_entries_in_range(session, entry->subjoin, + key, iter == NULL ? NULL : iter->child)); if (iter != NULL && - WT_CURJOIN_ITER_CONSUMED(iter->child)) { - WT_ERR(__curjoin_iter_bump(iter)); - ret = WT_NOTFOUND; - } - return (ret); + WT_CURJOIN_ITER_CONSUMED(iter->child)) + return (WT_NOTFOUND); + /* There's nothing more to do for this node. */ + return (0); } if (entry->index != NULL) { /* diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index 5cf67d8600f..d0679a9fb38 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -1508,30 +1508,30 @@ __wt_page_release(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) * memory_page_max setting, when we see many deleted items, and when we * are attempting to scan without trashing the cache. * - * Fast checks if flag indicates no evict, session can't perform slow - * operation, eviction is disabled for this handle, operation or - * tree, then perform a general check if eviction will be possible. + * Checkpoint should not queue pages for urgent eviction if they require + * dirty eviction: there is a special exemption that allows checkpoint + * to evict dirty pages in a tree that is being checkpointed, and no + * other thread can help with that. Checkpoints don't rely on this code + * for dirty eviction: that is handled explicitly in __wt_sync_file. * - * Checkpoint should not queue pages for urgent eviction if it cannot - * evict them immediately: there is a special exemption that allows - * checkpoint to evict dirty pages in a tree that is being - * checkpointed, and no other thread can help with that. + * If the operation has disabled eviction or splitting, or the session + * is preventing from reconciling, then just queue the page for urgent + * eviction. Otherwise, attempt to release and evict it. */ page = ref->page; - if (!LF_ISSET(WT_READ_NO_EVICT) && - __wt_session_can_wait(session) && - WT_READGEN_EVICT_SOON(page->read_gen) && + if (WT_READGEN_EVICT_SOON(page->read_gen) && btree->evict_disabled == 0 && - __wt_page_can_evict(session, ref, &inmem_split)) { - if (!__wt_page_evict_clean(page) && - (LF_ISSET(WT_READ_NO_SPLIT) || (!inmem_split && - F_ISSET(session, WT_SESSION_NO_RECONCILE)))) { - if (!WT_SESSION_BTREE_SYNC(session)) - WT_IGNORE_RET_BOOL( - __wt_page_evict_urgent(session, ref)); - } else { - WT_RET_BUSY_OK(__wt_page_release_evict(session, ref, - flags)); + __wt_page_can_evict(session, ref, &inmem_split) && + (!WT_SESSION_IS_CHECKPOINT(session) || + __wt_page_evict_clean(page))) { + if (LF_ISSET(WT_READ_NO_EVICT) || + (inmem_split ? LF_ISSET(WT_READ_NO_SPLIT) : + F_ISSET(session, WT_SESSION_NO_RECONCILE))) + WT_IGNORE_RET_BOOL( + __wt_page_evict_urgent(session, ref)); + else { + WT_RET_BUSY_OK( + __wt_page_release_evict(session, ref, flags)); return (0); } } diff --git a/src/third_party/wiredtiger/src/include/gcc.h b/src/third_party/wiredtiger/src/include/gcc.h index a9d271ed0bd..7ee64cb663f 100644 --- a/src/third_party/wiredtiger/src/include/gcc.h +++ b/src/third_party/wiredtiger/src/include/gcc.h @@ -89,38 +89,40 @@ * swap) operations. */ -#ifdef __clang__ /* - * We avoid __sync_bool_compare_and_swap with due to problems with optimization - * with some versions of clang. See http://llvm.org/bugs/show_bug.cgi?id=21499 - * for details. + * We've hit optimization bugs with Clang 3.5 in the past when using the atomic + * builtins. See http://llvm.org/bugs/show_bug.cgi?id=21499 for details. */ -#define WT_ATOMIC_CAS(ptr, old, new) \ - (__sync_val_compare_and_swap(ptr, old, new) == (old)) -#else -#define WT_ATOMIC_CAS(ptr, old, new) \ - __sync_bool_compare_and_swap(ptr, old, new) +#if defined(__clang__) && \ + defined(__clang_major__) && defined(__clang_minor__) && \ + (((__clang_major__ == 3) && (__clang_minor__ <= 5)) || \ + (__clang_major__ < 3)) +#error "Clang versions 3.5 and earlier are unsupported by WiredTiger" #endif + +#define WT_ATOMIC_CAS(ptr, oldp, new) \ + __atomic_compare_exchange_n( \ + ptr, oldp, new, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) #define WT_ATOMIC_CAS_FUNC(name, vp_arg, old_arg, new_arg) \ static inline bool \ __wt_atomic_cas##name(vp_arg, old_arg, new_arg) \ { \ - return (WT_ATOMIC_CAS(vp, old, new)); \ + return (WT_ATOMIC_CAS(vp, &old, new)); \ } WT_ATOMIC_CAS_FUNC(8, uint8_t *vp, uint8_t old, uint8_t new) WT_ATOMIC_CAS_FUNC(16, uint16_t *vp, uint16_t old, uint16_t new) WT_ATOMIC_CAS_FUNC(32, uint32_t *vp, uint32_t old, uint32_t new) WT_ATOMIC_CAS_FUNC(v32, \ - volatile uint32_t *vp, volatile uint32_t old, volatile uint32_t new) + volatile uint32_t *vp, uint32_t old, volatile uint32_t new) WT_ATOMIC_CAS_FUNC(i32, int32_t *vp, int32_t old, int32_t new) WT_ATOMIC_CAS_FUNC(iv32, \ - volatile int32_t *vp, volatile int32_t old, volatile int32_t new) + volatile int32_t *vp, int32_t old, volatile int32_t new) WT_ATOMIC_CAS_FUNC(64, uint64_t *vp, uint64_t old, uint64_t new) WT_ATOMIC_CAS_FUNC(v64, \ - volatile uint64_t *vp, volatile uint64_t old, volatile uint64_t new) + volatile uint64_t *vp, uint64_t old, volatile uint64_t new) WT_ATOMIC_CAS_FUNC(i64, int64_t *vp, int64_t old, int64_t new) WT_ATOMIC_CAS_FUNC(iv64, \ - volatile int64_t *vp, volatile int64_t old, volatile int64_t new) + volatile int64_t *vp, int64_t old, volatile int64_t new) WT_ATOMIC_CAS_FUNC(size, size_t *vp, size_t old, size_t new) /* @@ -130,29 +132,24 @@ WT_ATOMIC_CAS_FUNC(size, size_t *vp, size_t old, size_t new) static inline bool __wt_atomic_cas_ptr(void *vp, void *old, void *new) { - return (WT_ATOMIC_CAS((void **)vp, old, new)); + return (WT_ATOMIC_CAS((void **)vp, &old, new)); } #define WT_ATOMIC_FUNC(name, ret, vp_arg, v_arg) \ static inline ret \ __wt_atomic_add##name(vp_arg, v_arg) \ { \ - return (__sync_add_and_fetch(vp, v)); \ + return (__atomic_add_fetch(vp, v, __ATOMIC_SEQ_CST)); \ } \ static inline ret \ __wt_atomic_fetch_add##name(vp_arg, v_arg) \ { \ - return (__sync_fetch_and_add(vp, v)); \ -} \ -static inline ret \ -__wt_atomic_store##name(vp_arg, v_arg) \ -{ \ - return (__sync_lock_test_and_set(vp, v)); \ + return (__atomic_fetch_add(vp, v, __ATOMIC_SEQ_CST)); \ } \ static inline ret \ __wt_atomic_sub##name(vp_arg, v_arg) \ { \ - return (__sync_sub_and_fetch(vp, v)); \ + return (__atomic_sub_fetch(vp, v, __ATOMIC_SEQ_CST)); \ } WT_ATOMIC_FUNC(8, uint8_t, uint8_t *vp, uint8_t v) WT_ATOMIC_FUNC(16, uint16_t, uint16_t *vp, uint16_t v) diff --git a/src/third_party/wiredtiger/src/include/lint.h b/src/third_party/wiredtiger/src/include/lint.h index 903b0238b37..5d7cee531c2 100644 --- a/src/third_party/wiredtiger/src/include/lint.h +++ b/src/third_party/wiredtiger/src/include/lint.h @@ -35,15 +35,6 @@ __wt_atomic_fetch_add##name(type *vp, type v) \ return (orig); \ } \ static inline ret \ -__wt_atomic_store##name(type *vp, type v) \ -{ \ - type orig; \ - \ - orig = *vp; \ - *vp = v; \ - return (orig); \ -} \ -static inline ret \ __wt_atomic_sub##name(type *vp, type v) \ { \ *vp -= v; \ diff --git a/src/third_party/wiredtiger/src/include/msvc.h b/src/third_party/wiredtiger/src/include/msvc.h index 1586dae22b8..f4d8dc942f6 100644 --- a/src/third_party/wiredtiger/src/include/msvc.h +++ b/src/third_party/wiredtiger/src/include/msvc.h @@ -45,11 +45,6 @@ __wt_atomic_fetch_add##name(type *vp, type v) \ return (_InterlockedExchangeAdd ## s((t *)(vp), (t)(v))); \ } \ static inline ret \ -__wt_atomic_store##name(type *vp, type v) \ -{ \ - return (_InterlockedExchange ## s((t *)(vp), (t)(v))); \ -} \ -static inline ret \ __wt_atomic_sub##name(type *vp, type v) \ { \ return (_InterlockedExchangeAdd ## s((t *)(vp), - (t)v) - (v)); \ diff --git a/src/third_party/wiredtiger/src/include/mutex.i b/src/third_party/wiredtiger/src/include/mutex.i index 15e7218dd28..660ee22ed96 100644 --- a/src/third_party/wiredtiger/src/include/mutex.i +++ b/src/third_party/wiredtiger/src/include/mutex.i @@ -68,7 +68,7 @@ __wt_spin_trylock(WT_SESSION_IMPL *session, WT_SPINLOCK *t) { WT_UNUSED(session); - return (__sync_lock_test_and_set(&t->lock, 1) == 0 ? 0 : EBUSY); + return (__atomic_test_and_set(&t->lock, __ATOMIC_ACQUIRE) ? 0 : EBUSY); } /* @@ -82,7 +82,7 @@ __wt_spin_lock(WT_SESSION_IMPL *session, WT_SPINLOCK *t) WT_UNUSED(session); - while (__sync_lock_test_and_set(&t->lock, 1)) { + while (__atomic_test_and_set(&t->lock, __ATOMIC_ACQUIRE)) { for (i = 0; t->lock && i < WT_SPIN_COUNT; i++) WT_PAUSE(); if (t->lock) @@ -99,7 +99,7 @@ __wt_spin_unlock(WT_SESSION_IMPL *session, WT_SPINLOCK *t) { WT_UNUSED(session); - __sync_lock_release(&t->lock); + __atomic_clear(&t->lock, __ATOMIC_RELEASE); } #elif SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX || \ diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 7cba583b2b4..030b8fcb563 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -266,6 +266,8 @@ typedef enum { NEXT, PREV, SEARCH, SEARCH_NEAR } read_operation; typedef struct { thread_op op; /* Operation */ + uint64_t opid; /* Operation ID */ + uint64_t keyno; /* Row number */ uint64_t ts; /* Read/commit timestamp */ @@ -311,6 +313,8 @@ typedef struct { WT_ITEM *lastkey, _lastkey; bool repeatable_reads; /* if read ops repeatable */ + bool repeatable_wrap; /* if circular buffer wrapped */ + uint64_t opid; /* Operation ID */ uint64_t read_ts; /* read timestamp */ uint64_t commit_ts; /* commit timestamp */ SNAP_OPS *snap, *snap_first, snap_list[512]; @@ -352,6 +356,7 @@ void print_item(const char *, WT_ITEM *); void print_item_data(const char *, const uint8_t *, size_t); int read_row_worker(WT_CURSOR *, uint64_t, WT_ITEM *, WT_ITEM *, bool); uint32_t rng(WT_RAND_STATE *); +void snap_init(TINFO *, uint64_t, bool); void snap_repeat_single(WT_CURSOR *, TINFO *); int snap_repeat_txn(WT_CURSOR *, TINFO *); void snap_repeat_update(TINFO *, bool); diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index a27dec3dd0c..7adfb795694 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -338,11 +338,10 @@ begin_transaction_ts(TINFO *tinfo, u_int *iso_configp) buf, sizeof(buf), "read_timestamp=%" PRIx64, ts)); ret = session->timestamp_transaction(session, buf); if (ret == 0) { - tinfo->read_ts = ts; - tinfo->repeatable_reads = true; + snap_init(tinfo, ts, true); logop(session, "begin snapshot read-ts=%" PRIu64 " (repeatable)", - tinfo->read_ts); + ts); return; } if (ret != EINVAL) @@ -371,11 +370,9 @@ begin_transaction_ts(TINFO *tinfo, u_int *iso_configp) testutil_check(pthread_rwlock_unlock(&g.ts_lock)); - tinfo->read_ts = ts; - tinfo->repeatable_reads = false; - - logop(session, "begin snapshot read-ts=%" PRIu64 " (not repeatable)", - tinfo->read_ts); + snap_init(tinfo, ts, false); + logop(session, + "begin snapshot read-ts=%" PRIu64 " (not repeatable)", ts); } /* @@ -415,9 +412,7 @@ begin_transaction(TINFO *tinfo, u_int *iso_configp) wiredtiger_begin_transaction(session, config); - tinfo->read_ts = WT_TS_NONE; - tinfo->repeatable_reads = false; - + snap_init(tinfo, WT_TS_NONE, false); logop(session, "begin %s", log); } @@ -719,8 +714,6 @@ ops(void *arg) begin_transaction_ts(tinfo, &iso_config); else begin_transaction(tinfo, &iso_config); - - tinfo->snap_first = tinfo->snap; intxn = true; } @@ -899,7 +892,7 @@ remove_instead_of_truncate: */ greater_than = mmrand(&tinfo->rnd, 0, 1) == 1; range = g.rows < 20 ? - 1 : mmrand(&tinfo->rnd, 1, (u_int)g.rows / 20); + 0 : mmrand(&tinfo->rnd, 0, (u_int)g.rows / 20); tinfo->last = tinfo->keyno; if (greater_than) { if (g.c_reverse) { diff --git a/src/third_party/wiredtiger/test/format/snap.c b/src/third_party/wiredtiger/test/format/snap.c index a1853c56db9..9442f1fdeb3 100644 --- a/src/third_party/wiredtiger/test/format/snap.c +++ b/src/third_party/wiredtiger/test/format/snap.c @@ -29,6 +29,22 @@ #include "format.h" /* + * snap_init -- + * Initialize the repeatable operation tracking. + */ +void +snap_init(TINFO *tinfo, uint64_t read_ts, bool repeatable_reads) +{ + ++tinfo->opid; + + tinfo->snap_first = tinfo->snap; + + tinfo->read_ts = read_ts; + tinfo->repeatable_reads = repeatable_reads; + tinfo->repeatable_wrap = false; +} + +/* * snap_track -- * Add a single snapshot isolation returned value to the list. */ @@ -40,6 +56,7 @@ snap_track(TINFO *tinfo, thread_op op) snap = tinfo->snap; snap->op = op; + snap->opid = tinfo->opid; snap->keyno = tinfo->keyno; snap->ts = WT_TS_NONE; snap->repeatable = false; @@ -63,15 +80,17 @@ snap_track(TINFO *tinfo, thread_op op) memcpy(snap->vdata, ip->data, snap->vsize = ip->size); } + /* Move to the next slot, wrap at the end of the circular buffer. */ + if (++tinfo->snap >= &tinfo->snap_list[WT_ELEMENTS(tinfo->snap_list)]) + tinfo->snap = tinfo->snap_list; + /* - * Move to the next slot, wrap at the end of the circular buffer. - * * It's possible to pass this transaction's buffer starting point and - * start replacing our own entries. That's OK, we just skip earlier - * operations when we check. + * start replacing our own entries. If that happens, we can't repeat + * operations because we don't know which ones were previously modified. */ - if (++tinfo->snap >= &tinfo->snap_list[WT_ELEMENTS(tinfo->snap_list)]) - tinfo->snap = tinfo->snap_list; + if (tinfo->snap->opid == tinfo->opid) + tinfo->repeatable_wrap = true; } /* @@ -83,12 +102,22 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) { WT_DECL_RET; WT_ITEM *key, *value; + uint64_t keyno; uint8_t bitfield; key = tinfo->key; value = tinfo->value; /* + * Test just the first or last records in the truncate range; a key set + * to 0 flags a truncate from/to table beginning/end. + */ + if ((keyno = snap->keyno) == 0) { + keyno = snap->last; + testutil_assert(keyno != 0 && snap->op == TRUNCATE); + } + + /* * Retrieve the key/value pair by key. Row-store inserts have a unique * generated key we saved, else generate the key from the key number. */ @@ -100,10 +129,10 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) switch (g.type) { case FIX: case VAR: - cursor->set_key(cursor, snap->keyno); + cursor->set_key(cursor, keyno); break; case ROW: - key_gen(key, snap->keyno); + key_gen(key, keyno); cursor->set_key(cursor, key); break; } @@ -153,7 +182,7 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) testutil_die(ret, "snapshot-isolation: %" PRIu64 " search: " "expected {0x%02x}, found {0x%02x}", - snap->keyno, + keyno, snap->op == REMOVE ? 0 : *(uint8_t *)snap->vdata, ret == WT_NOTFOUND ? 0 : *(uint8_t *)value->data); /* NOTREACHED */ @@ -177,8 +206,7 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) /* NOTREACHED */ case VAR: fprintf(stderr, - "snapshot-isolation %" PRIu64 " search mismatch\n", - snap->keyno); + "snapshot-isolation %" PRIu64 " search mismatch\n", keyno); if (snap->op == REMOVE) fprintf(stderr, "expected {deleted}\n"); @@ -190,8 +218,7 @@ snap_verify(WT_CURSOR *cursor, TINFO *tinfo, SNAP_OPS *snap) print_item_data(" found", value->data, value->size); testutil_die(ret, - "snapshot-isolation: %" PRIu64 " search mismatch", - snap->keyno); + "snapshot-isolation: %" PRIu64 " search mismatch", keyno); /* NOTREACHED */ } @@ -209,7 +236,7 @@ snap_ts_clear(TINFO *tinfo, uint64_t ts) SNAP_OPS *snap; int count; - /* Check from the first operation to the last. */ + /* Check from the first slot to the last. */ for (snap = tinfo->snap_list, count = WT_ELEMENTS(tinfo->snap_list); count > 0; --count, ++snap) if (snap->repeatable && snap->ts <= ts) @@ -253,8 +280,7 @@ snap_repeat_ok_match(SNAP_OPS *current, SNAP_OPS *a) * committed successfully. */ static bool -snap_repeat_ok_commit( - TINFO *tinfo, SNAP_OPS *current, SNAP_OPS *first, SNAP_OPS *last) +snap_repeat_ok_commit(TINFO *tinfo, SNAP_OPS *current) { SNAP_OPS *p; @@ -266,13 +292,10 @@ snap_repeat_ok_commit( * do the repeatable read in that case.) */ for (p = current;;) { - /* - * Wrap at the end of the circular buffer; "last" is the element - * after the last element we want to test. - */ + /* Wrap at the end of the circular buffer. */ if (++p >= &tinfo->snap_list[WT_ELEMENTS(tinfo->snap_list)]) p = tinfo->snap_list; - if (p == last) + if (p->opid != tinfo->opid) break; if (!snap_repeat_ok_match(current, p)) @@ -282,21 +305,18 @@ snap_repeat_ok_commit( if (current->op != READ) return (true); for (p = current;;) { - /* - * Wrap at the beginning of the circular buffer; "first" is the - * last element we want to test. - */ - if (p == first) - return (true); + /* Wrap at the beginning of the circular buffer. */ if (--p < tinfo->snap_list) p = &tinfo->snap_list[ WT_ELEMENTS(tinfo->snap_list) - 1]; + if (p->opid != tinfo->opid) + break; if (!snap_repeat_ok_match(current, p)) return (false); } - /* NOTREACHED */ + return (true); } /* @@ -305,7 +325,7 @@ snap_repeat_ok_commit( * transaction has rolled back. */ static bool -snap_repeat_ok_rollback(TINFO *tinfo, SNAP_OPS *current, SNAP_OPS *first) +snap_repeat_ok_rollback(TINFO *tinfo, SNAP_OPS *current) { SNAP_OPS *p; @@ -318,21 +338,18 @@ snap_repeat_ok_rollback(TINFO *tinfo, SNAP_OPS *current, SNAP_OPS *first) * the read in that case. */ for (p = current;;) { - /* - * Wrap at the beginning of the circular buffer; "first" is the - * last element we want to test. - */ - if (p == first) - return (true); + /* Wrap at the beginning of the circular buffer. */ if (--p < tinfo->snap_list) p = &tinfo->snap_list[ WT_ELEMENTS(tinfo->snap_list) - 1]; + if (p->opid != tinfo->opid) + break; if (!snap_repeat_ok_match(current, p)) return (false); } - /* NOTREACHED */ + return (true); } /* @@ -342,31 +359,21 @@ snap_repeat_ok_rollback(TINFO *tinfo, SNAP_OPS *current, SNAP_OPS *first) int snap_repeat_txn(WT_CURSOR *cursor, TINFO *tinfo) { - SNAP_OPS *current, *stop; + SNAP_OPS *current; + + /* If we wrapped the buffer, we can't repeat operations. */ + if (tinfo->repeatable_wrap) + return (0); /* Check from the first operation we saved to the last. */ - for (current = tinfo->snap_first, stop = tinfo->snap;; ++current) { + for (current = tinfo->snap_first;; ++current) { /* Wrap at the end of the circular buffer. */ if (current >= &tinfo->snap_list[WT_ELEMENTS(tinfo->snap_list)]) current = tinfo->snap_list; - if (current == stop) + if (current->opid != tinfo->opid) break; - /* - * We don't test all of the records in a truncate range, only - * the first because that matches the rest of the isolation - * checks. If a truncate range was from the start of the table, - * switch to the record at the end. This is done in the first - * routine that considers if operations are repeatable, and the - * rest of those functions depend on it already being done. - */ - if (current->op == TRUNCATE && current->keyno == 0) { - current->keyno = current->last; - testutil_assert(current->keyno != 0); - } - - if (snap_repeat_ok_commit( - tinfo, current, tinfo->snap_first, stop)) + if (snap_repeat_ok_commit(tinfo, current)) WT_RET(snap_verify(cursor, tinfo, current)); } @@ -381,19 +388,18 @@ snap_repeat_txn(WT_CURSOR *cursor, TINFO *tinfo) void snap_repeat_update(TINFO *tinfo, bool committed) { - SNAP_OPS *start, *stop; + SNAP_OPS *current; - /* - * Check from the first operation we saved to the last. It's possible - * to update none at all if we did exactly the number of operations - * in the circular buffer, it will look like we didn't do any. That's - * OK, it's a big enough buffer that it's not going to matter. - */ - for (start = tinfo->snap_first, stop = tinfo->snap;; ++start) { + /* If we wrapped the buffer, we can't repeat operations. */ + if (tinfo->repeatable_wrap) + return; + + /* Check from the first operation we saved to the last. */ + for (current = tinfo->snap_first;; ++current) { /* Wrap at the end of the circular buffer. */ - if (start >= &tinfo->snap_list[WT_ELEMENTS(tinfo->snap_list)]) - start = tinfo->snap_list; - if (start == stop) + if (current >= &tinfo->snap_list[WT_ELEMENTS(tinfo->snap_list)]) + current = tinfo->snap_list; + if (current->opid != tinfo->opid) break; /* @@ -401,23 +407,23 @@ snap_repeat_update(TINFO *tinfo, bool committed) * timestamp chosen wasn't older than all concurrently running * uncommitted updates. */ - if (!tinfo->repeatable_reads && start->op == READ) + if (!tinfo->repeatable_reads && current->op == READ) continue; /* * Second, check based on the transaction resolution (the rules * are different if the transaction committed or rolled back). */ - start->repeatable = committed ? snap_repeat_ok_commit( - tinfo, start, tinfo->snap_first, stop) : - snap_repeat_ok_rollback(tinfo, start, tinfo->snap_first); + current->repeatable = committed ? + snap_repeat_ok_commit(tinfo, current) : + snap_repeat_ok_rollback(tinfo, current); /* * Repeat reads at the transaction's read timestamp and updates * at the commit timestamp. */ - if (start->repeatable) - start->ts = start->op == READ ? + if (current->repeatable) + current->ts = current->op == READ ? tinfo->read_ts : tinfo->commit_ts; } } diff --git a/src/third_party/wiredtiger/test/suite/test_compat02.py b/src/third_party/wiredtiger/test/suite/test_compat02.py index de5862513a8..a92c3f54300 100644 --- a/src/third_party/wiredtiger/test/suite/test_compat02.py +++ b/src/third_party/wiredtiger/test/suite/test_compat02.py @@ -133,6 +133,13 @@ class test_compat02(wttest.WiredTigerTestCase, suite_subprocess): # version. That configuration needs an existing database for it to be # useful. Test for success or failure based on the relative versions # configured. + + # Turn on checkpoint verbose to debug a rare occurence of a test + # hanging, most likely during the checkpoint of conn.close. + self.pr("Closing connection") + self.conn.reconfigure('verbose=(checkpoint)') + with self.expectedStdoutPattern('.'): + self.conn.close() compat_str = '' if (self.max_req != 'none'): compat_str += 'compatibility=(require_max="%s"),' % self.max_req @@ -140,7 +147,6 @@ class test_compat02(wttest.WiredTigerTestCase, suite_subprocess): compat_str += 'compatibility=(require_min="%s"),' % self.min_req if (self.rel != 'none'): compat_str += 'compatibility=(release="%s"),' % self.rel - self.conn.close() log_str = 'log=(enabled,file_max=%s,archive=false),' % self.logmax restart_config = log_str + compat_str self.pr("Restart conn " + restart_config) |