From c87e1c23421bf79614baf500fda6622bd90f674e Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Wed, 20 Apr 2022 11:39:06 +1000 Subject: Import wiredtiger: 59935f25151762defb0f0b645ce69d7e16ef1b65 from branch mongodb-5.0 ref: 3b6a65170a..59935f2515 for: 5.0.8 WT-7662 Format timed out with prepare-conflict WT-8260 Create a Python suite test to validate new EVENT_HANDLER JSON format WT-8708 Fix timestamp usage error in test/checkpoint WT-8924 Don't check against on disk time window if there is an insert list when checking for conflicts in row-store --- src/third_party/wiredtiger/import.data | 2 +- src/third_party/wiredtiger/src/include/txn_inline.h | 7 ++++++- src/third_party/wiredtiger/test/checkpoint/checkpointer.c | 12 +++++++----- .../wiredtiger/test/checkpoint/test_checkpoint.h | 4 ++-- src/third_party/wiredtiger/test/checkpoint/workers.c | 8 ++++---- src/third_party/wiredtiger/test/format/format.i | 10 ++++++---- src/third_party/wiredtiger/test/format/ops.c | 14 -------------- src/third_party/wiredtiger/test/suite/test_verbose01.py | 5 +++++ 8 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 94395d21a68..1cfb881916e 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.0", - "commit": "3b6a65170adf05883a55b5d0175547ab6e3c6912" + "commit": "59935f25151762defb0f0b645ce69d7e16ef1b65" } diff --git a/src/third_party/wiredtiger/src/include/txn_inline.h b/src/third_party/wiredtiger/src/include/txn_inline.h index 1201ff5326d..1a86a5cf808 100644 --- a/src/third_party/wiredtiger/src/include/txn_inline.h +++ b/src/third_party/wiredtiger/src/include/txn_inline.h @@ -1354,8 +1354,13 @@ __wt_txn_modify_check( * Check conflict against any on-page value if there is no update on the update chain except * aborted updates. Otherwise, we would have either already detected a conflict if we saw an * uncommitted update or determined that it would be safe to write if we saw a committed update. + * + * In the case of row-store we also need to check that the insert list is empty as the existence + * of it implies there is no on disk value for the given key. However we can still get a + * time-window from an unrelated on-disk value if we are not careful as the slot can still be + * set on the cursor b-tree. */ - if (!rollback && upd == NULL) { + if (!rollback && upd == NULL && (CUR2BT(cbt)->type != BTREE_ROW || cbt->ins == NULL)) { __wt_read_cell_time_window(cbt, &tw, &tw_found); if (tw_found) { if (WT_TIME_WINDOW_HAS_STOP(&tw)) { diff --git a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c index 1c8b5135d49..a07604a6520 100644 --- a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c +++ b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c @@ -44,10 +44,10 @@ set_stable(void) char buf[128]; if (g.race_timetamps) - testutil_check(__wt_snprintf( - buf, sizeof(buf), "stable_timestamp=%x,oldest_timestamp=%x", g.ts_stable, g.ts_stable)); + testutil_check(__wt_snprintf(buf, sizeof(buf), + "stable_timestamp=%" PRIx64 ",oldest_timestamp=%" PRIx64, g.ts_stable, g.ts_stable)); else - testutil_check(__wt_snprintf(buf, sizeof(buf), "stable_timestamp=%x", g.ts_stable)); + testutil_check(__wt_snprintf(buf, sizeof(buf), "stable_timestamp=%" PRIx64, g.ts_stable)); testutil_check(g.conn->set_timestamp(g.conn, buf)); } @@ -202,7 +202,9 @@ real_checkpointer(void) verify_ts = stable_ts; else verify_ts = __wt_random(&rnd) % (stable_ts - oldest_ts + 1) + oldest_ts; - WT_ORDERED_READ(g.ts_oldest, g.ts_stable); + __wt_writelock((WT_SESSION_IMPL *)session, &g.clock_lock); + g.ts_oldest = g.ts_stable; + __wt_writeunlock((WT_SESSION_IMPL *)session, &g.clock_lock); } /* Execute a checkpoint */ @@ -225,7 +227,7 @@ real_checkpointer(void) /* Advance the oldest timestamp to the most recently set stable timestamp. */ if (g.use_timestamps && g.ts_oldest != 0) { testutil_check(__wt_snprintf( - timestamp_buf, sizeof(timestamp_buf), "oldest_timestamp=%x", g.ts_oldest)); + timestamp_buf, sizeof(timestamp_buf), "oldest_timestamp=%" PRIx64, g.ts_oldest)); testutil_check(g.conn->set_timestamp(g.conn, timestamp_buf)); } /* Random value between 4 and 8 seconds. */ diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h index b3b65c5d828..93f76ec5e1c 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h @@ -71,8 +71,8 @@ typedef struct { bool hs_checkpoint_timing_stress; /* History store checkpoint timing stress */ bool reserved_txnid_timing_stress; /* Reserved transaction id timing stress */ bool checkpoint_slow_timing_stress; /* Checkpoint slow timing stress */ - u_int ts_oldest; /* Current oldest timestamp */ - u_int ts_stable; /* Current stable timestamp */ + uint64_t ts_oldest; /* Current oldest timestamp */ + uint64_t ts_stable; /* Current stable timestamp */ bool mixed_mode_deletes; /* Run with mixed mode deletes */ bool use_timestamps; /* Use txn timestamps */ bool race_timetamps; /* Async update to oldest timestamp */ diff --git a/src/third_party/wiredtiger/test/checkpoint/workers.c b/src/third_party/wiredtiger/test/checkpoint/workers.c index babc8731b0a..2f8a608b6f3 100644 --- a/src/third_party/wiredtiger/test/checkpoint/workers.c +++ b/src/third_party/wiredtiger/test/checkpoint/workers.c @@ -370,18 +370,18 @@ real_worker(void) next_rnd = __wt_random(&rnd); if (g.prepare && next_rnd % 2 == 0) { testutil_check(__wt_snprintf( - buf, sizeof(buf), "prepare_timestamp=%x", g.ts_stable + 1)); + buf, sizeof(buf), "prepare_timestamp=%" PRIx64, g.ts_stable + 1)); if ((ret = session->prepare_transaction(session, buf)) != 0) { __wt_readunlock((WT_SESSION_IMPL *)session, &g.clock_lock); (void)log_print_err("real_worker:prepare_transaction", ret, 1); goto err; } testutil_check(__wt_snprintf(buf, sizeof(buf), - "durable_timestamp=%x,commit_timestamp=%x", g.ts_stable + 3, - g.ts_stable + 1)); + "durable_timestamp=%" PRIx64 ",commit_timestamp=%" PRIx64, + g.ts_stable + 3, g.ts_stable + 1)); } else testutil_check(__wt_snprintf( - buf, sizeof(buf), "commit_timestamp=%x", g.ts_stable + 1)); + buf, sizeof(buf), "commit_timestamp=%" PRIx64, g.ts_stable + 1)); // Commit majority of times if (next_rnd % 49 != 0) { diff --git a/src/third_party/wiredtiger/test/format/format.i b/src/third_party/wiredtiger/test/format/format.i index 75cde988b0d..d2bcab9bf9d 100644 --- a/src/third_party/wiredtiger/test/format/format.i +++ b/src/third_party/wiredtiger/test/format/format.i @@ -26,6 +26,8 @@ * OTHER DEALINGS IN THE SOFTWARE. */ +#define FORMAT_PREPARE_TIMEOUT 120 + /* * read_op -- * Perform a read operation, waiting out prepare conflicts. @@ -48,7 +50,7 @@ read_op(WT_CURSOR *cursor, read_operation op, int *exactp) /* Ignore clock reset. */ __wt_seconds(NULL, &now); - testutil_assertfmt(now < start || now - start < 60, + testutil_assertfmt(now < start || now - start < FORMAT_PREPARE_TIMEOUT, "%s: timed out with prepare-conflict", "WT_CURSOR.next"); } break; @@ -58,7 +60,7 @@ read_op(WT_CURSOR *cursor, read_operation op, int *exactp) /* Ignore clock reset. */ __wt_seconds(NULL, &now); - testutil_assertfmt(now < start || now - start < 60, + testutil_assertfmt(now < start || now - start < FORMAT_PREPARE_TIMEOUT, "%s: timed out with prepare-conflict", "WT_CURSOR.prev"); } break; @@ -68,7 +70,7 @@ read_op(WT_CURSOR *cursor, read_operation op, int *exactp) /* Ignore clock reset. */ __wt_seconds(NULL, &now); - testutil_assertfmt(now < start || now - start < 60, + testutil_assertfmt(now < start || now - start < FORMAT_PREPARE_TIMEOUT, "%s: timed out with prepare-conflict", "WT_CURSOR.search"); } break; @@ -78,7 +80,7 @@ read_op(WT_CURSOR *cursor, read_operation op, int *exactp) /* Ignore clock reset. */ __wt_seconds(NULL, &now); - testutil_assertfmt(now < start || now - start < 60, + testutil_assertfmt(now < start || now - start < FORMAT_PREPARE_TIMEOUT, "%s: timed out with prepare-conflict", "WT_CURSOR.search_near"); } break; diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 6d079eb12aa..5647f1c4262 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -553,7 +553,6 @@ prepare_transaction(TINFO *tinfo) WT_DECL_RET; WT_SESSION *session; uint64_t ts; - uint32_t longwait, pause_ms; char buf[64]; session = tinfo->session; @@ -580,19 +579,6 @@ prepare_transaction(TINFO *tinfo) lock_writeunlock(session, &g.ts_lock); - /* - * Sometimes add a delay after prepare to induce extra memory stress. For 80% of the threads, - * there is never a delay, so there is always a dedicated set of threads trying to do work. For - * the other 20%, we'll sometimes delay. For these threads, 99% of the time, proceed without - * delay. The rest of the time, pause up to 5 seconds, weighted toward the smaller delays. - */ - if (tinfo->id % 5 == 0) { - longwait = mmrand(&tinfo->rnd, 0, 999); - if (longwait < 10) { - pause_ms = mmrand(&tinfo->rnd, 1, 10) << longwait; - __wt_sleep(0, (uint64_t)pause_ms * WT_THOUSAND); - } - } return (ret); } diff --git a/src/third_party/wiredtiger/test/suite/test_verbose01.py b/src/third_party/wiredtiger/test/suite/test_verbose01.py index 7bd7b542803..fdf335d46a9 100755 --- a/src/third_party/wiredtiger/test/suite/test_verbose01.py +++ b/src/third_party/wiredtiger/test/suite/test_verbose01.py @@ -64,6 +64,11 @@ class test_verbose_base(wttest.WiredTigerTestCase, suite_subprocess): else: self.assertEqual(len(verbose_messages), 0) + if len(output) >= self.nlines: + # If we've read the maximum number of characters, its likely that the last line is truncated ('...'). In this + # case, trim the last message as we can't parse it. + verbose_messages = verbose_messages[:-1] + # Test the contents of each verbose message, ensuring it satisfies the expected pattern. verb_pattern = re.compile('|'.join(patterns)) for line in verbose_messages: -- cgit v1.2.1