From 90787b49457d8a4f28b23985d56430a1a174454e Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Wed, 20 May 2020 16:43:26 +1000 Subject: Import wiredtiger: 7bf362af190a36a31589d3d78eb1cd1a5963b79d from branch mongodb-4.4 ref: 55d47c210c..7bf362af19 for: 4.5.1 WT-6056 Set DATA_CORRUPTION flag so we return TRY_SALVAGE on open WT-6129 Add test to ensure prepare transactions will not lead to cache stuck WT-6267 Fix not deleting prepared update when hs cursor is placed on another key --- src/third_party/wiredtiger/import.data | 2 +- src/third_party/wiredtiger/src/meta/meta_turtle.c | 2 + src/third_party/wiredtiger/src/txn/txn.c | 18 +- .../test/csuite/wt4156_metadata_salvage/main.c | 2 + .../wiredtiger/test/suite/test_prepare_hs04.py | 222 +++++++++++++++++++++ 5 files changed, 239 insertions(+), 7 deletions(-) create mode 100644 src/third_party/wiredtiger/test/suite/test_prepare_hs04.py diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 34221ab1fa5..2af6d667182 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "55d47c210ccb07a115c2e6f71ee90367c7c3ab77" + "commit": "7bf362af190a36a31589d3d78eb1cd1a5963b79d" } diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c index 119438fefcd..10605a71020 100644 --- a/src/third_party/wiredtiger/src/meta/meta_turtle.c +++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c @@ -381,6 +381,7 @@ err: */ if (ret == 0 || strcmp(key, WT_METADATA_COMPAT) == 0 || F_ISSET(S2C(session), WT_CONN_SALVAGE)) return (ret); + F_SET(S2C(session), WT_CONN_DATA_CORRUPTION); WT_RET_PANIC(session, WT_TRY_SALVAGE, "%s: fatal turtle file read error", WT_METADATA_TURTLE); } @@ -437,5 +438,6 @@ err: */ if (ret == 0) return (ret); + F_SET(conn, WT_CONN_DATA_CORRUPTION); WT_RET_PANIC(session, ret, "%s: fatal turtle file update error", WT_METADATA_TURTLE); } diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 769e54225a5..f20bfba99fb 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -631,17 +631,21 @@ __txn_append_hs_record(WT_SESSION_IMPL *session, WT_CURSOR *hs_cursor, WT_ITEM * WT_ERR(hs_cursor->get_key(hs_cursor, &hs_btree_id, hs_key, &hs_start_ts, &hs_counter)); - /* Stop before crossing over to the next btree */ - if (hs_btree_id != S2BT(session)->id) + /* Not found if we cross the tree boundary. */ + if (hs_btree_id != S2BT(session)->id) { + ret = WT_NOTFOUND; goto done; + } /* * Keys are sorted in an order, skip the ones before the desired key, and bail out if we have * crossed over the desired key and not found the record we are looking for. */ WT_ERR(__wt_compare(session, NULL, hs_key, key, &cmp)); - if (cmp != 0) + if (cmp != 0) { + ret = WT_NOTFOUND; goto done; + } /* * As part of the history store search, we never get an exact match based on our search criteria @@ -925,9 +929,11 @@ __txn_resolve_prepared_op(WT_SESSION_IMPL *session, WT_TXN_OP *op, bool commit, true); if (ret == 0) - WT_ERR(__txn_append_hs_record(session, hs_cursor, &op->u.op_row.key, cbt->ref->page, - upd, commit, &fix_upd, &upd_appended)); - else if (ret == WT_NOTFOUND && !commit) { + /* Not found if we cross the tree or key boundary. */ + WT_ERR_NOTFOUND_OK(__txn_append_hs_record(session, hs_cursor, &op->u.op_row.key, + cbt->ref->page, upd, commit, &fix_upd, &upd_appended), + true); + if (ret == WT_NOTFOUND && !commit) { /* * Allocate a tombstone so that when we reconcile the update chain we don't copy the * prepared cell, which is now associated with a rolled back prepare, and instead write diff --git a/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c b/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c index 0bc6adbbc00..aaa1b49f961 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c @@ -345,6 +345,8 @@ wt_open_corrupt(const char *sfx) WT_DECL_RET; char buf[1024]; + /* The child should not abort the test in the message handler. Set it here, don't inherit. */ + test_abort = false; if (sfx != NULL) testutil_check(__wt_snprintf(buf, sizeof(buf), "%s.%s", home, sfx)); else diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py new file mode 100644 index 00000000000..c5d3a9b7050 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from helper import copy_wiredtiger_home +import wiredtiger, wttest +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios +from wiredtiger import stat + +def timestamp_str(t): + return '%x' % t + +# test_prepare_hs04.py +# Read prepared updates from on-disk with ignore_prepare. +# Committing or aborting a prepared update when there exists a tombstone for that key already. +# +class test_prepare_hs04(wttest.WiredTigerTestCase): + # Force a small cache. + conn_config = 'cache_size=50MB,statistics=(fast)' + + # Create a small table. + uri = "table:test_prepare_hs04" + + nsessions = 3 + nkeys = 40 + nrows = 100 + + scenarios = make_scenarios([ + ('commit_transaction', dict(commit=True)), + ('rollback_transaction', dict(commit=False)) + ]) + + def get_stat(self, stat): + stat_cursor = self.session.open_cursor('statistics:') + val = stat_cursor[stat][2] + stat_cursor.close() + return val + + def search_keys_timestamp_and_ignore(self, ds, txn_config, expected_value, conflict=False): + cursor = self.session.open_cursor(self.uri) + + commit_key = "C" + self.session.begin_transaction(txn_config) + for i in range(1, self.nsessions * self.nkeys): + key = commit_key + ds.key(self.nrows + i) + cursor.set_key(key) + if conflict == True: + self.assertRaisesException(wiredtiger.WiredTigerError, lambda:cursor.search(), expected_value) + elif expected_value == None: + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + else: + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_value(), expected_value) + cursor.close() + self.session.commit_transaction() + + def prepare_updates(self, ds): + + # Commit some updates to get eviction and history store fired up + # Insert a key at timestamp 1 + commit_key = "C" + commit_value = b"bbbbb" * 100 + cursor = self.session.open_cursor(self.uri) + for i in range(1, self.nsessions * self.nkeys): + self.session.begin_transaction('isolation=snapshot') + key = commit_key + ds.key(self.nrows + i) + cursor.set_key(key) + cursor.set_value(commit_value) + self.assertEquals(cursor.insert(), 0) + self.session.commit_transaction('commit_timestamp=' + timestamp_str(1)) + cursor.close() + + # Call checkpoint + self.session.checkpoint() + + cursor = self.session.open_cursor(self.uri) + for i in range(1, self.nsessions * self.nkeys): + self.session.begin_transaction('isolation=snapshot') + key = commit_key + ds.key(self.nrows + i) + cursor.set_key(key) + self.assertEquals(cursor.remove(), 0) + self.session.commit_transaction('commit_timestamp=' + timestamp_str(10)) + cursor.close() + + hs_writes_start = self.get_stat(stat.conn.cache_write_hs) + # Have prepared updates in multiple sessions. This should ensure writing + # prepared updates to the data store + # Insert the same key at timestamp 20, but with prepare updates. + sessions = [0] * self.nsessions + cursors = [0] * self.nsessions + prepare_value = b"ccccc" * 100 + for j in range (0, self.nsessions): + sessions[j] = self.conn.open_session() + sessions[j].begin_transaction('isolation=snapshot') + cursors[j] = sessions[j].open_cursor(self.uri) + # Each session will update many consecutive keys. + start = (j * self.nkeys) + end = start + self.nkeys + for i in range(start, end): + cursors[j].set_key(commit_key + ds.key(self.nrows + i)) + cursors[j].set_value(prepare_value) + self.assertEquals(cursors[j].insert(), 0) + sessions[j].prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + + hs_writes = self.get_stat(stat.conn.cache_write_hs) - hs_writes_start + + # Assert if not writing anything to the history store. + self.assertGreaterEqual(hs_writes, 0) + + txn_config = 'read_timestamp=' + timestamp_str(5) + ',ignore_prepare=true' + # Search keys with timestamp 5, ignore_prepare=true and expect the cursor search to return 0 (key found) + self.search_keys_timestamp_and_ignore(ds, txn_config, commit_value) + + txn_config = 'read_timestamp=' + timestamp_str(20) + ',ignore_prepare=true' + # Search keys with timestamp 20, ignore_prepare=true, expect the cursor to return wiredtiger.WT_NOTFOUND + self.search_keys_timestamp_and_ignore(ds, txn_config, None) + + prepare_conflict_msg = '/conflict with a prepared update/' + txn_config = 'read_timestamp=' + timestamp_str(20) + ',ignore_prepare=false' + # Search keys with timestamp 20, ignore_prepare=false and expect the cursor the cursor search to return prepare conflict message + self.search_keys_timestamp_and_ignore(ds, txn_config, prepare_conflict_msg, True) + + # If commit is True then commit the transactions and simulate a crash which would + # eventualy rollback transactions. + if self.commit == True: + # Commit the prepared_transactions with timestamp 30. + for j in range (0, self.nsessions): + sessions[j].commit_transaction( + 'commit_timestamp=' + timestamp_str(30) + ',durable_timestamp=' + timestamp_str(30)) + + self.session.checkpoint() + + # Simulate a crash by copying to a new directory(RESTART). + copy_wiredtiger_home(".", "RESTART") + + # Open the new directory. + self.conn = self.setUpConnectionOpen("RESTART") + self.session = self.setUpSessionOpen(self.conn) + + # After simulating a crash, search for the keys inserted. + + txn_config = 'read_timestamp=' + timestamp_str(5) + ',ignore_prepare=false' + # Search keys with timestamp 5, ignore_prepare=false and expect the cursor value to be commit_value. + self.search_keys_timestamp_and_ignore(ds, txn_config, commit_value) + + txn_config = 'read_timestamp=' + timestamp_str(20) + ',ignore_prepare=true' + # Search keys with timestamp 20, ignore_prepare=true and expect the cursor search to return WT_NOTFOUND. + self.search_keys_timestamp_and_ignore(ds, txn_config, None) + + txn_config = 'read_timestamp=' + timestamp_str(20) + ',ignore_prepare=false' + # Search keys with timestamp 20, ignore_prepare=false and expect the cursor search to return WT_NOTFOUND. + self.search_keys_timestamp_and_ignore(ds, txn_config, None) + + # If commit is true then the commit_tramsactions was called and we will expect prepare_value. + if self.commit == True: + txn_config = 'read_timestamp=' + timestamp_str(30) + ',ignore_prepare=true' + # Search keys with timestamp 30, ignore_prepare=true and expect the cursor value to be prepare_value. + self.search_keys_timestamp_and_ignore(ds, txn_config, prepare_value) + else: + # Commit is false and we simulated a crash/restart which would have rolled-back the transactions, therefore we expect the + # cursor search to return WT_NOTFOUND. + txn_config = 'read_timestamp=' + timestamp_str(30) + ',ignore_prepare=true' + # Search keys with timestamp 30, ignore_prepare=true and expect the cursor value to return WT_NOTFOUND. + self.search_keys_timestamp_and_ignore(ds, txn_config, None) + + if self.commit == True: + txn_config = 'read_timestamp=' + timestamp_str(30) + ',ignore_prepare=false' + # Search keys with timestamp 30, ignore_prepare=false and expect the cursor value to be prepare_value. + self.search_keys_timestamp_and_ignore(ds, txn_config, prepare_value) + else: + # Commit is false and we simulated a crash/restart which would have rolled-back the transactions, therefore we expect the + # cursor search to return WT_NOTFOUND. + txn_config = 'read_timestamp=' + timestamp_str(30) + ',ignore_prepare=false' + # Search keys with timestamp 30, ignore_prepare=false and expect the cursor value to return WT_NOTFOUND. + self.search_keys_timestamp_and_ignore(ds, txn_config, None) + + def test_prepare_hs(self): + + ds = SimpleDataSet(self, self.uri, self.nrows, key_format="S", value_format='u') + ds.populate() + bigvalue = b"aaaaa" * 100 + + # Initially load huge data + cursor = self.session.open_cursor(self.uri) + for i in range(1, 10000): + cursor.set_key(ds.key(self.nrows + i)) + cursor.set_value(bigvalue) + self.assertEquals(cursor.insert(), 0) + cursor.close() + self.session.checkpoint() + + # We put prepared updates in multiple sessions so that we do not hang + # because of cache being full with uncommitted updates. + self.prepare_updates(ds) + +if __name__ == '__main__': + wttest.run() -- cgit v1.2.1