diff options
7 files changed, 234 insertions, 10 deletions
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index a51dde2abd6..473f3dfb664 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-4.4", - "commit": "f34a5afbb1d807ac90627555c3452fdec750b592" + "commit": "0aacf077324fea15db4202817e3b2f695232c691" } diff --git a/src/third_party/wiredtiger/src/cursor/cur_hs.c b/src/third_party/wiredtiger/src/cursor/cur_hs.c index 299c06a4fa7..aecc01ade8b 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_hs.c +++ b/src/third_party/wiredtiger/src/cursor/cur_hs.c @@ -1118,7 +1118,9 @@ __wt_curhs_open(WT_SESSION_IMPL *session, WT_CURSOR *owner, WT_CURSOR **cursorp) /* Open the file cursor for operations on the regular history store .*/ WT_ERR(__curhs_file_cursor_open(session, &hs_cursor->file_cursor)); - WT_ERR(__wt_cursor_init(cursor, WT_HS_URI, owner, NULL, cursorp)); + WT_WITH_BTREE(session, CUR2BT(hs_cursor->file_cursor), + ret = __wt_cursor_init(cursor, WT_HS_URI, owner, NULL, cursorp)); + WT_ERR(ret); WT_TIME_WINDOW_INIT(&hs_cursor->time_window); hs_cursor->btree_id = 0; WT_ERR(__wt_scr_alloc(session, 0, &hs_cursor->datastore_key)); diff --git a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c index ef5c3246e30..dc5748b184f 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_visibility.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_visibility.c @@ -483,6 +483,16 @@ __wt_rec_upd_select(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins, v */ WT_ASSERT(session, same_txn_valid_upd->type != WT_UPDATE_TOMBSTONE); upd_select->upd = upd = same_txn_valid_upd; + + } else if (same_txn_valid_upd != NULL && vpack != NULL && vpack->tw.prepare) { + /* + * The on-disk version is from an aborted prepare transaction. Therefore, use + * the update from the same transaction as the selected update. We are sure that + * the on-disk prepared update has been aborted because otherwise we would have + * chosen it as an update this tombstone can be applied to. + */ + WT_ASSERT(session, same_txn_valid_upd->type != WT_UPDATE_TOMBSTONE); + upd_select->upd = upd = same_txn_valid_upd; } } } diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index ee304fedff5..3e53737bdc1 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -346,17 +346,17 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *page /* * Do not include history store updates greater than on-disk data store version to construct - * a full update to restore. Include the most recent updates than the on-disk version - * shouldn't be problem as the on-disk version in history store is always a full update. It - * is better to not to include those updates as it unnecessarily increases the rollback to - * stable time. + * a full update to restore except when the on-disk update is prepared. Including more + * recent updates than the on-disk version shouldn't be problem as the on-disk version in + * history store is always a full update. It is better to not to include those updates as it + * unnecessarily increases the rollback to stable time. * * Comparing with timestamps here has no problem unlike in search flow where the timestamps * may be reset during reconciliation. RTS detects an on-disk update is unstable based on * the written proper timestamp, so comparing against it with history store shouldn't have * any problem. */ - if (hs_start_ts <= unpack->tw.start_ts) { + if (hs_start_ts <= unpack->tw.start_ts || unpack->tw.prepare) { if (type == WT_UPDATE_MODIFY) WT_ERR(__wt_modify_apply_item( session, S2BT(session)->value_format, &full_value, hs_value->data)); @@ -423,7 +423,7 @@ __rollback_ondisk_fixup_key(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *page __wt_timestamp_to_string(hs_durable_ts, ts_string[1]), __wt_timestamp_to_string(hs_stop_durable_ts, ts_string[2]), __wt_timestamp_to_string(rollback_timestamp, ts_string[3]), hs_tw->start_txn, type); - WT_ASSERT(session, hs_tw->start_ts <= unpack->tw.start_ts); + WT_ASSERT(session, unpack->tw.prepare || hs_tw->start_ts <= unpack->tw.start_ts); valid_update_found = true; break; } diff --git a/src/third_party/wiredtiger/test/format/t.c b/src/third_party/wiredtiger/test/format/t.c index 4cd239a528a..10f9763fa53 100644 --- a/src/third_party/wiredtiger/test/format/t.c +++ b/src/third_party/wiredtiger/test/format/t.c @@ -218,8 +218,6 @@ main(int argc, char *argv[]) * file, used when running checks. */ if (g.reopen) { - if (config != NULL) - testutil_die(EINVAL, "-c incompatible with -R"); if (access(g.home_config, R_OK) != 0) testutil_die(ENOENT, "%s", g.home_config); config = g.home_config; diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py new file mode 100644 index 00000000000..ae79bd247f5 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2020 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios +from wiredtiger import stat, WT_NOTFOUND + +def timestamp_str(t): + return '%x' % t + +# test_prepare_hs05.py +# Test that after aborting prepare transaction, correct update from the history store is restored. +class test_prepare_hs05(wttest.WiredTigerTestCase): + conn_config = 'cache_size=50MB' + session_config = 'isolation=snapshot' + + def test_check_prepare_abort_hs_restore(self): + uri = 'table:test_prepare_hs05' + create_params = 'key_format=S,value_format=S' + self.session.create(uri, create_params) + + value1 = 'a' * 5 + value2 = 'b' * 5 + value3 = 'c' * 5 + + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1)) + cursor = self.session.open_cursor(uri) + + key = 1 + + self.session.begin_transaction() + cursor[str(key)] = value1 + cursor.set_key(str(key)) + self.session.commit_transaction('commit_timestamp=' + timestamp_str(2)) + + # Commit update and remove operation in the same transaction. + self.session.begin_transaction() + cursor[str(key)] = value2 + cursor.set_key(str(key)) + cursor.remove() + self.session.commit_transaction('commit_timestamp=' + timestamp_str(3)) + + # Add a prepared update for the key. + self.session.begin_transaction() + cursor[str(key)] = value3 + self.session.prepare_transaction('prepare_timestamp='+ timestamp_str(4)) + + # Try to evict the page with prepared update. This will ensure that prepared update is + # written as the on-disk version and the older versions are moved to the history store. + session2 = self.conn.open_session() + session2.begin_transaction('ignore_prepare=true') + cursor2 = session2.open_cursor(uri, None, "debug=(release_evict=true)") + cursor2.set_key(str(key)) + self.assertEquals(cursor2.search(), WT_NOTFOUND) + cursor2.reset() + + # This should abort the prepared transaction. + self.session.rollback_transaction() + + self.session.checkpoint() + + # We should be able to read the older version of the key from the history store. + self.session.begin_transaction('read_timestamp='+timestamp_str(2)) + cursor.set_key(str(key)) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_value(), value1) + self.session.rollback_transaction() + + # The latest version should be marked deleted. + self.session.begin_transaction() + cursor.set_key(str(key)) + self.assertEqual(cursor.search(), WT_NOTFOUND) + self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py new file mode 100644 index 00000000000..1e898523c39 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from wiredtiger import stat +from wtscenario import make_scenarios +from helper import simulate_crash_restart +from wtdataset import SimpleDataSet +from test_rollback_to_stable01 import test_rollback_to_stable_base + +def timestamp_str(t): + return '%x' % t + +# test_rollback_to_stable21.py +# Test rollback to stable when an out of order prepared transaction is written to disk +class test_rollback_to_stable21(test_rollback_to_stable_base): + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + scenarios = make_scenarios(key_format_values) + + def conn_config(self): + config = 'cache_size=250MB,statistics=(all),statistics_log=(json,on_close,wait=1)' + return config + + def test_rollback_to_stable(self): + nrows = 1000 + + # Prepare transactions for column store table is not yet supported. + if self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + + # Create a table without logging. + uri = "table:rollback_to_stable21" + ds = SimpleDataSet( + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + ds.populate() + + # Pin oldest and stable timestamps to 10. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + + ',stable_timestamp=' + timestamp_str(10)) + + valuea = 'a' * 400 + valueb = 'b' * 400 + + cursor = self.session.open_cursor(uri) + self.session.begin_transaction() + for i in range(1, nrows + 1): + cursor[i] = valuea + + self.session.commit_transaction('commit_timestamp=' + timestamp_str(30)) + + self.session.begin_transaction() + for i in range(1, nrows + 1): + cursor[i] = valueb + + cursor.reset() + cursor.close() + self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) + + s = self.conn.open_session() + s.begin_transaction('ignore_prepare = true') + # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. + evict_cursor = s.open_cursor(uri, None, "debug=(release_evict)") + + for i in range(1, nrows + 1): + evict_cursor.set_key(i) + self.assertEquals(evict_cursor.search(), 0) + self.assertEqual(evict_cursor.get_value(), valuea) + evict_cursor.reset() + + s.rollback_transaction() + self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) + s.checkpoint() + + # Rollback the prepared transaction + self.session.rollback_transaction() + + # Simulate a server crash and restart. + self.pr("restart") + simulate_crash_restart(self, ".", "RESTART") + self.pr("restart complete") + + self.check(valuea, uri, nrows, 40) + + stat_cursor = self.session.open_cursor('statistics:', None, None) + hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] + stat_cursor.close() + + self.assertGreater(hs_removed, 0) |