diff options
author | Luke Chen <luke.chen@mongodb.com> | 2021-03-22 15:42:50 +1100 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-03-22 05:02:38 +0000 |
commit | 1da30d832cfd99849f1b400a1f257d5480ca8435 (patch) | |
tree | 86f12061cc609d26cb587df75f05107147f105cc | |
parent | d6d9fa4de0491f3b71ae4c3dbd4b5d1c1cbcbcff (diff) | |
download | mongo-1da30d832cfd99849f1b400a1f257d5480ca8435.tar.gz |
Import wiredtiger: f6216f19aaf7e66d1617331f69aebe335f0892fa from branch mongodb-5.0
ref: 61593ced6e..f6216f19aa
for: 4.9.0
WT-7269 Enable column store scenarios to all RTS existing tests
20 files changed, 404 insertions, 180 deletions
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index f5ef0074b04..ba494314b98 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.0", - "commit": "61593ced6e0b285360370ec7e7f6ccd0add4671e" + "commit": "f6216f19aaf7e66d1617331f69aebe335f0892fa" } diff --git a/src/third_party/wiredtiger/src/include/btree_inline.h b/src/third_party/wiredtiger/src/include/btree_inline.h index 4ca8db351b4..41510592410 100644 --- a/src/third_party/wiredtiger/src/include/btree_inline.h +++ b/src/third_party/wiredtiger/src/include/btree_inline.h @@ -1791,8 +1791,15 @@ static inline int __wt_bt_col_var_cursor_walk_txn_read(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_PAGE *page, WT_CELL_UNPACK_KV *unpack, WT_COL *cip) { + WT_UPDATE *upd; + + upd = NULL; + + if (cbt->ins) + upd = cbt->ins->upd; + cbt->slot = WT_COL_SLOT(page, cip); - WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, NULL, unpack)); + WT_RET(__wt_txn_read(session, cbt, NULL, cbt->recno, upd, unpack)); if (cbt->upd_value->type == WT_UPDATE_INVALID || cbt->upd_value->type == WT_UPDATE_TOMBSTONE) return (0); diff --git a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c index b95ca634d1f..239eeeb9c01 100644 --- a/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c +++ b/src/third_party/wiredtiger/src/txn/txn_rollback_to_stable.c @@ -735,6 +735,9 @@ __rollback_abort_col_var(WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t r if ((ins = WT_COL_APPEND(page)) != NULL) __rollback_abort_insert_list(session, ins, rollback_timestamp); + /* Mark the page as dirty to reconcile the page. */ + if (page->modify) + __wt_page_modify_set(session, page); return (0); } @@ -755,6 +758,10 @@ __rollback_abort_col_fix(WT_SESSION_IMPL *session, WT_PAGE *page, wt_timestamp_t /* Review the append list */ if ((ins = WT_COL_APPEND(page)) != NULL) __rollback_abort_insert_list(session, ins, rollback_timestamp); + + /* Mark the page as dirty to reconcile the page. */ + if (page->modify) + __wt_page_modify_set(session, page); } /* @@ -919,6 +926,10 @@ __rollback_abort_row_leaf(WT_SESSION_IMPL *session, WT_REF *ref, wt_timestamp_t */ if (!F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) WT_RET(__rollback_abort_row_reconciled_page(session, page, rollback_timestamp)); + + /* Mark the page as dirty to reconcile the page. */ + if (page->modify) + __wt_page_modify_set(session, page); return (0); } diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py index 7f3c8b23237..b3bf62f03ef 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py @@ -39,16 +39,16 @@ def timestamp_str(t): # test_rollback_to_stable01.py # Shared base class used by rollback to stable tests. class test_rollback_to_stable_base(wttest.WiredTigerTestCase): - def large_updates(self, uri, value, ds, nrows, commit_ts): + def large_updates(self, uri, value, ds, nrows, prepare, commit_ts): # Update a large number of records. session = self.session cursor = session.open_cursor(uri) - for i in range(0, nrows): + for i in range(1, nrows + 1): session.begin_transaction() cursor[ds.key(i)] = value if commit_ts == 0: session.commit_transaction() - elif self.prepare: + elif prepare: session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) @@ -57,19 +57,19 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) cursor.close() - def large_modifies(self, uri, value, ds, location, nbytes, nrows, commit_ts): + def large_modifies(self, uri, value, ds, location, nbytes, nrows, prepare, commit_ts): # Load a slight modification. session = self.session cursor = session.open_cursor(uri) session.begin_transaction() - for i in range(0, nrows): + for i in range(1, nrows + 1): cursor.set_key(i) mods = [wiredtiger.Modify(value, location, nbytes)] self.assertEqual(cursor.modify(mods), 0) if commit_ts == 0: session.commit_transaction() - elif self.prepare: + elif prepare: session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) @@ -78,17 +78,17 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): session.commit_transaction('commit_timestamp=' + timestamp_str(commit_ts)) cursor.close() - def large_removes(self, uri, ds, nrows, commit_ts): + def large_removes(self, uri, ds, nrows, prepare, commit_ts): # Remove a large number of records. session = self.session cursor = session.open_cursor(uri) - for i in range(0, nrows): + for i in range(1, nrows + 1): session.begin_transaction() cursor.set_key(i) cursor.remove() if commit_ts == 0: session.commit_transaction() - elif self.prepare: + elif prepare: session.prepare_transaction('prepare_timestamp=' + timestamp_str(commit_ts-1)) session.timestamp_transaction('commit_timestamp=' + timestamp_str(commit_ts)) session.timestamp_transaction('durable_timestamp=' + timestamp_str(commit_ts+1)) @@ -115,6 +115,11 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): class test_rollback_to_stable01(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + in_memory_values = [ ('no_inmem', dict(in_memory=False)), ('inmem', dict(in_memory=True)) @@ -125,7 +130,7 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(in_memory_values, prepare_values) + scenarios = make_scenarios(key_format_values, in_memory_values, prepare_values) def conn_config(self): config = 'cache_size=50MB,statistics=(all)' @@ -138,10 +143,14 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 10000 + # Prepare transactions for column store table is not yet supported. + if self.prepare and self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + # Create a table without logging. uri = "table:rollback_to_stable01" ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 1. @@ -149,12 +158,12 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): ',stable_timestamp=' + timestamp_str(1)) valuea = "aaaaa" * 100 - self.large_updates(uri, valuea, ds, nrows, 10) + self.large_updates(uri, valuea, ds, nrows, self.prepare, 10) # Check that all updates are seen. self.check(valuea, uri, nrows, 10) # Remove all keys with newer timestamp. - self.large_removes(uri, ds, nrows, 20) + self.large_removes(uri, ds, nrows, self.prepare, 20) # Check that the no keys should be visible. self.check(valuea, uri, 0, 20) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py index 3b6f643cbd1..20b5d7d6a41 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py @@ -42,6 +42,11 @@ def timestamp_str(t): class test_rollback_to_stable02(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + in_memory_values = [ ('no_inmem', dict(in_memory=False)), ('inmem', dict(in_memory=True)) @@ -52,7 +57,7 @@ class test_rollback_to_stable02(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(in_memory_values, prepare_values) + scenarios = make_scenarios(key_format_values, in_memory_values, prepare_values) def conn_config(self): config = 'cache_size=100MB,statistics=(all)' @@ -65,10 +70,14 @@ class test_rollback_to_stable02(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 10000 + # Prepare transactions for column store table is not yet supported. + if self.prepare and self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + # Create a table without logging. uri = "table:rollback_to_stable02" ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 1. @@ -79,19 +88,19 @@ class test_rollback_to_stable02(test_rollback_to_stable_base): valueb = "bbbbb" * 100 valuec = "ccccc" * 100 valued = "ddddd" * 100 - self.large_updates(uri, valuea, ds, nrows, 10) + self.large_updates(uri, valuea, ds, nrows, self.prepare, 10) # Check that all updates are seen. self.check(valuea, uri, nrows, 10) - self.large_updates(uri, valueb, ds, nrows, 20) + self.large_updates(uri, valueb, ds, nrows, self.prepare, 20) # Check that the new updates are only seen after the update timestamp. self.check(valueb, uri, nrows, 20) - self.large_updates(uri, valuec, ds, nrows, 30) + self.large_updates(uri, valuec, ds, nrows, self.prepare, 30) # Check that the new updates are only seen after the update timestamp. self.check(valuec, uri, nrows, 30) - self.large_updates(uri, valued, ds, nrows, 40) + self.large_updates(uri, valued, ds, nrows, self.prepare, 40) # Check that the new updates are only seen after the update timestamp. self.check(valued, uri, nrows, 40) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py index 05fff48b2b6..a0e2d4337b2 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py @@ -42,6 +42,11 @@ def timestamp_str(t): class test_rollback_to_stable01(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + in_memory_values = [ ('no_inmem', dict(in_memory=False)), ('inmem', dict(in_memory=True)) @@ -52,7 +57,7 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(in_memory_values, prepare_values) + scenarios = make_scenarios(key_format_values, in_memory_values, prepare_values) def conn_config(self): config = 'cache_size=4GB,statistics=(all)' @@ -65,10 +70,14 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 + # Prepare transactions for column store table is not yet supported. + if self.prepare and self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + # Create a table without logging. uri = "table:rollback_to_stable03" ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 1. @@ -78,15 +87,15 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): valuea = "aaaaa" * 100 valueb = "bbbbb" * 100 valuec = "ccccc" * 100 - self.large_updates(uri, valuea, ds, nrows, 10) + self.large_updates(uri, valuea, ds, nrows, self.prepare, 10) # Check that all updates are seen. self.check(valuea, uri, nrows, 10) - self.large_updates(uri, valueb, ds, nrows, 20) + self.large_updates(uri, valueb, ds, nrows, self.prepare, 20) # Check that all updates are seen. self.check(valueb, uri, nrows, 20) - self.large_updates(uri, valuec, ds, nrows, 30) + self.large_updates(uri, valuec, ds, nrows, self.prepare, 30) # Check that all updates are seen. self.check(valuec, uri, nrows, 30) @@ -117,10 +126,10 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): self.assertEqual(keys_removed, 0) self.assertEqual(keys_restored, 0) if self.in_memory: + self.assertEqual(upd_aborted, nrows) self.assertEqual(hs_removed, 0) else: - self.assertEqual(hs_removed, nrows) - self.assertEqual(upd_aborted, nrows) + self.assertGreaterEqual(upd_aborted + hs_removed, nrows) self.assertGreater(pages_visited, 0) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py index 1ff704be2ee..f865a3975f7 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py @@ -43,6 +43,11 @@ def mod_val(value, char, location, nbytes=1): class test_rollback_to_stable04(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + in_memory_values = [ ('no_inmem', dict(in_memory=False)), ('inmem', dict(in_memory=True)) @@ -53,7 +58,7 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(in_memory_values, prepare_values) + scenarios = make_scenarios(key_format_values, in_memory_values, prepare_values) def conn_config(self): config = 'cache_size=500MB,statistics=(all)' @@ -66,10 +71,14 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 + # Prepare transactions for column store table is not yet supported. + if self.prepare and self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + # Create a table without logging. uri = "table:rollback_to_stable04" ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 10. @@ -91,19 +100,19 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): value_modZ = mod_val(value_modY, 'Z', 7) # Perform a combination of modifies and updates. - self.large_updates(uri, value_a, ds, nrows, 20) - self.large_modifies(uri, 'Q', ds, 0, 1, nrows, 30) - self.large_modifies(uri, 'R', ds, 1, 1, nrows, 40) - self.large_modifies(uri, 'S', ds, 2, 1, nrows, 50) - self.large_updates(uri, value_b, ds, nrows, 60) - self.large_updates(uri, value_c, ds, nrows, 70) - self.large_modifies(uri, 'T', ds, 3, 1, nrows, 80) - self.large_updates(uri, value_d, ds, nrows, 90) - self.large_modifies(uri, 'W', ds, 4, 1, nrows, 100) - self.large_updates(uri, value_a, ds, nrows, 110) - self.large_modifies(uri, 'X', ds, 5, 1, nrows, 120) - self.large_modifies(uri, 'Y', ds, 6, 1, nrows, 130) - self.large_modifies(uri, 'Z', ds, 7, 1, nrows, 140) + self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) + self.large_modifies(uri, 'Q', ds, 0, 1, nrows, self.prepare, 30) + self.large_modifies(uri, 'R', ds, 1, 1, nrows, self.prepare, 40) + self.large_modifies(uri, 'S', ds, 2, 1, nrows, self.prepare, 50) + self.large_updates(uri, value_b, ds, nrows, self.prepare, 60) + self.large_updates(uri, value_c, ds, nrows, self.prepare, 70) + self.large_modifies(uri, 'T', ds, 3, 1, nrows, self.prepare, 80) + self.large_updates(uri, value_d, ds, nrows, self.prepare, 90) + self.large_modifies(uri, 'W', ds, 4, 1, nrows, self.prepare, 100) + self.large_updates(uri, value_a, ds, nrows, self.prepare, 110) + self.large_modifies(uri, 'X', ds, 5, 1, nrows, self.prepare, 120) + self.large_modifies(uri, 'Y', ds, 6, 1, nrows, self.prepare, 130) + self.large_modifies(uri, 'Z', ds, 7, 1, nrows, self.prepare, 140) # Verify data is visible and correct. self.check(value_a, uri, nrows, 20) @@ -139,6 +148,7 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] + hs_sweep = stat_cursor[stat.conn.txn_rts_sweep_hs_keys][2] keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2] pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2] @@ -150,11 +160,10 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): self.assertEqual(keys_restored, 0) self.assertGreater(pages_visited, 0) if self.in_memory: - self.assertGreaterEqual(upd_aborted, nrows * 11) - self.assertGreaterEqual(hs_removed, 0) + self.assertEqual(upd_aborted, nrows * 11) + self.assertEqual(hs_removed + hs_sweep, 0) else: - self.assertGreaterEqual(upd_aborted, 0) - self.assertGreaterEqual(hs_removed, nrows * 11) + self.assertGreaterEqual(upd_aborted + hs_removed + hs_sweep, nrows * 11) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py index 56cf9a0a123..5604196241c 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py @@ -42,6 +42,11 @@ def timestamp_str(t): class test_rollback_to_stable05(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + in_memory_values = [ ('no_inmem', dict(in_memory=False)), ('inmem', dict(in_memory=True)) @@ -52,7 +57,7 @@ class test_rollback_to_stable05(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(in_memory_values, prepare_values) + scenarios = make_scenarios(key_format_values, in_memory_values, prepare_values) def conn_config(self): config = 'cache_size=50MB,statistics=(all)' @@ -65,10 +70,14 @@ class test_rollback_to_stable05(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 + # Prepare transactions for column store table is not yet supported. + if self.prepare and self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + # Create two tables without logging. uri_1 = "table:rollback_to_stable05_1" ds_1 = SimpleDataSet( - self, uri_1, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri_1, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds_1.populate() uri_2 = "table:rollback_to_stable05_2" @@ -80,33 +89,33 @@ class test_rollback_to_stable05(test_rollback_to_stable_base): valueb = "bbbbb" * 100 valuec = "ccccc" * 100 valued = "ddddd" * 100 - self.large_updates(uri_1, valuea, ds_1, nrows, 0) + self.large_updates(uri_1, valuea, ds_1, nrows, self.prepare, 0) self.check(valuea, uri_1, nrows, 0) - self.large_updates(uri_2, valuea, ds_2, nrows, 0) + self.large_updates(uri_2, valuea, ds_2, nrows, self.prepare, 0) self.check(valuea, uri_2, nrows, 0) # Start a long running transaction and keep it open. session_2 = self.conn.open_session() session_2.begin_transaction('isolation=snapshot') - self.large_updates(uri_1, valueb, ds_1, nrows, 0) + self.large_updates(uri_1, valueb, ds_1, nrows, self.prepare, 0) self.check(valueb, uri_1, nrows, 0) - self.large_updates(uri_1, valuec, ds_1, nrows, 0) + self.large_updates(uri_1, valuec, ds_1, nrows, self.prepare, 0) self.check(valuec, uri_1, nrows, 0) - self.large_updates(uri_1, valued, ds_1, nrows, 0) + self.large_updates(uri_1, valued, ds_1, nrows, self.prepare, 0) self.check(valued, uri_1, nrows, 0) # Add updates to the another table. - self.large_updates(uri_2, valueb, ds_2, nrows, 0) + self.large_updates(uri_2, valueb, ds_2, nrows, self.prepare, 0) self.check(valueb, uri_2, nrows, 0) - self.large_updates(uri_2, valuec, ds_2, nrows, 0) + self.large_updates(uri_2, valuec, ds_2, nrows, self.prepare, 0) self.check(valuec, uri_2, nrows, 0) - self.large_updates(uri_2, valued, ds_2, nrows, 0) + self.large_updates(uri_2, valued, ds_2, nrows, self.prepare, 0) self.check(valued, uri_2, nrows, 0) # Checkpoint to ensure that all the data is flushed. diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py index bdfc532d723..cd2c35783ce 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py @@ -40,6 +40,11 @@ def timestamp_str(t): class test_rollback_to_stable06(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + in_memory_values = [ ('no_inmem', dict(in_memory=False)), ('inmem', dict(in_memory=True)) @@ -50,7 +55,7 @@ class test_rollback_to_stable06(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(in_memory_values, prepare_values) + scenarios = make_scenarios(key_format_values, in_memory_values, prepare_values) def conn_config(self): config = 'cache_size=50MB,statistics=(all)' @@ -63,10 +68,14 @@ class test_rollback_to_stable06(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 + # Prepare transactions for column store table is not yet supported. + if self.prepare and self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + # Create a table without logging. uri = "table:rollback_to_stable06" ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 10. @@ -79,10 +88,10 @@ class test_rollback_to_stable06(test_rollback_to_stable_base): value_d = "ddddd" * 100 # Perform several updates. - self.large_updates(uri, value_a, ds, nrows, 20) - self.large_updates(uri, value_b, ds, nrows, 30) - self.large_updates(uri, value_c, ds, nrows, 40) - self.large_updates(uri, value_d, ds, nrows, 50) + self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) + self.large_updates(uri, value_b, ds, nrows, self.prepare, 30) + self.large_updates(uri, value_c, ds, nrows, self.prepare, 40) + self.large_updates(uri, value_d, ds, nrows, self.prepare, 50) # Verify data is visible and correct. self.check(value_a, uri, nrows, 20) @@ -118,8 +127,7 @@ class test_rollback_to_stable06(test_rollback_to_stable_base): self.assertEqual(upd_aborted, nrows * 4) self.assertEqual(hs_removed, 0) else: - self.assertGreaterEqual(upd_aborted, 0) - self.assertGreaterEqual(hs_removed, nrows * 3) + self.assertGreaterEqual(upd_aborted + hs_removed + keys_removed, nrows * 4) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py index f666460fd1f..9c9e39d16b9 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py @@ -44,12 +44,17 @@ def timestamp_str(t): class test_rollback_to_stable07(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + prepare_values = [ ('no_prepare', dict(prepare=False)), ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(prepare_values) + scenarios = make_scenarios(key_format_values, prepare_values) def conn_config(self): config = 'cache_size=5MB,statistics=(all),log=(enabled=true)' @@ -58,10 +63,14 @@ class test_rollback_to_stable07(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 + # Prepare transactions for column store table is not yet supported. + if self.prepare and self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + # Create a table without logging. uri = "table:rollback_to_stable07" ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 10. @@ -74,10 +83,10 @@ class test_rollback_to_stable07(test_rollback_to_stable_base): value_d = "ddddd" * 100 # Perform several updates. - self.large_updates(uri, value_d, ds, nrows, 20) - self.large_updates(uri, value_c, ds, nrows, 30) - self.large_updates(uri, value_b, ds, nrows, 40) - self.large_updates(uri, value_a, ds, nrows, 50) + self.large_updates(uri, value_d, ds, nrows, self.prepare, 20) + self.large_updates(uri, value_c, ds, nrows, self.prepare, 30) + self.large_updates(uri, value_b, ds, nrows, self.prepare, 40) + self.large_updates(uri, value_a, ds, nrows, self.prepare, 50) # Verify data is visible and correct. self.check(value_d, uri, nrows, 20) @@ -92,9 +101,9 @@ class test_rollback_to_stable07(test_rollback_to_stable_base): self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) # Perform additional updates. - self.large_updates(uri, value_b, ds, nrows, 60) - self.large_updates(uri, value_c, ds, nrows, 70) - self.large_updates(uri, value_d, ds, nrows, 80) + self.large_updates(uri, value_b, ds, nrows, self.prepare, 60) + self.large_updates(uri, value_c, ds, nrows, self.prepare, 70) + self.large_updates(uri, value_d, ds, nrows, self.prepare, 80) # Checkpoint to ensure the data is flushed to disk. self.session.checkpoint() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py index e16d39446df..05153034362 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py @@ -40,6 +40,11 @@ def timestamp_str(t): class test_rollback_to_stable08(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + in_memory_values = [ ('no_inmem', dict(in_memory=False)), ('inmem', dict(in_memory=True)) @@ -50,7 +55,7 @@ class test_rollback_to_stable08(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(in_memory_values, prepare_values) + scenarios = make_scenarios(key_format_values, in_memory_values, prepare_values) def conn_config(self): config = 'cache_size=50MB,statistics=(all)' @@ -63,10 +68,14 @@ class test_rollback_to_stable08(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 10000 + # Prepare transactions for column store table is not yet supported. + if self.prepare and self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + # Create a table without logging. uri = "table:rollback_to_stable08" ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 10. @@ -79,10 +88,10 @@ class test_rollback_to_stable08(test_rollback_to_stable_base): value_d = "ddddd" * 100 # Perform several updates. - self.large_updates(uri, value_a, ds, nrows, 20) - self.large_updates(uri, value_b, ds, nrows, 30) - self.large_updates(uri, value_c, ds, nrows, 40) - self.large_updates(uri, value_d, ds, nrows, 50) + self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) + self.large_updates(uri, value_b, ds, nrows, self.prepare, 30) + self.large_updates(uri, value_c, ds, nrows, self.prepare, 40) + self.large_updates(uri, value_d, ds, nrows, self.prepare, 50) # Verify data is visible and correct. self.check(value_a, uri, nrows, 20) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py index 2ca9f2b4881..d0a45f0522d 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py @@ -69,8 +69,7 @@ class test_rollback_to_stable09(test_rollback_to_stable_base): self.pr('create table') session = self.session session.begin_transaction() - session.create(self.uri, 'key_format=5s,value_format=HQ,' + - 'columns=(country,year,population),' ) + session.create(self.uri, 'key_format=5s,value_format=HQ,columns=(country,year,population)') if commit_ts == 0: session.commit_transaction() elif self.prepare: diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py index c1a7b8c0ef6..5d1986d3b46 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py @@ -69,12 +69,17 @@ def retry_rollback(self, name, txn_session, code): class test_rollback_to_stable10(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + prepare_values = [ ('no_prepare', dict(prepare=False)), ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(prepare_values) + scenarios = make_scenarios(key_format_values, prepare_values) def conn_config(self): config = 'cache_size=6MB,statistics=(all),statistics_log=(json,on_close,wait=1),log=(enabled=true),timing_stress_for_test=[history_store_checkpoint_delay]' @@ -83,11 +88,15 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 + # Prepare transactions for column store table is not yet supported. + if self.prepare and self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + # Create a table without logging. self.pr("create/populate tables") uri_1 = "table:rollback_to_stable10_1" ds_1 = SimpleDataSet( - self, uri_1, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri_1, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds_1.populate() # Create another table without logging. @@ -109,15 +118,15 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): # Perform several updates. self.pr("large updates") - self.large_updates(uri_1, value_d, ds_1, nrows, 20) - self.large_updates(uri_1, value_c, ds_1, nrows, 30) - self.large_updates(uri_1, value_b, ds_1, nrows, 40) - self.large_updates(uri_1, value_a, ds_1, nrows, 50) + self.large_updates(uri_1, value_d, ds_1, nrows, self.prepare, 20) + self.large_updates(uri_1, value_c, ds_1, nrows, self.prepare, 30) + self.large_updates(uri_1, value_b, ds_1, nrows, self.prepare, 40) + self.large_updates(uri_1, value_a, ds_1, nrows, self.prepare, 50) - self.large_updates(uri_2, value_d, ds_2, nrows, 20) - self.large_updates(uri_2, value_c, ds_2, nrows, 30) - self.large_updates(uri_2, value_b, ds_2, nrows, 40) - self.large_updates(uri_2, value_a, ds_2, nrows, 50) + self.large_updates(uri_2, value_d, ds_2, nrows, self.prepare, 20) + self.large_updates(uri_2, value_c, ds_2, nrows, self.prepare, 30) + self.large_updates(uri_2, value_b, ds_2, nrows, self.prepare, 40) + self.large_updates(uri_2, value_a, ds_2, nrows, self.prepare, 50) # Verify data is visible and correct. self.check(value_d, uri_1, nrows, 20) @@ -147,13 +156,13 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): # Rollbacks may occur when checkpoint is running, so retry as needed. self.pr("updates") retry_rollback(self, 'update ds1, e', None, - lambda: self.large_updates(uri_1, value_e, ds_1, nrows, 70)) + lambda: self.large_updates(uri_1, value_e, ds_1, nrows, self.prepare, 70)) retry_rollback(self, 'update ds2, e', None, - lambda: self.large_updates(uri_2, value_e, ds_2, nrows, 70)) + lambda: self.large_updates(uri_2, value_e, ds_2, nrows, self.prepare, 70)) retry_rollback(self, 'update ds1, f', None, - lambda: self.large_updates(uri_1, value_f, ds_1, nrows, 80)) + lambda: self.large_updates(uri_1, value_f, ds_1, nrows, self.prepare, 80)) retry_rollback(self, 'update ds2, f', None, - lambda: self.large_updates(uri_2, value_f, ds_2, nrows, 80)) + lambda: self.large_updates(uri_2, value_f, ds_2, nrows, self.prepare, 80)) finally: done.set() ckpt.join() @@ -224,15 +233,15 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): # Perform several updates. self.pr("large updates") - self.large_updates(uri_1, value_d, ds_1, nrows, 20) - self.large_updates(uri_1, value_c, ds_1, nrows, 30) - self.large_updates(uri_1, value_b, ds_1, nrows, 40) - self.large_updates(uri_1, value_a, ds_1, nrows, 50) - - self.large_updates(uri_2, value_d, ds_2, nrows, 20) - self.large_updates(uri_2, value_c, ds_2, nrows, 30) - self.large_updates(uri_2, value_b, ds_2, nrows, 40) - self.large_updates(uri_2, value_a, ds_2, nrows, 50) + self.large_updates(uri_1, value_d, ds_1, nrows, self.prepare, 20) + self.large_updates(uri_1, value_c, ds_1, nrows, self.prepare, 30) + self.large_updates(uri_1, value_b, ds_1, nrows, self.prepare, 40) + self.large_updates(uri_1, value_a, ds_1, nrows, self.prepare, 50) + + self.large_updates(uri_2, value_d, ds_2, nrows, self.prepare, 20) + self.large_updates(uri_2, value_c, ds_2, nrows, self.prepare, 30) + self.large_updates(uri_2, value_b, ds_2, nrows, self.prepare, 40) + self.large_updates(uri_2, value_a, ds_2, nrows, self.prepare, 50) # Verify data is visible and correct. self.check(value_d, uri_1, nrows, 20) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py index a6d1f8703d4..59014d5aac9 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py @@ -41,12 +41,17 @@ def timestamp_str(t): class test_rollback_to_stable11(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + prepare_values = [ ('no_prepare', dict(prepare=False)), ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(prepare_values) + scenarios = make_scenarios(key_format_values, prepare_values) def conn_config(self): config = 'cache_size=1MB,statistics=(all),log=(archive=false,enabled=true)' @@ -55,10 +60,14 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1 + # Prepare transactions for column store table is not yet supported. + if self.prepare and self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + # Create a table without logging. uri = "table:rollback_to_stable11" ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 10. @@ -71,10 +80,10 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): value_d = "ddddd" * 100 # Perform several updates. - self.large_updates(uri, value_a, ds, nrows, 20) - self.large_updates(uri, value_a, ds, nrows, 20) - self.large_updates(uri, value_a, ds, nrows, 20) - self.large_updates(uri, value_b, ds, nrows, 20) + self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) + self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) + self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) + self.large_updates(uri, value_b, ds, nrows, self.prepare, 20) # Verify data is visible and correct. self.check(value_b, uri, nrows, 20) @@ -95,10 +104,10 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): self.check(value_b, uri, nrows, 20) # Perform several updates. - self.large_updates(uri, value_c, ds, nrows, 30) - self.large_updates(uri, value_c, ds, nrows, 30) - self.large_updates(uri, value_c, ds, nrows, 30) - self.large_updates(uri, value_d, ds, nrows, 30) + self.large_updates(uri, value_c, ds, nrows, self.prepare, 30) + self.large_updates(uri, value_c, ds, nrows, self.prepare, 30) + self.large_updates(uri, value_c, ds, nrows, self.prepare, 30) + self.large_updates(uri, value_d, ds, nrows, self.prepare, 30) # Verify data is visible and correct. self.check(value_d, uri, nrows, 30) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py index 0c0887371b4..cd0211f2d12 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py @@ -41,12 +41,17 @@ def timestamp_str(t): class test_rollback_to_stable12(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + prepare_values = [ ('no_prepare', dict(prepare=False)), ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(prepare_values) + scenarios = make_scenarios(key_format_values, prepare_values) def conn_config(self): config = 'cache_size=500MB,statistics=(all),log=(enabled=true)' @@ -55,6 +60,10 @@ class test_rollback_to_stable12(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000000 + # Prepare transactions for column store table is not yet supported. + if self.prepare and self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + # Create a table without logging. uri = "table:rollback_to_stable12" ds = SimpleDataSet( @@ -69,7 +78,7 @@ class test_rollback_to_stable12(test_rollback_to_stable_base): value_b = "bbbbb" * 100 # Perform several updates. - self.large_updates(uri, value_a, ds, nrows, 20) + self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) # Verify data is visible and correct. self.check(value_a, uri, nrows, 20) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py index 89c9b732ac5..66f66d62874 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py @@ -41,12 +41,17 @@ def timestamp_str(t): class test_rollback_to_stable13(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + prepare_values = [ ('no_prepare', dict(prepare=False)), ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(prepare_values) + scenarios = make_scenarios(key_format_values, prepare_values) def conn_config(self): config = 'cache_size=500MB,statistics=(all),log=(enabled=true)' @@ -55,10 +60,14 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1000 + # Prepare transactions for column store table is not yet supported. + if self.prepare and self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + # Create a table without logging. uri = "table:rollback_to_stable13" ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='split_pct=50,log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format="S", config='split_pct=50,log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 10. @@ -69,13 +78,13 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): value_b = "bbbbb" * 100 # Perform several updates. - self.large_updates(uri, value_a, ds, nrows, 20) + self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) # Perform several removes. - self.large_removes(uri, ds, nrows, 30) + self.large_removes(uri, ds, nrows, self.prepare, 30) # Perform several updates. - self.large_updates(uri, value_b, ds, nrows, 60) + self.large_updates(uri, value_b, ds, nrows, self.prepare, 60) # Verify data is visible and correct. self.check(value_a, uri, nrows, 20) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py index f14feeda4cc..20d0f9f7744 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py @@ -75,12 +75,17 @@ def retry_rollback(self, name, txn_session, code): class test_rollback_to_stable14(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + prepare_values = [ ('no_prepare', dict(prepare=False)), ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(prepare_values) + scenarios = make_scenarios(key_format_values, prepare_values) def conn_config(self): config = 'cache_size=8MB,statistics=(all),statistics_log=(json,on_close,wait=1),log=(enabled=true),timing_stress_for_test=[history_store_checkpoint_delay]' @@ -89,11 +94,15 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 1500 + # Prepare transactions for column store table is not yet supported. + if self.prepare and self.key_format == 'r': + self.skipTest('Prepare transactions for column store table is not yet supported') + # Create a table without logging. self.pr("create/populate table") uri = "table:rollback_to_stable14" ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 10. @@ -109,11 +118,11 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): # Perform a combination of modifies and updates. self.pr("large updates and modifies") - self.large_updates(uri, value_a, ds, nrows, 20) - self.large_modifies(uri, 'Q', ds, 0, 1, nrows, 30) - self.large_modifies(uri, 'R', ds, 1, 1, nrows, 40) - self.large_modifies(uri, 'S', ds, 2, 1, nrows, 50) - self.large_modifies(uri, 'T', ds, 3, 1, nrows, 60) + self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) + self.large_modifies(uri, 'Q', ds, 0, 1, nrows, self.prepare, 30) + self.large_modifies(uri, 'R', ds, 1, 1, nrows, self.prepare, 40) + self.large_modifies(uri, 'S', ds, 2, 1, nrows, self.prepare, 50) + self.large_modifies(uri, 'T', ds, 3, 1, nrows, self.prepare, 60) # Verify data is visible and correct. self.check(value_a, uri, nrows, 20) @@ -139,13 +148,13 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): # Rollbacks may occur when checkpoint is running, so retry as needed. self.pr("modifies") retry_rollback(self, 'modify ds1, W', None, - lambda: self.large_modifies(uri, 'W', ds, 4, 1, nrows, 70)) + lambda: self.large_modifies(uri, 'W', ds, 4, 1, nrows, self.prepare, 70)) retry_rollback(self, 'modify ds1, X', None, - lambda: self.large_modifies(uri, 'X', ds, 5, 1, nrows, 80)) + lambda: self.large_modifies(uri, 'X', ds, 5, 1, nrows, self.prepare, 80)) retry_rollback(self, 'modify ds1, Y', None, - lambda: self.large_modifies(uri, 'Y', ds, 6, 1, nrows, 90)) + lambda: self.large_modifies(uri, 'Y', ds, 6, 1, nrows, self.prepare, 90)) retry_rollback(self, 'modify ds1, Z', None, - lambda: self.large_modifies(uri, 'Z', ds, 7, 1, nrows, 100)) + lambda: self.large_modifies(uri, 'Z', ds, 7, 1, nrows, self.prepare, 100)) finally: done.set() ckpt.join() @@ -207,17 +216,17 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): # Perform a combination of modifies and updates. self.pr("large updates and modifies") - self.large_updates(uri, value_a, ds, nrows, 20) - self.large_modifies(uri, 'Q', ds, 0, 1, nrows, 30) + self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) + self.large_modifies(uri, 'Q', ds, 0, 1, nrows, self.prepare, 30) # prepare cannot use same timestamp always, so use a different timestamps that are aborted. if self.prepare: - self.large_modifies(uri, 'R', ds, 1, 1, nrows, 51) - self.large_modifies(uri, 'S', ds, 2, 1, nrows, 55) - self.large_modifies(uri, 'T', ds, 3, 1, nrows, 60) + self.large_modifies(uri, 'R', ds, 1, 1, nrows, self.prepare, 51) + self.large_modifies(uri, 'S', ds, 2, 1, nrows, self.prepare, 55) + self.large_modifies(uri, 'T', ds, 3, 1, nrows, self.prepare, 60) else: - self.large_modifies(uri, 'R', ds, 1, 1, nrows, 60) - self.large_modifies(uri, 'S', ds, 2, 1, nrows, 60) - self.large_modifies(uri, 'T', ds, 3, 1, nrows, 60) + self.large_modifies(uri, 'R', ds, 1, 1, nrows, self.prepare, 60) + self.large_modifies(uri, 'S', ds, 2, 1, nrows, self.prepare, 60) + self.large_modifies(uri, 'T', ds, 3, 1, nrows, self.prepare, 60) # Verify data is visible and correct. self.check(value_a, uri, nrows, 20) @@ -237,13 +246,13 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): # Rollbacks may occur when checkpoint is running, so retry as needed. self.pr("modifies") retry_rollback(self, 'modify ds1, W', None, - lambda: self.large_modifies(uri, 'W', ds, 4, 1, nrows, 70)) + lambda: self.large_modifies(uri, 'W', ds, 4, 1, nrows, self.prepare, 70)) retry_rollback(self, 'modify ds1, X', None, - lambda: self.large_modifies(uri, 'X', ds, 5, 1, nrows, 80)) + lambda: self.large_modifies(uri, 'X', ds, 5, 1, nrows, self.prepare, 80)) retry_rollback(self, 'modify ds1, Y', None, - lambda: self.large_modifies(uri, 'Y', ds, 6, 1, nrows, 90)) + lambda: self.large_modifies(uri, 'Y', ds, 6, 1, nrows, self.prepare, 90)) retry_rollback(self, 'modify ds1, Z', None, - lambda: self.large_modifies(uri, 'Z', ds, 7, 1, nrows, 100)) + lambda: self.large_modifies(uri, 'Z', ds, 7, 1, nrows, self.prepare, 100)) finally: done.set() ckpt.join() @@ -303,17 +312,17 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): # Perform a combination of modifies and updates. self.pr("large updates and modifies") - self.large_updates(uri, value_a, ds, nrows, 20) - self.large_modifies(uri, 'Q', ds, len(value_a), 1, nrows, 30) + self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) + self.large_modifies(uri, 'Q', ds, len(value_a), 1, nrows, self.prepare, 30) # prepare cannot use same timestamp always, so use a different timestamps that are aborted. if self.prepare: - self.large_modifies(uri, 'R', ds, len(value_modQ), 1, nrows, 51) - self.large_modifies(uri, 'S', ds, len(value_modR), 1, nrows, 55) - self.large_modifies(uri, 'T', ds, len(value_modS), 1, nrows, 60) + self.large_modifies(uri, 'R', ds, len(value_modQ), 1, nrows, self.prepare, 51) + self.large_modifies(uri, 'S', ds, len(value_modR), 1, nrows, self.prepare, 55) + self.large_modifies(uri, 'T', ds, len(value_modS), 1, nrows, self.prepare, 60) else: - self.large_modifies(uri, 'R', ds, len(value_modQ), 1, nrows, 60) - self.large_modifies(uri, 'S', ds, len(value_modR), 1, nrows, 60) - self.large_modifies(uri, 'T', ds, len(value_modS), 1, nrows, 60) + self.large_modifies(uri, 'R', ds, len(value_modQ), 1, nrows, self.prepare, 60) + self.large_modifies(uri, 'S', ds, len(value_modR), 1, nrows, self.prepare, 60) + self.large_modifies(uri, 'T', ds, len(value_modS), 1, nrows, self.prepare, 60) # Verify data is visible and correct. self.check(value_a, uri, nrows, 20) @@ -333,13 +342,13 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): # Rollbacks may occur when checkpoint is running, so retry as needed. self.pr("modifies") retry_rollback(self, 'modify ds1, W', None, - lambda: self.large_modifies(uri, 'W', ds, len(value_modT), 1, nrows, 70)) + lambda: self.large_modifies(uri, 'W', ds, len(value_modT), 1, nrows, self.prepare, 70)) retry_rollback(self, 'modify ds1, X', None, - lambda: self.large_modifies(uri, 'X', ds, len(value_modT) + 1, 1, nrows, 80)) + lambda: self.large_modifies(uri, 'X', ds, len(value_modT) + 1, 1, nrows, self.prepare, 80)) retry_rollback(self, 'modify ds1, Y', None, - lambda: self.large_modifies(uri, 'Y', ds, len(value_modT) + 2, 1, nrows, 90)) + lambda: self.large_modifies(uri, 'Y', ds, len(value_modT) + 2, 1, nrows, self.prepare, 90)) retry_rollback(self, 'modify ds1, Z', None, - lambda: self.large_modifies(uri, 'Z', ds, len(value_modT) + 3, 1, nrows, 100)) + lambda: self.large_modifies(uri, 'Z', ds, len(value_modT) + 3, 1, nrows, self.prepare, 100)) finally: done.set() ckpt.join() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable15.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable15.py index 5f2af4eb612..03a6aa396dd 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable15.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable15.py @@ -40,7 +40,6 @@ def timestamp_str(t): # update-list for both fixed length and variable length column store. # Eviction is set to false, so that everything persists in memory. class test_rollback_to_stable15(wttest.WiredTigerTestCase): - conn_config = 'cache_size=200MB,statistics=(all),debug_mode=(eviction=false)' session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), @@ -52,7 +51,19 @@ class test_rollback_to_stable15(wttest.WiredTigerTestCase): # Variable length ('variable', dict(value_format='i')), ] - scenarios = make_scenarios(key_format_values, value_format_values) + in_memory_values = [ + ('no_inmem', dict(in_memory=False)), + ('inmem', dict(in_memory=True)) + ] + scenarios = make_scenarios(key_format_values, value_format_values, in_memory_values) + + def conn_config(self): + config = 'cache_size=200MB,statistics=(all),debug_mode=(eviction=false)' + if self.in_memory: + config += ',in_memory=true' + else: + config += ',in_memory=false' + return config def check(self, check_value, uri, nrows, read_ts): session = self.session @@ -72,9 +83,9 @@ class test_rollback_to_stable15(wttest.WiredTigerTestCase): # Create a table. uri = "table:rollback_to_stable15" nrows = 2000 - create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + create_params = 'log=(enabled=false),key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, create_params) - cursor = self.session.open_cursor(uri) + cursor = self.session.open_cursor(uri) # Pin oldest and stable to timestamp 1. self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) + diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py index 1f88c845723..2539a4b88d0 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py @@ -30,27 +30,61 @@ from helper import simulate_crash_restart import wiredtiger, wttest from wiredtiger import stat from wtdataset import SimpleDataSet +from wtscenario import make_scenarios def timestamp_str(t): return '%x' % t # test_rollback_to_stable16.py -# Test that rollback to stable removes updates present on disk for variable length column store. +# Test that rollback to stable removes updates present on disk for column store. class test_rollback_to_stable16(wttest.WiredTigerTestCase): - conn_config = 'cache_size=200MB,statistics=(all)' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + value_format_values = [ + # Fixed length + #FIXME: WT-7304 Fixed length column store failing on rollback on disk values + # ('fixed', dict(value_format='8t')), + # Variable length + ('variable', dict(value_format='S')), + ] + + in_memory_values = [ + ('no_inmem', dict(in_memory=False)), + ('inmem', dict(in_memory=True)) + ] + + scenarios = make_scenarios(key_format_values, value_format_values, in_memory_values) + + def conn_config(self): + config = 'cache_size=200MB,statistics=(all)' + if self.in_memory: + config += ',in_memory=true' + else: + config += ',in_memory=false' + return config + def insert_update_data(self, uri, value, start_row, nrows, timestamp): cursor = self.session.open_cursor(uri) for i in range(start_row, start_row + nrows): self.session.begin_transaction() - cursor[i] = value + str(i) + if self.value_format == 'S': + cursor[i] = value + str(i) + else: + cursor[i] = value self.session.commit_transaction('commit_timestamp=' + timestamp_str(timestamp)) cursor.close() def check(self, check_value, uri, nrows, start_row, read_ts): session = self.session - session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) + if read_ts == 0: + session.begin_transaction() + else: + session.begin_transaction('read_timestamp=' + timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 @@ -59,10 +93,12 @@ class test_rollback_to_stable16(wttest.WiredTigerTestCase): ret = cursor.search() if check_value is None: if ret != wiredtiger.WT_NOTFOUND: - self.tty(f'value = {cursor.get_value()}') self.assertTrue(ret == wiredtiger.WT_NOTFOUND) else: - self.assertEqual(cursor.get_value(), check_value + str(count + start_row)) + if self.value_format == 'S': + self.assertEqual(cursor.get_value(), check_value + str(count + start_row)) + else: + self.assertEqual(cursor.get_value(), check_value) count += 1 session.commit_transaction() @@ -78,9 +114,12 @@ class test_rollback_to_stable16(wttest.WiredTigerTestCase): nrows = 200 start_row = 1 ts = [2,5,7,9] - values = ["aaaa", "bbbb", "cccc", "dddd"] + if self.value_format == 'S': + values = ["aaaa", "bbbb", "cccc", "dddd"] + else: + values = [0x01, 0x02, 0x03, 0x04] - create_params = 'key_format=r,value_format=S' + create_params = 'log=(enabled=false),key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, create_params) # Pin oldest and stable to timestamp 1. @@ -93,16 +132,27 @@ class test_rollback_to_stable16(wttest.WiredTigerTestCase): self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) - self.session.checkpoint() - - # Rollback to stable done as part of recovery. - simulate_crash_restart(self,".", "RESTART") + if not self.in_memory: + # Checkpoint to ensure that all the updates are flushed to disk. + self.session.checkpoint() + # Rollback to stable done as part of recovery. + simulate_crash_restart(self,".", "RESTART") + else: + # Manually call rollback_to_stable for in memory keys/values. + self.conn.rollback_to_stable() self.check(values[0], uri, nrows, 1, 2) self.check(values[1], uri, nrows, 201, 5) self.check(None, uri, nrows, 401, 7) self.check(None, uri, nrows, 601, 9) + stat_cursor = self.session.open_cursor('statistics:', None, None) + upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] + keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] + stat_cursor.close() + + self.assertGreaterEqual(upd_aborted + keys_removed, (nrows*2) - 2) + self.session.close() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py index 10db33e0d1c..829e5d13680 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py @@ -31,6 +31,7 @@ from helper import simulate_crash_restart import wiredtiger, wttest from wiredtiger import stat from wtdataset import SimpleDataSet +from wtscenario import make_scenarios def timestamp_str(t): return '%x' % t @@ -39,9 +40,28 @@ def timestamp_str(t): # Test that rollback to stable handles updates present on history store and data store for variable # length column store. class test_rollback_to_stable17(wttest.WiredTigerTestCase): - conn_config = 'cache_size=200MB,statistics=(all)' session_config = 'isolation=snapshot' + key_format_values = [ + ('column', dict(key_format='r')), + ('integer_row', dict(key_format='i')), + ] + + in_memory_values = [ + ('no_inmem', dict(in_memory=False)), + ('inmem', dict(in_memory=True)) + ] + + scenarios = make_scenarios(key_format_values, in_memory_values) + + def conn_config(self): + config = 'cache_size=200MB,statistics=(all)' + if self.in_memory: + config += ',in_memory=true' + else: + config += ',in_memory=false' + return config + def insert_update_data(self, uri, value, start_row, end_row, timestamp): cursor = self.session.open_cursor(uri) for i in range(start_row, end_row): @@ -72,7 +92,7 @@ class test_rollback_to_stable17(wttest.WiredTigerTestCase): ts = [2,5,7,9] values = ["aaaa", "bbbb", "cccc", "dddd"] - create_params = 'key_format=r,value_format=S' + create_params = 'log=(enabled=false),key_format=r,value_format=S' self.session.create(uri, create_params) # Pin oldest and stable to timestamp 1. @@ -86,11 +106,14 @@ class test_rollback_to_stable17(wttest.WiredTigerTestCase): # Set the stable timestamp to 5. self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5)) - # Checkpoint to ensure that all the updates are flushed to disk. - self.session.checkpoint() - - # Rollback to stable done as part of recovery. - simulate_crash_restart(self, ".", "RESTART") + if not self.in_memory: + # Checkpoint to ensure that all the updates are flushed to disk. + self.session.checkpoint() + # Rollback to stable done as part of recovery. + simulate_crash_restart(self,".", "RESTART") + else: + # Manually call rollback_to_stable for in memory keys/values. + self.conn.rollback_to_stable() # Check that keys at timestamps 2 and 5 have the correct values they were updated with. self.check(values[0], uri, nrows - 1, 2) @@ -100,6 +123,13 @@ class test_rollback_to_stable17(wttest.WiredTigerTestCase): self.check(values[1], uri, nrows - 1, 7) self.check(values[1], uri, nrows - 1, 9) + stat_cursor = self.session.open_cursor('statistics:', None, None) + upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] + hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] + stat_cursor.close() + + self.assertGreaterEqual(upd_aborted + hs_removed, (nrows*2) - 2) + self.session.close() if __name__ == '__main__': |