diff options
| author | Luke Chen <luke.chen@mongodb.com> | 2021-11-15 17:17:19 +1100 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-11-15 06:45:10 +0000 |
| commit | 5ae1743d30a508fbdbb63eb85bcd628c2123a624 (patch) | |
| tree | 50dbe37b7571355464823b69b99f8a6126980928 /src/third_party/wiredtiger/test/suite | |
| parent | bf73aa9e17d57086333ae545e02dd9172a14c944 (diff) | |
| download | mongo-5ae1743d30a508fbdbb63eb85bcd628c2123a624.tar.gz | |
Import wiredtiger: 6eeb4f7e61e7b34b7667f9fdd20e6ed5c55bd498 from branch mongodb-master
ref: 180c9bfa72..6eeb4f7e61
for: 5.2.0
WT-8287 Implement timestamping and history for fixed-length column store.
Diffstat (limited to 'src/third_party/wiredtiger/test/suite')
132 files changed, 3864 insertions, 1414 deletions
diff --git a/src/third_party/wiredtiger/test/suite/test_bulk02.py b/src/third_party/wiredtiger/test/suite/test_bulk02.py index d41291fd367..cd81f2cbb8b 100644 --- a/src/third_party/wiredtiger/test/suite/test_bulk02.py +++ b/src/third_party/wiredtiger/test/suite/test_bulk02.py @@ -43,18 +43,24 @@ class test_bulkload_checkpoint(wttest.WiredTigerTestCase, suite_subprocess): ('file', dict(uri='file:data')), ('table', dict(uri='table:data')), ] + configs = [ + ('fix', dict(keyfmt='r', valfmt='8t')), + ('var', dict(keyfmt='r', valfmt='S')), + ('row', dict(keyfmt='S', valfmt='S')), + ] ckpt_type = [ ('named', dict(ckpt_type='named')), ('unnamed', dict(ckpt_type='unnamed')), ] - scenarios = make_scenarios(types, ckpt_type) + scenarios = make_scenarios(types, configs, ckpt_type) # Bulk-load handles are skipped by checkpoints. # Named and unnamed checkpoint versions. def test_bulkload_checkpoint(self): # Open a bulk cursor and insert a few records. - self.session.create(self.uri, 'key_format=S,value_format=S') + config = 'key_format={},value_format={}'.format(self.keyfmt, self.valfmt) + self.session.create(self.uri, config) cursor = self.session.open_cursor(self.uri, None, 'bulk') for i in range(1, 10): cursor[simple_key(cursor, i)] = simple_value(cursor, i) @@ -82,6 +88,11 @@ class test_bulkload_backup(wttest.WiredTigerTestCase, suite_subprocess): ('file', dict(uri='file:data')), ('table', dict(uri='table:data')), ] + configs = [ + ('fix', dict(keyfmt='r', valfmt='8t')), + ('var', dict(keyfmt='r', valfmt='S')), + ('row', dict(keyfmt='S', valfmt='S')), + ] ckpt_type = [ ('named', dict(ckpt_type='named')), ('none', dict(ckpt_type='none')), @@ -91,7 +102,7 @@ class test_bulkload_backup(wttest.WiredTigerTestCase, suite_subprocess): ('different', dict(session_type='different')), ('same', dict(session_type='same')), ] - scenarios = make_scenarios(types, ckpt_type, session_type) + scenarios = make_scenarios(types, configs, ckpt_type, session_type) # Backup a set of chosen tables/files using the wt backup command. # The only files are bulk-load files, so they shouldn't be copied. @@ -108,7 +119,8 @@ class test_bulkload_backup(wttest.WiredTigerTestCase, suite_subprocess): def test_bulk_backup(self): # Open a bulk cursor and insert a few records. - self.session.create(self.uri, 'key_format=S,value_format=S') + config = 'key_format={},value_format={}'.format(self.keyfmt, self.valfmt) + self.session.create(self.uri, config) cursor = self.session.open_cursor(self.uri, None, 'bulk') for i in range(1, 10): cursor[simple_key(cursor, i)] = simple_value(cursor, i) diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint02.py b/src/third_party/wiredtiger/test/suite/test_checkpoint02.py index b1da367d99a..dc1662262f7 100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint02.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint02.py @@ -39,9 +39,10 @@ from wtscenario import make_scenarios # Run background checkpoints repeatedly while doing inserts and other # operations in another thread class test_checkpoint02(wttest.WiredTigerTestCase): - key_format_values = [ - ('column', dict(key_format='r')), - ('u32_row', dict(key_format='L')), + format_values = [ + ('column_fix', dict(key_format='r', value_format='8t')), + ('column', dict(key_format='r', value_format='S')), + ('u32_row', dict(key_format='L', value_format='S')), ] size_values = [ @@ -49,12 +50,19 @@ class test_checkpoint02(wttest.WiredTigerTestCase): ('table-10', dict(uri='table:test',dsize=10,nops=50000,nthreads=30)) ] - scenarios = make_scenarios(key_format_values, size_values) + scenarios = make_scenarios(format_values, size_values) def test_checkpoint02(self): done = threading.Event() self.session.create(self.uri, - "key_format=" + self.key_format + ",value_format=S") + "key_format={},value_format={}".format(self.key_format, self.value_format)) + + if self.value_format == '8t': + self.nops *= 2 + my_data = 97 + else: + my_data = 'a' * self.dsize + ckpt = checkpoint_thread(self.conn, done) work_queue = queue.Queue() opthreads = [] @@ -63,7 +71,6 @@ class test_checkpoint02(wttest.WiredTigerTestCase): uris = list() uris.append(self.uri) - my_data = 'a' * self.dsize for i in range(self.nops): if i % 191 == 0 and i != 0: work_queue.put_nowait(('b', i + 1, my_data)) diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint03.py b/src/third_party/wiredtiger/test/suite/test_checkpoint03.py index 106aaeaa00d..121ece8304c 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint03.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint03.py @@ -45,12 +45,13 @@ class test_checkpoint03(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename session_config = 'isolation=snapshot, ' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column-fix', dict(key_format='r', value_format='8t')), + ('column', dict(key_format='r', value_format='i')), + ('integer_row', dict(key_format='i', value_format='i')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def get_stat(self, stat): stat_cursor = self.session.open_cursor('statistics:') @@ -60,7 +61,8 @@ class test_checkpoint03(wttest.WiredTigerTestCase, suite_subprocess): def test_checkpoint_writes_to_hs(self): # Create a basic table. - self.session.create(self.uri, 'key_format={},value_format=i'.format(self.key_format)) + config = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(self.uri, config) self.session.begin_transaction() self.conn.set_timestamp('oldest_timestamp=1') diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint06.py b/src/third_party/wiredtiger/test/suite/test_checkpoint06.py index 91cd80667c6..1c5bdf48af4 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint06.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint06.py @@ -37,18 +37,24 @@ class test_checkpoint06(wttest.WiredTigerTestCase): conn_config = 'create,cache_size=50MB' session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column-fix', dict(key_format='r', value_format='8t')), + ('column', dict(key_format='r', value_format='S')), + ('integer_row', dict(key_format='i', value_format='S')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test_rollback_truncation_in_checkpoint(self): self.uri = 'table:ckpt06' - self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) + config = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(self.uri, config) + + if self.value_format == '8t': + value = 72 + else: + value = "abcdefghijklmnopqrstuvwxyz" * 3 - value = "abcdefghijklmnopqrstuvwxyz" * 3 self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) self.session.begin_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot01.py b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot01.py index 0bc38abb5d2..63dabc178d8 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot01.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot01.py @@ -42,10 +42,16 @@ from wiredtiger import stat # class test_checkpoint_snapshot01(wttest.WiredTigerTestCase): + uri = "table:test_checkpoint_snapshot01" conn_config = 'cache_size=50MB' - # Create a table. - uri = "table:test_checkpoint_snapshot01" + format_values = [ + ('column-fix', dict(key_format='r', value_format='8t')), + ('column', dict(key_format='r', value_format='u')), + ('string_row', dict(key_format='S', value_format='u')), + ] + + scenarios = make_scenarios(format_values) nsessions = 5 nkeys = 40 @@ -53,9 +59,15 @@ class test_checkpoint_snapshot01(wttest.WiredTigerTestCase): def test_checkpoint_snapshot(self): - ds = SimpleDataSet(self, self.uri, self.nrows, key_format="S", value_format='u') + # Create a table. + ds = SimpleDataSet(self, self.uri, self.nrows, \ + key_format=self.key_format, value_format=self.value_format) ds.populate() - value = b"aaaaa" * 100 + + if self.value_format == '8t': + value = 86 + else: + value = b"aaaaa" * 100 sessions = [0] * self.nsessions cursors = [0] * self.nsessions diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py index 9f796b28089..13d3a7c4292 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot02.py @@ -42,19 +42,30 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): # Create a table. uri = "table:test_checkpoint_snapshot02" - nrows = 1000 - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column_fix', dict(key_format='r', value_format='8t')), + ('column', dict(key_format='r', value_format='S')), + ('integer_row', dict(key_format='i', value_format='S')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def conn_config(self): config = 'cache_size=10MB,statistics=(all),statistics_log=(json,on_close,wait=1),log=(enabled=true),timing_stress_for_test=[checkpoint_slow]' return config + def moresetup(self): + if self.value_format == '8t': + # Rig to use more than one page; otherwise the inconsistent checkpoint assertions fail. + self.extraconfig = ',leaf_page_max=4096' + self.nrows = 5000 + self.valuea = 97 + else: + self.extraconfig = '' + self.nrows = 1000 + self.valuea = "aaaaa" * 100 + def large_updates(self, uri, value, ds, nrows, commit_ts): # Update a large number of records. session = self.session @@ -68,7 +79,14 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() - def check(self, check_value, uri, nrows, read_ts): + def check(self, check_value, uri, nrows, read_ts, more_invisible_rows_exist): + # In FLCS the existence of the invisible extra set of rows causes the table to + # extend under them. Until that's fixed, expect (not just allow) those rows to + # exist and demand that they read back as zero and not as check_value. When it + # is fixed (so the end of the table updates transactionally) the special-case + # logic can just be removed. + flcs_tolerance = more_invisible_rows_exist and self.value_format == '8t' + session = self.session if read_ts == 0: session.begin_transaction() @@ -77,19 +95,24 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): cursor = session.open_cursor(uri) count = 0 for k, v in cursor: - self.assertEqual(v, check_value) + if flcs_tolerance and count >= nrows: + self.assertEqual(v, 0) + else: + self.assertEqual(v, check_value) count += 1 session.commit_transaction() - self.assertEqual(count, nrows) + self.assertEqual(count, nrows * 2 if flcs_tolerance else nrows) def test_checkpoint_snapshot(self): + self.moresetup() - ds = SimpleDataSet(self, self.uri, 0, key_format=self.key_format, value_format="S",config='log=(enabled=false)') + ds = SimpleDataSet(self, self.uri, 0, \ + key_format=self.key_format, value_format=self.value_format, \ + config='log=(enabled=false)'+self.extraconfig) ds.populate() - valuea = "aaaaa" * 100 - self.large_updates(self.uri, valuea, ds, self.nrows, 0) - self.check(valuea, self.uri, self.nrows, 0) + self.large_updates(self.uri, self.valuea, ds, self.nrows, 0) + self.check(self.valuea, self.uri, self.nrows, 0, False) session1 = self.conn.open_session() session1.begin_transaction() @@ -97,7 +120,7 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): for i in range(self.nrows+1, (self.nrows*2)+1): cursor1.set_key(ds.key(i)) - cursor1.set_value(valuea) + cursor1.set_value(self.valuea) self.assertEqual(cursor1.insert(), 0) # Create a checkpoint thread @@ -117,7 +140,7 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): simulate_crash_restart(self, ".", "RESTART") # Check the table contains the last checkpointed value. - self.check(valuea, self.uri, self.nrows, 0) + self.check(self.valuea, self.uri, self.nrows, 0, True) stat_cursor = self.session.open_cursor('statistics:', None, None) inconsistent_ckpt = stat_cursor[stat.conn.txn_rts_inconsistent_ckpt][2] @@ -128,17 +151,19 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): self.assertGreaterEqual(keys_removed, 0) def test_checkpoint_snapshot_with_timestamp(self): + self.moresetup() - ds = SimpleDataSet(self, self.uri, 0, key_format="S", value_format="S",config='log=(enabled=false)') + ds = SimpleDataSet(self, self.uri, 0, \ + key_format=self.key_format, value_format=self.value_format, \ + config='log=(enabled=false)'+self.extraconfig) ds.populate() - valuea = "aaaaa" * 100 # Pin oldest and stable timestamps to 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - self.large_updates(self.uri, valuea, ds, self.nrows, 20) - self.check(valuea, self.uri, self.nrows, 20) + self.large_updates(self.uri, self.valuea, ds, self.nrows, 20) + self.check(self.valuea, self.uri, self.nrows, 20, False) session1 = self.conn.open_session() session1.begin_transaction() @@ -146,7 +171,7 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): for i in range(self.nrows+1, (self.nrows*2)+1): cursor1.set_key(ds.key(i)) - cursor1.set_value(valuea) + cursor1.set_value(self.valuea) self.assertEqual(cursor1.insert(), 0) session1.timestamp_transaction('commit_timestamp=' + self.timestamp_str(30)) @@ -170,7 +195,7 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): simulate_crash_restart(self, ".", "RESTART") # Check the table contains the last checkpointed value. - self.check(valuea, self.uri, self.nrows, 30) + self.check(self.valuea, self.uri, self.nrows, 30, True) stat_cursor = self.session.open_cursor('statistics:', None, None) inconsistent_ckpt = stat_cursor[stat.conn.txn_rts_inconsistent_ckpt][2] @@ -181,11 +206,12 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): self.assertGreaterEqual(keys_removed, 0) def test_checkpoint_snapshot_with_txnid_and_timestamp(self): + self.moresetup() - ds = SimpleDataSet(self, self.uri, 0, key_format="S", value_format="S",config='log=(enabled=false)') + ds = SimpleDataSet(self, self.uri, 0, \ + key_format=self.key_format, value_format=self.value_format, \ + config='log=(enabled=false)'+self.extraconfig) ds.populate() - valuea = "aaaaa" * 100 - valueb = "bbbbb" * 100 # Pin oldest and stable timestamps to 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + @@ -194,8 +220,8 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): session1 = self.conn.open_session() session1.begin_transaction() - self.large_updates(self.uri, valuea, ds, self.nrows, 20) - self.check(valuea, self.uri, self.nrows, 20) + self.large_updates(self.uri, self.valuea, ds, self.nrows, 20) + self.check(self.valuea, self.uri, self.nrows, 20, False) session2 = self.conn.open_session() session2.begin_transaction() @@ -203,7 +229,7 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): for i in range((self.nrows+1), (self.nrows*2)+1): cursor2.set_key(ds.key(i)) - cursor2.set_value(valuea) + cursor2.set_value(self.valuea) self.assertEqual(cursor2.insert(), 0) session1.timestamp_transaction('commit_timestamp=' + self.timestamp_str(30)) @@ -228,7 +254,7 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): simulate_crash_restart(self, ".", "RESTART") # Check the table contains the last checkpointed value. - self.check(valuea, self.uri, self.nrows, 30) + self.check(self.valuea, self.uri, self.nrows, 30, True) stat_cursor = self.session.open_cursor('statistics:', None, None) inconsistent_ckpt = stat_cursor[stat.conn.txn_rts_inconsistent_ckpt][2] @@ -240,7 +266,7 @@ class test_checkpoint_snapshot02(wttest.WiredTigerTestCase): simulate_crash_restart(self, "RESTART", "RESTART2") # Check the table contains the last checkpointed value. - self.check(valuea, self.uri, self.nrows, 30) + self.check(self.valuea, self.uri, self.nrows, 30, True) stat_cursor = self.session.open_cursor('statistics:', None, None) inconsistent_ckpt = stat_cursor[stat.conn.txn_rts_inconsistent_ckpt][2] diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot03.py b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot03.py index 293b16efaf5..d1823c50c3f 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot03.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot03.py @@ -47,6 +47,14 @@ class test_checkpoint_snapshot03(wttest.WiredTigerTestCase): uri = "table:test_checkpoint_snapshot03" nrows = 500000 + format_values = [ + ('column_fix', dict(key_format='r', value_format='8t')), + ('column', dict(key_format='r', value_format='S')), + ('string_row', dict(key_format='S', value_format='S')), + ] + + scenarios = make_scenarios(format_values) + def conn_config(self): config = 'cache_size=250MB,statistics=(all),statistics_log=(json,on_close,wait=1),log=(enabled=true)' return config @@ -55,35 +63,53 @@ class test_checkpoint_snapshot03(wttest.WiredTigerTestCase): # Update a large number of records. session = self.session cursor = session.open_cursor(uri) - for i in range(0, nrows): + for i in range(1, nrows + 1): session.begin_transaction() cursor[ds.key(i)] = value session.commit_transaction() cursor.close() def check(self, check_value, uri, nrows): + # In FLCS the existence of the invisible extra row causes the table to extend + # under it. Until that's fixed, expect (not just allow) this row to exist and + # and demand it reads back as zero and not as check_value. When this behavior + # is fixed (so the end of the table updates transactionally) the special-case + # logic can just be removed. + flcs_tolerance = self.value_format == '8t' + session = self.session session.begin_transaction() cursor = session.open_cursor(uri) count = 0 for k, v in cursor: - self.assertEqual(v, check_value) + if flcs_tolerance and count >= nrows: + self.assertEqual(v, 0) + else: + self.assertEqual(v, check_value) count += 1 session.commit_transaction() - self.assertEqual(count, nrows) + self.assertEqual(count, nrows + 1 if flcs_tolerance else nrows) def test_checkpoint_snapshot(self): - ds = SimpleDataSet(self, self.uri, 0, key_format="S", value_format="S",config='log=(enabled=false),leaf_page_max=4k') + ds = SimpleDataSet(self, self.uri, 0, \ + key_format=self.key_format, value_format=self.value_format, \ + config='log=(enabled=false),leaf_page_max=4k') ds.populate() - valuea = "aaaaa" * 100 - valueb = "bbbbb" * 100 - valuec = "ccccc" * 100 + + if self.value_format == '8t': + valuea = 97 + valueb = 98 + valuec = 99 + else: + valuea = "aaaaa" * 100 + valueb = "bbbbb" * 100 + valuec = "ccccc" * 100 session1 = self.conn.open_session() session1.begin_transaction() cursor1 = session1.open_cursor(self.uri) - for i in range(self.nrows, self.nrows + 1): + for i in range(self.nrows + 1, self.nrows + 2): cursor1.set_key(ds.key(i)) cursor1.set_value(valueb) self.assertEqual(cursor1.insert(), 0) @@ -96,19 +122,20 @@ class test_checkpoint_snapshot03(wttest.WiredTigerTestCase): self.reopen_conn() # Check the table contains the last checkpointed value. + self.session.breakpoint() self.check(valuea, self.uri, self.nrows) session1 = self.conn.open_session() session1.begin_transaction() cursor1 = session1.open_cursor(self.uri) - for i in range(self.nrows, self.nrows + 1): + for i in range(self.nrows + 1, self.nrows + 2): cursor1.set_key(ds.key(i)) cursor1.set_value(valueb) self.assertEqual(cursor1.insert(), 0) self.session.begin_transaction() cursor = self.session.open_cursor(self.uri) - for i in range(0, 1): + for i in range(1, 2): cursor.set_key(ds.key(i)) cursor.set_value(valuec) self.assertEqual(cursor.update(), 0) diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot04.py b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot04.py index 0bc908c26fc..baa262d89d4 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot04.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint_snapshot04.py @@ -42,12 +42,18 @@ class test_checkpoint_snapshot04(backup_base): uri = "table:test_checkpoint_snapshot04" nrows = 5000 + format_values = [ + ('column_fix', dict(key_format='r', value_format='8t')), + ('column', dict(key_format='r', value_format='S')), + ('string_row', dict(key_format='S', value_format='S')), + ] + target_backup = [ ('full', dict(target=False)), ('target', dict(target=True)) ] - scenarios = make_scenarios(target_backup) + scenarios = make_scenarios(format_values, target_backup) def conn_config(self): config = 'cache_size=200MB' @@ -57,33 +63,49 @@ class test_checkpoint_snapshot04(backup_base): # Update a large number of records. session = self.session cursor = session.open_cursor(uri) - for i in range(0, nrows): + for i in range(1, nrows + 1): session.begin_transaction() cursor[ds.key(i)] = value session.commit_transaction() cursor.close() def check(self, check_value, uri, nrows): + # In FLCS the existence of the invisible extra row causes the table to extend + # under it. Until that's fixed, expect (not just allow) this row to exist and + # and demand it reads back as zero and not as check_value. When this behavior + # is fixed (so the end of the table updates transactionally) the special-case + # logic can just be removed. + flcs_tolerance = self.value_format == '8t' + session = self.session session.begin_transaction() cursor = session.open_cursor(uri) count = 0 for k, v in cursor: - self.assertEqual(v, check_value) + if flcs_tolerance and count >= nrows: + self.assertEqual(v, 0) + else: + self.assertEqual(v, check_value) count += 1 session.commit_transaction() - self.assertEqual(count, nrows) + self.assertEqual(count, nrows + 1 if flcs_tolerance else nrows) def test_checkpoint_snapshot(self): - ds = SimpleDataSet(self, self.uri, 0, key_format="S", value_format="S") + ds = SimpleDataSet(self, self.uri, 0, \ + key_format=self.key_format, value_format=self.value_format) ds.populate() - valuea = "aaaaa" * 100 - valueb = "bbbbb" * 100 + + if self.value_format == '8t': + valuea = 97 + valueb = 98 + else: + valuea = "aaaaa" * 100 + valueb = "bbbbb" * 100 session1 = self.conn.open_session() session1.begin_transaction() cursor1 = session1.open_cursor(self.uri) - for i in range(self.nrows, self.nrows + 1): + for i in range(self.nrows + 1, self.nrows + 2): cursor1.set_key(ds.key(i)) cursor1.set_value(valueb) self.assertEqual(cursor1.insert(), 0) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor02.py b/src/third_party/wiredtiger/test/suite/test_cursor02.py index 163f9c777b1..cdef24eba6d 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor02.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor02.py @@ -43,7 +43,7 @@ class test_cursor02(TestCursorTracker): ('row', dict(tablekind='row', uri='table')), ('lsm-row', dict(tablekind='row', uri='lsm')), ('col', dict(tablekind='col', uri='table')), - #('fix', dict(tablekind='fix')) + ('fix', dict(tablekind='fix', uri='table')) ]) def create_session_and_cursor(self, ninitialentries): diff --git a/src/third_party/wiredtiger/test/suite/test_cursor03.py b/src/third_party/wiredtiger/test/suite/test_cursor03.py index 96c9c785b63..eea01312e5d 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor03.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor03.py @@ -44,7 +44,7 @@ class test_cursor03(TestCursorTracker): ('row', dict(tablekind='row', keysize=None, valsize=None, uri='table')), ('lsm-row', dict(tablekind='row', keysize=None, valsize=None, uri='lsm')), ('col', dict(tablekind='col', keysize=None, valsize=None, uri='table')), - #('fix', dict(tablekind='fix', keysize=None, valsize=None)) + ('fix', dict(tablekind='fix', keysize=None, valsize=None, uri='table')), ('row.val10k', dict(tablekind='row', keysize=None, valsize=[10, 10000], uri='table')), ('col.val10k', dict(tablekind='col', keysize=None, valsize=[10, 10000], uri='table')), ('row.keyval10k', dict(tablekind='row', keysize=[10,10000], valsize=[10, 10000], uri='table')), diff --git a/src/third_party/wiredtiger/test/suite/test_cursor05.py b/src/third_party/wiredtiger/test/suite/test_cursor05.py index 1c4c7481afc..ebc95fb797f 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor05.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor05.py @@ -27,6 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest +from wtscenario import make_scenarios # test_cursor05.py # Test cursors at the point where a cursor is first initialized, and when it @@ -35,22 +36,56 @@ class test_cursor05(wttest.WiredTigerTestCase): """ Test basic operations """ - nentries = 2 + + type_values = [ + ('row', dict(usecolumns=False, usefixed=False)), + ('col', dict(usecolumns=True, usefixed=False)), + ('fix', dict(usecolumns=True, usefixed=True)), + ] + + nentries_values = [ + ('empty', dict(nentries=0)), + ('nonempty', dict(nentries=3)) + ] + + colgroups_values = [ + ('no_colgroups', dict(colgroups=None)), + ('two_colgroups', dict(colgroups=["(S1,i2)","(S3,i4)"])), + ('four_colgroups', dict(colgroups=["(S1)","(i2)","(S3)","(i4)"])), + ] + + # For fix, skip cases that won't use it. (The 8t column has to be standing alone.) + def checkfix(name, d): + if d['usefixed'] and (d['colgroups'] is None or len(d['colgroups']) < 4): + return False + return True + + scenarios = make_scenarios(type_values, nentries_values, colgroups_values, include=checkfix) + + def makekey(self, i): + if self.usecolumns: + return i + 1 + else: + return (i, 'key' + str(i)) def populate(self, count): """ Populate the given number of entries. """ cursor = self.session.open_cursor('table:main', None, None) for i in range(0, count): - cursor[(i, 'key' + str(i))] = ('val' + str(i), i, 'val' + str(i), i) + cursor[self.makekey(i)] = ('val' + str(i), i, 'val' + str(i), i) cursor.close() def check_iterate_forward(self, cursor, expectcount): """ Use the cursor to iterate and check for the expected entries. """ i = 0 - for ikey, skey, s1, i2, s3, i4 in cursor: - #print 'forward: ' + str([ikey, skey, s1, i2, s3, i4]) - self.assertEqual(ikey, i) - self.assertEqual(skey, 'key' + str(i)) + for row in cursor: + if self.usecolumns: + key, s1, i2, s3, i4 = row + else: + ikey, skey, s1, i2, s3, i4 = row + key = (ikey, skey) + #print 'forward: ' + str([key, s1, i2, s3, i4]) + self.assertEqual(key, self.makekey(i)) self.assertEqual(s1, 'val' + str(i)) self.assertEqual(i2, i) self.assertEqual(s3, 'val' + str(i)) @@ -63,11 +98,14 @@ class test_cursor05(wttest.WiredTigerTestCase): i = expectcount while cursor.prev() == 0: i -= 1 - (ikey, skey) = cursor.get_keys() + if self.usecolumns: + key = cursor.get_key() + else: + [ikey, skey] = cursor.get_keys() + key = (ikey, skey) (s1, i2, s3, i4) = cursor.get_values() - #print 'backward: ' + str([ikey, skey, s1, i2, s3, i4]) - self.assertEqual(ikey, i) - self.assertEqual(skey, 'key' + str(i)) + #print 'backward: ' + str([key, s1, i2, s3, i4]) + self.assertEqual(key, self.makekey(i)) self.assertEqual(s1, 'val' + str(i)) self.assertEqual(i2, i) self.assertEqual(s3, 'val' + str(i)) @@ -108,7 +146,7 @@ class test_cursor05(wttest.WiredTigerTestCase): # Do something that leaves the cursor in an uninitialized spot if expectcount > 0: n = expectcount - 1 - s1, i2, s3, i4 = cursor[(n, 'key' + str(n))] + s1, i2, s3, i4 = cursor[self.makekey(n)] self.assertEqual(s1, 'val' + str(n)) self.assertEqual(i2, n) self.assertEqual(s3, 'val' + str(n)) @@ -146,13 +184,34 @@ class test_cursor05(wttest.WiredTigerTestCase): cursor.close() - def common_test(self, nentries, hascolgroups): - cgstr = ',colgroups=(c1,c2)' if hascolgroups else '' - self.session.create('table:main', 'key_format=iS,value_format=SiSi,' - 'columns=(ikey,Skey,S1,i2,S3,i4)' + cgstr) - if hascolgroups: - self.session.create("colgroup:main:c1", "columns=(S1,i2)") - self.session.create("colgroup:main:c2", "columns=(S3,i4)") + def test_cursor(self): + usecolumns = self.usecolumns + usefixed = self.usefixed + nentries = self.nentries + colgroups = self.colgroups + + key_format = 'key_format={}'.format('r' if usecolumns else 'iS') + value_format = ',value_format={}'.format('S8tS8t' if usefixed else 'SiSi') + columns = ',columns=({},S1,i2,S3,i4)'.format('rkey' if usecolumns else 'ikey,Skey') + + if colgroups is None: + cgstr = '' + else: + cgstr = ',colgroups=(' + for i in range(1, len(colgroups) + 1): + if i > 1: + cgstr += ',' + cgstr += 'c{}'.format(i) + cgstr += ')' + + config = key_format + value_format + columns + cgstr + self.session.create('table:main', config) + + if colgroups is not None: + i = 1 + for cg in colgroups: + self.session.create("colgroup:main:c{}".format(i), "columns={}".format(cg)) + i += 1 self.populate(nentries) self.check_entries(0, nentries, True) self.check_entries(1, nentries, True) @@ -161,17 +220,5 @@ class test_cursor05(wttest.WiredTigerTestCase): self.check_entries(1, nentries, False) self.check_entries(2, nentries, False) - def test_without_colgroups(self): - self.common_test(3, False) - - def test_with_colgroups(self): - self.common_test(3, True) - - def test_empty_without_colgroups(self): - self.common_test(0, False) - - def test_empty_with_colgroups(self): - self.common_test(0, True) - if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor06.py b/src/third_party/wiredtiger/test/suite/test_cursor06.py index 54ec1519a86..1efa311ecd4 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor06.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor06.py @@ -39,21 +39,23 @@ from wtscenario import make_scenarios class test_cursor06(wttest.WiredTigerTestCase): name = 'reconfigure' scenarios = make_scenarios([ - ('file-r', dict(type='file:', keyfmt='r', dataset=SimpleDataSet)), - ('file-S', dict(type='file:', keyfmt='S', dataset=SimpleDataSet)), - ('lsm-S', dict(type='lsm:', keyfmt='S', dataset=SimpleDataSet)), - ('table-r', dict(type='table:', keyfmt='r', dataset=SimpleDataSet)), - ('table-S', dict(type='table:', keyfmt='S', dataset=SimpleDataSet)), - ('table-r-complex', dict(type='table:', keyfmt='r', + ('file-f', dict(type='file:', keyfmt='r', valfmt='8t', dataset=SimpleDataSet)), + ('file-r', dict(type='file:', keyfmt='r', valfmt='S', dataset=SimpleDataSet)), + ('file-S', dict(type='file:', keyfmt='S', valfmt='S', dataset=SimpleDataSet)), + ('lsm-S', dict(type='lsm:', keyfmt='S', valfmt='S', dataset=SimpleDataSet)), + ('table-f', dict(type='table:', keyfmt='r', valfmt='8t', dataset=SimpleDataSet)), + ('table-r', dict(type='table:', keyfmt='r', valfmt='S', dataset=SimpleDataSet)), + ('table-S', dict(type='table:', keyfmt='S', valfmt='S', dataset=SimpleDataSet)), + ('table-r-complex', dict(type='table:', keyfmt='r', valfmt=None, dataset=ComplexDataSet)), - ('table-S-complex', dict(type='table:', keyfmt='S', + ('table-S-complex', dict(type='table:', keyfmt='S', valfmt=None, dataset=ComplexDataSet)), - ('table-S-complex-lsm', dict(type='table:', keyfmt='S', + ('table-S-complex-lsm', dict(type='table:', keyfmt='S', valfmt=None, dataset=ComplexLSMDataSet)), ]) def populate(self, uri): - self.ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + self.ds = self.dataset(self, uri, 100, key_format=self.keyfmt, value_format=self.valfmt) self.ds.populate() def set_kv(self, cursor): diff --git a/src/third_party/wiredtiger/test/suite/test_cursor09.py b/src/third_party/wiredtiger/test/suite/test_cursor09.py index b2bfc55f16c..a7961b1ae82 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor09.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor09.py @@ -34,16 +34,18 @@ from wtscenario import make_scenarios # JIRA WT-2217: insert resets key/value "set". class test_cursor09(wttest.WiredTigerTestCase): scenarios = make_scenarios([ - ('file-r', dict(type='file:', keyfmt='r', dataset=SimpleDataSet)), - ('file-S', dict(type='file:', keyfmt='S', dataset=SimpleDataSet)), - ('lsm-S', dict(type='lsm:', keyfmt='S', dataset=SimpleDataSet)), - ('table-r', dict(type='table:', keyfmt='r', dataset=SimpleDataSet)), - ('table-S', dict(type='table:', keyfmt='S', dataset=SimpleDataSet)), - ('table-r-complex', dict(type='table:', keyfmt='r', + ('file-f', dict(type='file:', keyfmt='r', valfmt='8t', dataset=SimpleDataSet)), + ('file-r', dict(type='file:', keyfmt='r', valfmt='S', dataset=SimpleDataSet)), + ('file-S', dict(type='file:', keyfmt='S', valfmt='S', dataset=SimpleDataSet)), + ('lsm-S', dict(type='lsm:', keyfmt='S', valfmt='S', dataset=SimpleDataSet)), + ('table-f', dict(type='table:', keyfmt='r', valfmt='8t', dataset=SimpleDataSet)), + ('table-r', dict(type='table:', keyfmt='r', valfmt='S', dataset=SimpleDataSet)), + ('table-S', dict(type='table:', keyfmt='S', valfmt='S', dataset=SimpleDataSet)), + ('table-r-complex', dict(type='table:', keyfmt='r', valfmt=None, dataset=ComplexDataSet)), - ('table-S-complex', dict(type='table:', keyfmt='S', + ('table-S-complex', dict(type='table:', keyfmt='S', valfmt=None, dataset=ComplexDataSet)), - ('table-S-complex-lsm', dict(type='table:', keyfmt='S', + ('table-S-complex-lsm', dict(type='table:', keyfmt='S', valfmt=None, dataset=ComplexLSMDataSet)), ]) @@ -52,7 +54,7 @@ class test_cursor09(wttest.WiredTigerTestCase): def test_cursor09(self): uri = self.type + 'cursor09' - ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + ds = self.dataset(self, uri, 100, key_format=self.keyfmt, value_format=self.valfmt) ds.populate() cursor = self.session.open_cursor(uri, None, None) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor10.py b/src/third_party/wiredtiger/test/suite/test_cursor10.py index 5f62a84b93c..1b3cf5cc45d 100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor10.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor10.py @@ -33,7 +33,7 @@ from wtscenario import make_scenarios # Cursors with projections. class test_cursor10(wttest.WiredTigerTestCase): """ - Test cursor search and search_near + Test cursor search """ table_name1 = 'test_cursor10' nentries = 20 diff --git a/src/third_party/wiredtiger/test/suite/test_cursor11.py b/src/third_party/wiredtiger/test/suite/test_cursor11.py index 03d1e50f38e..67be17fda19 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor11.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor11.py @@ -37,9 +37,10 @@ from wtscenario import make_scenarios class test_cursor11(wttest.WiredTigerTestCase): keyfmt = [ - ('integer', dict(keyfmt='i')), - ('recno', dict(keyfmt='r')), - ('string', dict(keyfmt='S')), + ('integer', dict(keyfmt='i', valfmt='S')), + ('recno', dict(keyfmt='r', valfmt='S')), + ('recno-fix', dict(keyfmt='r', valfmt='8t')), + ('string', dict(keyfmt='S', valfmt='S')), ] types = [ ('file', dict(uri='file', ds=SimpleDataSet)), @@ -50,18 +51,22 @@ class test_cursor11(wttest.WiredTigerTestCase): ('table-simple', dict(uri='table', ds=SimpleDataSet)), ('table-simple-lsm', dict(uri='table', ds=SimpleLSMDataSet)), ] - scenarios = make_scenarios(types, keyfmt) - def skip(self): - return self.keyfmt == 'r' and \ - (self.ds.is_lsm() or self.uri == 'lsm') + # Discard invalid or unhelpful scenario combinations. + def keep(name, d): + if d['keyfmt'] == 'r' and (d['ds'].is_lsm() or d['uri'] == 'lsm'): + return False + if d['valfmt'] == '8t' and d['keyfmt'] != 'r': + return False + if d['valfmt'] == '8t' and d['ds'] == ComplexDataSet: + return False + return True + + scenarios = make_scenarios(types, keyfmt, include=keep) # Do a remove using the cursor after setting a position, and confirm # the key and position remain set but no value. def test_cursor_remove_with_position(self): - if self.skip(): - return - # Build an object. uri = self.uri + ':test_cursor11' ds = self.ds(self, uri, 50, key_format=self.keyfmt) @@ -84,9 +89,6 @@ class test_cursor11(wttest.WiredTigerTestCase): # Do a remove using the cursor without setting a position, and confirm # no key, value or position remains. def test_cursor_remove_without_position(self): - if self.skip(): - return - # Build an object. uri = self.uri + ':test_cursor11' ds = self.ds(self, uri, 50, key_format=self.keyfmt) @@ -108,9 +110,6 @@ class test_cursor11(wttest.WiredTigerTestCase): # Do a remove using the key after also setting a position, and confirm # no key, value or position remains. def test_cursor_remove_with_key_and_position(self): - if self.skip(): - return - # Build an object. uri = self.uri + ':test_cursor11' ds = self.ds(self, uri, 50, key_format=self.keyfmt) @@ -133,9 +132,6 @@ class test_cursor11(wttest.WiredTigerTestCase): # Do an insert and confirm no key, value or position remains. def test_cursor_insert(self): - if self.skip(): - return - # Build an object. uri = self.uri + ':test_cursor11' ds = self.ds(self, uri, 50, key_format=self.keyfmt) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor13.py b/src/third_party/wiredtiger/test/suite/test_cursor13.py index 75ac0b9a890..d9708d58fde 100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor13.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor13.py @@ -36,6 +36,10 @@ from helper import confirm_does_not_exist from suite_random import suite_random # Cursor caching tests +# +# This test uses only row-store (key_format='S') but the cursor-caching code has been reviewed +# for dependence on the access method and found to be access-method independent, so rearranging +# it to also test column-store is not necessary. class test_cursor13_base(wttest.WiredTigerTestCase): conn_config = 'statistics=(fast)' stat_cursor_cache = 0 diff --git a/src/third_party/wiredtiger/test/suite/test_cursor14.py b/src/third_party/wiredtiger/test/suite/test_cursor14.py index ceedd85a64f..ab746471da2 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor14.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor14.py @@ -34,23 +34,25 @@ from wtscenario import make_scenarios # Test that more than 64K cursors can be opened on a data source class test_cursor14(wttest.WiredTigerTestCase): scenarios = make_scenarios([ - ('file-r', dict(type='file:', keyfmt='r', dataset=SimpleDataSet)), - ('file-S', dict(type='file:', keyfmt='S', dataset=SimpleDataSet)), - ('lsm-S', dict(type='lsm:', keyfmt='S', dataset=SimpleDataSet)), - ('table-r', dict(type='table:', keyfmt='r', dataset=SimpleDataSet)), - ('table-S', dict(type='table:', keyfmt='S', dataset=SimpleDataSet)), - ('table-r-complex', dict(type='table:', keyfmt='r', + ('file-f', dict(type='file:', keyfmt='r', valfmt='8t', dataset=SimpleDataSet)), + ('file-r', dict(type='file:', keyfmt='r', valfmt='S', dataset=SimpleDataSet)), + ('file-S', dict(type='file:', keyfmt='S', valfmt='S', dataset=SimpleDataSet)), + ('lsm-S', dict(type='lsm:', keyfmt='S', valfmt='S', dataset=SimpleDataSet)), + ('table-f', dict(type='table:', keyfmt='r', valfmt='8t', dataset=SimpleDataSet)), + ('table-r', dict(type='table:', keyfmt='r', valfmt='S', dataset=SimpleDataSet)), + ('table-S', dict(type='table:', keyfmt='S', valfmt='S', dataset=SimpleDataSet)), + ('table-r-complex', dict(type='table:', keyfmt='r', valfmt=None, dataset=ComplexDataSet)), - ('table-S-complex', dict(type='table:', keyfmt='S', + ('table-S-complex', dict(type='table:', keyfmt='S', valfmt=None, dataset=ComplexDataSet)), - ('table-S-complex-lsm', dict(type='table:', keyfmt='S', + ('table-S-complex-lsm', dict(type='table:', keyfmt='S', valfmt=None, dataset=ComplexLSMDataSet)), ]) def test_cursor14(self): uri = self.type + 'cursor14' - ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + ds = self.dataset(self, uri, 100, key_format=self.keyfmt, value_format=self.valfmt) ds.populate() for i in range(66000): diff --git a/src/third_party/wiredtiger/test/suite/test_cursor16.py b/src/third_party/wiredtiger/test/suite/test_cursor16.py index 2954c8c0a76..ec08da4e7f2 100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor16.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor16.py @@ -32,6 +32,7 @@ import wttest from wiredtiger import stat +from wtscenario import make_scenarios class test_cursor16(wttest.WiredTigerTestCase): tablename = 'test_cursor16' @@ -41,6 +42,22 @@ class test_cursor16(wttest.WiredTigerTestCase): conn_config = 'cache_cursors=true,statistics=(fast),in_memory=true' + scenarios = make_scenarios([ + ('fix', dict(keyfmt='r',valfmt='8t')), + ('var', dict(keyfmt='r',valfmt='S')), + ('row', dict(keyfmt='S',valfmt='S')), + ]) + + def getkey(self, n): + if self.keyfmt == 'r': + return n + 1 + return str(n) + + def getval(self, n): + if self.valfmt == '8t': + return n + return str(n) + # Returns the number of cursors cached def cached_stats(self): stat_cursor = self.session.open_cursor('statistics:', None, None) @@ -55,14 +72,15 @@ class test_cursor16(wttest.WiredTigerTestCase): for i in range(0, self.uri_count): uri = self.uri_prefix + '-' + str(i) uris.append(uri) - self.session.create(uri, 'key_format=S,value_format=S') + config = 'key_format={},value_format={}'.format(self.keyfmt, self.valfmt) + self.session.create(uri, config) cursor = self.session.open_cursor(uri) # We keep the cursors open in the main session, so there # will always be a reference to their dhandle, and cached # cursors won't get swept. cursors.append(cursor) for j in range(0, 10): - cursor[str(j)] = str(j) + cursor[self.getkey(j)] = self.getval(j) self.assertEqual(0, self.cached_stats()) @@ -76,7 +94,7 @@ class test_cursor16(wttest.WiredTigerTestCase): for uri in uris: cursor = session.open_cursor(uri) # spot check, and leaves the cursor positioned - self.assertEqual(cursor['3'],'3') + self.assertEqual(cursor[self.getkey(3)], self.getval(3)) cursor.close() #self.tty('max cursors cached=' + str(self.cached_stats())) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_compare.py b/src/third_party/wiredtiger/test/suite/test_cursor_compare.py index 1c4e532903e..9c5d27d9eaa 100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor_compare.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_compare.py @@ -40,13 +40,28 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): ('table', dict(type='table:', lsm=False, dataset=ComplexDataSet)) ] keyfmt = [ - ('integer', dict(keyfmt='i')), - ('recno', dict(keyfmt='r')), - ('string', dict(keyfmt='S')) + ('integer', dict(keyfmt='i', valfmt='S')), + ('recno', dict(keyfmt='r', valfmt='S')), + ('recno-fix', dict(keyfmt='r', valfmt='8t')), + ('string', dict(keyfmt='S', valfmt='S')) ] - # Skip record number keys with LSM. - scenarios = filter_scenarios(make_scenarios(types, keyfmt), - lambda name, d: not (d['lsm'] and d['keyfmt'] == 'r')) + + # Discard invalid or unhelpful scenario combinations. + def keep(name, d): + if d['keyfmt'] == 'r': + # Skip record number keys with LSM. + if d['lsm']: + return False + # Skip complex data sets with FLCS. + if d['valfmt'] == '8t' and d['dataset'] != SimpleDataSet: + return False + else: + # Skip byte data with row-store. + if d['valfmt'] == '8t': + return False + return True + + scenarios = make_scenarios(types, keyfmt, include=keep) def test_cursor_comparison(self): uri = self.type + 'compare' diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_pin.py b/src/third_party/wiredtiger/test/suite/test_cursor_pin.py index c9b8d0f74b3..7fc2f0c72d2 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_pin.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_pin.py @@ -38,15 +38,16 @@ class test_cursor_pin(wttest.WiredTigerTestCase): nentries = 10000 config = 'allocation_size=512,leaf_page_max=512' scenarios = make_scenarios([ - ('recno', dict(keyfmt='r')), - ('string', dict(keyfmt='S')), + ('recno-fix', dict(keyfmt='r', valfmt='8t')), + ('recno', dict(keyfmt='r', valfmt='S')), + ('string', dict(keyfmt='S', valfmt='S')), ]) # Create a multi-page file, confirm that a simple search to the local # page works, followed by a search to a different page. def test_smoke(self): ds = SimpleDataSet(self, self.uri, self.nentries, - config=self.config, key_format=self.keyfmt) + config=self.config, key_format=self.keyfmt, value_format=self.valfmt) ds.populate() self.reopen_conn() c = self.session.open_cursor(self.uri, None) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_random.py b/src/third_party/wiredtiger/test/suite/test_cursor_random.py index 896001506b4..3b64532c036 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_random.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_random.py @@ -188,13 +188,18 @@ class test_cursor_random(wttest.WiredTigerTestCase): # Check that opening a random cursor on column-store returns not-supported. class test_cursor_random_column(wttest.WiredTigerTestCase): - scenarios = make_scenarios([ + type_values = [ ('file', dict(uri='file:random')), ('table', dict(uri='table:random')) - ]) + ] + valfmt_values = [ + ('string', dict(valfmt='S')), + ('fix', dict(valfmt='8t')), + ] + scenarios = make_scenarios(type_values, valfmt_values) def test_cursor_random_column(self): - self.session.create(self.uri, 'key_format=r,value_format=S') + self.session.create(self.uri, 'key_format=r,value_format={}'.format(self.valfmt)) msg = '/next_random .* not supported/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.open_cursor(self.uri, None, "next_random=true"), msg) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_tracker.py b/src/third_party/wiredtiger/test/suite/test_cursor_tracker.py index 5536afafb3c..7a94baf89aa 100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor_tracker.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_tracker.py @@ -79,6 +79,7 @@ import wiredtiger, wttest class TestCursorTracker(wttest.WiredTigerTestCase): table_name1 = 'test_cursor' DELETED = 0xffffffffffffffff + DELETED_VERSION = 0xffff TRACE_API = False # a print output for each WT API call def config_string(self): @@ -142,6 +143,7 @@ class TestCursorTracker(wttest.WiredTigerTestCase): raise Exception('cur_initial_conditions: npairs too big') self.tablekind = tablekind self.isrow = (tablekind == 'row') + self.isfix = (tablekind == 'fix') self.setup_encoders_decoders() self.bitlist = [(x << 32) for x in range(npairs)] self.vers = dict((x << 32, 0) for x in range(npairs)) @@ -245,10 +247,11 @@ class TestCursorTracker(wttest.WiredTigerTestCase): return ((maj << 32) | (min << 16)) def encode_value_fix(self, bits): + [maj, min, ver] = self.bits_to_triple(bits) + if ver == self.DELETED_VERSION: + return 0 # can only encode only 8 bits - maj = ((bits >> 32) & 0xff) - min = (bits >> 16) & 0xff - return (maj ^ min) + return (maj ^ min) % 256 def decode_value_fix(self, s): return int(s) @@ -256,7 +259,7 @@ class TestCursorTracker(wttest.WiredTigerTestCase): def setpos(self, newpos, isforward): length = len(self.bitlist) while newpos >= 0 and newpos < length: - if not self.isrow and self.bitlist[newpos] == self.DELETED: + if not self.isrow and not self.isfix and self.bitlist[newpos] == self.DELETED: if isforward: newpos = newpos + 1 else: @@ -326,6 +329,9 @@ class TestCursorTracker(wttest.WiredTigerTestCase): self.bitlist.pop(self.curpos) self.setpos(self.curpos - 1, True) self.nopos = True + elif self.isfix: + [major, minor, version] = self.bits_to_triple(self.bitlist[self.curpos]) + self.bitlist[self.curpos] = self.triple_to_bits(major, minor, self.DELETED_VERSION) else: self.bitlist[self.curpos] = self.DELETED self.curremoved = True @@ -442,7 +448,7 @@ class TestCursorTracker(wttest.WiredTigerTestCase): return str(self.bits_to_triple(self.decode_key(k))) + ' = ' + str(k) def _cursor_value_to_string(self, v): - return str(self.bits_to_triple(self.decode_value(v))) + ' = ' + v + return str(self.bits_to_triple(self.decode_value(v))) + ' = ' + str(v) def _dumpcursor(self, cursor): print('cursor') diff --git a/src/third_party/wiredtiger/test/suite/test_dupc.py b/src/third_party/wiredtiger/test/suite/test_dupc.py index 81d7f2dc868..68265518c38 100644 --- a/src/third_party/wiredtiger/test/suite/test_dupc.py +++ b/src/third_party/wiredtiger/test/suite/test_dupc.py @@ -41,10 +41,12 @@ class test_duplicate_cursor(wttest.WiredTigerTestCase): nentries = 1000 scenarios = make_scenarios([ - ('file-r', dict(uri='file:', fmt='r')), - ('file-S', dict(uri='file:', fmt='S')), - ('table-r', dict(uri='table:', fmt='r')), - ('table-S', dict(uri='table:', fmt='S')) + ('file-f', dict(uri='file:', keyfmt='r', valfmt='8t')), + ('file-r', dict(uri='file:', keyfmt='r', valfmt='S')), + ('file-S', dict(uri='file:', keyfmt='S', valfmt='S')), + ('table-f', dict(uri='table:', keyfmt='r', valfmt='8t')), + ('table-r', dict(uri='table:', keyfmt='r', valfmt='S')), + ('table-S', dict(uri='table:', keyfmt='S', valfmt='S')) ]) # Iterate through an object, duplicate the cursor and checking that it @@ -71,14 +73,16 @@ class test_duplicate_cursor(wttest.WiredTigerTestCase): uri = self.uri + self.name # A simple, one-file file or table object. - ds = SimpleDataSet(self, uri, self.nentries, key_format=self.fmt) + ds = SimpleDataSet(self, uri, self.nentries, \ + key_format=self.keyfmt, value_format=self.valfmt) ds.populate() self.iterate(uri, ds) self.session.drop(uri, None) # A complex, multi-file table object. - if self.uri == "table:": - ds = ComplexDataSet(self, uri, self.nentries, key_format=self.fmt) + if self.uri == "table:" and self.valfmt != '8t': + ds = ComplexDataSet(self, uri, self.nentries, \ + key_format=self.keyfmt, value_format=self.valfmt) ds.populate() self.iterate(uri, ds) self.session.drop(uri, None) diff --git a/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py b/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py index dfad0d8869d..a07199cc717 100644 --- a/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_rollback_to_stable.py @@ -38,10 +38,11 @@ from wtscenario import make_scenarios class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' - keyfmt = [ - ('row-string', dict(keyfmt='S')), - ('row-int', dict(keyfmt='i')), - ('column-store', dict(keyfmt='r')), + format_values = [ + ('row-string', dict(keyfmt='S', valfmt='S')), + ('row-int', dict(keyfmt='i', valfmt='S')), + ('column', dict(keyfmt='r', valfmt='S')), + ('column-fix', dict(keyfmt='r', valfmt='8t')), ] types = [ ('file', dict(uri='file', ds=SimpleDataSet)), @@ -54,20 +55,18 @@ class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subproces ('isolation_default', dict(isolation='')), ('isolation_snapshot', dict(isolation='snapshot')) ] - scenarios = make_scenarios(types, keyfmt, iso_types) - def skip(self): - return self.keyfmt == 'r' and \ - (self.ds.is_lsm() or self.uri == 'lsm') + def keep(name, d): + return d['keyfmt'] != 'r' or (d['uri'] != 'lsm' and not d['ds'].is_lsm()) + + scenarios = make_scenarios(types, format_values, iso_types, include=keep) # Test durable timestamp. def test_durable_rollback_to_stable(self): - if self.skip(): - return # Build an object. uri = self.uri + ':test_durable_rollback_to_stable' - ds = self.ds(self, uri, 50, key_format=self.keyfmt) + ds = self.ds(self, uri, 50, key_format=self.keyfmt, value_format=self.valfmt) ds.populate() session = self.conn.open_session(self.session_config) diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts01.py b/src/third_party/wiredtiger/test/suite/test_durable_ts01.py index cd20fbf6b35..cea0ae3196e 100644 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts01.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts01.py @@ -37,10 +37,11 @@ from wtscenario import make_scenarios class test_durable_ts01(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' - keyfmt = [ - ('row-string', dict(keyfmt='S')), - ('row-int', dict(keyfmt='i')), - ('column-store', dict(keyfmt='r')), + format_values = [ + ('row-string', dict(keyfmt='S', valfmt='S')), + ('row-int', dict(keyfmt='i', valfmt='S')), + ('column', dict(keyfmt='r', valfmt='S')), + ('column-fix', dict(keyfmt='r', valfmt='8t')), ] types = [ ('file', dict(uri='file', ds=SimpleDataSet)), @@ -53,20 +54,18 @@ class test_durable_ts01(wttest.WiredTigerTestCase): ('isolation_default', dict(isolation='')), ('isolation_snapshot', dict(isolation='snapshot')) ] - scenarios = make_scenarios(types, keyfmt, iso_types) - def skip(self): - return self.keyfmt == 'r' and \ - (self.ds.is_lsm() or self.uri == 'lsm') + def keep(name, d): + return d['keyfmt'] != 'r' or (d['uri'] != 'lsm' and not d['ds'].is_lsm()) + + scenarios = make_scenarios(types, format_values, iso_types, include=keep) # Test durable timestamp. def test_durable_ts01(self): - if self.skip(): - return # Build an object. uri = self.uri + ':test_durable_ts01' - ds = self.ds(self, uri, 50, key_format=self.keyfmt) + ds = self.ds(self, uri, 50, key_format=self.keyfmt, value_format=self.valfmt) ds.populate() session = self.conn.open_session(self.session_config) diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts02.py b/src/third_party/wiredtiger/test/suite/test_durable_ts02.py index 88dea1291a2..3a5f8aabaef 100644 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts02.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts02.py @@ -36,10 +36,11 @@ from wtscenario import make_scenarios class test_durable_ts03(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' - keyfmt = [ - ('row-string', dict(keyfmt='S')), - ('row-int', dict(keyfmt='i')), - ('column-store', dict(keyfmt='r')), + format_values = [ + ('row-string', dict(keyfmt='S', valfmt='S')), + ('row-int', dict(keyfmt='i', valfmt='S')), + ('column', dict(keyfmt='r', valfmt='S')), + ('column-fix', dict(keyfmt='r', valfmt='8t')), ] types = [ ('file', dict(uri='file', ds=SimpleDataSet)), @@ -52,20 +53,18 @@ class test_durable_ts03(wttest.WiredTigerTestCase): ('isolation_default', dict(isolation='')), ('isolation_snapshot', dict(isolation='snapshot')) ] - scenarios = make_scenarios(types, keyfmt, iso_types) - def skip(self): - return self.keyfmt == 'r' and \ - (self.ds.is_lsm() or self.uri == 'lsm') + def keep(name, d): + return d['keyfmt'] != 'r' or (d['uri'] != 'lsm' and not d['ds'].is_lsm()) + + scenarios = make_scenarios(types, format_values, iso_types, include=keep) # Test durable timestamp. def test_durable_ts03(self): - if self.skip(): - return # Build an object. uri = self.uri + ':test_durable_ts03' - ds = self.ds(self, uri, 50, key_format=self.keyfmt) + ds = self.ds(self, uri, 50, key_format=self.keyfmt, value_format=self.valfmt) ds.populate() session = self.conn.open_session(self.session_config) diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py index ea3fe771169..4f0d66ab8ae 100755 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py @@ -36,20 +36,26 @@ class test_durable_ts03(wttest.WiredTigerTestCase): conn_config = 'cache_size=10MB' session_config = 'isolation=snapshot' - key_format_values = [ - ('integer-row', dict(key_format='i')), - ('column', dict(key_format='r')), + format_values = [ + ('integer-row', dict(key_format='i', value_format='u')), + ('column', dict(key_format='r', value_format='u')), + ('column-fix', dict(key_format='r', value_format='8t')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test_durable_ts03(self): # Create a table. uri = 'table:test_durable_ts03' nrows = 3000 - self.session.create(uri, 'key_format={},value_format=u'.format(self.key_format)) - valueA = b"aaaaa" * 100 - valueB = b"bbbbb" * 100 - valueC = b"ccccc" * 100 + self.session.create(uri, 'key_format={},value_format={}'.format(self.key_format, self.value_format)) + if self.value_format == '8t': + valueA = 97 + valueB = 98 + valueC = 99 + else: + valueA = b"aaaaa" * 100 + valueB = b"bbbbb" * 100 + valueC = b"ccccc" * 100 # Start with setting a stable and oldest timestamp. self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1) + \ diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt07.py b/src/third_party/wiredtiger/test/suite/test_encrypt07.py index 40da03637be..d4ca530eead 100755 --- a/src/third_party/wiredtiger/test/suite/test_encrypt07.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt07.py @@ -32,6 +32,8 @@ import os, run, string, codecs import wiredtiger, wttest + +# If removing this, update test_salvage to not reference here. import test_salvage # Run the regular salvage test, but with encryption on @@ -56,12 +58,17 @@ class test_encrypt07(test_salvage.test_salvage): def rot13(self, s): return codecs.encode(s, 'rot_13') - # overrides test_salvage.damage. + # Supplement test_salvage.moreinit. # When we're looking in the file for our 'unique' set of bytes, # (to find a physical spot to damage) we'll need to search for # the rot13 encrypted string. - def damage(self, tablename): - self.damage_inner(tablename, self.rot13(self.unique).encode()) + def moreinit(self): + super().moreinit() + self.uniquebytes = self.rot13(self.uniquebytes.decode()).encode() + + # overrides test_salvage.damage. + #def damage(self, tablename): + # self.damage_inner(tablename, self.rot13(self.unique).encode()) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_flcs01.py b/src/third_party/wiredtiger/test/suite/test_flcs01.py new file mode 100644 index 00000000000..9969f70fd0f --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_flcs01.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_flcs01.py +# +# Test various cases of deleting values and expecting them to read back as 0. +# +# FUTURE: it would be nice to pin the page to prevent reconciliation until we +# evict it explicitly, to make sure that the first section of the test exercises +# in-memory update records. (Testing on an in-memory database does not have that +# effect.) +class test_flcs01(wttest.WiredTigerTestCase): + session_config = 'isolation=snapshot' + conn_config = 'in_memory=false' + + # Evict the page to force reconciliation. + def evict(self, uri, key, check_value): + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + self.session.begin_transaction() + v = evict_cursor[key] + self.assertEqual(v, check_value) + self.assertEqual(evict_cursor.reset(), 0) + self.session.rollback_transaction() + evict_cursor.close() + + def check_next(self, cursor, k, expected_val): + cursor.set_key(k - 1) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.next(), 0) + self.assertEqual(cursor.get_key(), k) + self.assertEqual(cursor.get_value(), expected_val) + cursor.reset() + + def check_prev(self, cursor, k, expected_val): + cursor.set_key(k + 1) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.prev(), 0) + self.assertEqual(cursor.get_key(), k) + self.assertEqual(cursor.get_value(), expected_val) + cursor.reset() + + def check_prev_fail(self, cursor, k): + cursor.set_key(k + 1) + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + cursor.reset() + + # Delete a value and read it back in the same transaction. + def delete_readback_abort(self, cursor, k): + self.session.begin_transaction() + cursor.set_key(k) + self.assertEqual(cursor.remove(), 0) + cursor.reset() + v = cursor[k] + self.assertEqual(v, 0) + cursor.reset() + self.check_next(cursor, k, 0) + self.check_prev(cursor, k, 0) + self.session.rollback_transaction() + self.session.begin_transaction() + v = cursor[k] + self.assertEqual(v, k) + cursor.reset() + self.session.rollback_transaction() + + # Delete a value and read it back from a different transaction. + def delete_readback_commit(self, cursor, k): + self.session.begin_transaction() + cursor.set_key(k) + self.assertEqual(cursor.remove(), 0) + cursor.reset() + self.session.commit_transaction() + + self.session.begin_transaction() + v = cursor[k] + self.assertEqual(v, 0) + cursor.reset() + self.check_next(cursor, k, 0) + self.check_prev(cursor, k, 0) + self.session.rollback_transaction() + + def test_flcs(self): + uri = "table:test_flcs01" + nrows = 44 + ds = SimpleDataSet( + self, uri, nrows, key_format='r', value_format='6t', config='leaf_page_max=4096') + ds.populate() + + updatekey1 = 33 + updatekey2 = 37 + updatekey3 = 21 + updatekey4 = 11 + appendkey1 = nrows + 10 + appendkey2 = nrows + 17 + + # Write a few records. + cursor = self.session.open_cursor(uri) + self.session.begin_transaction() + for i in range(1, nrows + 1): + cursor[i] = i + self.session.commit_transaction() + + # Delete one of the values and read it back in the same transaction. + self.delete_readback_abort(cursor, updatekey1) + + # Append a value, delete it, and read it back. + self.session.begin_transaction() + cursor.set_key(appendkey1) + cursor.set_value(appendkey1) + self.assertEqual(cursor.update(), 0) + cursor.reset() + v = cursor[appendkey1] + self.assertEqual(v, appendkey1) + cursor.reset() + cursor.set_key(appendkey1) + self.assertEqual(cursor.remove(), 0) + cursor.reset() + v = cursor[appendkey1] + self.assertEqual(v, 0) + cursor.reset() + self.check_next(cursor, appendkey1, 0) + self.check_prev_fail(cursor, appendkey1) + self.session.rollback_transaction() + + # Doing that might or might not have extended the table. Accept either behavior, + # at least for now. + self.session.begin_transaction() + cursor.set_key(appendkey1) + result = cursor.search() + if result != wiredtiger.WT_NOTFOUND: + self.assertEqual(result, 0) + v = cursor.get_value() + self.assertEqual(v, 0) + self.session.rollback_transaction() + + # Now, delete one of the values and read it back from a different transaction. + self.delete_readback_commit(cursor, updatekey2) + + # Do the same with an append. + self.session.begin_transaction() + cursor.set_key(appendkey2) + cursor.set_value(appendkey2) + self.assertEqual(cursor.update(), 0) + cursor.reset() + v = cursor[appendkey2] + self.assertEqual(v, appendkey2) + cursor.reset() + cursor.set_key(appendkey2) + self.assertEqual(cursor.remove(), 0) + cursor.reset() + self.session.commit_transaction() + + # This should definitely have extended the table. + self.session.begin_transaction() + v = cursor[appendkey2] + self.assertEqual(v, 0) + cursor.reset() + self.check_next(cursor, appendkey2, 0) + self.check_prev_fail(cursor, appendkey2) + self.session.rollback_transaction() + + # Evict the page to force reconciliation. + self.evict(uri, 1, 1) + + # The committed zeros should still be there. + self.session.begin_transaction() + v = cursor[updatekey2] + self.assertEqual(v, 0) + cursor.reset() + self.check_next(cursor, updatekey2, 0) + self.check_prev(cursor, updatekey2, 0) + self.session.rollback_transaction() + + self.session.begin_transaction() + v = cursor[appendkey2] + self.assertEqual(v, 0) + cursor.reset() + self.check_next(cursor, appendkey2, 0) + self.check_prev_fail(cursor, appendkey2) + self.session.rollback_transaction() + + # Now try both on-page deletes again. + self.delete_readback_abort(cursor, updatekey3) + self.delete_readback_commit(cursor, updatekey4) diff --git a/src/third_party/wiredtiger/test/suite/test_flcs02.py b/src/third_party/wiredtiger/test/suite/test_flcs02.py new file mode 100644 index 00000000000..e320b3f3500 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_flcs02.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_flcs01.py +# +# Test various cases of deleting values and expecting them to read back as 0, +# in the presence of timestamps and history. +# +# FUTURE: it would be nice to pin the page to prevent reconciliation until we +# evict it explicitly, to make sure that the first section of the test exercises +# in-memory update records. (Testing on an in-memory database does not have that +# effect.) +class test_flcs01(wttest.WiredTigerTestCase): + session_config = 'isolation=snapshot' + conn_config = 'in_memory=false' + + prepare_values = [ + ('no_prepare', dict(do_prepare=False)), + ('prepare', dict(do_prepare=True)) + ] + + scenarios = make_scenarios(prepare_values) + + # Evict the page to force reconciliation. + def evict(self, uri, key, check_value): + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + self.session.begin_transaction() + v = evict_cursor[key] + self.assertEqual(v, check_value) + self.assertEqual(evict_cursor.reset(), 0) + self.session.rollback_transaction() + evict_cursor.close() + + def check_next(self, cursor, k, expected_val): + cursor.set_key(k - 1) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.next(), 0) + self.assertEqual(cursor.get_key(), k) + self.assertEqual(cursor.get_value(), expected_val) + cursor.reset() + + def check_prev(self, cursor, k, expected_val): + cursor.set_key(k + 1) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.prev(), 0) + self.assertEqual(cursor.get_key(), k) + self.assertEqual(cursor.get_value(), expected_val) + cursor.reset() + + def check_prev_fail(self, cursor, k): + cursor.set_key(k + 1) + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + cursor.reset() + + # Delete a value and read it back in the same transaction. + def delete_readback_abort(self, cursor, k, readts): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(readts)) + cursor.set_key(k) + self.assertEqual(cursor.remove(), 0) + cursor.reset() + v = cursor[k] + self.assertEqual(v, 0) + cursor.reset() + self.check_next(cursor, k, 0) + self.check_prev(cursor, k, 0) + self.session.rollback_transaction() + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(readts - 5)) + v = cursor[k] + self.assertEqual(v, k) + cursor.reset() + self.session.rollback_transaction() + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(readts)) + v = cursor[k] + self.assertEqual(v, k) + cursor.reset() + self.session.rollback_transaction() + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(readts + 5)) + v = cursor[k] + self.assertEqual(v, k) + cursor.reset() + self.session.rollback_transaction() + + # Delete a value and read it back from a different transaction. + def delete_readback_commit(self, cursor, k, readts, committs): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(readts)) + cursor.set_key(k) + self.assertEqual(cursor.remove(), 0) + cursor.reset() + if self.do_prepare: + preparets = committs - 1 + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(preparets)) + durable = ',durable_timestamp=' + self.timestamp_str(committs + 1) + else: + durable = '' + commit = 'commit_timestamp=' + self.timestamp_str(committs) + durable + self.session.commit_transaction(commit) + + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(committs - 5)) + v = cursor[k] + self.assertEqual(v, k) + cursor.reset() + self.session.rollback_transaction() + + if self.do_prepare: + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(preparets)) + v = cursor[k] + self.assertEqual(v, k) + cursor.reset() + self.session.rollback_transaction() + + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(committs)) + v = cursor[k] + self.assertEqual(v, 0) + cursor.reset() + self.check_next(cursor, k, 0) + self.check_prev(cursor, k, 0) + self.session.rollback_transaction() + + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(committs + 5)) + v = cursor[k] + self.assertEqual(v, 0) + cursor.reset() + self.check_next(cursor, k, 0) + self.check_prev(cursor, k, 0) + self.session.rollback_transaction() + + def test_flcs(self): + uri = "table:test_flcs02" + nrows = 44 + ds = SimpleDataSet( + self, uri, 0, key_format='r', value_format='6t', config='leaf_page_max=4096') + ds.populate() + + updatekey1 = 33 + updatekey2 = 37 + updatekey3 = 21 + updatekey4 = 11 + appendkey1 = nrows + 10 + appendkey2 = nrows + 17 + + # Pin oldest and stable to timestamp 1. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) + + # Write a few records at time 10. + cursor = self.session.open_cursor(uri) + self.session.begin_transaction() + for i in range(1, nrows + 1): + cursor[i] = i + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) + + # Doing that might or might not have extended the table before time 10. + # Accept either behavior, at least for now. + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) + cursor.set_key(1) + result = cursor.search() + if result != wiredtiger.WT_NOTFOUND: + self.assertEqual(result, 0) + v = cursor.get_value() + self.assertEqual(v, 0) + self.session.rollback_transaction() + + # Delete one of the values and read it back in the same transaction, at time 20. + self.delete_readback_abort(cursor, updatekey1, 20) + + # Append a value, delete it, and read it back, at time 20. + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) + cursor.set_key(appendkey1) + cursor.set_value(appendkey1) + self.assertEqual(cursor.update(), 0) + cursor.reset() + v = cursor[appendkey1] + self.assertEqual(v, appendkey1) + cursor.reset() + cursor.set_key(appendkey1) + self.assertEqual(cursor.remove(), 0) + cursor.reset() + v = cursor[appendkey1] + self.assertEqual(v, 0) + cursor.reset() + self.check_next(cursor, appendkey1, 0) + self.check_prev_fail(cursor, appendkey1) + self.session.rollback_transaction() + + # Doing that might or might not have extended the table, including in the past. + # Accept either behavior, at least for now. + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(10)) + cursor.set_key(appendkey1) + result = cursor.search() + if result != wiredtiger.WT_NOTFOUND: + self.assertEqual(result, 0) + v = cursor.get_value() + self.assertEqual(v, 0) + self.session.rollback_transaction() + + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) + cursor.set_key(appendkey1) + result = cursor.search() + if result != wiredtiger.WT_NOTFOUND: + self.assertEqual(result, 0) + v = cursor.get_value() + self.assertEqual(v, 0) + self.session.rollback_transaction() + + # Now, delete one of the values and read it back from a different transaction. + self.delete_readback_commit(cursor, updatekey2, 20, 30) + + # Do the same with an append. + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(40)) + cursor.set_key(appendkey2) + cursor.set_value(appendkey2) + self.assertEqual(cursor.update(), 0) + cursor.reset() + v = cursor[appendkey2] + self.assertEqual(v, appendkey2) + cursor.reset() + cursor.set_key(appendkey2) + self.assertEqual(cursor.remove(), 0) + cursor.reset() + if self.do_prepare: + self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(49)) + durable = ',durable_timestamp=' + self.timestamp_str(51) + else: + durable = '' + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(50) + durable) + + # This might have extended the table in the past. + # Accept either behavior, at least for now. + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) + cursor.set_key(appendkey2) + result = cursor.search() + if result != wiredtiger.WT_NOTFOUND: + self.assertEqual(result, 0) + v = cursor.get_value() + self.assertEqual(v, 0) + self.session.rollback_transaction() + + # This should definitely have extended the table in the present. + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(50)) + v = cursor[appendkey2] + self.assertEqual(v, 0) + cursor.reset() + self.check_next(cursor, appendkey2, 0) + self.check_prev_fail(cursor, appendkey2) + self.session.rollback_transaction() + + # Evict the page to force reconciliation. + self.evict(uri, 1, 1) + + # The committed zeros should still be there. + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(50)) + v = cursor[updatekey2] + self.assertEqual(v, 0) + cursor.reset() + self.check_next(cursor, updatekey2, 0) + self.check_prev(cursor, updatekey2, 0) + self.session.rollback_transaction() + + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(50)) + v = cursor[appendkey2] + self.assertEqual(v, 0) + cursor.reset() + self.check_next(cursor, appendkey2, 0) + self.check_prev_fail(cursor, appendkey2) + self.session.rollback_transaction() + + # Now try both on-page deletes again. + self.delete_readback_abort(cursor, updatekey3, 60) + self.delete_readback_commit(cursor, updatekey4, 60, 70) diff --git a/src/third_party/wiredtiger/test/suite/test_flcs03.py b/src/third_party/wiredtiger/test/suite/test_flcs03.py new file mode 100644 index 00000000000..9ace77ddbf6 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_flcs03.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet +from wtscenario import make_scenarios + +# test_flcs03.py +# +# Test various cases associated with the location of the end of the table. +# +# FUTURE: it would be nice to pin the page to prevent reconciliation in various +# places, to make sure that we're testing on in-memory update records when we intend +# to be. (We do test on an in-memory database, but that isn't sufficient by itself; +# pages are still reconciled and that can eliminate the update configuration we're +# trying to test. +class test_flcs03(wttest.WiredTigerTestCase): + session_config = 'isolation=snapshot' + + in_memory_values = [ + ('no_inmem', dict(in_memory=False)), + ('inmem', dict(in_memory=True)) + ] + + # Test scenarios (all appends, obviously): + # - pending uncommitted updates, before or after reconciliation + # - prepared updates, before or after reconciliation + # - committed updates that are deletes, before or after reconciliation + # - aborted updates, right away while the update records still exist + # - aborted updates, if reconciled right away while the update records still exist + # - aborted updates after the update records have been G/C'd + # - aborted updates, if reconciled after the update records have been G/C'd + # (this last is not that interesting but it follows naturally from the scenario generation) + # + # The case of reading at a time before some committed updates is tested + # on every run using the initial update set. + operation_values = [ + ('uncommitted', dict(op='uncommitted', prepare=False, gc=False)), + ('prepared', dict(op='uncommitted', prepare=True, gc=False)), + ('committed_deletes', dict(op='deleted', prepare=False, gc=False)), + ('committed_prepared_deletes', dict(op='deleted', prepare=True, gc=False)), + ('aborted', dict(op='aborted', prepare=False, gc=False)), + ('aborted_GC', dict(op='aborted', prepare=False, gc=True)), + ] + + reconcile_values = [ + ('no_reconcile', dict(reconcile=False)), + ('reconcile', dict(reconcile=True)), + ] + + scenarios = make_scenarios(in_memory_values, operation_values, reconcile_values) + + def conn_config(self): + if self.in_memory: + return 'in_memory=true' + else: + return 'in_memory=false' + + # Evict the page to force reconciliation. + def evict(self, uri, key, check_value): + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + self.session.begin_transaction() + v = evict_cursor[key] + self.assertEqual(v, check_value) + self.assertEqual(evict_cursor.reset(), 0) + self.session.rollback_transaction() + evict_cursor.close() + + def check_end(self, cursor, hint, ts, expected_last_key): + readtime = 'read_timestamp=' + self.timestamp_str(ts) + self.session.begin_transaction(readtime + ',ignore_prepare=true') + cursor.reset() + if hint is not None: + cursor.set_key(hint) + self.assertEqual(cursor.search(), 0) + last = None + while cursor.next() != wiredtiger.WT_NOTFOUND: + last = cursor.get_key() + self.assertEqual(last, expected_last_key) + cursor.reset() + self.assertEqual(cursor.prev(), 0) + last = cursor.get_key() + self.assertEqual(last, expected_last_key) + self.session.rollback_transaction() + + def check_end_empty(self, cursor, ts): + self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts)) + cursor.reset() + self.assertEqual(cursor.next(), wiredtiger.WT_NOTFOUND) + cursor.reset() + self.assertEqual(cursor.prev(), wiredtiger.WT_NOTFOUND) + self.session.rollback_transaction() + + def test_flcs(self): + uri = "table:test_flcs03" + nrows = 10 + ds = SimpleDataSet( + self, uri, 0, key_format='r', value_format='6t', config='leaf_page_max=4096') + ds.populate() + + # Pin oldest and stable to timestamp 1. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + + ',stable_timestamp=' + self.timestamp_str(1)) + + cursor = self.session.open_cursor(uri) + + # The table should end without any keys. + self.check_end_empty(cursor, 5) + + # Write a few records at time 10. + self.session.begin_transaction() + for i in range(1, nrows + 1): + cursor[i] = i + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) + + # Current behavior: this extends the table before the commit as well as at/after. + self.check_end(cursor, None, 5, nrows) + self.check_end(cursor, None, 10, nrows) + self.check_end(cursor, None, 11, nrows) + + # Make another session to do the operation in. + session2 = self.conn.open_session() + cursor2 = session2.open_cursor(uri) + + append_key = nrows * 2 + append_key_lite = nrows + 2 + + # Do the requested operation. + if self.op == 'uncommitted': + session2.begin_transaction('read_timestamp=' + self.timestamp_str(15)) + cursor2[append_key] = 20 + elif self.op == 'deleted': + session2.begin_transaction('read_timestamp=' + self.timestamp_str(15)) + cursor2[append_key] = 21 + cursor2.set_key(append_key) + self.assertEqual(cursor2.remove(), 0) + else: + self.assertEqual(self.op, 'aborted') + session2.begin_transaction('read_timestamp=' + self.timestamp_str(15)) + cursor2[append_key] = 22 + session2.rollback_transaction() + + if self.prepare: + session2.prepare_transaction('prepare_timestamp=' + self.timestamp_str(19)) + + if self.op == 'deleted': + committime = 'commit_timestamp=' + self.timestamp_str(20) + if self.prepare: + durabletime = ',durable_timestamp=' + self.timestamp_str(21) + else: + durabletime = '' + session2.commit_transaction(committime + durabletime) + + if self.gc: + session2.begin_transaction('read_timestamp=' + self.timestamp_str(25)) + # Churn the append list enough to clean out the previous aborted updates. + for i in range(0, 20): + cursor2[append_key_lite] = 30 + i + cursor2[append_key_lite + 1] = 31 + i + session2.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) + + if self.reconcile: + # This will all fit on one page, so we only need to evict once. + self.evict(uri, nrows, nrows) + + # For aborted without G/C, we could lose the aborted update records at any point, so + # do the most interesting checks first. + # + # Current behavior for all cases is that the table extends out to the append key, + # including in the past. + # + # Without a way to prevent reconciliation it's not entirely clear that the aborted + # with G/C case doesn't get reconciled (which currently physically extends the table + # in a way that cannot be rolled back) before the updates are tossed. If that happens, + # the table definitely extends. If it doesn't, it might not. Maybe. + if self.op == 'aborted' and not self.gc: + self.check_end(cursor, nrows-1, 30, append_key) + self.check_end(cursor, nrows-1, 10, append_key) + self.check_end(cursor, None, 5, append_key) + else: + self.check_end(cursor, None, 5, append_key) + self.check_end(cursor, None, 10, append_key) + self.check_end(cursor, None, 19, append_key) + self.check_end(cursor, None, 20, append_key) + self.check_end(cursor, None, 21, append_key) + self.check_end(cursor, None, 30, append_key) diff --git a/src/third_party/wiredtiger/test/suite/test_flcs04.py b/src/third_party/wiredtiger/test/suite/test_flcs04.py new file mode 100644 index 00000000000..556ee03dcb3 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_flcs04.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet + +# test_flcs04.py +# +# Make sure modify fails cleanly on FLCS tables. + +class test_flcs04(wttest.WiredTigerTestCase): + session_config = 'isolation=snapshot' + conn_config = 'in_memory=false' + + def test_flcs(self): + uri = "table:test_flcs04" + nrows = 10 + ds = SimpleDataSet( + self, uri, nrows, key_format='r', value_format='6t', config='leaf_page_max=4096') + ds.populate() + + + cursor = self.session.open_cursor(uri) + self.session.begin_transaction() + + cursor.set_key(5) + mods = [wiredtiger.Modify('Q', 100, 1)] + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: cursor.modify(mods), + "/WT_CURSOR.modify only supported for/") + + self.session.rollback_transaction() + cursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_hs01.py b/src/third_party/wiredtiger/test/suite/test_hs01.py index 52c68cb699d..ffa35c144b7 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs01.py +++ b/src/third_party/wiredtiger/test/suite/test_hs01.py @@ -38,12 +38,13 @@ from wtscenario import make_scenarios class test_hs01(wttest.WiredTigerTestCase): conn_config = 'cache_size=200MB,statistics=(all)' session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('row_integer', dict(key_format='i')), - ('row_string', dict(key_format='S')) + format_values = [ + ('column', dict(key_format='r', value_format='u')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('row_integer', dict(key_format='i', value_format='u')), + ('row_string', dict(key_format='S', value_format='u')) ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def get_stat(self, stat): stat_cursor = self.session.open_cursor('statistics:') @@ -70,10 +71,16 @@ class test_hs01(wttest.WiredTigerTestCase): # Hence, we begin/commit transaction manually. session.begin_transaction() cursor.set_key(ds.key(i)) - mods = [] - mod = wiredtiger.Modify('A', offset, 1) - mods.append(mod) - self.assertEqual(cursor.modify(mods), 0) + + # FLCS doesn't allow modify (it doesn't make sense) so just update to 'j' then 'k'. + if self.value_format == '8t': + cursor.set_value(106 + offset) + self.assertEqual(cursor.update(), 0) + else: + mods = [] + mod = wiredtiger.Modify('A', offset, 1) + mods.append(mod) + self.assertEqual(cursor.modify(mods), 0) if timestamp == True: session.commit_transaction('commit_timestamp=' + self.timestamp_str(i + 1)) @@ -100,12 +107,23 @@ class test_hs01(wttest.WiredTigerTestCase): def test_hs(self): # Create a small table. uri = "table:test_hs01" - ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format='u') + ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format=self.value_format) ds.populate() + if self.value_format == '8t': + bigvalue = 97 + bigvalue2 = 98 + bigvalue3 = 107 + bigvalue4 = 100 + else: + bigvalue = b"aaaaa" * 100 + bigvalue2 = b"ccccc" * 100 + bigvalue3 = b"ccccc" * 100 + bigvalue3 = b'AA' + bigvalue3[2:] + bigvalue4 = b"ddddd" * 100 + # Initially insert a lot of data. nrows = 10000 - bigvalue = b"aaaaa" * 100 cursor = self.session.open_cursor(uri) for i in range(1, nrows): cursor.set_key(ds.key(i)) @@ -116,7 +134,6 @@ class test_hs01(wttest.WiredTigerTestCase): # Scenario: 1 # Check to see if the history store is working with the old reader. - bigvalue2 = b"ccccc" * 100 # Open session 2. session2 = self.conn.open_session() session2.begin_transaction('isolation=snapshot') @@ -135,12 +152,10 @@ class test_hs01(wttest.WiredTigerTestCase): # Scenario: 2 # Check to see the history store working with modify operations. - bigvalue3 = b"ccccc" * 100 - bigvalue3 = b'AA' + bigvalue3[2:] # Open session 2. session2 = self.conn.open_session() session2.begin_transaction('isolation=snapshot') - # Apply two modify operations (session1)- replacing the first two items with 'A'. + # Apply two modify operations (session1)- replacing the first two letters with 'A'. self.large_modifies(self.session, uri, 0, ds, nrows) self.large_modifies(self.session, uri, 1, ds, nrows) @@ -160,7 +175,6 @@ class test_hs01(wttest.WiredTigerTestCase): # Scenario: 3 # Check to see if the history store is working with the old timestamp. - bigvalue4 = b"ddddd" * 100 self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) self.large_updates(self.session, uri, bigvalue4, ds, nrows, timestamp=True) diff --git a/src/third_party/wiredtiger/test/suite/test_hs02.py b/src/third_party/wiredtiger/test/suite/test_hs02.py index 42cd3b327ce..0388646cfb8 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs02.py +++ b/src/third_party/wiredtiger/test/suite/test_hs02.py @@ -38,11 +38,12 @@ class test_hs02(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' - key_format_values = [ - ('string-row', dict(key_format='S')), - ('column', dict(key_format='r')) + format_values = [ + ('string-row', dict(key_format='S', value_format='S')), + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')) ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def large_updates(self, uri, value, ds, nrows, commit_ts): # Update a large number of records, we'll hang if the history store table isn't working. @@ -54,16 +55,25 @@ class test_hs02(wttest.WiredTigerTestCase): session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() - def check(self, check_value, uri, nrows, read_ts): + # expect[] is a list of (value, count) pairs to expect while scanning the table. + def check(self, uri, expect, read_ts): session = self.session session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) + entry = 0 + (check_value, check_count) = expect[entry] count = 0 for k, v in cursor: + if count >= check_count: + self.assertLess(entry, len(expect), "Too many rows returned by cursor") + entry += 1 + (check_value, check_count) = expect[entry] + count = 0 self.assertEqual(v, check_value) count += 1 session.rollback_transaction() - self.assertEqual(count, nrows) + # If this fails, the cursor didn't return enough rows. + self.assertEqual(count, check_count) def test_hs(self): nrows = 10000 @@ -72,30 +82,41 @@ class test_hs02(wttest.WiredTigerTestCase): # behavior. uri = "table:las02_main" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() uri2 = "table:las02_extra" - ds2 = SimpleDataSet(self, uri2, 0, key_format=self.key_format, value_format="S") + ds2 = SimpleDataSet( + self, uri2, 0, key_format=self.key_format, value_format=self.value_format) ds2.populate() + if self.value_format == '8t': + bigvalue = 97 + bigvalue2 = 100 + else: + bigvalue = "aaaaa" * 100 + bigvalue2 = "ddddd" * 100 + # Pin oldest and stable to timestamp 1. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) - bigvalue = "aaaaa" * 100 self.large_updates(uri, bigvalue, ds, nrows // 3, 1) # Check that all updates are seen - self.check(bigvalue, uri, nrows // 3, 1) + self.check(uri, [(bigvalue, nrows // 3)], 1) # Check to see the history store working with old timestamp - bigvalue2 = "ddddd" * 100 self.large_updates(uri, bigvalue2, ds, nrows, 100) # Check that the new updates are only seen after the update timestamp - self.check(bigvalue, uri, nrows // 3, 1) - self.check(bigvalue2, uri, nrows, 100) + expect_1 = [(bigvalue, nrows // 3)] + # In FLCS zeros appear under uncommitted/non-visible updates at the end of the table. + if self.value_format == '8t': + expect_1.append((0, nrows - nrows // 3)) + self.check(uri, expect_1, 1) + self.check(uri, [(bigvalue2, nrows)], 100) # Force out most of the pages by updating a different tree self.large_updates(uri2, bigvalue, ds2, nrows, 100) @@ -109,11 +130,15 @@ class test_hs02(wttest.WiredTigerTestCase): self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(200)) # Check that the truncate is visible after commit - self.check(bigvalue2, uri, nrows // 2, 200) + if self.value_format == '8t': + expect_200 = [(0, nrows // 2), (bigvalue2, nrows // 2)] + else: + expect_200 = [(bigvalue2, nrows // 2)] + self.check(uri, expect_200, 200) # Repeat earlier checks - self.check(bigvalue, uri, nrows // 3, 1) - self.check(bigvalue2, uri, nrows, 100) + self.check(uri, expect_1, 1) + self.check(uri, [(bigvalue2, nrows)], 100) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_hs03.py b/src/third_party/wiredtiger/test/suite/test_hs03.py index 54a54bf1f0f..768b8598a1a 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs03.py +++ b/src/third_party/wiredtiger/test/suite/test_hs03.py @@ -38,12 +38,13 @@ class test_hs03(wttest.WiredTigerTestCase): # Force a small cache. conn_config = 'cache_size=50MB,statistics=(fast)' session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer-row', dict(key_format='i')), - ('string-row', dict(key_format='S')) + format_values = [ + ('column', dict(key_format='r', value_format='u')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer-row', dict(key_format='i', value_format='u')), + ('string-row', dict(key_format='S', value_format='u')) ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def get_stat(self, stat): stat_cursor = self.session.open_cursor('statistics:') @@ -65,9 +66,15 @@ class test_hs03(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test_hs03" nrows = 100 - ds = SimpleDataSet(self, uri, nrows, key_format=self.key_format, value_format='u') + ds = SimpleDataSet(self, uri, nrows, key_format=self.key_format, value_format=self.value_format) ds.populate() - bigvalue = b"aaaaa" * 100 + + if self.value_format == '8t': + bigvalue = 97 + bigvalue2 = 100 + else: + bigvalue = b"aaaaa" * 100 + bigvalue2 = b"ddddd" * 100 # Initially load huge data. cursor = self.session.open_cursor(uri) @@ -77,7 +84,6 @@ class test_hs03(wttest.WiredTigerTestCase): self.session.checkpoint() # Check to see the history store working with old timestamp. - bigvalue2 = b"ddddd" * 100 self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) hs_writes_start = self.get_stat(stat.conn.cache_write_hs) self.large_updates(self.session, uri, bigvalue2, ds, nrows, 10000) diff --git a/src/third_party/wiredtiger/test/suite/test_hs05.py b/src/third_party/wiredtiger/test/suite/test_hs05.py index 58cbfefccd7..021ffc35538 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs05.py +++ b/src/third_party/wiredtiger/test/suite/test_hs05.py @@ -42,12 +42,13 @@ class test_hs05(wttest.WiredTigerTestCase): conn_config += 'eviction_updates_target=95,eviction_updates_trigger=100' session_config = 'isolation=snapshot' stable = 1 - key_format_values = [ - ('column', dict(key_format='r')), - ('integer-row', dict(key_format='i')), - ('string-row', dict(key_format='S')) + format_values = [ + ('column', dict(key_format='r', value_format='u')), + ('column-fix', dict(key_format='r', value_format='8t')), + ('integer-row', dict(key_format='i', value_format='u')), + ('string-row', dict(key_format='S', value_format='u')) ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def get_stat(self, stat): stat_cursor = self.session.open_cursor('statistics:') @@ -75,9 +76,13 @@ class test_hs05(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test_hs05" nrows = 100 - ds = SimpleDataSet(self, uri, nrows, key_format=self.key_format, value_format='u') + ds = SimpleDataSet(self, uri, nrows, key_format=self.key_format, value_format=self.value_format) ds.populate() - bigvalue = b"aaaaa" * 100 + + if self.value_format == '8t': + bigvalue = 97 + else: + bigvalue = b"aaaaa" * 100 # Initially load huge data. # Add 10000 items that have a 500b value that is about 50Mb that @@ -99,7 +104,10 @@ class test_hs05(wttest.WiredTigerTestCase): valstr='abcdefghijklmnopqrstuvwxyz' loop_start = self.get_stat(stat.conn.cache_hs_score) for i in range(1, 9): - bigvalue2 = valstr[i].encode() * 50 + if self.value_format == '8t': + bigvalue2 = 105 + i + else: + bigvalue2 = valstr[i].encode() * 50 self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(self.stable)) entries_start = self.get_stat(stat.conn.cache_hs_insert) score_start = self.get_stat(stat.conn.cache_hs_score) @@ -127,7 +135,10 @@ class test_hs05(wttest.WiredTigerTestCase): self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(self.stable)) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(self.stable)) for i in range(9, 11): - bigvalue2 = valstr[i].encode() * 50 + if self.value_format == '8t': + bigvalue2 = 105 + i + else: + bigvalue2 = valstr[i].encode() * 50 self.pr("Update iteration with oldest: " + str(i) + " Value: " + str(bigvalue2)) self.large_updates(self.session, uri, bigvalue2, ds, nrows, nrows) self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(self.stable)) diff --git a/src/third_party/wiredtiger/test/suite/test_hs06.py b/src/third_party/wiredtiger/test/suite/test_hs06.py index 1c2f18f09f2..918e1dd9fa8 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs06.py +++ b/src/third_party/wiredtiger/test/suite/test_hs06.py @@ -43,12 +43,13 @@ class test_hs06(wttest.WiredTigerTestCase): # Force a small cache. conn_config = 'cache_size=50MB,statistics=(fast)' session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer-row', dict(key_format='i')), - ('string-row', dict(key_format='S')) + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), + ('integer-row', dict(key_format='i', value_format='S')), + ('string-row', dict(key_format='S', value_format='S')) ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) nrows = 2000 def get_stat(self, stat): @@ -68,11 +69,15 @@ class test_hs06(wttest.WiredTigerTestCase): def test_hs_reads(self): # Create a small table. uri = "table:test_hs06" - create_params = 'key_format={},value_format=S'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, create_params) - value1 = 'a' * 500 - value2 = 'b' * 500 + if self.value_format == '8t': + value1 = 97 + value2 = 98 + else: + value1 = 'a' * 500 + value2 = 'b' * 500 # Load 1Mb of data. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) @@ -131,9 +136,13 @@ class test_hs06(wttest.WiredTigerTestCase): # WT-5336 causing the read at timestamp 4 returning the value committed at timestamp 5 or 3 def test_hs_modify_reads(self): + # FLCS doesn't support modify, so just skip over this test. + if self.value_format == '8t': + return + # Create a small table. uri = "table:test_hs06" - create_params = 'key_format={},value_format=S'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, create_params) # Create initial large values. @@ -178,13 +187,13 @@ class test_hs06(wttest.WiredTigerTestCase): expected = str().join(expected) # Whenever we request something of timestamp 3, this should be a modify - # op. We should looking forwards in the history store until we find the + # op. We should be looking forwards in the history store until we find the # newest whole update (timestamp 4). # # t5: value1 (full update on page) - # t4: full update in las - # t3: (reverse delta in las) <= We're querying for t4 so we begin here. - # t2: value2 (full update in las) + # t4: full update in hs + # t3: (reverse delta in hs) <= We're querying for t4 so we begin here. + # t2: value2 (full update in hs) self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3)) for i in range(1, self.nrows): self.assertEqual(cursor[self.create_key(i)], expected) @@ -209,11 +218,15 @@ class test_hs06(wttest.WiredTigerTestCase): def test_hs_prepare_reads(self): # Create a small table. uri = "table:test_hs06" - create_params = 'key_format={},value_format=S'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, create_params) - value1 = 'a' * 500 - value2 = 'b' * 500 + if self.value_format == '8t': + value1 = 97 + value2 = 98 + else: + value1 = 'a' * 500 + value2 = 'b' * 500 self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) @@ -262,13 +275,19 @@ class test_hs06(wttest.WiredTigerTestCase): def test_hs_multiple_updates(self): # Create a small table. uri = "table:test_hs06" - create_params = 'key_format={},value_format=S'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, create_params) - value1 = 'a' * 500 - value2 = 'b' * 500 - value3 = 'c' * 500 - value4 = 'd' * 500 + if self.value_format == '8t': + value1 = 97 + value2 = 98 + value3 = 99 + value4 = 100 + else: + value1 = 'a' * 500 + value2 = 'b' * 500 + value3 = 'c' * 500 + value4 = 'd' * 500 # Load 1Mb of data. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) @@ -299,9 +318,13 @@ class test_hs06(wttest.WiredTigerTestCase): self.session.rollback_transaction() def test_hs_multiple_modifies(self): + # FLCS doesn't support modify, so just skip over this test. + if self.value_format == '8t': + return + # Create a small table. uri = "table:test_hs06" - create_params = 'key_format={},value_format=S'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, create_params) value1 = 'a' * 500 @@ -344,9 +367,13 @@ class test_hs06(wttest.WiredTigerTestCase): self.session.rollback_transaction() def test_hs_instantiated_modify(self): + # FLCS doesn't support modify, so just skip over this test. + if self.value_format == '8t': + return + # Create a small table. uri = "table:test_hs06" - create_params = 'key_format={},value_format=S'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, create_params) value1 = 'a' * 500 @@ -405,9 +432,13 @@ class test_hs06(wttest.WiredTigerTestCase): self.session.rollback_transaction() def test_hs_modify_stable_is_base_update(self): + # FLCS doesn't support modify, so just skip over this test. + if self.value_format == '8t': + return + # Create a small table. uri = "table:test_hs06" - create_params = 'key_format={},value_format=S'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, create_params) value1 = 'a' * 500 @@ -467,9 +498,13 @@ class test_hs06(wttest.WiredTigerTestCase): self.session.rollback_transaction() def test_hs_rec_modify(self): + # FLCS doesn't support modify, so just skip over this test. + if self.value_format == '8t': + return + # Create a small table. uri = "table:test_hs06" - create_params = 'key_format={},value_format=S'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, create_params) value1 = 'a' * 500 diff --git a/src/third_party/wiredtiger/test/suite/test_hs07.py b/src/third_party/wiredtiger/test/suite/test_hs07.py index 7e2562e5420..549868b5ef3 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs07.py +++ b/src/third_party/wiredtiger/test/suite/test_hs07.py @@ -40,11 +40,12 @@ class test_hs07(wttest.WiredTigerTestCase): 'eviction_updates_target=80,log=(enabled)') session_config = 'isolation=snapshot' - key_format_values = ( - ('column', dict(key_format='r')), - ('integer-row', dict(key_format='i')) + format_values = ( + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), + ('integer-row', dict(key_format='i', value_format='S')) ) - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def large_updates(self, uri, value, ds, nrows, commit_ts): # Update a large number of records, we'll hang if the history store table isn't working. @@ -74,19 +75,26 @@ class test_hs07(wttest.WiredTigerTestCase): # behavior. uri = "table:las07_main" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() uri2 = "table:las07_extra" - ds2 = SimpleDataSet(self, uri2, 0, key_format=self.key_format, value_format="S") + ds2 = SimpleDataSet( + self, uri2, 0, key_format=self.key_format, value_format=self.value_format) ds2.populate() + if self.value_format == '8t': + bigvalue = 97 + bigvalue2 = 100 + else: + bigvalue = "aaaaa" * 100 + bigvalue2 = "ddddd" * 100 + # Pin oldest and stable to timestamp 1. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) - bigvalue = "aaaaa" * 100 - bigvalue2 = "ddddd" * 100 self.large_updates(uri, bigvalue, ds, nrows, 1) # Check that all updates are seen @@ -113,24 +121,36 @@ class test_hs07(wttest.WiredTigerTestCase): self.session.begin_transaction() for i in range(1, nrows): cursor.set_key(i) - mods = [wiredtiger.Modify('A', 10, 1)] - self.assertEqual(cursor.modify(mods), 0) + if self.value_format == '8t': + cursor.set_value(105) + cursor.update() + else: + mods = [wiredtiger.Modify('A', 10, 1)] + self.assertEqual(cursor.modify(mods), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(110)) # Load a slight modification with a later timestamp. self.session.begin_transaction() for i in range(1, nrows): cursor.set_key(i) - mods = [wiredtiger.Modify('B', 20, 1)] - self.assertEqual(cursor.modify(mods), 0) + if self.value_format == '8t': + cursor.set_value(106) + cursor.update() + else: + mods = [wiredtiger.Modify('B', 20, 1)] + self.assertEqual(cursor.modify(mods), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(120)) # Load a slight modification with a later timestamp. self.session.begin_transaction() for i in range(1, nrows): cursor.set_key(i) - mods = [wiredtiger.Modify('C', 30, 1)] - self.assertEqual(cursor.modify(mods), 0) + if self.value_format == '8t': + cursor.set_value(107) + cursor.update() + else: + mods = [wiredtiger.Modify('C', 30, 1)] + self.assertEqual(cursor.modify(mods), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(130)) cursor.close() @@ -158,24 +178,36 @@ class test_hs07(wttest.WiredTigerTestCase): self.session.begin_transaction() for i in range(1, nrows): cursor.set_key(i) - mods = [wiredtiger.Modify('A', 10, 1)] - self.assertEqual(cursor.modify(mods), 0) + if self.value_format == '8t': + cursor.set_value(105) + cursor.update() + else: + mods = [wiredtiger.Modify('A', 10, 1)] + self.assertEqual(cursor.modify(mods), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(210)) # Load a slight modification with a later timestamp. self.session.begin_transaction() for i in range(1, nrows): cursor.set_key(i) - mods = [wiredtiger.Modify('B', 20, 1)] - self.assertEqual(cursor.modify(mods), 0) + if self.value_format == '8t': + cursor.set_value(106) + cursor.update() + else: + mods = [wiredtiger.Modify('B', 20, 1)] + self.assertEqual(cursor.modify(mods), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(220)) # Load a slight modification with a later timestamp. self.session.begin_transaction() for i in range(1, nrows): cursor.set_key(i) - mods = [wiredtiger.Modify('C', 30, 1)] - self.assertEqual(cursor.modify(mods), 0) + if self.value_format == '8t': + cursor.set_value(107) + cursor.update() + else: + mods = [wiredtiger.Modify('C', 30, 1)] + self.assertEqual(cursor.modify(mods), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(230)) cursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_hs09.py b/src/third_party/wiredtiger/test/suite/test_hs09.py index 1b7993239b2..3980e0b275c 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs09.py +++ b/src/third_party/wiredtiger/test/suite/test_hs09.py @@ -42,12 +42,13 @@ class test_hs09(wttest.WiredTigerTestCase): conn_config = 'cache_size=20MB' session_config = 'isolation=snapshot' uri = "table:test_hs09" - key_format_values = [ - ('column', dict(key_format='r')), - ('integer-row', dict(key_format='i')), - ('string-row', dict(key_format='S')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), + ('integer-row', dict(key_format='i', value_format='S')), + ('string-row', dict(key_format='S', value_format='S')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) nrows = 1000 def create_key(self, i): @@ -91,12 +92,17 @@ class test_hs09(wttest.WiredTigerTestCase): def test_uncommitted_updates_not_written_to_hs(self): # Create a small table. - create_params = 'key_format={},value_format=S'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(self.uri, create_params) - value1 = 'a' * 500 - value2 = 'b' * 500 - value3 = 'c' * 500 + if self.value_format == '8t': + value1 = 97 + value2 = 98 + value3 = 99 + else: + value1 = 'a' * 500 + value2 = 'b' * 500 + value3 = 'c' * 500 # Load 500KB of data. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) @@ -121,12 +127,17 @@ class test_hs09(wttest.WiredTigerTestCase): def test_prepared_updates_not_written_to_hs(self): # Create a small table. - create_params = 'key_format={},value_format=S'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(self.uri, create_params) - value1 = 'a' * 500 - value2 = 'b' * 500 - value3 = 'c' * 500 + if self.value_format == '8t': + value1 = 97 + value2 = 98 + value3 = 99 + else: + value1 = 'a' * 500 + value2 = 'b' * 500 + value3 = 'c' * 500 # Load 1MB of data. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) @@ -156,11 +167,15 @@ class test_hs09(wttest.WiredTigerTestCase): def test_write_newest_version_to_data_store(self): # Create a small table. - create_params = 'key_format={},value_format=S'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(self.uri, create_params) - value1 = 'a' * 500 - value2 = 'b' * 500 + if self.value_format == '8t': + value1 = 97 + value2 = 98 + else: + value1 = 'a' * 500 + value2 = 'b' * 500 # Load 500KB of data. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) @@ -180,11 +195,15 @@ class test_hs09(wttest.WiredTigerTestCase): def test_write_deleted_version_to_data_store(self): # Create a small table. - create_params = 'key_format={},value_format=S'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(self.uri, create_params) - value1 = 'a' * 500 - value2 = 'b' * 500 + if self.value_format == '8t': + value1 = 97 + value2 = 98 + else: + value1 = 'a' * 500 + value2 = 'b' * 500 # Load 500KB of data. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) @@ -208,7 +227,10 @@ class test_hs09(wttest.WiredTigerTestCase): self.assertEqual(cursor.remove(), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4)) - self.check_ckpt_hs(value2, value1, 2, 3) + # For FLCS, the deleted records should read back as 0. For non-FLCS, no deleted + # records should be seen so none should be compared to 0, and if any are the + # resulting Python type error means something's wrong. + self.check_ckpt_hs(0, value1, 2, 3) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_hs11.py b/src/third_party/wiredtiger/test/suite/test_hs11.py index fd2afc0d977..6188633dfc7 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs11.py +++ b/src/third_party/wiredtiger/test/suite/test_hs11.py @@ -35,16 +35,17 @@ from wiredtiger import stat class test_hs11(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,statistics=(all)' session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer-row', dict(key_format='i')), - ('string-row', dict(key_format='S')) + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), + ('integer-row', dict(key_format='i', value_format='S')), + ('string-row', dict(key_format='S', value_format='S')), ] update_type_values = [ ('deletion', dict(update_type='deletion')), ('update', dict(update_type='update')) ] - scenarios = make_scenarios(key_format_values, update_type_values) + scenarios = make_scenarios(format_values, update_type_values) nrows = 10000 def create_key(self, i): @@ -60,11 +61,15 @@ class test_hs11(wttest.WiredTigerTestCase): def test_non_ts_updates_clears_hs(self): uri = 'table:test_hs11' - create_params = 'key_format={},value_format=S'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, create_params) - value1 = 'a' * 500 - value2 = 'b' * 500 + if self.value_format == '8t': + value1 = 97 + value2 = 98 + else: + value1 = 'a' * 500 + value2 = 'b' * 500 # Apply a series of updates from timestamps 1-4. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) @@ -103,7 +108,11 @@ class test_hs11(wttest.WiredTigerTestCase): if i % 2 == 0: if self.update_type == 'deletion': cursor.set_key(self.create_key(i)) - self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + if self.value_format == '8t': + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_value(), 0) + else: + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) else: self.assertEqual(cursor[self.create_key(i)], value2) else: @@ -116,11 +125,15 @@ class test_hs11(wttest.WiredTigerTestCase): def test_ts_updates_donot_clears_hs(self): uri = 'table:test_hs11' - create_params = 'key_format={},value_format=S'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, create_params) - value1 = 'a' * 500 - value2 = 'b' * 500 + if self.value_format == '8t': + value1 = 97 + value2 = 98 + else: + value1 = 'a' * 500 + value2 = 'b' * 500 # Apply a series of updates from timestamps 1-4. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) @@ -157,7 +170,11 @@ class test_hs11(wttest.WiredTigerTestCase): for i in range(1, self.nrows): if i % 2 == 0: cursor.set_key(self.create_key(i)) - self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + if self.value_format == '8t': + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_value(), 0) + else: + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) else: self.assertEqual(cursor[self.create_key(i)], value1) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs14.py b/src/third_party/wiredtiger/test/suite/test_hs14.py index 7c9d0241d82..ad2d98ad7b1 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs14.py +++ b/src/third_party/wiredtiger/test/suite/test_hs14.py @@ -35,11 +35,12 @@ from wtscenario import make_scenarios class test_hs14(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('string-row', dict(key_format='S')) + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), + ('string-row', dict(key_format='S', value_format='S')) ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def create_key(self, i): if self.key_format == 'S': @@ -48,17 +49,25 @@ class test_hs14(wttest.WiredTigerTestCase): def test_hs14(self): uri = 'table:test_hs14' - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + config = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, config) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) nrows = 10000 - value1 = 'a' * 500 - value2 = 'b' * 500 - value3 = 'c' * 500 - value4 = 'd' * 500 - value5 = 'e' * 500 + if self.value_format == '8t': + value1 = 97 + value2 = 98 + value3 = 99 + value4 = 100 + value5 = 101 + else: + value1 = 'a' * 500 + value2 = 'b' * 500 + value3 = 'c' * 500 + value4 = 'd' * 500 + value5 = 'e' * 500 for i in range(1, nrows): self.session.begin_transaction() @@ -103,7 +112,11 @@ class test_hs14(wttest.WiredTigerTestCase): self.session.begin_transaction('read_timestamp=' + self.timestamp_str(9)) for i in range(1, nrows): cursor.set_key(self.create_key(i)) - self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + if self.value_format == '8t': + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_value(), 0) + else: + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) self.session.rollback_transaction() end = time.time() diff --git a/src/third_party/wiredtiger/test/suite/test_hs15.py b/src/third_party/wiredtiger/test/suite/test_hs15.py index 9dded0b2d7d..3628f79ea0f 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs15.py +++ b/src/third_party/wiredtiger/test/suite/test_hs15.py @@ -39,11 +39,12 @@ from wtscenario import make_scenarios class test_hs15(wttest.WiredTigerTestCase): conn_config = 'cache_size=5MB' session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('string-row', dict(key_format='S')) + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), + ('string-row', dict(key_format='S', value_format='S')) ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def create_key(self, i): if self.key_format == 'S': @@ -52,12 +53,18 @@ class test_hs15(wttest.WiredTigerTestCase): def test_hs15(self): uri = 'table:test_hs15' - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, format) cursor = self.session.open_cursor(uri) - value1 = 'a' * 500 - value2 = 'b' * 500 - value3 = 'c' * 500 + if self.value_format == '8t': + value1 = 97 + value2 = 98 + value3 = 99 + else: + value1 = 'a' * 500 + value2 = 'b' * 500 + value3 = 'c' * 500 # Insert an update without timestamp self.session.begin_transaction() @@ -70,11 +77,15 @@ class test_hs15(wttest.WiredTigerTestCase): cursor[self.create_key(i)] = value2 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) - # Do a modify and an update with timestamps + # Do a modify and an update with timestamps (for FLCS, just modifies) self.session.begin_transaction() cursor.set_key(self.create_key(1)) - mods = [wiredtiger.Modify('B', 100, 1)] - self.assertEqual(cursor.modify(mods), 0) + if self.value_format == '8t': + cursor.set_value(66) + self.assertEqual(cursor.update(), 0) + else: + mods = [wiredtiger.Modify('B', 100, 1)] + self.assertEqual(cursor.modify(mods), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) self.session.begin_transaction() @@ -97,9 +108,12 @@ class test_hs15(wttest.WiredTigerTestCase): cursor[self.create_key(i)] = value3 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) - expected = list(value1) - expected[100] = 'B' - expected = str().join(expected) + if self.value_format == '8t': + expected = 66 # 'B' + else: + expected = list(value1) + expected[100] = 'B' + expected = str().join(expected) self.session.begin_transaction('read_timestamp=' + self.timestamp_str(1)) self.assertEqual(cursor[self.create_key(1)], expected) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs16.py b/src/third_party/wiredtiger/test/suite/test_hs16.py index 1a5ed51128c..fca22579713 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs16.py +++ b/src/third_party/wiredtiger/test/suite/test_hs16.py @@ -34,11 +34,12 @@ from wtscenario import make_scenarios class test_hs16(wttest.WiredTigerTestCase): conn_config = 'cache_size=5MB' session_config = 'isolation=snapshot' - key_format_values = ( - ('column', dict(key_format='r')), - ('string-row', dict(key_format='S')) + format_values = ( + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), + ('string-row', dict(key_format='S', value_format='S')) ) - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def create_key(self,i): if self.key_format == 'S': @@ -47,34 +48,45 @@ class test_hs16(wttest.WiredTigerTestCase): def test_hs16(self): uri = 'table:test_hs16' - create_params = 'key_format={}, value_format=S'.format(self.key_format) + create_params = 'key_format={}, value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, create_params) cursor = self.session.open_cursor(uri) + if self.value_format == '8t': + valuea = 97 # 'a' + valueb = 98 # 'b' + valuec = 99 # 'c' + valued = 100 # 'd' + else: + valuea = 'a' + valueb = 'b' + valuec = 'c' + valued = 'd' + # Insert an update without timestamp self.session.begin_transaction() - cursor[self.create_key(1)] = 'a' + cursor[self.create_key(1)] = valuea self.session.commit_transaction() # Update an update at timestamp 1 self.session.begin_transaction() - cursor[self.create_key(1)] = 'b' + cursor[self.create_key(1)] = valueb self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) # Open anther session to make the next update without timestamp non-globally visible session2 = self.setUpSessionOpen(self.conn) cursor2 = session2.open_cursor(uri) session2.begin_transaction() - cursor[self.create_key(2)] = 'a' + cursor[self.create_key(2)] = valuea # Update an update without timestamp self.session.begin_transaction() - cursor[self.create_key(1)] = 'c' + cursor[self.create_key(1)] = valuec self.session.commit_transaction() # Update an update at timestamp 2 self.session.begin_transaction() - cursor[self.create_key(1)] = 'd' + cursor[self.create_key(1)] = valued self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Do a checkpoint, it should not panic diff --git a/src/third_party/wiredtiger/test/suite/test_hs18.py b/src/third_party/wiredtiger/test/suite/test_hs18.py index 00fa6a96f5a..b47bb2b66ed 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs18.py +++ b/src/third_party/wiredtiger/test/suite/test_hs18.py @@ -34,11 +34,12 @@ from wtscenario import make_scenarios class test_hs18(wttest.WiredTigerTestCase): conn_config = 'cache_size=5MB,eviction=(threads_max=1)' session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('string-row', dict(key_format='S')) + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), + ('string-row', dict(key_format='S', value_format='S')) ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def create_key(self, i): if self.key_format == 'S': @@ -66,17 +67,26 @@ class test_hs18(wttest.WiredTigerTestCase): def test_base_scenario(self): uri = 'table:test_base_scenario' - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, format) session2 = self.setUpSessionOpen(self.conn) cursor = self.session.open_cursor(uri) cursor2 = session2.open_cursor(uri) - value0 = 'f' * 500 - value1 = 'a' * 500 - value2 = 'b' * 500 - value3 = 'c' * 500 - value4 = 'd' * 500 - value5 = 'e' * 500 + if self.value_format == '8t': + value0 = 102 + value1 = 97 + value2 = 98 + value3 = 99 + value4 = 100 + value5 = 101 + else: + value0 = 'f' * 500 + value1 = 'a' * 500 + value2 = 'b' * 500 + value3 = 'c' * 500 + value4 = 'd' * 500 + value5 = 'e' * 500 # Insert an update at timestamp 3 self.session.begin_transaction() @@ -124,18 +134,26 @@ class test_hs18(wttest.WiredTigerTestCase): # Test that we don't get the wrong value if we read with a timestamp originally. def test_read_timestamp_weirdness(self): uri = 'table:test_hs18' - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, format) cursor = self.session.open_cursor(uri) session2 = self.setUpSessionOpen(self.conn) cursor2 = session2.open_cursor(uri) session3 = self.setUpSessionOpen(self.conn) cursor3 = session3.open_cursor(uri) - value1 = 'a' * 500 - value2 = 'b' * 500 - value3 = 'c' * 500 - value4 = 'd' * 500 - value5 = 'e' * 500 + if self.value_format == '8t': + value1 = 97 + value2 = 98 + value3 = 99 + value4 = 100 + value5 = 101 + else: + value1 = 'a' * 500 + value2 = 'b' * 500 + value3 = 'c' * 500 + value4 = 'd' * 500 + value5 = 'e' * 500 # Insert an update at timestamp 3 self.session.begin_transaction() @@ -187,15 +205,24 @@ class test_hs18(wttest.WiredTigerTestCase): # Test that forces us to ignore tombstone in order to not remove the first non timestamped updated. def test_ignore_tombstone(self): uri = 'table:test_ignore_tombstone' - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, format) session2 = self.setUpSessionOpen(self.conn) cursor = self.session.open_cursor(uri) cursor2 = session2.open_cursor(uri) - value0 = 'A' * 500 - value1 = 'a' * 500 - value2 = 'b' * 500 - value3 = 'c' * 500 - value4 = 'd' * 500 + + if self.value_format == '8t': + value0 = 65 + value1 = 97 + value2 = 98 + value3 = 99 + value4 = 100 + else: + value0 = 'A' * 500 + value1 = 'a' * 500 + value2 = 'b' * 500 + value3 = 'c' * 500 + value4 = 'd' * 500 # Insert an update without a timestamp self.session.begin_transaction() @@ -240,7 +267,8 @@ class test_hs18(wttest.WiredTigerTestCase): # Test older readers for each of the updates moved to the history store. def test_multiple_older_readers(self): uri = 'table:test_multiple_older_readers' - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, format) cursor = self.session.open_cursor(uri) # The ID of the session corresponds the value it should see. @@ -250,7 +278,10 @@ class test_hs18(wttest.WiredTigerTestCase): for i in range(0, 5): sessions.append(self.setUpSessionOpen(self.conn)) cursors.append(sessions[i].open_cursor(uri)) - values.append(str(i) * 10) + if self.value_format == '8t': + values.append(i + 48) + else: + values.append(str(i) * 10) # Insert an update at timestamp 3 self.session.begin_transaction() @@ -306,7 +337,8 @@ class test_hs18(wttest.WiredTigerTestCase): def test_multiple_older_readers_with_multiple_mixed_mode(self): uri = 'table:test_multiple_older_readers' - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, format) cursor = self.session.open_cursor(uri) # The ID of the session corresponds the value it should see. @@ -316,7 +348,10 @@ class test_hs18(wttest.WiredTigerTestCase): for i in range(0, 9): sessions.append(self.setUpSessionOpen(self.conn)) cursors.append(sessions[i].open_cursor(uri)) - values.append(str(i) * 10) + if self.value_format == '8t': + values.append(i + 48) + else: + values.append(str(i) * 10) # Insert an update at timestamp 3 self.session.begin_transaction() @@ -414,8 +449,13 @@ class test_hs18(wttest.WiredTigerTestCase): cursors[i].reset() def test_modifies(self): + # FLCS doesn't support modify, so just skip this case. + if self.value_format == '8t': + return + uri = 'table:test_modifies' - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, format) cursor = self.session.open_cursor(uri) session_ts_reader = self.setUpSessionOpen(self.conn) cursor_ts_reader = session_ts_reader.open_cursor(uri) diff --git a/src/third_party/wiredtiger/test/suite/test_hs21.py b/src/third_party/wiredtiger/test/suite/test_hs21.py index 7e8755430ca..ff43c04355c 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs21.py +++ b/src/third_party/wiredtiger/test/suite/test_hs21.py @@ -47,12 +47,13 @@ class test_hs21(wttest.WiredTigerTestCase): numfiles = 10 nrows = 1000 - key_format_values = [ - ('column', dict(key_format='r', key1=1, key2=2)), - ('string-row', dict(key_format='S', key1=str(0), key2=str(1))), + format_values = [ + ('column', dict(key_format='r', key1=1, key2=2, value_format='S')), + ('column-fix', dict(key_format='r', key1=1, key2=2, value_format='8t')), + ('string-row', dict(key_format='S', key1=str(0), key2=str(1), value_format='S')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def large_updates(self, uri, value, ds, nrows, commit_ts): # Update a large number of records, we'll hang if the history store table isn't working. @@ -64,24 +65,24 @@ class test_hs21(wttest.WiredTigerTestCase): session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts)) cursor.close() - def check(self, session, check_value, uri, nrows, read_ts=-1): + def check(self, session, check_value, uri, nrows, read_ts=-1, flcs_nrows=None): + if self.value_format != '8t': + flcs_nrows = None + # Validate we read an expected value (optionally at a given read timestamp). if read_ts != -1: session.begin_transaction('read_timestamp=' + self.timestamp_str(read_ts)) cursor = session.open_cursor(uri) count = 0 for k, v in cursor: - self.assertEqual(v, check_value) + if flcs_nrows is not None and count >= nrows: + self.assertEqual(v, 0) + else: + self.assertEqual(v, check_value) count += 1 if read_ts != -1: session.rollback_transaction() - if count != nrows: - self.prout("Oops") - self.prout("value: " + str(check_value)) - self.prout("count: " + str(count)) - self.prout("nrows: " + str(nrows)) - self.prout("read_ts: " + str(read_ts)) - self.assertEqual(count, nrows) + self.assertEqual(count, flcs_nrows if flcs_nrows is not None else nrows) cursor.close() def parse_run_write_gen(self, uri): @@ -102,8 +103,13 @@ class test_hs21(wttest.WiredTigerTestCase): def test_hs(self): active_files = [] - value1 = 'a' * 500 - value2 = 'd' * 500 + + if self.value_format == '8t': + value1 = 97 + value2 = 100 + else: + value1 = 'a' * 500 + value2 = 'd' * 500 # Set up 'numfiles' with 'numrows' entries. We want to create a number of files that # contain active history (content newer than the oldest timestamp). @@ -112,7 +118,8 @@ class test_hs21(wttest.WiredTigerTestCase): file_uri = 'file:%s.%d.wt' % (self.file_name, f) # Create a small table. ds = SimpleDataSet( - self, table_uri, 0, key_format=self.key_format, value_format='S', config='log=(enabled=false)') + self, table_uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() # Checkpoint to ensure we write the files metadata checkpoint value. self.session.checkpoint() @@ -145,7 +152,7 @@ class test_hs21(wttest.WiredTigerTestCase): # Load more data with a later timestamp. self.large_updates(ds.uri, value2, ds, self.nrows, 100) # Check that the new updates are only seen after the update timestamp. - self.check(self.session, value1, ds.uri, self.nrows // 2, 2) + self.check(self.session, value1, ds.uri, self.nrows // 2, 2, flcs_nrows=self.nrows) self.check(self.session, value2, ds.uri, self.nrows, 100) # Our sweep scan interval is every 1 second and the amount of idle time needed for a handle to be closed is 2 seconds. @@ -191,7 +198,7 @@ class test_hs21(wttest.WiredTigerTestCase): # handles have been closed. for (_, ds) in active_files: # Check that all updates at timestamp 2 are seen. - self.check(session_read, value1, ds.uri, self.nrows // 2) + self.check(session_read, value1, ds.uri, self.nrows // 2, flcs_nrows=self.nrows) session_read.rollback_transaction() # Perform a series of checks over our files to ensure that our transactions have been written diff --git a/src/third_party/wiredtiger/test/suite/test_hs22.py b/src/third_party/wiredtiger/test/suite/test_hs22.py index 97a65ace10a..b453c43602a 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs22.py +++ b/src/third_party/wiredtiger/test/suite/test_hs22.py @@ -36,16 +36,18 @@ class test_hs22(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r', key1=1, key2=2)), - ('string-row', dict(key_format='S', key1=str(0), key2=str(1))), + format_values = [ + ('column', dict(key_format='r', key1=1, key2=2, value_format='S')), + ('column-fix', dict(key_format='r', key1=1, key2=2, value_format='8t')), + ('string-row', dict(key_format='S', key1=str(0), key2=str(1), value_format='S')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test_onpage_out_of_order_timestamp_update(self): uri = 'table:test_hs22' - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, format) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) @@ -53,8 +55,12 @@ class test_hs22(wttest.WiredTigerTestCase): key1 = self.key1 key2 = self.key2 - value1 = 'a' - value2 = 'b' + if self.value_format == '8t': + value1 = 97 # 'a' + value2 = 98 # 'b' + else: + value1 = 'a' + value2 = 'b' # Insert a key. self.session.begin_transaction() @@ -103,7 +109,8 @@ class test_hs22(wttest.WiredTigerTestCase): def test_out_of_order_timestamp_update_newer_than_tombstone(self): uri = 'table:test_hs22' - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, format) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) @@ -111,8 +118,12 @@ class test_hs22(wttest.WiredTigerTestCase): key1 = self.key1 key2 = self.key2 - value1 = 'a' - value2 = 'b' + if self.value_format == '8t': + value1 = 97 # 'a' + value2 = 98 # 'b' + else: + value1 = 'a' + value2 = 'b' # Insert a key. self.session.begin_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs23.py b/src/third_party/wiredtiger/test/suite/test_hs23.py index d6bc1ec3d70..d3898722620 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs23.py +++ b/src/third_party/wiredtiger/test/suite/test_hs23.py @@ -36,27 +36,36 @@ class test_hs23(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r', key=1)), - ('string-row', dict(key_format='S', key=str(0))), + format_values = [ + ('column', dict(key_format='r', key=1, value_format='S')), + ('column-fix', dict(key_format='r', key=1, value_format='8t')), + ('string-row', dict(key_format='S', key=str(0), value_format='S')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test(self): uri = 'table:test_hs23' - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, format) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) key = self.key - value1 = 'a' - value2 = 'b' - value3 = 'c' - value4 = 'd' - value5 = 'e' + if self.value_format == '8t': + value1 = 97 # 'a' + value2 = 98 # 'b' + value3 = 99 # 'c' + value4 = 100 # 'd' + value5 = 101 # 'e' + else: + value1 = 'a' + value2 = 'b' + value3 = 'c' + value4 = 'd' + value5 = 'e' # Insert a key. self.session.begin_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_hs24.py b/src/third_party/wiredtiger/test/suite/test_hs24.py index 47929688071..31543fbb2cd 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs24.py +++ b/src/third_party/wiredtiger/test/suite/test_hs24.py @@ -33,9 +33,10 @@ from wtscenario import make_scenarios # test_hs24.py # Test that out of order timestamp fix racing with checkpointing the history store doesn't create inconsistent checkpoint. class test_hs24(wttest.WiredTigerTestCase): - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] checkpoint_stress_scenarios = [ @@ -43,7 +44,7 @@ class test_hs24(wttest.WiredTigerTestCase): ('history_store_checkpoint_delay_stress', dict(checkpoint_stress='history_store_checkpoint_delay')), ] - scenarios = make_scenarios(key_format_values, checkpoint_stress_scenarios) + scenarios = make_scenarios(format_values, checkpoint_stress_scenarios) def conn_config(self): return 'timing_stress_for_test=({})'.format(self.checkpoint_stress) @@ -52,12 +53,22 @@ class test_hs24(wttest.WiredTigerTestCase): uri = 'table:test_hs24' numrows = 2000 - value1 = 'a' * 500 - value2 = 'b' * 500 - value3 = 'c' * 500 - value4 = 'd' * 500 + def moresetup(self): + self.format = 'key_format={},value_format={}'. format(self.key_format, self.value_format) + if self.value_format == '8t': + self.value1 = 97 + self.value2 = 98 + self.value3 = 99 + self.value4 = 100 + else: + self.value1 = 'a' * 500 + self.value2 = 'b' * 500 + self.value3 = 'c' * 500 + self.value4 = 'd' * 500 + def test_zero_ts(self): - self.session.create(self.uri, 'key_format={},value_format=S'. format(self.key_format)) + self.moresetup() + self.session.create(self.uri, self.format) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) for i in range(1, self.numrows + 1): @@ -110,7 +121,8 @@ class test_hs24(wttest.WiredTigerTestCase): session.close() def test_zero_commit(self): - self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) + self.moresetup() + self.session.create(self.uri, self.format) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) for i in range(1, self.numrows + 1): @@ -153,7 +165,8 @@ class test_hs24(wttest.WiredTigerTestCase): session.close() def test_out_of_order_ts(self): - self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) + self.moresetup() + self.session.create(self.uri, self.format) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(self.uri) for i in range(1, self.numrows + 1): diff --git a/src/third_party/wiredtiger/test/suite/test_hs25.py b/src/third_party/wiredtiger/test/suite/test_hs25.py index a1a18928864..514440c350c 100644 --- a/src/third_party/wiredtiger/test/suite/test_hs25.py +++ b/src/third_party/wiredtiger/test/suite/test_hs25.py @@ -36,43 +36,54 @@ class test_hs25(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' uri = 'table:test_hs25' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test_insert_updates_hs(self): self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) - self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(self.uri, format) s = self.conn.open_session() + if self.value_format == '8t': + valuea = 97 + valueb = 98 + valuec = 99 + else: + valuea = 'a' + valueb = 'b' + valuec = 'c' + # Update the first key. cursor1 = self.session.open_cursor(self.uri) self.session.begin_transaction() - cursor1[1] = 'a' + cursor1[1] = valuea self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Update the second key. self.session.begin_transaction() - cursor1[2] = 'a' + cursor1[2] = valuea self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) self.session.begin_transaction() - cursor1[2] = 'b' + cursor1[2] = valueb self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Prepared update on the first key. self.session.begin_transaction() - cursor1[1] = 'b' - cursor1[1] = 'c' + cursor1[1] = valueb + cursor1[1] = valuec self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(4)) # Run eviction cursor. s.begin_transaction('ignore_prepare=true') evict_cursor = s.open_cursor(self.uri, None, 'debug=(release_evict)') - self.assertEqual(evict_cursor[1], 'a') - self.assertEqual(evict_cursor[2], 'b') + self.assertEqual(evict_cursor[1], valuea) + self.assertEqual(evict_cursor[2], valueb) s.rollback_transaction() self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare04.py b/src/third_party/wiredtiger/test/suite/test_prepare04.py index aa130622ab5..1771613d88c 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare04.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare04.py @@ -49,6 +49,7 @@ class test_prepare04(wttest.WiredTigerTestCase, suite_subprocess): types = [ ('col', dict(extra_config=',log=(enabled=false),key_format=r')), + ('col-fix', dict(extra_config=',log=(enabled=false),key_format=r,value_format=8t')), ('lsm', dict(extra_config=',log=(enabled=false),type=lsm')), ('row', dict(extra_config=',log=(enabled=false)')), ] diff --git a/src/third_party/wiredtiger/test/suite/test_prepare05.py b/src/third_party/wiredtiger/test/suite/test_prepare05.py index c92a079bf6c..922fcb8a241 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare05.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare05.py @@ -39,15 +39,17 @@ class test_prepare05(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='i')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='i')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test_timestamp_api(self): - self.session.create(self.uri, 'key_format={},value_format=i'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(self.uri, format) c = self.session.open_cursor(self.uri) # It is illegal to set a prepare timestamp older than oldest timestamp. diff --git a/src/third_party/wiredtiger/test/suite/test_prepare06.py b/src/third_party/wiredtiger/test/suite/test_prepare06.py index b4e65f183b0..5348f11ef1e 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare06.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare06.py @@ -39,15 +39,17 @@ class test_prepare06(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='i')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='i')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test_timestamp_api(self): - self.session.create(self.uri, 'key_format={},value_format=i'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(self.uri, format) c = self.session.open_cursor(self.uri) # It is illegal to set the prepare timestamp older than the oldest diff --git a/src/third_party/wiredtiger/test/suite/test_prepare07.py b/src/third_party/wiredtiger/test/suite/test_prepare07.py index 49d3544cf40..d0986efce94 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare07.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare07.py @@ -41,20 +41,20 @@ class test_prepare07(wttest.WiredTigerTestCase): # Force a small cache. conn_config = 'cache_size=50MB' - key_format_values = [ - ('column', dict(key_format='r')), - ('string-row', dict(key_format='S')), + format_values = [ + ('column', dict(key_format='r', value_format='u')), + ('column-fix', dict(key_format='r', value_format='8t')), + ('string-row', dict(key_format='S', value_format='u')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) - def older_prepare_updates(self, uri, ds, nrows, value_a): + def older_prepare_updates(self, uri, ds, nrows, value_a, value_b): # Commit some updates along with a prepared update, which is not resolved. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(100)) self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(100)) # Commit some updates. - value_b = b"bbbbb" * 100 cursor = self.session.open_cursor(uri) self.session.begin_transaction('isolation=snapshot') cursor.set_key(ds.key(nrows + 1)) @@ -148,9 +148,16 @@ class test_prepare07(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test" nrows = 100 - ds = SimpleDataSet(self, uri, nrows, key_format=self.key_format, value_format='u') + ds = SimpleDataSet( + self, uri, nrows, key_format=self.key_format, value_format=self.value_format) ds.populate() - value_a = b"aaaaa" * 100 + + if self.value_format == '8t': + value_a = 97 + value_b = 98 + else: + value_a = b"aaaaa" * 100 + value_b = b"bbbbb" * 100 # Initially load huge data cursor = self.session.open_cursor(uri) @@ -164,7 +171,7 @@ class test_prepare07(wttest.WiredTigerTestCase): # Check if txn_visible_all is working properly, when an active oldest # transaction is a prepared transaction and the oldest timestamp # advances beyond the prepared timestamp. - self.older_prepare_updates(uri, ds, nrows, value_a) + self.older_prepare_updates(uri, ds, nrows, value_a, value_b) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare08.py b/src/third_party/wiredtiger/test/suite/test_prepare08.py index 05ad8ab287e..9598ca1f81a 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare08.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare08.py @@ -39,12 +39,13 @@ class test_prepare08(wttest.WiredTigerTestCase): # Force a small cache. conn_config = 'cache_size=10MB,eviction_dirty_trigger=80,eviction_updates_trigger=80' - key_format_values = [ - ('column', dict(key_format='r')), - ('string-row', dict(key_format='S')), + format_values = [ + ('column', dict(key_format='r', value_format='u')), + ('column-fix', dict(key_format='r', value_format='8t')), + ('string-row', dict(key_format='S', value_format='u')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def updates(self, ds, uri, nrows, value, ts): cursor = self.session.open_cursor(uri) @@ -87,18 +88,27 @@ class test_prepare08(wttest.WiredTigerTestCase): # Create a small table. uri_1 = "table:test_prepare08_1" - ds_1 = SimpleDataSet(self, uri_1, 0, key_format=self.key_format, value_format='u') + ds_1 = SimpleDataSet( + self, uri_1, 0, key_format=self.key_format, value_format=self.value_format) ds_1.populate() uri_2 = "table:test_prepare08_2" - ds_2 = SimpleDataSet(self, uri_2, 0, key_format=self.key_format, value_format='u') + ds_2 = SimpleDataSet( + self, uri_2, 0, key_format=self.key_format, value_format=self.value_format) ds_2.populate() - value_a = b"aaaaa" * 100 - value_b = b"bbbbb" * 100 - value_c = b"ccccc" * 100 - value_d = b"ddddd" * 100 - value_e = b"eeeee" * 100 + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + value_d = 100 + value_e = 101 + else: + value_a = b"aaaaa" * 100 + value_b = b"bbbbb" * 100 + value_c = b"ccccc" * 100 + value_d = b"ddddd" * 100 + value_e = b"eeeee" * 100 # Commit some updates along with a prepared update, which is not resolved. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) @@ -155,19 +165,28 @@ class test_prepare08(wttest.WiredTigerTestCase): # Create a small table. uri_1 = "table:test_prepare10_1" - ds_1 = SimpleDataSet(self, uri_1, 0, key_format=self.key_format, value_format='u') + ds_1 = SimpleDataSet( + self, uri_1, 0, key_format=self.key_format, value_format=self.value_format) ds_1.populate() # Create another small table. uri_2 = "table:test_prepare10_2" - ds_2 = SimpleDataSet(self, uri_2, 0, key_format=self.key_format, value_format='u') + ds_2 = SimpleDataSet( + self, uri_2, 0, key_format=self.key_format, value_format=self.value_format) ds_2.populate() - value_a = b"aaaaa" * 100 - value_b = b"bbbbb" * 100 - value_c = b"ccccc" * 100 - value_d = b"ddddd" * 100 - value_e = b"eeeee" * 100 + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + value_d = 100 + value_e = 101 + else: + value_a = b"aaaaa" * 100 + value_b = b"bbbbb" * 100 + value_c = b"ccccc" * 100 + value_d = b"ddddd" * 100 + value_e = b"eeeee" * 100 # Commit some updates along with a prepared update, which is not resolved. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) @@ -229,19 +248,28 @@ class test_prepare08(wttest.WiredTigerTestCase): # Create a small table. uri_1 = "table:test_prepare10_1" - ds_1 = SimpleDataSet(self, uri_1, 0, key_format=self.key_format, value_format='u') + ds_1 = SimpleDataSet( + self, uri_1, 0, key_format=self.key_format, value_format=self.value_format) ds_1.populate() # Create another small table. uri_2 = "table:test_prepare10_2" - ds_2 = SimpleDataSet(self, uri_2, 0, key_format=self.key_format, value_format='u') + ds_2 = SimpleDataSet( + self, uri_2, 0, key_format=self.key_format, value_format=self.value_format) ds_2.populate() - value_a = b"aaaaa" * 100 - value_b = b"bbbbb" * 100 - value_c = b"ccccc" * 100 - value_d = b"ddddd" * 100 - value_e = b"eeeee" * 100 + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + value_d = 100 + value_e = 101 + else: + value_a = b"aaaaa" * 100 + value_b = b"bbbbb" * 100 + value_c = b"ccccc" * 100 + value_d = b"ddddd" * 100 + value_e = b"eeeee" * 100 # Commit some updates along with a prepared update, which is not resolved. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare09.py b/src/third_party/wiredtiger/test/suite/test_prepare09.py index 8aaacf1c3fb..14acd175c57 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare09.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare09.py @@ -39,19 +39,26 @@ class test_prepare09(wttest.WiredTigerTestCase): conn_config = 'cache_size=2MB' session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test_prepared_update_is_aborted_correctly_with_on_disk_value(self): uri = "table:test_prepare09" - create_params = 'value_format=S,key_format={}'.format(self.key_format) - value1 = 'a' * 10000 - value2 = 'b' * 10000 - value3 = 'c' * 10000 + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + + if self.value_format == '8t': + value1 = 97 + value2 = 98 + value3 = 99 + else: + value1 = 'a' * 10000 + value2 = 'b' * 10000 + value3 = 'c' * 10000 self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) @@ -90,11 +97,19 @@ class test_prepare09(wttest.WiredTigerTestCase): def test_prepared_update_is_aborted_correctly(self): uri = "table:test_prepare09" - create_params = 'value_format=S,key_format=i' - value1 = 'a' * 10000 - value2 = 'b' * 10000 - value3 = 'e' * 10000 - value4 = 'd' * 10000 + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + + if self.value_format == '8t': + value1 = 97 + value2 = 98 + value3 = 99 + value4 = 100 + else: + value1 = 'a' * 10000 + value2 = 'b' * 10000 + value3 = 'e' * 10000 + value4 = 'd' * 10000 + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) @@ -119,8 +134,14 @@ class test_prepare09(wttest.WiredTigerTestCase): self.assertEqual(self.session.rollback_transaction(), 0) # Search for key one, we should get not found. + # (Except for FLCS, where for now at least the table extends itself under uncommitted + # updates, so we expect to see 0.) cursor.set_key(1) - self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + if self.value_format == '8t': + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_value(), 0) + else: + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare10.py b/src/third_party/wiredtiger/test/suite/test_prepare10.py index a4eec9bfcc4..2dacafbf7be 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare10.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare10.py @@ -40,12 +40,13 @@ class test_prepare10(wttest.WiredTigerTestCase): conn_config = 'cache_size=10MB,eviction_dirty_trigger=80,eviction_updates_trigger=80' session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('string-row', dict(key_format='S')), + format_values = [ + ('column', dict(key_format='r', value_format='u')), + ('column-fix', dict(key_format='r', value_format='8t')), + ('string-row', dict(key_format='S', value_format='u')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def updates(self, ds, uri, nrows, value, ts): cursor = self.session.open_cursor(uri) @@ -81,7 +82,12 @@ class test_prepare10(wttest.WiredTigerTestCase): self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts)) for i in range(1, nrows): cursor.set_key(ds.key(i)) - self.assertEquals(cursor.search(), wiredtiger.WT_NOTFOUND) + if self.value_format == '8t': + # In FLCS, deleted values read back as 0. + self.assertEquals(cursor.search(), 0) + self.assertEquals(cursor.get_value(), 0) + else: + self.assertEquals(cursor.search(), wiredtiger.WT_NOTFOUND) self.session.commit_transaction() cursor.close() @@ -89,12 +95,17 @@ class test_prepare10(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test_prepare10" nrows = 1000 - ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format='u') + ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format=self.value_format) ds.populate() - value_a = b"aaaaa" * 100 - value_b = b"bbbbb" * 100 - value_c = b"ccccc" * 100 + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + else: + value_a = b"aaaaa" * 100 + value_b = b"bbbbb" * 100 + value_c = b"ccccc" * 100 # Commit some updates along with a prepared update, which is not resolved. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) @@ -131,7 +142,12 @@ class test_prepare10(wttest.WiredTigerTestCase): session3.begin_transaction() for i in range(1, nrows): cursor3.set_key(ds.key(i)) - self.assertEquals(cursor3.search(), wiredtiger.WT_NOTFOUND) + if self.value_format == '8t': + # In FLCS deleted records read back as 0. + self.assertEquals(cursor3.search(), 0) + self.assertEquals(cursor3.get_value(), 0) + else: + self.assertEquals(cursor3.search(), wiredtiger.WT_NOTFOUND) session3.commit_transaction() # Reset the cursor. @@ -169,7 +185,12 @@ class test_prepare10(wttest.WiredTigerTestCase): # session3 still can't see a value for i in range(1, nrows): cursor3.set_key(ds.key(i)) - self.assertEquals(cursor3.search(), wiredtiger.WT_NOTFOUND) + if self.value_format == '8t': + # In FLCS, deleted records read back as 0. + self.assertEquals(cursor3.search(), 0) + self.assertEquals(cursor3.get_value(), 0) + else: + self.assertEquals(cursor3.search(), wiredtiger.WT_NOTFOUND) session3.commit_transaction() # close sessions. diff --git a/src/third_party/wiredtiger/test/suite/test_prepare11.py b/src/third_party/wiredtiger/test/suite/test_prepare11.py index 5770d9ad2f2..beaf69bca46 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare11.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare11.py @@ -35,9 +35,10 @@ class test_prepare11(wttest.WiredTigerTestCase): conn_config = 'cache_size=2MB' session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r', key1=17)), - ('string-row', dict(key_format='S', key1='key1')), + format_values = [ + ('column', dict(key_format='r', key1=17, value_format='S')), + ('column-fix', dict(key_format='r', key1=17, value_format='8t')), + ('string-row', dict(key_format='S', key1='key1', value_format='S')), ] commit_values = [ @@ -45,20 +46,29 @@ class test_prepare11(wttest.WiredTigerTestCase): ('rollback', dict(commit=False)), ] - scenarios = make_scenarios(key_format_values, commit_values) + scenarios = make_scenarios(format_values, commit_values) def test_prepare_update_rollback(self): uri = "table:test_prepare11" - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, format) + + if self.value_format == '8t': + value_x = 120 + value_y = 121 + else: + value_x = 'xxxx' + value_y = 'yyyy' + self.session.begin_transaction("isolation=snapshot") # In the scenario where we have a reserved update in between two updates, the key repeated # flag won't get set and we'll call resolve prepared op on both prepared updates. c = self.session.open_cursor(uri, None) - c[self.key1] = 'xxxx' + c[self.key1] = value_x c.set_key(self.key1) c.reserve() - c[self.key1] = 'yyyy' + c[self.key1] = value_y self.session.prepare_transaction('prepare_timestamp=10') if self.commit: self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(20)) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare12.py b/src/third_party/wiredtiger/test/suite/test_prepare12.py index f559bde9c1b..b852545024c 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare12.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare12.py @@ -39,35 +39,46 @@ class test_prepare12(wttest.WiredTigerTestCase): conn_config = 'cache_size=2MB' session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test_prepare_update_restore(self): uri = "table:test_prepare12" - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, format) + + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_aaa = 65 + else: + value_a = 'a' + value_b = 'b' + value_aaa = 'a' * 500 # Prepare a transaction cursor = self.session.open_cursor(uri, None) self.session.begin_transaction() - cursor[1] = 'a' + cursor[1] = value_a self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(1)) # Insert an uncommitted key session2 = self.conn.open_session(None) cursor2 = session2.open_cursor(uri, None) session2.begin_transaction() - cursor2[2] = 'b' + cursor2[2] = value_b # Insert a bunch of other content to fill the database to trigger eviction. session3 = self.conn.open_session(None) cursor3 = session3.open_cursor(uri, None) for i in range(3, 101): session3.begin_transaction() - cursor3[i] = 'a' * 500 + cursor3[i] = value_aaa session3.commit_transaction() # Commit the prepared update @@ -75,4 +86,4 @@ class test_prepare12(wttest.WiredTigerTestCase): # Read the prepared update self.session.begin_transaction('read_timestamp=' + self.timestamp_str(2)) - self.assertEqual(cursor[1], 'a') + self.assertEqual(cursor[1], value_a) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare13.py b/src/third_party/wiredtiger/test/suite/test_prepare13.py index fbb735ce4f1..1b6b85a37fc 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare13.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare13.py @@ -40,24 +40,30 @@ class test_prepare13(wttest.WiredTigerTestCase): # Force a small cache. conn_config = 'cache_size=10MB' - key_format_values = [ - ('column', dict(key_format='r')), - ('string-row', dict(key_format='S')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), + ('string-row', dict(key_format='S', value_format='S')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test_prepare(self): nrows = 20000 + + if self.value_format == '8t': + replacement_value = 199 + else: + replacement_value = "replacement_value" + # Pin oldest and stable to timestamp 1. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) # Create a large table with lots of pages. uri = "table:test_prepare13" - key_format_str = "key_format=" + self.key_format - config = 'allocation_size=512,leaf_page_max=512,{},value_format=S'.format(key_format_str) - self.session.create(uri, config) + config = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, 'allocation_size=512,leaf_page_max=512,' + config) cursor = self.session.open_cursor(uri) for i in range(1, nrows): cursor[simple_key(cursor, i)] = simple_value(cursor, i) @@ -66,7 +72,7 @@ class test_prepare13(wttest.WiredTigerTestCase): # Prepare a record. self.session.begin_transaction() cursor = self.session.open_cursor(uri) - cursor[simple_key(cursor, 1000)] = "replacement_value" + cursor[simple_key(cursor, 1000)] = replacement_value cursor.close() self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(10)) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare14.py b/src/third_party/wiredtiger/test/suite/test_prepare14.py index 71737907526..d6127bb7f9c 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare14.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare14.py @@ -43,12 +43,13 @@ class test_prepare14(wttest.WiredTigerTestCase): ('inmem', dict(in_memory=True)) ] - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] - scenarios = make_scenarios(in_memory_values, key_format_values) + scenarios = make_scenarios(in_memory_values, format_values) def conn_config(self): config = 'cache_size=50MB' @@ -61,15 +62,19 @@ class test_prepare14(wttest.WiredTigerTestCase): def test_prepare14(self): # Create a table without logging. uri = "table:prepare14" - create_config = 'allocation_size=512,key_format={},value_format=S'.format(self.key_format) + create_config = 'allocation_size=512,key_format={},value_format={}'.format( + self.key_format, self.value_format) self.session.create(uri, create_config) + if self.value_format == '8t': + value = 97 # 'a' + else: + value = 'a' + # Pin oldest and stable timestamps to 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - value = 'a' - # Perform an update and a remove. s = self.conn.open_session() cursor = s.open_cursor(uri) @@ -87,7 +92,12 @@ class test_prepare14(wttest.WiredTigerTestCase): # Search for the key so we position our cursor on the page that we want to evict. self.session.begin_transaction("ignore_prepare = true") evict_cursor.set_key(key) - self.assertEquals(evict_cursor.search(), WT_NOTFOUND) + if self.value_format == '8t': + # In FLCS deleted values read back as 0. + self.assertEquals(evict_cursor.search(), 0) + self.assertEquals(evict_cursor.get_value(), 0) + else: + self.assertEquals(evict_cursor.search(), WT_NOTFOUND) evict_cursor.reset() evict_cursor.close() self.session.commit_transaction() @@ -95,5 +105,10 @@ class test_prepare14(wttest.WiredTigerTestCase): self.session.begin_transaction("ignore_prepare = true") cursor2 = self.session.open_cursor(uri) cursor2.set_key(key) - self.assertEquals(cursor2.search(), WT_NOTFOUND) + if self.value_format == '8t': + # In FLCS deleted values read back as 0. + self.assertEquals(cursor2.search(), 0) + self.assertEquals(cursor2.get_value(), 0) + else: + self.assertEquals(cursor2.search(), WT_NOTFOUND) self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare15.py b/src/third_party/wiredtiger/test/suite/test_prepare15.py index 1f8fb825042..d3a49d1a857 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare15.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare15.py @@ -40,9 +40,10 @@ class test_prepare15(wttest.WiredTigerTestCase): ('inmem', dict(in_memory=True)) ] - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] txn_end_values = [ @@ -50,7 +51,7 @@ class test_prepare15(wttest.WiredTigerTestCase): ('rollback', dict(commit=False)), ] - scenarios = make_scenarios(in_memory_values, key_format_values, txn_end_values) + scenarios = make_scenarios(in_memory_values, format_values, txn_end_values) def conn_config(self): config = 'cache_size=50MB' @@ -63,15 +64,19 @@ class test_prepare15(wttest.WiredTigerTestCase): def test_prepare_hs_update_and_tombstone(self): # Create a table without logging. uri = "table:prepare15" - create_config = 'key_format={},value_format=S'.format(self.key_format) + create_config = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, create_config) # Pin oldest and stable timestamps to 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - valuea = 'a' - valueb = 'b' + if self.value_format == '8t': + valuea = 97 # 'a' + valueb = 98 # 'b' + else: + valuea = 'a' + valueb = 'b' # Perform an update and remove. cursor = self.session.open_cursor(uri) @@ -101,7 +106,12 @@ class test_prepare15(wttest.WiredTigerTestCase): # Search for the key so we position our cursor on the page that we want to evict. self.session.begin_transaction('ignore_prepare = true') evict_cursor.set_key(1) - self.assertEquals(evict_cursor.search(), WT_NOTFOUND) + if self.value_format == '8t': + # In FLCS, deleted values read back as 0. + self.assertEquals(evict_cursor.search(), 0) + self.assertEquals(evict_cursor.get_value(), 0) + else: + self.assertEquals(evict_cursor.search(), WT_NOTFOUND) evict_cursor.reset() evict_cursor.close() self.session.commit_transaction() @@ -119,7 +129,12 @@ class test_prepare15(wttest.WiredTigerTestCase): # Search for the key so we position our cursor on the page that we want to evict. self.session.begin_transaction() evict_cursor.set_key(1) - self.assertEquals(evict_cursor.search(), WT_NOTFOUND) + if self.value_format == '8t': + # In FLCS, deleted values read back as 0. + self.assertEquals(evict_cursor.search(), 0) + self.assertEquals(evict_cursor.get_value(), 0) + else: + self.assertEquals(evict_cursor.search(), WT_NOTFOUND) evict_cursor.reset() evict_cursor.close() self.session.commit_transaction() @@ -134,16 +149,21 @@ class test_prepare15(wttest.WiredTigerTestCase): def test_prepare_hs_update(self): # Create a table without logging. uri = "table:prepare15" - create_config = 'key_format={},value_format=S'.format(self.key_format) + create_config = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, create_config) # Pin oldest and stable timestamps to 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - valuea = 'a' - valueb = 'b' - valuec = 'c' + if self.value_format == '8t': + valuea = 97 # 'a' + valueb = 98 # 'b' + valuec = 99 # 'c' + else: + valuea = 'a' + valueb = 'b' + valuec = 'c' # Perform an update. cursor = self.session.open_cursor(uri) @@ -218,7 +238,12 @@ class test_prepare15(wttest.WiredTigerTestCase): cursor2 = self.session.open_cursor(uri) cursor2.set_key(1) if self.commit: - self.assertEquals(cursor2.search(), WT_NOTFOUND) + if self.value_format == '8t': + # In FLCS, deleted values read back as 0. + self.assertEquals(cursor2.search(), 0) + self.assertEquals(cursor2.get_value(), 0) + else: + self.assertEquals(cursor2.search(), WT_NOTFOUND) else: self.assertEquals(cursor2.search(), 0) self.assertEqual(cursor2.get_value(), valuea) @@ -227,14 +252,17 @@ class test_prepare15(wttest.WiredTigerTestCase): def test_prepare_no_hs(self): # Create a table without logging. uri = "table:prepare15" - create_config = 'key_format={},value_format=S'.format(self.key_format) + create_config = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, create_config) # Pin oldest and stable timestamps to 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - value = 'a' + if self.value_format == '8t': + value = 97 # 'a' + else: + value = 'a' # Perform an update and remove. s = self.conn.open_session() @@ -252,7 +280,12 @@ class test_prepare15(wttest.WiredTigerTestCase): # Search for the key so we position our cursor on the page that we want to evict. self.session.begin_transaction("ignore_prepare = true") evict_cursor.set_key(1) - self.assertEquals(evict_cursor.search(), WT_NOTFOUND) + if self.value_format == '8t': + # In FLCS, deleted values read back as 0. + self.assertEquals(evict_cursor.search(), 0) + self.assertEquals(evict_cursor.get_value(), 0) + else: + self.assertEquals(evict_cursor.search(), WT_NOTFOUND) evict_cursor.reset() evict_cursor.close() self.session.commit_transaction() @@ -267,5 +300,10 @@ class test_prepare15(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor2 = self.session.open_cursor(uri) cursor2.set_key(1) - self.assertEquals(cursor2.search(), WT_NOTFOUND) + if self.value_format == '8t': + # In FLCS, deleted values read back as 0. + self.assertEquals(cursor2.search(), 0) + self.assertEquals(cursor2.get_value(), 0) + else: + self.assertEquals(cursor2.search(), WT_NOTFOUND) self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare16.py b/src/third_party/wiredtiger/test/suite/test_prepare16.py index f6e05aadfa9..6d22b28e06d 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare16.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare16.py @@ -39,9 +39,10 @@ class test_prepare16(wttest.WiredTigerTestCase): ('inmem', dict(in_memory=True)) ] - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('string_row', dict(key_format='S', value_format='S')), ] txn_end_values = [ @@ -49,7 +50,7 @@ class test_prepare16(wttest.WiredTigerTestCase): ('rollback', dict(commit=False)), ] - scenarios = make_scenarios(in_memory_values, key_format_values, txn_end_values) + scenarios = make_scenarios(in_memory_values, format_values, txn_end_values) def conn_config(self): config = 'cache_size=250MB' @@ -59,24 +60,33 @@ class test_prepare16(wttest.WiredTigerTestCase): config += ',in_memory=false' return config + def make_key(self, i): + if self.key_format == 'r': + return i + return str(i) + def test_prepare(self): nrows = 1000 # Create a table without logging. uri = "table:prepare16" - create_config = 'allocation_size=512,key_format=S,value_format=S,leaf_page_max=512,leaf_value_max=64MB' + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + create_config = 'allocation_size=512,leaf_page_max=512,leaf_value_max=64MB,' + format self.session.create(uri, create_config) + if self.value_format == '8t': + valuea = 97 + else: + valuea = 'a' * 400 + # Pin oldest and stable timestamps to 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - valuea = 'a' * 400 - cursor = self.session.open_cursor(uri) self.session.begin_transaction() for i in range(1, nrows + 1): - cursor[str(i)] = valuea + cursor[self.make_key(i)] = valuea cursor.reset() cursor.close() @@ -88,8 +98,13 @@ class test_prepare16(wttest.WiredTigerTestCase): evict_cursor = s.open_cursor(uri, None, "debug=(release_evict)") for i in range(1, nrows + 1): - evict_cursor.set_key(str(i)) - self.assertEquals(evict_cursor.search(), WT_NOTFOUND) + evict_cursor.set_key(self.make_key(i)) + # In FLCS (at least for now) uncommitted values extend the table with zeros. + if self.value_format == '8t': + self.assertEquals(evict_cursor.search(), 0) + self.assertEquals(evict_cursor.get_value(), 0) + else: + self.assertEquals(evict_cursor.search(), WT_NOTFOUND) evict_cursor.reset() if self.commit: @@ -106,10 +121,14 @@ class test_prepare16(wttest.WiredTigerTestCase): self.session.begin_transaction('read_timestamp=' + self.timestamp_str(20)) cursor = self.session.open_cursor(uri) for i in range(1, nrows + 1): - cursor.set_key(str(i)) + cursor.set_key(self.make_key(i)) if self.commit: self.assertEquals(cursor.search(), 0) self.assertEqual(cursor.get_value(), valuea) + elif self.value_format == '8t': + # In FLCS (at least for now) uncommitted values extend the table with zeros. + self.assertEquals(cursor.search(), 0) + self.assertEquals(cursor.get_value(), 0) else: self.assertEquals(cursor.search(), WT_NOTFOUND) self.session.commit_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare17.py b/src/third_party/wiredtiger/test/suite/test_prepare17.py index ae514b3865f..eb1e33ab594 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare17.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare17.py @@ -38,21 +38,29 @@ class test_prepare17(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' uri = 'table:test_prepare17' nrows = 1000 - value1 = 'aaaaa' - value2 = 'bbbbb' - key_format_values = [ - ('integer-row', dict(key_format='i')), - ('column', dict(key_format='r')), + format_values = [ + ('integer-row', dict(key_format='i', value_format='S')), + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), ] update = [ ('prepare', dict(prepare=True)), ('non-prepare', dict(prepare=False)), ] - scenarios = make_scenarios(key_format_values, update) + scenarios = make_scenarios(format_values, update) + + def moresetup(self): + if self.value_format == '8t': + self.value1 = 97 + self.value2 = 98 + else: + self.value1 = 'aaaaa' + self.value2 = 'bbbbb' def test_prepare(self): - create_params = 'key_format={},value_format=S'.format(self.key_format) + self.moresetup() + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(self.uri, create_params) cursor = self.session.open_cursor(self.uri) @@ -87,10 +95,11 @@ class test_prepare17(wttest.WiredTigerTestCase): self.session.checkpoint() def test_prepare_insert_remove(self): + self.moresetup() if not self.prepare: return - create_params = 'key_format={},value_format=S'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(self.uri, create_params) cursor = self.session.open_cursor(self.uri) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py b/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py index d62ee17b2a2..a2b8eb97cd8 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_conflict.py @@ -35,19 +35,25 @@ from wtdataset import simple_key, simple_value from wtscenario import make_scenarios class test_prepare_conflict(wttest.WiredTigerTestCase): - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test_prepare(self): # Create a large table with lots of pages. uri = "table:test_prepare_conflict" - key_format = 'key_format=' + self.key_format - config = 'allocation_size=512,leaf_page_max=512,{},value_format=S'.format(key_format) - self.session.create(uri, config) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, 'allocation_size=512,leaf_page_max=512,' + format) + + if self.value_format == '8t': + replacement_value = 199 + else: + replacement_value = "replacement_value" + cursor = self.session.open_cursor(uri) for i in range(1, 80000): cursor[simple_key(cursor, i)] = simple_value(cursor, i) @@ -70,7 +76,7 @@ class test_prepare_conflict(wttest.WiredTigerTestCase): # Modify a record on a fast-truncate page. cursor = self.session.open_cursor(uri) - cursor[simple_key(cursor, 40000)] = "replacement_value" + cursor[simple_key(cursor, 40000)] = replacement_value cursor.close() # Prepare and commit the transaction. diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py b/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py index 579927bd38e..07192fa9888 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_cursor01.py @@ -39,9 +39,10 @@ from wtscenario import make_scenarios # WT_CURSOR navigation (next/prev) tests with prepared transactions class test_prepare_cursor01(wttest.WiredTigerTestCase): - keyfmt = [ - ('row-store', dict(keyfmt='i')), - ('column-store', dict(keyfmt='r')), + fmt = [ + ('row-store', dict(keyfmt='i', valfmt='S')), + ('column-store', dict(keyfmt='r', valfmt='S')), + ('fixed-length-column-store', dict(keyfmt='r', valfmt='8t')), ] types = [ ('table-simple', dict(uri='table', ds=SimpleDataSet)), @@ -51,11 +52,11 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): ('isolation_read_committed', dict(isolation='read-committed')), ('isolation_snapshot', dict(isolation='snapshot')) ] - scenarios = make_scenarios(types, keyfmt, iso_types) - def skip(self): - return self.keyfmt == 'r' and \ - (self.ds.is_lsm() or self.uri == 'lsm') + def keep(name, d): + return d['keyfmt'] != 'r' or (d['uri'] != 'lsm' and not d['ds'].is_lsm()) + + scenarios = make_scenarios(types, fmt, iso_types, include=keep) # Test cursor navigate (next/prev) with prepared transactions. # Cursor navigate with timestamp reads and non-timestamped reads. @@ -64,8 +65,6 @@ class test_prepare_cursor01(wttest.WiredTigerTestCase): # after cursor : with timestamp after commit timestamp. # Cursor with out read timestamp behaviour should be same after cursor behavior. def test_cursor_navigate_prepare_transaction(self): - if self.skip(): - return # Build an object. uri = self.uri + ':test_prepare_cursor01' diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_cursor02.py b/src/third_party/wiredtiger/test/suite/test_prepare_cursor02.py index 4353ffabf70..09321f9a1b6 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_cursor02.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_cursor02.py @@ -37,8 +37,9 @@ class test_prepare_cursor02(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' keyfmt = [ - ('row-store', dict(keyfmt='i')), - ('column-store', dict(keyfmt='r')), + ('row-store', dict(keyfmt='i', valfmt='S')), + ('column-store', dict(keyfmt='r', valfmt='S')), + ('fixed-length-column-store', dict(keyfmt='r', valfmt='8t')), ] types = [ ('table-simple', dict(uri='table', ds=SimpleDataSet)), @@ -46,18 +47,12 @@ class test_prepare_cursor02(wttest.WiredTigerTestCase): scenarios = make_scenarios(types, keyfmt) - def skip(self): - return self.keyfmt == 'r' and \ - (self.ds.is_lsm() or self.uri == 'lsm') - # Test cursor navigate (next/prev) with prepared transactions. def test_cursor_navigate_prepare_transaction(self): - if self.skip(): - return # Build an object. uri = self.uri + ':test_prepare_cursor02' - ds = self.ds(self, uri, 0, key_format=self.keyfmt) + ds = self.ds(self, uri, 0, key_format=self.keyfmt, value_format=self.valfmt) ds.populate() session = self.session diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py index 909f534982e..ddb57cf6496 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs01.py @@ -37,12 +37,13 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): # Force a small cache. conn_config = 'cache_size=50MB,eviction_updates_trigger=95,eviction_updates_target=80' - key_format_values = [ - ('column', dict(key_format='r')), - ('string-row', dict(key_format='S')), + format_values = [ + ('column', dict(key_format='r', value_format='u')), + ('column-fix', dict(key_format='r', value_format='8t')), + ('string-row', dict(key_format='S', value_format='u')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def check(self, uri, ds, nrows, nsessions, nkeys, read_ts, expected_value, not_expected_value): cursor = self.session.open_cursor(uri) @@ -74,8 +75,14 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): # Start with setting a stable timestamp to pin history in cache self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) + if self.value_format == '8t': + bigvalue1 = 98 + bigvalue2 = 99 + else: + bigvalue1 = b"bbbbb" * 100 + bigvalue2 = b"ccccc" * 100 + # Commit some updates to get eviction and history store fired up - bigvalue1 = b"bbbbb" * 100 cursor = self.session.open_cursor(uri) for i in range(1, nsessions * nkeys): self.session.begin_transaction('isolation=snapshot') @@ -88,7 +95,6 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): # prepared updates to the history store sessions = [0] * nsessions cursors = [0] * nsessions - bigvalue2 = b"ccccc" * 100 for j in range (0, nsessions): sessions[j] = self.conn.open_session() sessions[j].begin_transaction('isolation=snapshot') @@ -121,9 +127,14 @@ class test_prepare_hs01(wttest.WiredTigerTestCase): # Create a small table. uri = "table:test_prepare_hs01" nrows = 100 - ds = SimpleDataSet(self, uri, nrows, key_format=self.key_format, value_format='u') + ds = SimpleDataSet( + self, uri, nrows, key_format=self.key_format, value_format=self.value_format) ds.populate() - bigvalue = b"aaaaa" * 100 + + if self.value_format == '8t': + bigvalue = 97 + else: + bigvalue = b"aaaaa" * 100 # Initially load huge data cursor = self.session.open_cursor(uri) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs02.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs02.py index 6fbd416e94b..3790dbbadcc 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs02.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs02.py @@ -42,9 +42,10 @@ class test_prepare_hs02(wttest.WiredTigerTestCase, suite_subprocess): txn_config = 'isolation=snapshot' types = [ - ('col', dict(s_config='value_format=i,log=(enabled=false),key_format=r')), + ('col', dict(s_config='key_format=r,value_format=i,log=(enabled=false)')), + ('col-fix', dict(s_config='key_format=r,value_format=8t,log=(enabled=false)')), ('row', dict(s_config='key_format=i,value_format=i,log=(enabled=false)')), - ('lsm', dict(s_config='key_format=i, value_format=i,log=(enabled=false),type=lsm')), + ('lsm', dict(s_config='key_format=i,value_format=i,log=(enabled=false),type=lsm')), ] # Transaction end types diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs03.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs03.py index c2d6ca66d4e..5f6b6588b54 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs03.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs03.py @@ -53,12 +53,13 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): ('dont_corrupt_table', dict(corrupt=False)) ] - key_format_values = [ - ('column', dict(key_format='r')), - ('string-row', dict(key_format='S')), + format_values = [ + ('column', dict(key_format='r', value_format='u')), + ('column-fix', dict(key_format='r', value_format='8t')), + ('string-row', dict(key_format='S', value_format='u')), ] - scenarios = make_scenarios(corrupt_values, key_format_values) + scenarios = make_scenarios(corrupt_values, format_values) def corrupt_table(self): tablename="test_prepare_hs03.wt" @@ -92,8 +93,14 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): return val def prepare_updates(self, ds, nrows, nsessions, nkeys): + if self.value_format == '8t': + commit_value = 98 + prepare_value = 99 + else: + commit_value = b"bbbbb" * 100 + prepare_value = b"ccccc" * 100 + # Commit some updates to get eviction and history store fired up - commit_value = b"bbbbb" * 100 cursor = self.session.open_cursor(self.uri) for i in range(1, nsessions * nkeys): self.session.begin_transaction('isolation=snapshot') @@ -116,7 +123,6 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): # prepared updates to the history store sessions = [0] * nsessions cursors = [0] * nsessions - prepare_value = b"ccccc" * 100 for j in range (0, nsessions): sessions[j] = self.conn.open_session() sessions[j].begin_transaction('isolation=snapshot') @@ -199,9 +205,14 @@ class test_prepare_hs03(wttest.WiredTigerTestCase): def test_prepare_hs(self): nrows = 100 - ds = SimpleDataSet(self, self.uri, nrows, key_format=self.key_format, value_format='u') + ds = SimpleDataSet( + self, self.uri, nrows, key_format=self.key_format, value_format=self.value_format) ds.populate() - bigvalue = b"aaaaa" * 100 + + if self.value_format == '8t': + bigvalue = 97 + else: + bigvalue = b"aaaaa" * 100 # Initially load huge data cursor = self.session.open_cursor(self.uri) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py index fa74584ab0c..fd12cfe270f 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs04.py @@ -52,13 +52,14 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): ('rollback_transaction', dict(commit=False)) ] - key_format_values = [ + format_values = [ # Note: commit_key must exceed nrows to give behavior comparable to the row case. - ('column', dict(key_format='r', commit_key=1000)), - ('string-row', dict(key_format='S', commit_key='C')), + ('column', dict(key_format='r', commit_key=1000, value_format='u')), + ('column-fix', dict(key_format='r', commit_key=1000, value_format='8t')), + ('string-row', dict(key_format='S', commit_key='C', value_format='u')), ] - scenarios = make_scenarios(commit_values, key_format_values) + scenarios = make_scenarios(commit_values, format_values) def get_stat(self, stat): stat_cursor = self.session.open_cursor('statistics:') @@ -77,7 +78,12 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): if conflict == True: self.assertRaisesException(wiredtiger.WiredTigerError, lambda:cursor.search(), expected_value) elif expected_value == None: - self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + if self.value_format == '8t': + # In FLCS, deleted values read back as 0. + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_value(), 0) + else: + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) else: self.assertEqual(cursor.search(), 0) self.assertEqual(cursor.get_value(), expected_value) @@ -86,14 +92,20 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): def prepare_updates(self, ds): + commit_key = self.commit_key + if self.value_format == '8t': + commit_value = 98 + prepare_value = 99 + else: + commit_value = b"bbbbb" * 100 + prepare_value = b"ccccc" * 100 + # Set oldest and stable timestamp for the database. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1)) # Commit some updates to get eviction and history store fired up. # Insert a key at timestamp 1. - commit_key = self.commit_key - commit_value = b"bbbbb" * 100 cursor = self.session.open_cursor(self.uri) for i in range(1, self.nsessions * self.nkeys): self.session.begin_transaction('isolation=snapshot') @@ -124,7 +136,6 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): # the data store. Insert the same key at timestamp 20, but with prepare updates. sessions = [0] * self.nsessions cursors = [0] * self.nsessions - prepare_value = b"ccccc" * 100 for j in range (0, self.nsessions): sessions[j] = self.conn.open_session() sessions[j].begin_transaction('isolation=snapshot') @@ -216,9 +227,14 @@ class test_prepare_hs04(wttest.WiredTigerTestCase): def test_prepare_hs(self): - ds = SimpleDataSet(self, self.uri, self.nrows, key_format=self.key_format, value_format='u') + ds = SimpleDataSet( + self, self.uri, self.nrows, key_format=self.key_format, value_format=self.value_format) ds.populate() - bigvalue = b"aaaaa" * 100 + + if self.value_format == '8t': + bigvalue = 97 + else: + bigvalue = b"aaaaa" * 100 # Initially load huge data cursor = self.session.open_cursor(self.uri) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py b/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py index cc6f4a2f05d..a668bd100e2 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_hs05.py @@ -36,21 +36,27 @@ class test_prepare_hs05(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r', key=1)), - ('string-row', dict(key_format='S', key=str(1))), + format_values = [ + ('column', dict(key_format='r', key=1, value_format='S')), + ('column-fix', dict(key_format='r', key=1, value_format='8t')), + ('string-row', dict(key_format='S', key=str(1), value_format='S')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test_check_prepare_abort_hs_restore(self): uri = 'table:test_prepare_hs05' - create_params = 'key_format={},value_format=S'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, create_params) - value1 = 'a' * 5 - value2 = 'b' * 5 - value3 = 'c' * 5 + if self.value_format == '8t': + value1 = 97 + value2 = 98 + value3 = 99 + else: + value1 = 'a' * 5 + value2 = 'b' * 5 + value3 = 'c' * 5 self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) @@ -80,7 +86,12 @@ class test_prepare_hs05(wttest.WiredTigerTestCase): session2.begin_transaction('ignore_prepare=true') cursor2 = session2.open_cursor(uri, None, "debug=(release_evict=true)") cursor2.set_key(key) - self.assertEquals(cursor2.search(), WT_NOTFOUND) + if self.value_format == '8t': + # In FLCS, deleted values read back as 0. + self.assertEquals(cursor2.search(), 0) + self.assertEquals(cursor2.get_value(), 0) + else: + self.assertEquals(cursor2.search(), WT_NOTFOUND) cursor2.reset() # This should abort the prepared transaction. @@ -98,5 +109,10 @@ class test_prepare_hs05(wttest.WiredTigerTestCase): # The latest version should be marked deleted. self.session.begin_transaction() cursor.set_key(key) - self.assertEqual(cursor.search(), WT_NOTFOUND) + if self.value_format == '8t': + # In FLCS, deleted values read back as 0. + self.assertEquals(cursor.search(), 0) + self.assertEquals(cursor.get_value(), 0) + else: + self.assertEqual(cursor.search(), WT_NOTFOUND) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_reserve.py b/src/third_party/wiredtiger/test/suite/test_reserve.py index 1bd57000afb..b99d1465da3 100644 --- a/src/third_party/wiredtiger/test/suite/test_reserve.py +++ b/src/third_party/wiredtiger/test/suite/test_reserve.py @@ -37,10 +37,11 @@ from wtscenario import make_scenarios # Test WT_CURSOR.reserve. class test_reserve(wttest.WiredTigerTestCase): - keyfmt = [ - ('integer', dict(keyfmt='i')), - ('recno', dict(keyfmt='r')), - ('string', dict(keyfmt='S')), + format_values = [ + ('integer', dict(keyfmt='i', valfmt='S')), + ('recno', dict(keyfmt='r', valfmt='S')), + ('fix', dict(keyfmt='r', valfmt='8t')), + ('string', dict(keyfmt='S', valfmt='S')), ] types = [ ('file', dict(uri='file', ds=SimpleDataSet)), @@ -51,19 +52,21 @@ class test_reserve(wttest.WiredTigerTestCase): ('table-simple', dict(uri='table', ds=SimpleDataSet)), ('table-simple-lsm', dict(uri='table', ds=SimpleLSMDataSet)), ] - scenarios = make_scenarios(types, keyfmt) - def skip(self): - return self.keyfmt == 'r' and \ - (self.ds.is_lsm() or self.uri == 'lsm') + def keep(name, d): + if d['keyfmt'] == 'r' and (d['uri'] == 'lsm' or d['ds'].is_lsm()): + return False + # The complex data sets have their own built-in value schemas that are not FLCS. + if d['valfmt'] == '8t' and d['ds'] == ComplexDataSet: + return False + return True - def test_reserve(self): - if self.skip(): - return + scenarios = make_scenarios(types, format_values, include=keep) + def test_reserve(self): uri = self.uri + ':test_reserve' - ds = self.ds(self, uri, 500, key_format=self.keyfmt) + ds = self.ds(self, uri, 500, key_format=self.keyfmt, value_format=self.valfmt) ds.populate() s = self.conn.open_session() c = s.open_cursor(uri, None) @@ -137,12 +140,10 @@ class test_reserve(wttest.WiredTigerTestCase): # Test cursor.reserve will fail if a key has not yet been set. def test_reserve_without_key(self): - if self.skip(): - return uri = self.uri + ':test_reserve_without_key' - ds = self.ds(self, uri, 10, key_format=self.keyfmt) + ds = self.ds(self, uri, 10, key_format=self.keyfmt, value_format=self.valfmt) ds.populate() s = self.conn.open_session() c = s.open_cursor(uri, None) @@ -153,12 +154,10 @@ class test_reserve(wttest.WiredTigerTestCase): # Test cursor.reserve will fail if there's no running transaction. def test_reserve_without_txn(self): - if self.skip(): - return uri = self.uri + ':test_reserve_without_txn' - ds = self.ds(self, uri, 10, key_format=self.keyfmt) + ds = self.ds(self, uri, 10, key_format=self.keyfmt, value_format=self.valfmt) ds.populate() s = self.conn.open_session() c = s.open_cursor(uri, None) @@ -169,12 +168,10 @@ class test_reserve(wttest.WiredTigerTestCase): # Test cursor.reserve returns a value on success. def test_reserve_returns_value(self): - if self.skip(): - return uri = self.uri + ':test_reserve_returns_value' - ds = self.ds(self, uri, 10, key_format=self.keyfmt) + ds = self.ds(self, uri, 10, key_format=self.keyfmt, value_format=self.valfmt) ds.populate() s = self.conn.open_session() c = s.open_cursor(uri, None) @@ -185,12 +182,10 @@ class test_reserve(wttest.WiredTigerTestCase): # Test cursor.reserve fails on non-standard cursors. def test_reserve_not_supported(self): - if self.skip(): - return uri = self.uri + ':test_reserve_not_supported' s = self.conn.open_session() - s.create(uri, 'key_format=' + self.keyfmt + ",value_format=S") + s.create(uri, 'key_format=' + self.keyfmt + ",value_format=" + self.valfmt) list = [ "bulk", "dump=json" ] for l in list: diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py index 5d1fe8029cd..0def121250b 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable01.py @@ -36,6 +36,9 @@ from time import sleep # test_rollback_to_stable01.py # Shared base class used by rollback to stable tests. +# +# Note: this class now expects self.value_format to have been set for some of the +# operations (those that need to specialize themselves for FLCS). class test_rollback_to_stable_base(wttest.WiredTigerTestCase): def retry_rollback(self, name, txn_session, code): retry_limit = 100 @@ -95,8 +98,14 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): session.begin_transaction() for i in range(1, nrows + 1): cursor.set_key(i) - mods = [wiredtiger.Modify(value, location, nbytes)] - self.assertEqual(cursor.modify(mods), 0) + # FLCS doesn't support modify (for obvious reasons) so just update. + # Use the first character of the passed-in value. + if self.value_format == '8t': + cursor.set_value(bytes(value, encoding='utf-8')[0]) + self.assertEqual(cursor.update(), 0) + else: + mods = [wiredtiger.Modify(value, location, nbytes)] + self.assertEqual(cursor.modify(mods), 0) if commit_ts == 0: session.commit_transaction() @@ -139,7 +148,12 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): session.rollback_transaction() raise(e) - def check(self, check_value, uri, nrows, read_ts): + def check(self, check_value, uri, nrows, flcs_extrarows, read_ts): + # In FLCS, deleted values read back as 0, and (at least for now) uncommitted appends + # cause zeros to appear under them. If flcs_extrarows isn't None, expect that many + # rows of zeros following the regular data. + flcs_tolerance = self.value_format == '8t' and flcs_extrarows is not None + session = self.session if read_ts == 0: session.begin_transaction() @@ -148,10 +162,13 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): cursor = session.open_cursor(uri) count = 0 for k, v in cursor: - self.assertEqual(v, check_value) + if flcs_tolerance and count >= nrows: + self.assertEqual(v, 0) + else: + self.assertEqual(v, check_value) count += 1 session.commit_transaction() - self.assertEqual(count, nrows) + self.assertEqual(count, nrows + flcs_extrarows if flcs_tolerance else nrows) cursor.close() def evict_cursor(self, uri, nrows, check_value): @@ -170,9 +187,10 @@ class test_rollback_to_stable_base(wttest.WiredTigerTestCase): class test_rollback_to_stable01(test_rollback_to_stable_base): session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] in_memory_values = [ @@ -185,7 +203,7 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(key_format_values, in_memory_values, prepare_values) + scenarios = make_scenarios(format_values, in_memory_values, prepare_values) def conn_config(self): config = 'cache_size=50MB,statistics=(all)' @@ -201,22 +219,27 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable01" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() + if self.value_format == '8t': + valuea = 97 + else: + valuea = "aaaaa" * 100 + # Pin oldest and stable to timestamp 1. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) - valuea = "aaaaa" * 100 self.large_updates(uri, valuea, ds, nrows, self.prepare, 10) # Check that all updates are seen. - self.check(valuea, uri, nrows, 10) + self.check(valuea, uri, nrows, None, 10) # Remove all keys with newer timestamp. self.large_removes(uri, ds, nrows, self.prepare, 20) # Check that the no keys should be visible. - self.check(valuea, uri, 0, 20) + self.check(valuea, uri, 0, nrows, 20) # Pin stable to timestamp 20 if prepare otherwise 10. if self.prepare: @@ -229,7 +252,8 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): self.conn.rollback_to_stable() # Check that the new updates are only seen after the update timestamp. - self.check(valuea, uri, nrows, 20) + self.session.breakpoint() + self.check(valuea, uri, nrows, None, 20) stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py index 8c27f5395e7..a041bb925cb 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable02.py @@ -39,9 +39,20 @@ from test_rollback_to_stable01 import test_rollback_to_stable_base class test_rollback_to_stable02(test_rollback_to_stable_base): session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + # For FLCS, set the page size down. Otherwise for the in-memory scenarios we get enough + # updates on the page that the in-memory page footprint exceeds the default maximum + # in-memory size, and that in turn leads to pathological behavior where the page gets + # force-evicted over and over again trying to resolve/condense the updates. But they + # don't (for in-memory, they can't be moved to the history store) so this leads to a + # semi-livelock state that makes the test some 20x slower than it needs to be. + # + # FUTURE: it would be better if the system adjusted on its own, but it's not critical + # and this workload (with every entry on the page modified repeatedly) isn't much like + # anything that happens in production. + format_values = [ + ('column', dict(key_format='r', value_format='S', extraconfig='')), + ('column_fix', dict(key_format='r', value_format='8t', extraconfig=',leaf_page_max=4096')), + ('integer_row', dict(key_format='i', value_format='S', extraconfig='')), ] in_memory_values = [ @@ -54,7 +65,7 @@ class test_rollback_to_stable02(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(key_format_values, in_memory_values, prepare_values) + scenarios = make_scenarios(format_values, in_memory_values, prepare_values) def conn_config(self): config = 'cache_size=100MB,statistics=(all)' @@ -70,32 +81,40 @@ class test_rollback_to_stable02(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable02" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)' + self.extraconfig) ds.populate() + if self.value_format == '8t': + valuea = 97 + valueb = 98 + valuec = 99 + valued = 100 + else: + valuea = "aaaaa" * 100 + valueb = "bbbbb" * 100 + valuec = "ccccc" * 100 + valued = "ddddd" * 100 + # Pin oldest and stable to timestamp 1. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) - valuea = "aaaaa" * 100 - valueb = "bbbbb" * 100 - valuec = "ccccc" * 100 - valued = "ddddd" * 100 self.large_updates(uri, valuea, ds, nrows, self.prepare, 10) # Check that all updates are seen. - self.check(valuea, uri, nrows, 10) + self.check(valuea, uri, nrows, None, 10) self.large_updates(uri, valueb, ds, nrows, self.prepare, 20) # Check that the new updates are only seen after the update timestamp. - self.check(valueb, uri, nrows, 20) + self.check(valueb, uri, nrows, None, 20) self.large_updates(uri, valuec, ds, nrows, self.prepare, 30) # Check that the new updates are only seen after the update timestamp. - self.check(valuec, uri, nrows, 30) + self.check(valuec, uri, nrows, None, 30) self.large_updates(uri, valued, ds, nrows, self.prepare, 40) # Check that the new updates are only seen after the update timestamp. - self.check(valued, uri, nrows, 40) + self.check(valued, uri, nrows, None, 40) # Pin stable to timestamp 30 if prepare otherwise 20. if self.prepare: @@ -103,14 +122,16 @@ class test_rollback_to_stable02(test_rollback_to_stable_base): else: self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) # Checkpoint to ensure that all the data is flushed. + self.session.breakpoint() if not self.in_memory: self.session.checkpoint() self.conn.rollback_to_stable() # Check that the new updates are only seen after the update timestamp. - self.check(valueb, uri, nrows, 40) - self.check(valueb, uri, nrows, 20) - self.check(valuea, uri, nrows, 10) + self.session.breakpoint() + self.check(valueb, uri, nrows, None, 40) + self.check(valueb, uri, nrows, None, 20) + self.check(valuea, uri, nrows, None, 10) stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py index c7260db1166..715e0fd23c3 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable03.py @@ -39,9 +39,10 @@ from test_rollback_to_stable01 import test_rollback_to_stable_base class test_rollback_to_stable01(test_rollback_to_stable_base): session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] in_memory_values = [ @@ -54,7 +55,7 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(key_format_values, in_memory_values, prepare_values) + scenarios = make_scenarios(format_values, in_memory_values, prepare_values) def conn_config(self): config = 'cache_size=4GB,statistics=(all)' @@ -70,27 +71,34 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable03" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() + if self.value_format == '8t': + valuea = 97 + valueb = 98 + valuec = 99 + else: + valuea = "aaaaa" * 100 + valueb = "bbbbb" * 100 + valuec = "ccccc" * 100 + # Pin oldest and stable to timestamp 1. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) - valuea = "aaaaa" * 100 - valueb = "bbbbb" * 100 - valuec = "ccccc" * 100 self.large_updates(uri, valuea, ds, nrows, self.prepare, 10) # Check that all updates are seen. - self.check(valuea, uri, nrows, 10) + self.check(valuea, uri, nrows, None, 10) self.large_updates(uri, valueb, ds, nrows, self.prepare, 20) # Check that all updates are seen. - self.check(valueb, uri, nrows, 20) + self.check(valueb, uri, nrows, None, 20) self.large_updates(uri, valuec, ds, nrows, self.prepare, 30) # Check that all updates are seen. - self.check(valuec, uri, nrows, 30) + self.check(valuec, uri, nrows, None, 30) # Pin stable to timestamp 30 if prepare otherwise 20. if self.prepare: @@ -103,8 +111,8 @@ class test_rollback_to_stable01(test_rollback_to_stable_base): self.conn.rollback_to_stable() # Check that the old updates are only seen even with the update timestamp. - self.check(valueb, uri, nrows, 20) - self.check(valuea, uri, nrows, 10) + self.check(valueb, uri, nrows, None, 20) + self.check(valuea, uri, nrows, None, 10) stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py index 2342163a327..0528548b94c 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable04.py @@ -40,9 +40,10 @@ def mod_val(value, char, location, nbytes=1): class test_rollback_to_stable04(test_rollback_to_stable_base): session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] in_memory_values = [ @@ -55,7 +56,7 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(key_format_values, in_memory_values, prepare_values) + scenarios = make_scenarios(format_values, in_memory_values, prepare_values) def conn_config(self): config = 'cache_size=500MB,statistics=(all)' @@ -71,27 +72,44 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable04" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() + if self.value_format == '8t': + value_a = 97 # 'a' + value_b = 98 # 'b' + value_c = 99 # 'c' + value_d = 100 # 'd' + + # No modifies in FLCS; do ordinary updates instead. + value_modQ = 81 # 'Q' + value_modR = 82 # 'R' + value_modS = 83 # 'S' + value_modT = 84 # 'T' + value_modW = 87 # 'W' + value_modX = 88 # 'X' + value_modY = 89 # 'Y' + value_modZ = 90 # 'Z' + else: + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + value_c = "ccccc" * 100 + value_d = "ddddd" * 100 + + value_modQ = mod_val(value_a, 'Q', 0) + value_modR = mod_val(value_modQ, 'R', 1) + value_modS = mod_val(value_modR, 'S', 2) + value_modT = mod_val(value_c, 'T', 3) + value_modW = mod_val(value_d, 'W', 4) + value_modX = mod_val(value_a, 'X', 5) + value_modY = mod_val(value_modX, 'Y', 6) + value_modZ = mod_val(value_modY, 'Z', 7) + # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - value_c = "ccccc" * 100 - value_d = "ddddd" * 100 - - value_modQ = mod_val(value_a, 'Q', 0) - value_modR = mod_val(value_modQ, 'R', 1) - value_modS = mod_val(value_modR, 'S', 2) - value_modT = mod_val(value_c, 'T', 3) - value_modW = mod_val(value_d, 'W', 4) - value_modX = mod_val(value_a, 'X', 5) - value_modY = mod_val(value_modX, 'Y', 6) - value_modZ = mod_val(value_modY, 'Z', 7) - # Perform a combination of modifies and updates. self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) self.large_modifies(uri, 'Q', ds, 0, 1, nrows, self.prepare, 30) @@ -108,19 +126,19 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): self.large_modifies(uri, 'Z', ds, 7, 1, nrows, self.prepare, 140) # Verify data is visible and correct. - self.check(value_a, uri, nrows, 20) - self.check(value_modQ, uri, nrows, 30) - self.check(value_modR, uri, nrows, 40) - self.check(value_modS, uri, nrows, 50) - self.check(value_b, uri, nrows, 60) - self.check(value_c, uri, nrows, 70) - self.check(value_modT, uri, nrows, 80) - self.check(value_d, uri, nrows, 90) - self.check(value_modW, uri, nrows, 100) - self.check(value_a, uri, nrows, 110) - self.check(value_modX, uri, nrows, 120) - self.check(value_modY, uri, nrows, 130) - self.check(value_modZ, uri, nrows, 140) + self.check(value_a, uri, nrows, None, 20) + self.check(value_modQ, uri, nrows, None, 30) + self.check(value_modR, uri, nrows, None, 40) + self.check(value_modS, uri, nrows, None, 50) + self.check(value_b, uri, nrows, None, 60) + self.check(value_c, uri, nrows, None, 70) + self.check(value_modT, uri, nrows, None, 80) + self.check(value_d, uri, nrows, None, 90) + self.check(value_modW, uri, nrows, None, 100) + self.check(value_a, uri, nrows, None, 110) + self.check(value_modX, uri, nrows, None, 120) + self.check(value_modY, uri, nrows, None, 130) + self.check(value_modZ, uri, nrows, None, 140) # Pin stable to timestamp 40 if prepare otherwise 30. if self.prepare: @@ -134,9 +152,9 @@ class test_rollback_to_stable04(test_rollback_to_stable_base): self.conn.rollback_to_stable() # Check that the correct data is seen at and after the stable timestamp. - self.check(value_modQ, uri, nrows, 30) - self.check(value_modQ, uri, nrows, 150) - self.check(value_a, uri, nrows, 20) + self.check(value_modQ, uri, nrows, None, 30) + self.check(value_modQ, uri, nrows, None, 150) + self.check(value_a, uri, nrows, None, 20) stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py index 4a32b589b18..92a63d273e1 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable05.py @@ -39,9 +39,10 @@ from test_rollback_to_stable01 import test_rollback_to_stable_base class test_rollback_to_stable05(test_rollback_to_stable_base): session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] in_memory_values = [ @@ -54,7 +55,7 @@ class test_rollback_to_stable05(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(key_format_values, in_memory_values, prepare_values) + scenarios = make_scenarios(format_values, in_memory_values, prepare_values) def conn_config(self): config = 'cache_size=50MB,statistics=(all)' @@ -70,46 +71,55 @@ class test_rollback_to_stable05(test_rollback_to_stable_base): # Create two tables without logging. uri_1 = "table:rollback_to_stable05_1" ds_1 = SimpleDataSet( - self, uri_1, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri_1, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds_1.populate() uri_2 = "table:rollback_to_stable05_2" ds_2 = SimpleDataSet( - self, uri_2, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri_2, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds_2.populate() - valuea = "aaaaa" * 100 - valueb = "bbbbb" * 100 - valuec = "ccccc" * 100 - valued = "ddddd" * 100 + if self.value_format == '8t': + valuea = 97 + valueb = 98 + valuec = 99 + valued = 100 + else: + valuea = "aaaaa" * 100 + valueb = "bbbbb" * 100 + valuec = "ccccc" * 100 + valued = "ddddd" * 100 + self.large_updates(uri_1, valuea, ds_1, nrows, self.prepare, 0) - self.check(valuea, uri_1, nrows, 0) + self.check(valuea, uri_1, nrows, None, 0) self.large_updates(uri_2, valuea, ds_2, nrows, self.prepare, 0) - self.check(valuea, uri_2, nrows, 0) + self.check(valuea, uri_2, nrows, None, 0) # Start a long running transaction and keep it open. session_2 = self.conn.open_session() session_2.begin_transaction('isolation=snapshot') self.large_updates(uri_1, valueb, ds_1, nrows, self.prepare, 0) - self.check(valueb, uri_1, nrows, 0) + self.check(valueb, uri_1, nrows, None, 0) self.large_updates(uri_1, valuec, ds_1, nrows, self.prepare, 0) - self.check(valuec, uri_1, nrows, 0) + self.check(valuec, uri_1, nrows, None, 0) self.large_updates(uri_1, valued, ds_1, nrows, self.prepare, 0) - self.check(valued, uri_1, nrows, 0) + self.check(valued, uri_1, nrows, None, 0) # Add updates to the another table. self.large_updates(uri_2, valueb, ds_2, nrows, self.prepare, 0) - self.check(valueb, uri_2, nrows, 0) + self.check(valueb, uri_2, nrows, None, 0) self.large_updates(uri_2, valuec, ds_2, nrows, self.prepare, 0) - self.check(valuec, uri_2, nrows, 0) + self.check(valuec, uri_2, nrows, None, 0) self.large_updates(uri_2, valued, ds_2, nrows, self.prepare, 0) - self.check(valued, uri_2, nrows, 0) + self.check(valued, uri_2, nrows, None, 0) # Checkpoint to ensure that all the data is flushed. if not self.in_memory: @@ -120,8 +130,8 @@ class test_rollback_to_stable05(test_rollback_to_stable_base): session_2.close() self.conn.rollback_to_stable() - self.check(valued, uri_1, nrows, 0) - self.check(valued, uri_2, nrows, 0) + self.check(valued, uri_1, nrows, None, 0) + self.check(valued, uri_2, nrows, None, 0) stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py index 7b702ca7b5e..eb2a3c16d02 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable06.py @@ -37,9 +37,10 @@ from test_rollback_to_stable01 import test_rollback_to_stable_base class test_rollback_to_stable06(test_rollback_to_stable_base): session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] in_memory_values = [ @@ -52,7 +53,7 @@ class test_rollback_to_stable06(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(key_format_values, in_memory_values, prepare_values) + scenarios = make_scenarios(format_values, in_memory_values, prepare_values) def conn_config(self): config = 'cache_size=50MB,statistics=(all)' @@ -68,18 +69,25 @@ class test_rollback_to_stable06(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable06" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + value_d = 100 + else: + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + value_c = "ccccc" * 100 + value_d = "ddddd" * 100 + # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - value_c = "ccccc" * 100 - value_d = "ddddd" * 100 - # Perform several updates. self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) self.large_updates(uri, value_b, ds, nrows, self.prepare, 30) @@ -87,10 +95,10 @@ class test_rollback_to_stable06(test_rollback_to_stable_base): self.large_updates(uri, value_d, ds, nrows, self.prepare, 50) # Verify data is visible and correct. - self.check(value_a, uri, nrows, 20) - self.check(value_b, uri, nrows, 30) - self.check(value_c, uri, nrows, 40) - self.check(value_d, uri, nrows, 50) + self.check(value_a, uri, nrows, None, 20) + self.check(value_b, uri, nrows, None, 30) + self.check(value_c, uri, nrows, None, 40) + self.check(value_d, uri, nrows, None, 50) # Checkpoint to ensure the data is flushed, then rollback to the stable timestamp. if not self.in_memory: @@ -98,10 +106,12 @@ class test_rollback_to_stable06(test_rollback_to_stable_base): self.conn.rollback_to_stable() # Check that all keys are removed. - self.check(value_a, uri, 0, 20) - self.check(value_b, uri, 0, 30) - self.check(value_c, uri, 0, 40) - self.check(value_d, uri, 0, 50) + # (For FLCS, at least for now, they will read back as 0, meaning deleted, rather + # than disappear.) + self.check(value_a, uri, 0, nrows, 20) + self.check(value_b, uri, 0, nrows, 30) + self.check(value_c, uri, 0, nrows, 40) + self.check(value_d, uri, 0, nrows, 50) stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py index f745d0a3592..ef59be0a089 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable07.py @@ -41,9 +41,10 @@ from wtscenario import make_scenarios class test_rollback_to_stable07(test_rollback_to_stable_base): session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] prepare_values = [ @@ -51,7 +52,7 @@ class test_rollback_to_stable07(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(key_format_values, prepare_values) + scenarios = make_scenarios(format_values, prepare_values) def conn_config(self): config = 'cache_size=5MB,statistics=(all),log=(enabled=true)' @@ -63,18 +64,25 @@ class test_rollback_to_stable07(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable07" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + value_d = 100 + else: + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + value_c = "ccccc" * 100 + value_d = "ddddd" * 100 + # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - value_c = "ccccc" * 100 - value_d = "ddddd" * 100 - # Perform several updates. self.large_updates(uri, value_d, ds, nrows, self.prepare, 20) self.large_updates(uri, value_c, ds, nrows, self.prepare, 30) @@ -82,10 +90,10 @@ class test_rollback_to_stable07(test_rollback_to_stable_base): self.large_updates(uri, value_a, ds, nrows, self.prepare, 50) # Verify data is visible and correct. - self.check(value_d, uri, nrows, 20) - self.check(value_c, uri, nrows, 30) - self.check(value_b, uri, nrows, 40) - self.check(value_a, uri, nrows, 50) + self.check(value_d, uri, nrows, None, 20) + self.check(value_c, uri, nrows, None, 30) + self.check(value_b, uri, nrows, None, 40) + self.check(value_a, uri, nrows, None, 50) # Pin stable to timestamp 50 if prepare otherwise 40. if self.prepare: @@ -102,18 +110,18 @@ class test_rollback_to_stable07(test_rollback_to_stable_base): self.session.checkpoint() # Verify additional update data is visible and correct. - self.check(value_b, uri, nrows, 60) - self.check(value_c, uri, nrows, 70) - self.check(value_d, uri, nrows, 80) + self.check(value_b, uri, nrows, None, 60) + self.check(value_c, uri, nrows, None, 70) + self.check(value_d, uri, nrows, None, 80) # Simulate a server crash and restart. simulate_crash_restart(self, ".", "RESTART") # Check that the correct data is seen at and after the stable timestamp. - self.check(value_b, uri, nrows, 40) - self.check(value_b, uri, nrows, 80) - self.check(value_c, uri, nrows, 30) - self.check(value_d, uri, nrows, 20) + self.check(value_b, uri, nrows, None, 40) + self.check(value_b, uri, nrows, None, 80) + self.check(value_c, uri, nrows, None, 30) + self.check(value_d, uri, nrows, None, 20) stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] @@ -135,10 +143,10 @@ class test_rollback_to_stable07(test_rollback_to_stable_base): simulate_crash_restart(self, "RESTART", "RESTART2") # Check that the correct data is seen at and after the stable timestamp. - self.check(value_b, uri, nrows, 40) - self.check(value_b, uri, nrows, 80) - self.check(value_c, uri, nrows, 30) - self.check(value_d, uri, nrows, 20) + self.check(value_b, uri, nrows, None, 40) + self.check(value_b, uri, nrows, None, 80) + self.check(value_c, uri, nrows, None, 30) + self.check(value_d, uri, nrows, None, 20) stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py index e5c3c5bc9ae..5eca9a16f6b 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable08.py @@ -37,9 +37,10 @@ from test_rollback_to_stable01 import test_rollback_to_stable_base class test_rollback_to_stable08(test_rollback_to_stable_base): session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] in_memory_values = [ @@ -52,7 +53,7 @@ class test_rollback_to_stable08(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(key_format_values, in_memory_values, prepare_values) + scenarios = make_scenarios(format_values, in_memory_values, prepare_values) def conn_config(self): config = 'cache_size=50MB,statistics=(all)' @@ -68,18 +69,25 @@ class test_rollback_to_stable08(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable08" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + value_d = 100 + else: + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + value_c = "ccccc" * 100 + value_d = "ddddd" * 100 + # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - value_c = "ccccc" * 100 - value_d = "ddddd" * 100 - # Perform several updates. self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) self.large_updates(uri, value_b, ds, nrows, self.prepare, 30) @@ -87,10 +95,10 @@ class test_rollback_to_stable08(test_rollback_to_stable_base): self.large_updates(uri, value_d, ds, nrows, self.prepare, 50) # Verify data is visible and correct. - self.check(value_a, uri, nrows, 20) - self.check(value_b, uri, nrows, 30) - self.check(value_c, uri, nrows, 40) - self.check(value_d, uri, nrows, 50) + self.check(value_a, uri, nrows, None, 20) + self.check(value_b, uri, nrows, None, 30) + self.check(value_c, uri, nrows, None, 40) + self.check(value_d, uri, nrows, None, 50) # Pin stable to timestamp 60 if prepare otherwise 50. if self.prepare: @@ -104,10 +112,10 @@ class test_rollback_to_stable08(test_rollback_to_stable_base): self.conn.rollback_to_stable() # Check that the correct data is seen. - self.check(value_a, uri, nrows, 20) - self.check(value_b, uri, nrows, 30) - self.check(value_c, uri, nrows, 40) - self.check(value_d, uri, nrows, 50) + self.check(value_a, uri, nrows, None, 20) + self.check(value_b, uri, nrows, None, 30) + self.check(value_c, uri, nrows, None, 40) + self.check(value_d, uri, nrows, None, 50) stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py index 70664a3d776..eaa5ce49b73 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable09.py @@ -38,6 +38,8 @@ from test_rollback_to_stable01 import test_rollback_to_stable_base class test_rollback_to_stable09(test_rollback_to_stable_base): session_config = 'isolation=snapshot' + # Don't bother testing FLCS tables as well as they're highly unlikely to + # behave differently at this level. colstore_values = [ ('column', dict(use_columns=True)), ('row', dict(use_columns=False)), diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py index cf4e43282da..6b22b72bd2c 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable10.py @@ -39,9 +39,11 @@ from wtthread import checkpoint_thread class test_rollback_to_stable10(test_rollback_to_stable_base): session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S', prepare_extraconfig='')), + ('column_fix', dict(key_format='r', value_format='8t', + prepare_extraconfig=',allocation_size=512,leaf_page_max=512')), + ('integer_row', dict(key_format='i', value_format='S', prepare_extraconfig='')), ] prepare_values = [ @@ -49,7 +51,7 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(key_format_values, prepare_values) + scenarios = make_scenarios(format_values, prepare_values) def conn_config(self): config = 'cache_size=25MB,statistics=(all),statistics_log=(json,on_close,wait=1),log=(enabled=true),timing_stress_for_test=[history_store_checkpoint_delay]' @@ -62,26 +64,36 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): self.pr("create/populate tables") uri_1 = "table:rollback_to_stable10_1" ds_1 = SimpleDataSet( - self, uri_1, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri_1, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds_1.populate() # Create another table without logging. uri_2 = "table:rollback_to_stable10_2" ds_2 = SimpleDataSet( - self, uri_2, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri_2, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds_2.populate() + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + value_d = 100 + value_e = 101 + value_f = 102 + else: + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + value_c = "ccccc" * 100 + value_d = "ddddd" * 100 + value_e = "eeeee" * 100 + value_f = "fffff" * 100 + # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - value_c = "ccccc" * 100 - value_d = "ddddd" * 100 - value_e = "eeeee" * 100 - value_f = "fffff" * 100 - # Perform several updates. self.pr("large updates") self.large_updates(uri_1, value_d, ds_1, nrows, self.prepare, 20) @@ -95,15 +107,15 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): self.large_updates(uri_2, value_a, ds_2, nrows, self.prepare, 50) # Verify data is visible and correct. - self.check(value_d, uri_1, nrows, 20) - self.check(value_c, uri_1, nrows, 30) - self.check(value_b, uri_1, nrows, 40) - self.check(value_a, uri_1, nrows, 50) + self.check(value_d, uri_1, nrows, None, 20) + self.check(value_c, uri_1, nrows, None, 30) + self.check(value_b, uri_1, nrows, None, 40) + self.check(value_a, uri_1, nrows, None, 50) - self.check(value_d, uri_2, nrows, 20) - self.check(value_c, uri_2, nrows, 30) - self.check(value_b, uri_2, nrows, 40) - self.check(value_a, uri_2, nrows, 50) + self.check(value_d, uri_2, nrows, None, 20) + self.check(value_c, uri_2, nrows, None, 30) + self.check(value_b, uri_2, nrows, None, 40) + self.check(value_a, uri_2, nrows, None, 50) # Pin stable to timestamp 60 if prepare otherwise 50. if self.prepare: @@ -145,18 +157,18 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): self.pr("restart complete") # Check that the correct data is seen at and after the stable timestamp. - self.check(value_a, uri_1, nrows, 50) - self.check(value_a, uri_1, nrows, 80) - self.check(value_b, uri_1, nrows, 40) - self.check(value_c, uri_1, nrows, 30) - self.check(value_d, uri_1, nrows, 20) + self.check(value_a, uri_1, nrows, None, 50) + self.check(value_a, uri_1, nrows, None, 80) + self.check(value_b, uri_1, nrows, None, 40) + self.check(value_c, uri_1, nrows, None, 30) + self.check(value_d, uri_1, nrows, None, 20) # Check that the correct data is seen at and after the stable timestamp. - self.check(value_c, uri_2, nrows, 30) - self.check(value_a, uri_2, nrows, 50) - self.check(value_a, uri_2, nrows, 80) - self.check(value_b, uri_2, nrows, 40) - self.check(value_d, uri_2, nrows, 20) + self.check(value_c, uri_2, nrows, None, 30) + self.check(value_a, uri_2, nrows, None, 50) + self.check(value_a, uri_2, nrows, None, 80) + self.check(value_b, uri_2, nrows, None, 40) + self.check(value_d, uri_2, nrows, None, 20) stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] @@ -183,25 +195,35 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): self.pr("create/populate tables") uri_1 = "table:rollback_to_stable10_1" ds_1 = SimpleDataSet( - self, uri_1, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri_1, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)' + self.prepare_extraconfig) ds_1.populate() # Create another table without logging. uri_2 = "table:rollback_to_stable10_2" ds_2 = SimpleDataSet( - self, uri_2, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri_2, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)' + self.prepare_extraconfig) ds_2.populate() + if self.value_format == '8t': + nrows *= 2 + value_a = 97 + value_b = 98 + value_c = 99 + value_d = 100 + value_e = 101 + else: + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + value_c = "ccccc" * 100 + value_d = "ddddd" * 100 + value_e = "eeeee" * 100 + # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - value_c = "ccccc" * 100 - value_d = "ddddd" * 100 - value_e = "eeeee" * 100 - # Perform several updates. self.pr("large updates") self.large_updates(uri_1, value_d, ds_1, nrows, self.prepare, 20) @@ -215,15 +237,15 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): self.large_updates(uri_2, value_a, ds_2, nrows, self.prepare, 50) # Verify data is visible and correct. - self.check(value_d, uri_1, nrows, 20) - self.check(value_c, uri_1, nrows, 30) - self.check(value_b, uri_1, nrows, 40) - self.check(value_a, uri_1, nrows, 50) + self.check(value_d, uri_1, nrows, None, 20) + self.check(value_c, uri_1, nrows, None, 30) + self.check(value_b, uri_1, nrows, None, 40) + self.check(value_a, uri_1, nrows, None, 50) - self.check(value_d, uri_2, nrows, 20) - self.check(value_c, uri_2, nrows, 30) - self.check(value_b, uri_2, nrows, 40) - self.check(value_a, uri_2, nrows, 50) + self.check(value_d, uri_2, nrows, None, 20) + self.check(value_c, uri_2, nrows, None, 30) + self.check(value_b, uri_2, nrows, None, 40) + self.check(value_a, uri_2, nrows, None, 50) # Pin stable to timestamp 60 if prepare otherwise 50. if self.prepare: @@ -231,6 +253,15 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): else: self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) + # Do an explicit checkpoint first, before starting the background checkpointer. + # Otherwise (depending on timing and load) because there's a lot to write for the + # first checkpoint there's a tendency for the background checkpointer to only + # manage to do the one checkpoint; and sometimes (especially on FLCS) it ends up + # not containing any of the concurrent updates, and then the test fails because + # RTS correctly notices it has no work to do and doesn't visit any of the pages + # or update anything in the history store. + self.session.checkpoint() + # Here's the update operations we'll perform, encapsulated so we can easily retry # it if we get a rollback. Rollbacks may occur when checkpoint is running. def prepare_range_updates(session, cursor, ds, value, nrows, prepare_config): @@ -249,8 +280,8 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): try: self.pr("start checkpoint") ckpt.start() - # Sleep for sometime so that checkpoint starts. - time.sleep(2) + # Sleep for some time so that checkpoint starts. + time.sleep(5) # Perform several updates in parallel with checkpoint. session_p1 = self.conn.open_session() @@ -304,18 +335,18 @@ class test_rollback_to_stable10(test_rollback_to_stable_base): self.assertGreater(cache_hs_ondisk, 0) # Check that the correct data is seen at and after the stable timestamp. - self.check(value_a, uri_1, nrows, 50) - self.check(value_a, uri_1, nrows, 80) - self.check(value_b, uri_1, nrows, 40) - self.check(value_c, uri_1, nrows, 30) - self.check(value_d, uri_1, nrows, 20) + self.check(value_a, uri_1, nrows, None, 50) + self.check(value_a, uri_1, nrows, None, 80) + self.check(value_b, uri_1, nrows, None, 40) + self.check(value_c, uri_1, nrows, None, 30) + self.check(value_d, uri_1, nrows, None, 20) # Check that the correct data is seen at and after the stable timestamp. - self.check(value_a, uri_2, nrows, 50) - self.check(value_a, uri_2, nrows, 80) - self.check(value_b, uri_2, nrows, 40) - self.check(value_c, uri_2, nrows, 30) - self.check(value_d, uri_2, nrows, 20) + self.check(value_a, uri_2, nrows, None, 50) + self.check(value_a, uri_2, nrows, None, 80) + self.check(value_b, uri_2, nrows, None, 40) + self.check(value_c, uri_2, nrows, None, 30) + self.check(value_d, uri_2, nrows, None, 20) stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py index f88d1ec0082..067d8c341f1 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable11.py @@ -38,9 +38,10 @@ from wtscenario import make_scenarios class test_rollback_to_stable11(test_rollback_to_stable_base): session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] prepare_values = [ @@ -48,7 +49,7 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(key_format_values, prepare_values) + scenarios = make_scenarios(format_values, prepare_values) def conn_config(self): config = 'cache_size=1MB,statistics=(all),log=(archive=false,enabled=true)' @@ -60,18 +61,25 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable11" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + value_d = 100 + else: + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + value_c = "ccccc" * 100 + value_d = "ddddd" * 100 + # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - value_c = "ccccc" * 100 - value_d = "ddddd" * 100 - # Perform several updates. self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) @@ -79,7 +87,7 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): self.large_updates(uri, value_b, ds, nrows, self.prepare, 20) # Verify data is visible and correct. - self.check(value_b, uri, nrows, 20) + self.check(value_b, uri, nrows, None, 20) # Pin stable to timestamp 30 if prepare otherwise 20. if self.prepare: @@ -94,7 +102,7 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): simulate_crash_restart(self, ".", "RESTART") # Check that the correct data is seen at and after the stable timestamp. - self.check(value_b, uri, nrows, 20) + self.check(value_b, uri, nrows, None, 20) # Perform several updates. self.large_updates(uri, value_c, ds, nrows, self.prepare, 30) @@ -103,7 +111,7 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): self.large_updates(uri, value_d, ds, nrows, self.prepare, 30) # Verify data is visible and correct. - self.check(value_d, uri, nrows, 30) + self.check(value_d, uri, nrows, None, 30) # Checkpoint to ensure that all the updates are flushed to disk. self.session.checkpoint() @@ -112,8 +120,8 @@ class test_rollback_to_stable11(test_rollback_to_stable_base): simulate_crash_restart(self, "RESTART", "RESTART2") # Check that the correct data is seen at and after the stable timestamp. - self.check(value_b, uri, nrows, 20) - self.check(value_b, uri, nrows, 40) + self.check(value_b, uri, nrows, None, 20) + self.check(value_b, uri, nrows, None, 40) stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py index 0ab66b1d991..372263a73f4 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable12.py @@ -38,9 +38,16 @@ from wtscenario import make_scenarios class test_rollback_to_stable12(test_rollback_to_stable_base): session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + # This test is slow, and the value of running it on every access method maybe somewhat + # questionable, since the code for skipping over subtrees during RTS is not dependent on + # access method. However, that relies on the aggregated timestamps in internal nodes + # being correct, which _is_ dependent on access method, so running all three versions is + # probably still worthwhile. However, if cutting back on test time becomes desirable it + # is probably reasonable to run only one of these unless -l is given. + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] prepare_values = [ @@ -48,7 +55,7 @@ class test_rollback_to_stable12(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(key_format_values, prepare_values) + scenarios = make_scenarios(format_values, prepare_values) def conn_config(self): config = 'cache_size=500MB,statistics=(all),log=(enabled=true)' @@ -60,21 +67,26 @@ class test_rollback_to_stable12(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable12" ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='split_pct=50,log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='split_pct=50,log=(enabled=false)') ds.populate() + if self.value_format == '8t': + value_a = 97 + value_b = 98 + else: + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - # Perform several updates. self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) # Verify data is visible and correct. - self.check(value_a, uri, nrows, 20) + self.check(value_a, uri, nrows, None, 20) # Pin stable to timestamp 30 if prepare otherwise 20. if self.prepare: @@ -102,7 +114,7 @@ class test_rollback_to_stable12(test_rollback_to_stable_base): simulate_crash_restart(self, ".", "RESTART") # Check that the correct data is seen at and after the stable timestamp. - self.check(value_a, uri, nrows, 30) + self.check(value_a, uri, nrows, None, 30) stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py index dddbd02c260..e4d27e53a87 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable13.py @@ -37,9 +37,10 @@ from wtscenario import make_scenarios class test_rollback_to_stable13(test_rollback_to_stable_base): session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] prepare_values = [ @@ -47,7 +48,7 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(key_format_values, prepare_values) + scenarios = make_scenarios(format_values, prepare_values) def conn_config(self): config = 'cache_size=50MB,statistics=(all),log=(enabled=true)' @@ -59,16 +60,21 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable13" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='split_pct=50,log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='split_pct=50,log=(enabled=false)') ds.populate() + if self.value_format == '8t': + value_a = 97 + value_b = 98 + else: + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - # Perform several updates. self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) @@ -79,9 +85,10 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): self.large_updates(uri, value_b, ds, nrows, self.prepare, 60) # Verify data is visible and correct. - self.check(value_a, uri, nrows, 20) - self.check(None, uri, 0, 30) - self.check(value_b, uri, nrows, 60) + # (In FLCS, the removed rows should read back as zero.) + self.check(value_a, uri, nrows, None, 20) + self.check(None, uri, 0, nrows, 30) + self.check(value_b, uri, nrows, None, 60) # Pin stable to timestamp 50 if prepare otherwise 40. if self.prepare: @@ -94,10 +101,10 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): simulate_crash_restart(self, ".", "RESTART") # Check that the correct data is seen at and after the stable timestamp. - self.check(None, uri, 0, 50) + self.check(None, uri, 0, nrows, 50) # Check that we restore the correct value from the history store. - self.check(value_a, uri, nrows, 20) + self.check(value_a, uri, nrows, None, 20) stat_cursor = self.session.open_cursor('statistics:', None, None) restored_tombstones = stat_cursor[stat.conn.txn_rts_hs_restore_tombstones][2] @@ -109,18 +116,25 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable13" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='split_pct=50,log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='split_pct=50,log=(enabled=false)') ds.populate() + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + value_d = 100 + else: + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + value_c = "ccccc" * 100 + value_d = "ddddd" * 100 + # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - value_c = "ccccc" * 100 - value_d = "ddddd" * 100 - # Perform several updates. self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) @@ -147,9 +161,10 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): self.large_updates(uri, value_d, ds, nrows, self.prepare, 60) # Verify data is visible and correct. - self.check(value_a, uri, nrows, 20) - self.check(None, uri, 0, 30) - self.check(value_d, uri, nrows, 60) + # (In FLCS, the removed rows should read back as zero.) + self.check(value_a, uri, nrows, None, 20) + self.check(None, uri, 0, nrows, 30) + self.check(value_d, uri, nrows, None, 60) # Pin stable to timestamp 50 if prepare otherwise 40. if self.prepare: @@ -162,10 +177,10 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): simulate_crash_restart(self, ".", "RESTART") # Check that the correct data is seen at and after the stable timestamp. - self.check(None, uri, 0, 50) + self.check(None, uri, 0, nrows, 50) # Check that we restore the correct value from the history store. - self.check(value_a, uri, nrows, 20) + self.check(value_a, uri, nrows, None, 20) stat_cursor = self.session.open_cursor('statistics:', None, None) restored_tombstones = stat_cursor[stat.conn.txn_rts_hs_restore_tombstones][2] @@ -177,17 +192,23 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable13" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='split_pct=50,log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='split_pct=50,log=(enabled=false)') ds.populate() + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + else: + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + value_c = "ccccc" * 100 + # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - value_c = "ccccc" * 100 - # Perform several updates. self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) @@ -217,18 +238,19 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): self.session.checkpoint() # Verify data is visible and correct. - self.check(value_a, uri, nrows, 20) - self.check(None, uri, 0, 40) - self.check(value_c, uri, nrows, 60) + # (In FLCS, the removed rows should read back as zero.) + self.check(value_a, uri, nrows, None, 20) + self.check(None, uri, 0, nrows, 40) + self.check(value_c, uri, nrows, None, 60) # Simulate a server crash and restart. simulate_crash_restart(self, ".", "RESTART") # Check that the correct data is seen at and after the stable timestamp. - self.check(None, uri, 0, 50) + self.check(None, uri, 0, nrows, 50) # Check that we restore the correct value from the history store. - self.check(value_a, uri, nrows, 20) + self.check(value_a, uri, nrows, None, 20) stat_cursor = self.session.open_cursor('statistics:', None, None) restored_tombstones = stat_cursor[stat.conn.txn_rts_hs_restore_tombstones][2] @@ -239,14 +261,23 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable13" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='split_pct=50,log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='split_pct=50,log=(enabled=false)') ds.populate() + + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + else: + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + value_c = "ccccc" * 100 + # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - value_c = "ccccc" * 100 + # Perform several updates. self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) # Perform several updates. @@ -261,10 +292,13 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): # Perform several updates and checkpoint. self.large_updates(uri, value_c, ds, nrows, self.prepare, 60) self.session.checkpoint() + # Verify data is visible and correct. - self.check(value_a, uri, nrows, 20) - self.check(None, uri, 0, 40) - self.check(value_c, uri, nrows, 60) + # (In FLCS, the removed rows should read back as zero.) + self.check(value_a, uri, nrows, None, 20) + self.check(None, uri, 0, nrows, 40) + self.check(value_c, uri, nrows, None, 60) + self.conn.rollback_to_stable() # Perform several updates and checkpoint. self.large_updates(uri, value_c, ds, nrows, self.prepare, 60) @@ -272,9 +306,9 @@ class test_rollback_to_stable13(test_rollback_to_stable_base): # Simulate a server crash and restart. simulate_crash_restart(self, ".", "RESTART") # Check that the correct data is seen at and after the stable timestamp. - self.check(None, uri, 0, 50) + self.check(None, uri, 0, nrows, 50) # Check that we restore the correct value from the history store. - self.check(value_a, uri, nrows, 20) + self.check(value_a, uri, nrows, None, 20) stat_cursor = self.session.open_cursor('statistics:', None, None) restored_tombstones = stat_cursor[stat.conn.txn_rts_hs_restore_tombstones][2] self.assertEqual(restored_tombstones, nrows) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py index 601302b0762..e213818ce96 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable14.py @@ -41,7 +41,9 @@ def append_val(value, char): return value + char # test_rollback_to_stable14.py -# Test the rollback to stable operation uses proper base update while restoring modifies from history store. +# Test the rollback to stable operation uses proper base update while restoring modifies from +# history store. Since FLCS inherently doesn't support modify, there's no need to run this on +# FLCS. (Note that self.value_format needs to exist anyway for the base class to use.) class test_rollback_to_stable14(test_rollback_to_stable_base): session_config = 'isolation=snapshot' @@ -49,6 +51,7 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): ('column', dict(key_format='r')), ('integer_row', dict(key_format='i')), ] + value_format='S' prepare_values = [ ('no_prepare', dict(prepare=False)), @@ -68,7 +71,8 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): self.pr("create/populate table") uri = "table:rollback_to_stable14" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 10. @@ -95,11 +99,11 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): self.large_modifies(uri, 'T', ds, 3, 1, nrows, self.prepare, 60) # Verify data is visible and correct. - self.check(value_a, uri, nrows, 20) - self.check(value_modQ, uri, nrows, 30) - self.check(value_modR, uri, nrows, 40) - self.check(value_modS, uri, nrows, 50) - self.check(value_modT, uri, nrows, 60) + self.check(value_a, uri, nrows, None, 20) + self.check(value_modQ, uri, nrows, None, 30) + self.check(value_modR, uri, nrows, None, 40) + self.check(value_modS, uri, nrows, None, 50) + self.check(value_modT, uri, nrows, None, 60) # Pin stable to timestamp 60 if prepare otherwise 50. if self.prepare: @@ -164,10 +168,10 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): self.assertGreaterEqual(hs_sweep, 0) # Check that the correct data is seen at and after the stable timestamp. - self.check(value_a, uri, nrows, 20) - self.check(value_modQ, uri, nrows, 30) - self.check(value_modR, uri, nrows, 40) - self.check(value_modS, uri, nrows, 50) + self.check(value_a, uri, nrows, None, 20) + self.check(value_modQ, uri, nrows, None, 30) + self.check(value_modR, uri, nrows, None, 40) + self.check(value_modS, uri, nrows, None, 50) # The test may output the following message in eviction under cache pressure. Ignore that. self.ignoreStdoutPatternIfExists("oldest pinned transaction ID rolled back for eviction") @@ -179,7 +183,8 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): self.pr("create/populate table") uri = "table:rollback_to_stable14" ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 10. @@ -212,9 +217,9 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): self.large_modifies(uri, 'T', ds, 3, 1, nrows, self.prepare, 60) # Verify data is visible and correct. - self.check(value_a, uri, nrows, 20) - self.check(value_modQ, uri, nrows, 30) - self.check(value_modT, uri, nrows, 60) + self.check(value_a, uri, nrows, None, 20) + self.check(value_modQ, uri, nrows, None, 30) + self.check(value_modT, uri, nrows, None, 60) self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) @@ -275,8 +280,8 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): self.assertGreaterEqual(hs_sweep, 0) # Check that the correct data is seen at and after the stable timestamp. - self.check(value_a, uri, nrows, 20) - self.check(value_modQ, uri, nrows, 30) + self.check(value_a, uri, nrows, None, 20) + self.check(value_modQ, uri, nrows, None, 30) # The test may output the following message in eviction under cache pressure. Ignore that. self.ignoreStdoutPatternIfExists("oldest pinned transaction ID rolled back for eviction") @@ -288,7 +293,8 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): self.pr("create/populate table") uri = "table:rollback_to_stable14" ds = SimpleDataSet( - self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 10. @@ -321,9 +327,9 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): self.large_modifies(uri, 'T', ds, len(value_modS), 1, nrows, self.prepare, 60) # Verify data is visible and correct. - self.check(value_a, uri, nrows, 20) - self.check(value_modQ, uri, nrows, 30) - self.check(value_modT, uri, nrows, 60) + self.check(value_a, uri, nrows, None, 20) + self.check(value_modQ, uri, nrows, None, 30) + self.check(value_modT, uri, nrows, None, 60) self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) @@ -382,8 +388,8 @@ class test_rollback_to_stable14(test_rollback_to_stable_base): self.assertGreaterEqual(hs_sweep, 0) # Check that the correct data is seen at and after the stable timestamp. - self.check(value_a, uri, nrows, 20) - self.check(value_modQ, uri, nrows, 30) + self.check(value_a, uri, nrows, None, 20) + self.check(value_modQ, uri, nrows, None, 30) # The test may output the following message in eviction under cache pressure. Ignore that. self.ignoreStdoutPatternIfExists("oldest pinned transaction ID rolled back for eviction") diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py index 4c52e52f50c..e371801a60a 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable16.py @@ -39,6 +39,7 @@ from wtscenario import make_scenarios # test_rollback_to_stable16.py # Test that rollback to stable removes updates present on disk for column store. +# (This test is now probably redundant with others, and could maybe be removed?) class test_rollback_to_stable16(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' @@ -49,8 +50,7 @@ class test_rollback_to_stable16(wttest.WiredTigerTestCase): value_format_values = [ # Fixed length - #FIXME: WT-7304 Fixed length column store failing on rollback on disk values - # ('fixed', dict(value_format='8t')), + ('fixed', dict(value_format='8t')), # Variable length ('variable', dict(value_format='S')), ] @@ -60,7 +60,14 @@ class test_rollback_to_stable16(wttest.WiredTigerTestCase): ('inmem', dict(in_memory=True)) ] - scenarios = make_scenarios(key_format_values, value_format_values, in_memory_values) + def keep(name, d): + if d['key_format'] == 'i' and d['value_format'] == '8t': + # Fixed-length format is only special for column-stores. + return False + return True + + scenarios = make_scenarios(key_format_values, value_format_values, in_memory_values, + include=keep) def conn_config(self): config = 'cache_size=200MB,statistics=(all)' @@ -145,8 +152,8 @@ class test_rollback_to_stable16(wttest.WiredTigerTestCase): self.check(values[0], uri, nrows, 1, 2) self.check(values[1], uri, nrows, 201, 5) - self.check(None, uri, nrows, 401, 7) - self.check(None, uri, nrows, 601, 9) + self.check(0 if self.value_format == '8t' else None, uri, nrows, 401, 7) + self.check(0 if self.value_format == '8t' else None, uri, nrows, 601, 9) stat_cursor = self.session.open_cursor('statistics:', None, None) upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py index 1697cd50c8b..f5e1fea379f 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable17.py @@ -34,14 +34,14 @@ from wtdataset import SimpleDataSet from wtscenario import make_scenarios # test_rollback_to_stable17.py -# Test that rollback to stable handles updates present on history store and data store for variable -# length column store. +# Test that rollback to stable handles updates present on history store and data store. class test_rollback_to_stable17(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] in_memory_values = [ @@ -49,7 +49,7 @@ class test_rollback_to_stable17(wttest.WiredTigerTestCase): ('inmem', dict(in_memory=True)) ] - scenarios = make_scenarios(key_format_values, in_memory_values) + scenarios = make_scenarios(format_values, in_memory_values) def conn_config(self): config = 'cache_size=200MB,statistics=(all)' @@ -87,10 +87,13 @@ class test_rollback_to_stable17(wttest.WiredTigerTestCase): nrows = 200 start_row = 1 ts = [2,5,7,9] - values = ["aaaa", "bbbb", "cccc", "dddd"] + if self.value_format == '8t': + values = [97, 98, 99, 100] + else: + values = ["aaaa", "bbbb", "cccc", "dddd"] - create_params = 'log=(enabled=false),key_format=r,value_format=S' - self.session.create(uri, create_params) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, format + ',log=(enabled=false)') # Pin oldest and stable to timestamp 1. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py index 01e0898a973..d341d27610a 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable18.py @@ -41,9 +41,10 @@ from wtscenario import make_scenarios class test_rollback_to_stable18(test_rollback_to_stable_base): session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] prepare_values = [ @@ -51,7 +52,7 @@ class test_rollback_to_stable18(test_rollback_to_stable_base): ('prepare', dict(prepare=True)) ] - scenarios = make_scenarios(key_format_values, prepare_values) + scenarios = make_scenarios(format_values, prepare_values) def conn_config(self): config = 'cache_size=50MB,in_memory=true,statistics=(all),log=(enabled=false),' \ @@ -64,15 +65,19 @@ class test_rollback_to_stable18(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable18" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() + if self.value_format == '8t': + value_a = 97 + else: + value_a = "aaaaa" * 100 + # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - value_a = "aaaaa" * 100 - # Perform several updates. self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) @@ -80,8 +85,8 @@ class test_rollback_to_stable18(test_rollback_to_stable_base): self.large_removes(uri, ds, nrows, self.prepare, 30) # Verify data is visible and correct. - self.check(value_a, uri, nrows, 20) - self.check(None, uri, 0, 30) + self.check(value_a, uri, nrows, None, 20) + self.check(None, uri, 0, nrows, 30) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") @@ -104,7 +109,7 @@ class test_rollback_to_stable18(test_rollback_to_stable_base): self.conn.rollback_to_stable() # Verify data is not visible. - self.check(value_a, uri, nrows, 30) + self.check(value_a, uri, nrows, None, 30) stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py index c07895c6316..cbf7c839433 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable19.py @@ -42,9 +42,10 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): ('inmem', dict(in_memory=True)) ] - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] restart_options = [ @@ -52,7 +53,7 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): ('crash', dict(crash=True)), ] - scenarios = make_scenarios(in_memory_values, key_format_values, restart_options) + scenarios = make_scenarios(in_memory_values, format_values, restart_options) def conn_config(self): config = 'cache_size=50MB,statistics=(all),log=(enabled=false),eviction_dirty_trigger=10,' \ @@ -69,15 +70,19 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable19" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() + if self.value_format == '8t': + valuea = 97 + else: + valuea = "aaaaa" * 100 + # Pin oldest and stable timestamps to 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - valuea = "aaaaa" * 100 - # Perform several updates and removes. s = self.conn.open_session() cursor = s.open_cursor(uri) @@ -95,16 +100,25 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): # Search for the key so we position our cursor on the page that we want to evict. self.session.begin_transaction("ignore_prepare = true") evict_cursor.set_key(1) - self.assertEquals(evict_cursor.search(), WT_NOTFOUND) + if self.value_format == '8t': + # In FLCS deleted values read back as 0. + self.assertEquals(evict_cursor.search(), 0) + self.assertEquals(evict_cursor.get_value(), 0) + else: + self.assertEquals(evict_cursor.search(), WT_NOTFOUND) evict_cursor.reset() evict_cursor.close() self.session.commit_transaction() - # Search to make sure the data is not visible + # Search to make sure the data is not visible (or, in FLCS, that it's zero) self.session.begin_transaction("ignore_prepare = true") cursor2 = self.session.open_cursor(uri) cursor2.set_key(1) - self.assertEquals(cursor2.search(), WT_NOTFOUND) + if self.value_format == '8t': + self.assertEquals(cursor2.search(), 0) + self.assertEquals(cursor2.get_value(), 0) + else: + self.assertEquals(cursor2.search(), WT_NOTFOUND) self.session.commit_transaction() cursor2.close() @@ -124,8 +138,8 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): s.rollback_transaction() # Verify data is not visible. - self.check(valuea, uri, 0, 20) - self.check(valuea, uri, 0, 30) + self.check(valuea, uri, 0, nrows, 20) + self.check(valuea, uri, 0, nrows, 30) stat_cursor = self.session.open_cursor('statistics:', None, None) upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] @@ -149,16 +163,21 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable19" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() + if self.value_format == '8t': + valuea = 97 + valueb = 98 + else: + valuea = "aaaaa" * 100 + valueb = "bbbbb" * 100 + # Pin oldest and stable timestamps to 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - valuea = "aaaaa" * 100 - valueb = "bbbbb" * 100 - # Perform several updates. self.large_updates(uri, valuea, ds, nrows, 0, 20) @@ -182,16 +201,25 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): # Search for the key so we position our cursor on the page that we want to evict. self.session.begin_transaction("ignore_prepare = true") evict_cursor.set_key(1) - self.assertEquals(evict_cursor.search(), WT_NOTFOUND) + if self.value_format == '8t': + # In FLCS deleted values read back as 0. + self.assertEquals(evict_cursor.search(), 0) + self.assertEquals(evict_cursor.get_value(), 0) + else: + self.assertEquals(evict_cursor.search(), WT_NOTFOUND) evict_cursor.reset() evict_cursor.close() self.session.commit_transaction() - # Search to make sure the data is not visible + # Search to make sure the data is not visible (or, in FLCS, that it's zero) self.session.begin_transaction("ignore_prepare = true") cursor2 = self.session.open_cursor(uri) cursor2.set_key(1) - self.assertEquals(cursor2.search(), WT_NOTFOUND) + if self.value_format == '8t': + self.assertEquals(cursor2.search(), 0) + self.assertEquals(cursor2.get_value(), 0) + else: + self.assertEquals(cursor2.search(), WT_NOTFOUND) self.session.commit_transaction() cursor2.close() @@ -211,9 +239,9 @@ class test_rollback_to_stable19(test_rollback_to_stable_base): s.rollback_transaction() # Verify data. - self.check(valuea, uri, nrows, 20) - self.check(valuea, uri, 0, 30) - self.check(valuea, uri, 0, 40) + self.check(valuea, uri, nrows, None, 20) + self.check(valuea, uri, 0, nrows, 30) + self.check(valuea, uri, 0, nrows, 40) stat_cursor = self.session.open_cursor('statistics:', None, None) upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable20.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable20.py index 8a2f44c5a36..6e4e68535c5 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable20.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable20.py @@ -39,12 +39,13 @@ from test_rollback_to_stable01 import test_rollback_to_stable_base class test_rollback_to_stable20(test_rollback_to_stable_base): session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def conn_config(self): config = 'cache_size=50MB,statistics=(all)' @@ -53,18 +54,22 @@ class test_rollback_to_stable20(test_rollback_to_stable_base): def test_rollback_to_stable(self): nrows = 10000 ntables = 100 - create_params = 'key_format=i,value_format=S' + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) uri = "table:rollback_to_stable20" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() + if self.value_format == '8t': + valuea = 97 + else: + valuea = "aaaaa" * 100 + # Pin oldest and stable timestamp to 1. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) - valuea = "aaaaa" * 100 - for i in range(0, ntables): uri = 'table:rollback_to_stable20_' + str(i) self.session.create(uri, create_params) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py index 5d1f01f43e5..0f6a1e2eea0 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable21.py @@ -40,12 +40,13 @@ from test_rollback_to_stable01 import test_rollback_to_stable_base # test_rollback_to_stable21.py # Test rollback to stable when an out of order prepared transaction is written to disk class test_rollback_to_stable21(test_rollback_to_stable_base): - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def conn_config(self): config = 'cache_size=250MB,statistics=(all),statistics_log=(json,on_close,wait=1)' @@ -57,16 +58,21 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable21" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() + if self.value_format == '8t': + valuea = 97 + valueb = 98 + else: + valuea = 'a' * 400 + valueb = 'b' * 400 + # Pin oldest and stable timestamps to 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - valuea = 'a' * 400 - valueb = 'b' * 400 - cursor = self.session.open_cursor(uri) self.session.begin_transaction() for i in range(1, nrows + 1): @@ -105,7 +111,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): simulate_crash_restart(self, ".", "RESTART") self.pr("restart complete") - self.check(valuea, uri, nrows, 40) + self.check(valuea, uri, nrows, None, 40) stat_cursor = self.session.open_cursor('statistics:', None, None) hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] @@ -119,16 +125,21 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable21" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() + if self.value_format == '8t': + valuea = 97 + valueb = 98 + else: + valuea = 'a' * 400 + valueb = 'b' * 400 + # Pin oldest and stable timestamps to 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - valuea = 'a' * 400 - valueb = 'b' * 400 - cursor = self.session.open_cursor(uri) self.session.begin_transaction() for i in range(1, nrows + 1): @@ -172,8 +183,8 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): simulate_crash_restart(self, ".", "RESTART") self.pr("restart complete") - self.check(valuea, uri, nrows, 30) - self.check(valuea, uri, 0, 40) + self.check(valuea, uri, nrows, None, 30) + self.check(valuea, uri, 0, nrows, 40) stat_cursor = self.session.open_cursor('statistics:', None, None) hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] @@ -189,16 +200,21 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable21" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() + if self.value_format == '8t': + valuea = 97 + valueb = 98 + else: + valuea = 'a' * 400 + valueb = 'b' * 400 + # Pin oldest and stable timestamps to 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - valuea = 'a' * 400 - valueb = 'b' * 400 - cursor = self.session.open_cursor(uri) self.session.begin_transaction() for i in range(1, nrows + 1): @@ -223,7 +239,11 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): for i in range(1, nrows + 1): evict_cursor.set_key(i) - self.assertEquals(evict_cursor.search(), WT_NOTFOUND) + if self.value_format == '8t': + self.assertEquals(evict_cursor.search(), 0) + self.assertEquals(evict_cursor.get_value(), 0) + else: + self.assertEquals(evict_cursor.search(), WT_NOTFOUND) evict_cursor.reset() s.rollback_transaction() @@ -238,7 +258,7 @@ class test_rollback_to_stable21(test_rollback_to_stable_base): simulate_crash_restart(self, ".", "RESTART") self.pr("restart complete") - self.check(valuea, uri, 0, 40) + self.check(valuea, uri, 0, nrows, 40) stat_cursor = self.session.open_cursor('statistics:', None, None) hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py index 7c85800b070..d89f37b217c 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable22.py @@ -34,6 +34,10 @@ from wtdataset import SimpleDataSet # history store eviction concurrently to a rollback to stable call. We'll do this by limiting # the cache size to 100MB and performing 100MB worth of inserts while periodically calling rollback # to stable. +# +# The machinery this tests is all underneath the access-method-specific parts of RTS (and the +# history store itself is always row-store) so it doesn't seem necessary or worthwhile to run +# this explicitly on VLCS or FLCS. class test_rollback_to_stable22(test_rollback_to_stable_base): conn_config = 'cache_size=100MB' session_config = 'isolation=snapshot' diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable23.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable23.py index e4ff9e4dfb4..0ae13de928f 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable23.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable23.py @@ -37,7 +37,9 @@ def mod_val(value, char, location, nbytes=1): # test_rollback_to_stable23.py # Test to verify that search operation uses proper base update while returning modifies from -# the history store after the on-disk update is removed by the rollback to stable. +# the history store after the on-disk update is removed by the rollback to stable. Since FLCS +# inherently doesn't support modify, there's no need to run this on FLCS. (Note that +# self.value_format needs to exist anyway for the base class to use.) class test_rollback_to_stable23(test_rollback_to_stable_base): session_config = 'isolation=snapshot' @@ -45,6 +47,7 @@ class test_rollback_to_stable23(test_rollback_to_stable_base): ('column', dict(key_format='r')), ('integer_row', dict(key_format='i')), ] + value_format='S' prepare_values = [ ('no_prepare', dict(prepare=False)), @@ -73,7 +76,8 @@ class test_rollback_to_stable23(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable23" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 10. @@ -95,11 +99,11 @@ class test_rollback_to_stable23(test_rollback_to_stable_base): self.large_modifies(uri, 'T', ds, 3, 1, nrows, self.prepare, 60) # Verify data is visible and correct. - self.check(value_a, uri, nrows, 20) - self.check(value_modQ, uri, nrows, 30) - self.check(value_modR, uri, nrows, 40) - self.check(value_modS, uri, nrows, 50) - self.check(value_modT, uri, nrows, 60) + self.check(value_a, uri, nrows, None, 20) + self.check(value_modQ, uri, nrows, None, 30) + self.check(value_modR, uri, nrows, None, 40) + self.check(value_modS, uri, nrows, None, 50) + self.check(value_modT, uri, nrows, None, 60) # Pin stable to timestamp 60 if prepare otherwise 50. if self.prepare: diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable24.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable24.py index ea690506dc9..ac067c1ce67 100644 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable24.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable24.py @@ -57,6 +57,8 @@ from wtscenario import make_scenarios # # Run this test on rows as well as columns to help make sure the test itself is valid (and # stays so over time...) +# +# Don't run it on FLCS because FLCS doesn't do RLE encoding so there's no point. class test_rollback_to_stable24(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' conn_config = 'in_memory=false' diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable26.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable26.py index d9cc9b9a5fa..2c2c8896487 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable26.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable26.py @@ -40,9 +40,10 @@ from wtthread import checkpoint_thread class test_rollback_to_stable26(test_rollback_to_stable_base): session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] hs_remove_values = [ @@ -55,7 +56,7 @@ class test_rollback_to_stable26(test_rollback_to_stable_base): ('prepare_remove', dict(prepare_remove=True)) ] - scenarios = make_scenarios(key_format_values, hs_remove_values, prepare_remove_values) + scenarios = make_scenarios(format_values, hs_remove_values, prepare_remove_values) def conn_config(self): config = 'cache_size=10MB,statistics=(all),timing_stress_for_test=[history_store_checkpoint_delay]' @@ -78,19 +79,27 @@ class test_rollback_to_stable26(test_rollback_to_stable_base): # Create a table without logging. uri = "table:rollback_to_stable26" ds = SimpleDataSet( - self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + value_d = 100 + value_e = 101 + else: + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + value_c = "ccccc" * 100 + value_d = "ddddd" * 100 + value_e = "eeeee" * 100 + # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - value_c = "ccccc" * 100 - value_d = "ddddd" * 100 - value_e = "eeeee" * 100 - self.large_updates(uri, value_a, ds, nrows, False, 20) self.large_updates(uri, value_b, ds, nrows, False, 30) @@ -109,8 +118,8 @@ class test_rollback_to_stable26(test_rollback_to_stable_base): prepare_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(50)) # Verify data is visible and correct. - self.check(value_a, uri, nrows, 20) - self.check(value_b, uri, nrows, 30) + self.check(value_a, uri, nrows, None, 20) + self.check(value_b, uri, nrows, None, 30) self.evict_cursor(uri, nrows) @@ -132,9 +141,9 @@ class test_rollback_to_stable26(test_rollback_to_stable_base): self.large_updates(uri, value_d, ds, nrows, False, 60) # Check that the correct data. - self.check(value_a, uri, nrows, 20) - self.check(value_b, uri, nrows, 30) - self.check(value_d, uri, nrows, 60) + self.check(value_a, uri, nrows, None, 20) + self.check(value_b, uri, nrows, None, 30) + self.check(value_d, uri, nrows, None, 60) # Simulate a server crash and restart. simulate_crash_restart(self, ".", "RESTART") @@ -150,17 +159,17 @@ class test_rollback_to_stable26(test_rollback_to_stable_base): self.assertEqual(hs_removed, nrows) # Check that the correct data. - self.check(value_a, uri, nrows, 20) - self.check(value_b, uri, nrows, 30) + self.check(value_a, uri, nrows, None, 20) + self.check(value_b, uri, nrows, None, 30) self.large_updates(uri, value_e, ds, nrows, False, 60) self.evict_cursor(uri, nrows) # Check that the correct data. - self.check(value_a, uri, nrows, 20) - self.check(value_b, uri, nrows, 30) - self.check(value_e, uri, nrows, 60) + self.check(value_a, uri, nrows, None, 20) + self.check(value_b, uri, nrows, None, 30) + self.check(value_e, uri, nrows, None, 60) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable28.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable28.py index 42f6861ca74..52da4177a2b 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable28.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable28.py @@ -29,6 +29,7 @@ import re from wiredtiger import stat from wtdataset import SimpleDataSet +from wtscenario import make_scenarios from helper import simulate_crash_restart from test_rollback_to_stable01 import test_rollback_to_stable_base @@ -42,9 +43,22 @@ class test_rollback_to_stable28(test_rollback_to_stable_base): conn_config = 'log=(enabled=true),statistics=(all)' # Recovery connection config: The debug mode is only effective on high cache pressure as WiredTiger can potentially decide # to do an update restore evict on a page when the cache pressure requirements are not met. - # This means setting eviction target low and cache size high. + # This means setting eviction target low and cache size low. conn_recon = ',eviction_updates_trigger=10,eviction_dirty_trigger=5,eviction_dirty_target=1,' \ - 'cache_size=10MB,debug_mode=(update_restore_evict=true),log=(recover=on)' + 'cache_size=1MB,debug_mode=(update_restore_evict=true),log=(recover=on)' + + format_values = [ + ('column', dict(key_format='r', value_format='S', extraconfig='')), + # Does not run reliably; does not always trigger update restore eviction and fails that + # assertion, even with small pages and more rows. Probably needs a lot more rows. For + # the moment we've concluded that the marginal benefit of running it on FLCS is small so + # just disabling the scenario seems to be the best way forward. + #('column_fix', dict(key_format='r', value_format='8t', + # extraconfig=',allocation_size=512,leaf_page_max=512')), + ('integer_row', dict(key_format='i', value_format='S', extraconfig='')), + ] + + scenarios = make_scenarios(format_values) def parse_write_gen(self, uri): meta_cursor = self.session.open_cursor('metadata:') @@ -75,22 +89,30 @@ class test_rollback_to_stable28(test_rollback_to_stable_base): nrows = 10000 # Create our table. - ds = SimpleDataSet(self, uri, 0, key_format='i', value_format='S',config='log=(enabled=false)') + ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)' + self.extraconfig) ds.populate() - value_a = 'a' * 500 - value_b = 'b' * 500 - value_c = 'c' * 500 - value_d = 'd' * 500 + if self.value_format == '8t': + nrows *= 2 + value_a = 97 + value_b = 98 + value_c = 99 + value_d = 100 + else: + value_a = 'a' * 500 + value_b = 'b' * 500 + value_c = 'c' * 500 + value_d = 'd' * 500 # Perform several updates. self.large_updates(uri, value_a, ds, nrows, False, 20) self.large_updates(uri, value_b, ds, nrows, False, 30) self.large_updates(uri, value_c, ds, nrows, False, 40) # Verify data is visible and correct. - self.check(value_a, uri, nrows, 20) - self.check(value_b, uri, nrows, 30) - self.check(value_c, uri, nrows, 40) + self.check(value_a, uri, nrows, None, 20) + self.check(value_b, uri, nrows, None, 30) + self.check(value_c, uri, nrows, None, 40) # Pin the stable timestamp to 40. We will be validating the state of the data post-stable timestamp # after we perform a recovery. @@ -102,9 +124,9 @@ class test_rollback_to_stable28(test_rollback_to_stable_base): self.large_updates(uri, value_b, ds, nrows, False, 70) # Verify additional updated data is visible and correct. - self.check(value_d, uri, nrows, 50) - self.check(value_a, uri, nrows, 60) - self.check(value_b, uri, nrows, 70) + self.check(value_d, uri, nrows, None, 50) + self.check(value_a, uri, nrows, None, 60) + self.check(value_b, uri, nrows, None, 70) # Checkpoint to ensure the data is flushed to disk. self.session.checkpoint() @@ -138,11 +160,11 @@ class test_rollback_to_stable28(test_rollback_to_stable_base): self.assertGreater(pages_update_restored, 0) # Check that after recovery, we see the correct data with respect to our previous stable timestamp (40). - self.check(value_c, uri, nrows, 40) - self.check(value_c, uri, nrows, 50) - self.check(value_c, uri, nrows, 60) - self.check(value_c, uri, nrows, 70) - self.check(value_b, uri, nrows, 30) - self.check(value_a, uri, nrows, 20) + self.check(value_c, uri, nrows, None, 40) + self.check(value_c, uri, nrows, None, 50) + self.check(value_c, uri, nrows, None, 60) + self.check(value_c, uri, nrows, None, 70) + self.check(value_b, uri, nrows, None, 30) + self.check(value_a, uri, nrows, None, 20) # Passing 0 results in opening a transaction with no read timestamp. - self.check(value_c, uri, nrows, 0) + self.check(value_c, uri, nrows, None, 0) diff --git a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable29.py b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable29.py index d40ef12d7c8..ac6d37f4e70 100755 --- a/src/third_party/wiredtiger/test/suite/test_rollback_to_stable29.py +++ b/src/third_party/wiredtiger/test/suite/test_rollback_to_stable29.py @@ -40,26 +40,34 @@ from test_rollback_to_stable01 import test_rollback_to_stable_base class test_rollback_to_stable29(test_rollback_to_stable_base): conn_config = 'cache_size=25MB,statistics=(all),statistics_log=(json,on_close,wait=1),log=(enabled=true)' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test_rollback_to_stable(self): uri = 'table:test_rollback_to_stable29' nrows = 100 + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + value_d = 100 + else: + value_a = 'a' * 100 + value_b = 'b' * 100 + value_c = 'c' * 100 + value_d = 'd' * 100 + # Create our table. - ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format='S',config='log=(enabled=false)') + ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() - value_a = 'a' * 100 - value_b = 'b' * 100 - value_c = 'c' * 100 - value_d = 'd' * 100 - # Pin oldest and stable to timestamp 1. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) @@ -67,19 +75,19 @@ class test_rollback_to_stable29(test_rollback_to_stable_base): self.large_updates(uri, value_a, ds, nrows, False, 10) self.large_removes(uri, ds, nrows, False, 30) self.large_updates(uri, value_b, ds, nrows, False, 40) - self.check(value_b, uri, nrows, 40) + self.check(value_b, uri, nrows, None, 40) self.large_updates(uri, value_c, ds, nrows, False, 50) - self.check(value_c, uri, nrows, 50) + self.check(value_c, uri, nrows, None, 50) self.evict_cursor(uri, nrows, value_c) # Insert an out of order update. self.session.breakpoint() self.large_updates(uri, value_d, ds, nrows, False, 20) - self.check(value_a, uri, nrows, 10) - self.check(value_d, uri, nrows, 40) - self.check(value_d, uri, nrows, 50) - self.check(value_d, uri, nrows, 20) + self.check(value_a, uri, nrows, None, 10) + self.check(value_d, uri, nrows, None, 40) + self.check(value_d, uri, nrows, None, 50) + self.check(value_d, uri, nrows, None, 20) # Pin stable to timestamp 10. self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) @@ -88,7 +96,7 @@ class test_rollback_to_stable29(test_rollback_to_stable_base): # Simulate a crash by copying to a new directory(RESTART). simulate_crash_restart(self, ".", "RESTART") - self.check(value_a, uri, nrows, 10) + self.check(value_a, uri, nrows, None, 10) stat_cursor = self.session.open_cursor('statistics:', None, None) hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] diff --git a/src/third_party/wiredtiger/test/suite/test_salvage.py b/src/third_party/wiredtiger/test/suite/test_salvage.py index 141a1c0ad22..c1859cc9e0b 100755 --- a/src/third_party/wiredtiger/test/suite/test_salvage.py +++ b/src/third_party/wiredtiger/test/suite/test_salvage.py @@ -29,27 +29,88 @@ import os, struct from suite_subprocess import suite_subprocess import wiredtiger, wttest +from wtscenario import make_scenarios # test_salvage.py # Utilities: wt salvage + +# Note that this class is reused by test_encrypt07; be sure to test any changes with +# that version as well. + class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_salvage.a' nentries = 1000 - session_params = 'key_format=S,value_format=S' - unique = 'SomeUniqueString' + + format_values = [ + ('string-row', dict(key_format='S', value_format='S')), + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), + ] + scenarios = make_scenarios(format_values) + + def moreinit(self): + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + if self.key_format == 'r': + # VLCS requires smaller pages or the workload fits on one page, which then gets + # zapped by the damage operation and the table comes out empty, which isn't what + # we want. + format += ',leaf_page_max=4096' + self.session_params = format + + if self.value_format == '8t': + # If the test starts failing weirdly, try picking a different byte. Don't forget to + # set value_modulus to some smaller value. It had better be < 127 so the mandatory + # Python trip through UTF-8 doesn't blow up. Currently it is set to 125, which at + # least gets to the salvage code; 126 apparently corrupts the root page and then + # nothing works. + self.unique = 125 + self.value_modulus = 113 + self.uniquebytes = bytes([self.unique]) + + else: + self.unique = 'SomeUniqueString' + self.uniquebytes = self.unique.encode() + + def firstkey(self): + if self.key_format == 'r': + return 1 + return '' + + def nextkey(self, key, i): + if self.key_format == 'r': + # Use key + i to create gaps. This makes the actual number of rows larger in FLCS, + # but since the rows are small that doesn't make the table excessively large. + return key + i + else: + return key + str(i) + + def uniqueval(self): + if self.value_format == '8t': + return self.unique + else: + return self.unique + '0' + + def ordinaryval(self, key): + if self.value_format == '8t': + # Pick something that won't overlap self.unique. + return key % self.value_modulus + elif self.key_format == 'r': + return str(key) + str(key) + else: + return key + key def populate(self, tablename): """ Insert some simple entries into the table """ cursor = self.session.open_cursor('table:' + tablename, None, None) - key = '' + key = self.firstkey() for i in range(0, self.nentries): - key += str(i) + key = self.nextkey(key, i) if i == self.nentries // 2: - val = self.unique + '0' + val = self.uniqueval() else: - val = key + key + val = self.ordinaryval(key) cursor[key] = val cursor.close() @@ -58,18 +119,30 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): Verify that items added by populate are still there """ cursor = self.session.open_cursor('table:' + tablename, None, None) - wantkey = '' + wantkey = self.firstkey() i = 0 + zeros = 0 for gotkey, gotval in cursor: - wantkey += str(i) + nextkey = self.nextkey(wantkey, i) + + # In FLCS the values between the keys we wrote will read as zero. Count them. + if gotkey < nextkey and self.value_format == '8t': + self.assertEqual(gotval, 0) + zeros += 1 + continue + wantkey = nextkey + if i == self.nentries // 2: - wantval = self.unique + '0' + wantval = self.uniqueval() else: - wantval = wantkey + wantkey + wantval = self.ordinaryval(wantkey) self.assertEqual(gotkey, wantkey) - self.assertTrue(gotval, wantval) + self.assertEqual(gotval, wantval) i += 1 self.assertEqual(i, self.nentries) + if self.value_format == '8t': + # We should have visited every key, so the total number of should match the last key. + self.assertEqual(self.nentries + zeros, wantkey) cursor.close() def check_damaged(self, tablename): @@ -79,18 +152,20 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): just that the ones that are here are correct. """ cursor = self.session.open_cursor('table:' + tablename, None, None) - wantkey = '' + wantkey = self.firstkey() i = -1 correct = 0 for gotkey, gotval in cursor: i += 1 - wantkey += str(i) + wantkey = self.nextkey(wantkey, i) if gotkey != wantkey: + # Note that if a chunk in the middle of the table got lost, + # this will never sync up again. continue if i == self.nentries // 2: - wantval = self.unique + '0' + wantval = self.uniqueval() else: - wantval = wantkey + wantkey + wantval = self.ordinaryval(wantkey) self.assertEqual(gotkey, wantkey) self.assertTrue(gotval, wantval) correct += 1 @@ -106,7 +181,7 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): cursor.close() def damage(self, tablename): - self.damage_inner(tablename, self.unique.encode()) + self.damage_inner(tablename, self.uniquebytes) def read_byte(self, fp): """ @@ -151,6 +226,7 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): """ Test salvage in a 'wt' process, using an empty table """ + self.moreinit() self.session.create('table:' + self.tablename, self.session_params) errfile = "salvageerr.out" self.runWt(["salvage", self.tablename + ".wt"], errfilename=errfile) @@ -161,6 +237,7 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): """ Test salvage in a 'wt' process, using a populated table. """ + self.moreinit() self.session.create('table:' + self.tablename, self.session_params) self.populate(self.tablename) errfile = "salvageerr.out" @@ -172,6 +249,7 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): """ Test salvage via API, using an empty table """ + self.moreinit() self.session.create('table:' + self.tablename, self.session_params) self.session.salvage('table:' + self.tablename, None) self.check_empty_table(self.tablename) @@ -180,6 +258,7 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): """ Test salvage via API, using a populated table. """ + self.moreinit() self.session.create('table:' + self.tablename, self.session_params) self.populate(self.tablename) self.session.salvage('file:' + self.tablename + ".wt", None) @@ -189,6 +268,7 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): """ Test salvage via API, on a damaged table. """ + self.moreinit() self.session.create('table:' + self.tablename, self.session_params) self.populate(self.tablename) self.damage(self.tablename) @@ -206,6 +286,7 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): """ Test salvage in a 'wt' process on a table that is purposely damaged. """ + self.moreinit() self.session.create('table:' + self.tablename, self.session_params) self.populate(self.tablename) self.damage(self.tablename) diff --git a/src/third_party/wiredtiger/test/suite/test_stat01.py b/src/third_party/wiredtiger/test/suite/test_stat01.py index 249b6a0d494..4cc962add0e 100755 --- a/src/third_party/wiredtiger/test/suite/test_stat01.py +++ b/src/third_party/wiredtiger/test/suite/test_stat01.py @@ -46,8 +46,9 @@ class test_stat01(wttest.WiredTigerTestCase): ('table', dict(uri='table:test_stat01.wt')) ] keyfmt = [ - ('recno', dict(keyfmt='r')), - ('string', dict(keyfmt='S')), + ('column', dict(keyfmt='r', valfmt='S')), + ('column-fix', dict(keyfmt='r', valfmt='8t')), + ('string-row', dict(keyfmt='S', valfmt='S')), ] scenarios = make_scenarios(types, keyfmt) @@ -89,8 +90,9 @@ class test_stat01(wttest.WiredTigerTestCase): # Test simple connection statistics. def test_basic_conn_stats(self): # Build an object and force some writes. - SimpleDataSet(self, self.uri, 1000, - config=self.config, key_format=self.keyfmt).populate() + ds = SimpleDataSet(self, self.uri, 1000, + config=self.config, key_format=self.keyfmt, value_format = self.valfmt) + ds.populate() self.session.checkpoint(None) # See that we can get a specific stat value by its key and verify its diff --git a/src/third_party/wiredtiger/test/suite/test_stat03.py b/src/third_party/wiredtiger/test/suite/test_stat03.py index ca1eaa5c83f..0cff6c99076 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat03.py +++ b/src/third_party/wiredtiger/test/suite/test_stat03.py @@ -41,11 +41,17 @@ from wtscenario import make_scenarios class test_stat_cursor_reset(wttest.WiredTigerTestCase): pfx = 'test_stat_cursor_reset' uri = [ - ('file-simple', dict(uri='file:' + pfx, dataset=SimpleDataSet)), - ('table-simple', dict(uri='table:' + pfx, dataset=SimpleDataSet)), - ('table-complex', dict(uri='table:' + pfx, dataset=ComplexDataSet)), + ('file-simple-row', dict(uri='file:' + pfx, dataset=SimpleDataSet, kf='S', vf='S')), + ('file-simple-var', dict(uri='file:' + pfx, dataset=SimpleDataSet, kf='r', vf='S')), + ('file-simple-fix', dict(uri='file:' + pfx, dataset=SimpleDataSet, kf='r', vf='8t')), + ('table-simple-row', dict(uri='table:' + pfx, dataset=SimpleDataSet, kf='S', vf='S')), + ('table-simple-var', dict(uri='table:' + pfx, dataset=SimpleDataSet, kf='r', vf='S')), + ('table-simple-fix', dict(uri='table:' + pfx, dataset=SimpleDataSet, kf='r', vf='8t')), + # The complex data sets ignore any passed-in value format. + ('table-complex-row', dict(uri='table:' + pfx, dataset=ComplexDataSet, kf='S', vf=None)), + ('table-complex-var', dict(uri='table:' + pfx, dataset=ComplexDataSet, kf='r', vf=None)), ('table-complex-lsm', dict(uri='table:' + pfx, - dataset=ComplexLSMDataSet)) + dataset=ComplexLSMDataSet, kf='S', vf=None)) ] scenarios = make_scenarios(uri) @@ -57,7 +63,7 @@ class test_stat_cursor_reset(wttest.WiredTigerTestCase): def test_stat_cursor_reset(self): n = 100 - ds = self.dataset(self, self.uri, n) + ds = self.dataset(self, self.uri, n, key_format=self.kf, value_format=self.vf) ds.populate() # The number of btree_entries reported is influenced by the diff --git a/src/third_party/wiredtiger/test/suite/test_stat04.py b/src/third_party/wiredtiger/test/suite/test_stat04.py index 4505547e9e6..8a8a9bcda4a 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat04.py +++ b/src/third_party/wiredtiger/test/suite/test_stat04.py @@ -37,10 +37,9 @@ from wiredtiger import stat class test_stat04(wttest.WiredTigerTestCase, suite_subprocess): uripfx = 'table:test_stat04.' - # Note: stats for fixed length bit fields (valuefmt='8t') - # do not include accurate counts for kv pairs. keyfmt = [ ('col', dict(keyfmt='r', valuefmt='S', storekind='col')), + ('fix', dict(keyfmt='r', valuefmt='8t', storekind='fix')), ('row', dict(keyfmt='S', valuefmt='S', storekind='row')), ] nentries = [ @@ -95,7 +94,7 @@ class test_stat04(wttest.WiredTigerTestCase, suite_subprocess): # Remove a number of entries, at each step checking that stats match. for i in range(0, self.nentries // 37): cursor.set_key(self.genkey(i*11 % self.nentries)) - if cursor.remove() == 0: + if cursor.remove() == 0 and self.valuefmt != '8t': count -= 1 self.checkcount(uri, count) cursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_stat05.py b/src/third_party/wiredtiger/test/suite/test_stat05.py index df5b6eb3d55..ff2c06c06f1 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat05.py +++ b/src/third_party/wiredtiger/test/suite/test_stat05.py @@ -39,16 +39,25 @@ class test_stat_cursor_config(wttest.WiredTigerTestCase): conn_config = 'statistics=(fast)' uri = [ - ('file', dict(uri='file:' + pfx, dataset=SimpleDataSet, cfg='')), - ('table', dict(uri='table:' + pfx, dataset=SimpleDataSet, cfg='')), - ('inmem', dict(uri='table:' + pfx, dataset=SimpleDataSet, cfg='', + ('file-row', dict(uri='file:' + pfx, dataset=SimpleDataSet, type='row', cfg='')), + ('file-var', dict(uri='file:' + pfx, dataset=SimpleDataSet, type='var', cfg='')), + ('file-fix', dict(uri='file:' + pfx, dataset=SimpleDataSet, type='fix', cfg='')), + ('table-row', dict(uri='table:' + pfx, dataset=SimpleDataSet, type='row', cfg='')), + ('table-var', dict(uri='table:' + pfx, dataset=SimpleDataSet, type='var', cfg='')), + ('table-fix', dict(uri='table:' + pfx, dataset=SimpleDataSet, type='fix', cfg='')), + ('inmem-row', dict(uri='table:' + pfx, dataset=SimpleDataSet, type='row', cfg='', conn_config = 'in_memory,statistics=(fast)')), - ('table-lsm', dict(uri='table:' + pfx, dataset=SimpleDataSet, + ('inmem-var', dict(uri='table:' + pfx, dataset=SimpleDataSet, type='var', cfg='', + conn_config = 'in_memory,statistics=(fast)')), + ('inmem-fix', dict(uri='table:' + pfx, dataset=SimpleDataSet, type='fix', cfg='', + conn_config = 'in_memory,statistics=(fast)')), + ('table-lsm', dict(uri='table:' + pfx, dataset=SimpleDataSet, type='lsm', cfg='lsm=(chunk_size=1MB,merge_min=2)', conn_config = 'statistics=(fast)')), - ('complex', dict(uri='table:' + pfx, dataset=ComplexDataSet, cfg='')), + ('complex-row', dict(uri='table:' + pfx, dataset=ComplexDataSet, type='row', cfg='')), + ('complex-var', dict(uri='table:' + pfx, dataset=ComplexDataSet, type='fix', cfg='')), ('complex-lsm', - dict(uri='table:' + pfx, dataset=ComplexLSMDataSet, + dict(uri='table:' + pfx, dataset=ComplexLSMDataSet, type='lsm', cfg='lsm=(chunk_size=1MB,merge_min=2)', conn_config = 'statistics=(fast)')), ] @@ -67,7 +76,22 @@ class test_stat_cursor_config(wttest.WiredTigerTestCase): # the cursor open succeeds. Insert enough data that LSM tables to need to # switch and merge. def test_stat_cursor_size(self): - ds = self.dataset(self, self.uri, 100, config=self.cfg) + if self.type == 'row': + key_format = 'S' + value_format = 'S' + elif self.type == 'var': + key_format = 'r' + value_format = 'S' + elif self.type == 'fix': + key_format = 'r' + value_format = '8t' + else: + self.assertEqual(self.type, 'lsm') + key_format = 'S' + value_format = 'S' + + ds = self.dataset( + self, self.uri, 100, key_format=key_format, value_format=value_format, config=self.cfg) ds.populate() self.openAndWalkStatCursor() cursor = self.session.open_cursor(self.uri, None) diff --git a/src/third_party/wiredtiger/test/suite/test_stat10.py b/src/third_party/wiredtiger/test/suite/test_stat10.py new file mode 100644 index 00000000000..f5c65a9e606 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_stat10.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import random +import wiredtiger, wttest +from wtscenario import make_scenarios +from wiredtiger import stat + +# test_stat10.py +# +# Check the per-table-type btree stats. +# +# For rows: +# btree_row_empty_values +# btree_overflow +# For VLCS: +# btree_column_deleted +# btree_column_rle +# btree_overflow +# For FLCS: +# btree_column_tws +# +# The other types' stats should remain zero. + +class test_stat10(wttest.WiredTigerTestCase): + uri = 'table:test_stat10' + conn_config = 'statistics=(all)' + session_config = 'isolation=snapshot' + + format_values = [ + ('column', dict(key_format='r', value_format='u')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('string_row', dict(key_format='S', value_format='u')), + ] + + oldest_values = [ + ('15', dict(oldest=15)), + ('25', dict(oldest=25)), + ('35', dict(oldest=35)), + ] + + stable_values = [ + ('15', dict(stable=15)), + ('25', dict(stable=25)), + ('35', dict(stable=35)), + ] + + def keep(name, d): + return d['oldest'] <= d['stable'] + + scenarios = make_scenarios(format_values, oldest_values, stable_values, include=keep) + + # Get a key. + def make_key(self, i): + if self.key_format == 'r': + return i + key = 'k' + str(i) + # Make a few keys overflow keys. + if i % 47 == 0: + key += 'blablabla' * 1000 + return key + + # Make an invariant value. + def make_base_value(self): + if self.value_format == '8t': + return 170 + return 'abcde' * 100 + + # Make a varying value. + def make_compound_value(self, i): + if self.value_format == '8t': + return i + val = str(i) + 'abcde' + if i % 61 == 0: + # Make an overflow value. + return val * 10000 + elif i % 67 == 0: + # Use an empty value. + return "" + return val * 100 + + def evict(self, k): + evict_cursor = self.session.open_cursor(self.uri, None, "debug=(release_evict)") + self.session.begin_transaction() + evict_cursor.set_key(k) + evict_cursor.search() + evict_cursor.reset() + evict_cursor.close() + self.session.rollback_transaction() + + def test_tree_stats(self): + format = "key_format={},value_format={}".format(self.key_format, self.value_format) + self.session.create(self.uri, format) + + nrows = 50 + + # Pin oldest and stable to timestamp 10. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + + ',stable_timestamp=' + self.timestamp_str(10)) + + cursor = self.session.open_cursor(self.uri) + + # Add 50 keys with identical values and 50 keys with different values, at time 20. + self.session.begin_transaction() + for i in range(1, nrows + 1): + cursor[self.make_key(i)] = self.make_base_value() + for i in range(nrows + 1, nrows * 2 + 1): + cursor[self.make_key(i)] = self.make_compound_value(i) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) + + # Delete two keys at time 30. + self.session.begin_transaction() + cursor.set_key(self.make_key(44)) + self.assertEqual(cursor.remove(), 0) + cursor.set_key(self.make_key(66)) + self.assertEqual(cursor.remove(), 0) + self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) + + cursor.close() + + # Move oldest and stable up as dictated by the scenario. + self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(self.oldest) + + ',stable_timestamp=' + self.timestamp_str(self.stable)) + + # Evict the page(s) so that e.g. VLCS RLE-encoding and FLCS timestamp handling take place. + for i in range(1, nrows * 2 + 1, nrows // 10): + self.evict(self.make_key(i)) + + # Read the stats. + statscursor = self.session.open_cursor('statistics:' + self.uri, None, 'statistics=(all)') + + entries = statscursor[stat.dsrc.btree_entries][2] + row_empty_values = statscursor[stat.dsrc.btree_row_empty_values][2] + column_deleted = statscursor[stat.dsrc.btree_column_deleted][2] + column_rle = statscursor[stat.dsrc.btree_column_rle][2] + column_tws = statscursor[stat.dsrc.btree_column_tws][2] + overflow = statscursor[stat.dsrc.btree_overflow][2] + + statscursor.close() + + # Validate the stats. + + # Note that the visibility of timestamped changes to the stats is not always + # very consistent. This behavior is not clearly specified and (AFAIK) not + # really intended to be; the purpose of checking it here is not to enforce the + # current behavior but to make sure the behavior doesn't change unexpectedly. + # I've kept the timestamp tests and the format tests separate to help clarify + # this. + + # entries: always 100 for FLCS; for RS and VLCS, when oldest passes 30 the two + # deleted values show up in the count. + if self.value_format == '8t': + self.assertEqual(entries, nrows * 2) + else: + if self.oldest > 30: + self.assertEqual(entries, nrows * 2 - 2) + else: + self.assertEqual(entries, nrows * 2) + + # row_empty_values: 1 for RS, otherwise 0; only appears when oldest passes 20 + if self.key_format == 'S': + if self.oldest > 20: + self.assertEqual(row_empty_values, 1) + else: + self.assertEqual(row_empty_values, 0) + else: + self.assertEqual(row_empty_values, 0) + + # column_deleted: for VLCS only; only appears when oldest passes 30. + if self.key_format == 'r' and self.value_format != '8t': + if self.oldest > 30: + self.assertEqual(column_deleted, 2) + else: + self.assertEqual(column_deleted, 0) + else: + self.assertEqual(column_deleted, 0) + + # column_rle: for VLCS only. + if self.key_format == 'r' and self.value_format != '8t': + # We deleted a key, so two RLE cells adding to nrows - 1 total. + self.assertEqual(column_rle, nrows - 3) + else: + self.assertEqual(column_rle, 0) + + # column_tws: for FLCS only. + if self.key_format == 'r' and self.value_format == '8t': + if self.oldest > 20: + # Only the deletions show. + self.assertEqual(column_tws, 2) + else: + self.assertEqual(column_tws, nrows * 2) + else: + self.assertEqual(column_tws, 0) + + # overflow: two keys and one value, so 3 for rows, 1 for VLCS, 0 for FLCS. + if self.key_format == 'S': + self.assertEqual(overflow, 3) + elif self.value_format == '8t': + self.assertEqual(overflow, 0) + else: + self.assertEqual(overflow, 1) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp02.py b/src/third_party/wiredtiger/test/suite/test_timestamp02.py index 4bc2c7d55ae..0dc0ce3b7e0 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp02.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp02.py @@ -41,6 +41,7 @@ class test_timestamp02(wttest.WiredTigerTestCase, suite_subprocess): scenarios = make_scenarios([ ('col', dict(extra_config=',key_format=r')), + ('col_fix', dict(extra_config=',key_format=r,value_format=8t')), ('lsm', dict(extra_config=',type=lsm')), ('row', dict(extra_config='')), ]) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp03.py b/src/third_party/wiredtiger/test/suite/test_timestamp03.py index a18d3fd86b0..0fb3f5be7de 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp03.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp03.py @@ -42,14 +42,28 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): table_nots_log = 'ts03_nots_logged' table_nots_nolog = 'ts03_nots_nologged' + # XXX neither the use_cg nor the use_index values are actually used. + # I've commented out the entries that are therefore duplicates; if/when that's fixed, put + # them back I guess. types = [ - ('file-row', dict(uri='file:', key_format='i', use_cg=False, use_index=False)), - ('file-col', dict(uri='file:', key_format='r', use_cg=False, use_index=False)), - ('lsm', dict(uri='lsm:', key_format='i', use_cg=False, use_index=False)), - ('table-row', dict(uri='table:', key_format='i', use_cg=False, use_index=False)), - ('table-row-index', dict(uri='table:', key_format='i', use_cg=False, use_index=True)), - ('table-col', dict(uri='table:', key_format='r', use_cg=False, use_index=False)), - ('table-col-cg', dict(uri='table:', key_format='r', use_cg=True, use_index=False)), + ('file-row', dict(uri='file:', key_format='i', value_format='S', + use_cg=False, use_index=False)), + ('file-col', dict(uri='file:', key_format='r', value_format='S', + use_cg=False, use_index=False)), + ('file-col-fix', dict(uri='file:', key_format='r', value_format='8t', + use_cg=False, use_index=False)), + ('lsm', dict(uri='lsm:', key_format='i', value_format='S', + use_cg=False, use_index=False)), + ('table-row', dict(uri='table:', key_format='i', value_format='S', + use_cg=False, use_index=False)), + #('table-row-index', dict(uri='table:', key_format='i', value_format='S', + # use_cg=False, use_index=True)), + ('table-col', dict(uri='table:', key_format='r', value_format='S', + use_cg=False, use_index=False)), + ('table-col-fix', dict(uri='table:', key_format='r', value_format='8t', + use_cg=False, use_index=False)), + #('table-col-cg', dict(uri='table:', key_format='r', value_format='S', + # use_cg=True, use_index=False)), ] ckpt = [ @@ -67,10 +81,16 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): scenarios = make_scenarios(types, ckpt, conncfg) - # Binary values. - value = u'\u0001\u0002abcd\u0003\u0004' - value2 = u'\u0001\u0002dcba\u0003\u0004' - value3 = u'\u0001\u0002cdef\u0003\u0004' + def moresetup(self): + # Binary values. + if self.value_format == '8t': + self.value = 4 + self.value2 = 5 + self.value3 = 6 + else: + self.value = u'\u0001\u0002abcd\u0003\u0004' + self.value2 = u'\u0001\u0002dcba\u0003\u0004' + self.value3 = u'\u0001\u0002cdef\u0003\u0004' # Check that a cursor (optionally started in a new transaction), sees the # expected values. @@ -105,6 +125,13 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): cur_ts_nolog = session.open_cursor(self.uri + self.table_ts_nolog, None) cur_nots_log = session.open_cursor(self.uri + self.table_nots_log, None) cur_nots_nolog = session.open_cursor(self.uri + self.table_nots_nolog, None) + + # In FLCS the values are bytes, which are numbers, but the tests below are via + # string inclusion rather than just equality of values. Not sure why that is, but + # I'm going to assume there's a reason for it and not change things. Compensate. + if self.value_format == '8t': + check_value = str(check_value) + # Count how many times the check_value is present in the # logged timestamp table. actual_ts_log = 0 @@ -154,6 +181,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): valcnt_nots_log, valcnt_nots_nolog) def test_timestamp03(self): + self.moresetup() uri_ts_log = self.uri + self.table_ts_log uri_ts_nolog = self.uri + self.table_ts_nolog uri_nots_log = self.uri + self.table_nots_log @@ -165,7 +193,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): # 3. Table is logged and does not use timestamps. # 4. Table is not logged and does not use timestamps. # - format = 'key_format={},value_format=S'.format(self.key_format) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri_ts_log, format) cur_ts_log = self.session.open_cursor(uri_ts_log) self.session.create(uri_ts_nolog, format + ',log=(enabled=false)') diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp04.py b/src/third_party/wiredtiger/test/suite/test_timestamp04.py index e2aa7609fa7..c35fdbc2a55 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp04.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp04.py @@ -54,8 +54,8 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Minimum cache_size requirement of lsm is 31MB. types = [ # FLCS does not yet work in a timestamp world. - #('col_fix', dict(empty=1, \ - # cacheSize='cache_size=20MB', extra_config=',key_format=r,value_format=8t')), + ('col_fix', dict(empty=1, \ + cacheSize='cache_size=20MB', extra_config=',key_format=r,value_format=8t')), ('lsm', dict(empty=0, cacheSize='cache_size=31MB', extra_config=',type=lsm')), ('row', dict(empty=0, cacheSize='cache_size=20MB', extra_config='',)), ('row-smallcache', dict(empty=0, cacheSize='cache_size=2MB', extra_config='',)), diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp05.py b/src/third_party/wiredtiger/test/suite/test_timestamp05.py index 852823231c7..05347ced247 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp05.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp05.py @@ -39,27 +39,30 @@ class test_timestamp05(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:ts05' session_config = 'isolation=snapshot' - key_format_values = [ - ('integer-row', dict(key_format='i')), - ('column', dict(key_format='r')), + format_values = [ + ('integer-row', dict(key_format='i', value_format='S')), + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test_create(self): s = self.session conn = self.conn + new_value = 71 if self.value_format == '8t' else 'new value' + # Start timestamps at 50 conn.set_timestamp('oldest_timestamp=50,stable_timestamp=50') # Commit at 100 s.begin_transaction() - s.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) + s.create(self.uri, 'key_format={},value_format={}'.format(self.key_format, self.value_format)) s.commit_transaction('commit_timestamp=' + self.timestamp_str(100)) # Make sure the tree is dirty c = s.open_cursor(self.uri) - c[200] = 'new value' + c[200] = new_value # Checkpoint at 50 s.checkpoint('use_timestamp=true') @@ -68,13 +71,16 @@ class test_timestamp05(wttest.WiredTigerTestCase, suite_subprocess): s = self.session conn = self.conn - s.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) + some_value = 66 if self.value_format == '8t' else 'some value' + new_value = 71 if self.value_format == '8t' else 'new value' + + s.create(self.uri, 'key_format={},value_format={}'.format(self.key_format, self.value_format)) c = s.open_cursor(self.uri, None, 'bulk') # Insert keys 1..100 each with timestamp=key, in some order nkeys = 100 for k in range(1, nkeys+1): - c[k] = 'some value' + c[k] = some_value # Start timestamps at 50 conn.set_timestamp('oldest_timestamp=50,stable_timestamp=50') @@ -86,7 +92,7 @@ class test_timestamp05(wttest.WiredTigerTestCase, suite_subprocess): # Make sure the tree is dirty c = s.open_cursor(self.uri) - c[200] = 'new value' + c[200] = new_value # Checkpoint at 50 s.checkpoint('use_timestamp=true') diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp06.py b/src/third_party/wiredtiger/test/suite/test_timestamp06.py index 27e32309fdc..d0e8f89ba6c 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp06.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp06.py @@ -41,8 +41,7 @@ class test_timestamp06(wttest.WiredTigerTestCase, suite_subprocess): table_ts_nolog = 'table:ts06_ts_nologged' types = [ - # FLCS does not yet work in a timestamp world. - #('col_fix', dict(empty=1, extra_config=',key_format=r,value_format=8t')), + ('col_fix', dict(empty=1, extra_config=',key_format=r,value_format=8t')), ('col_var', dict(empty=0, extra_config=',key_format=r')), ('lsm', dict(empty=0, extra_config=',type=lsm')), ('row', dict(empty=0, extra_config='',)), diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp07.py b/src/third_party/wiredtiger/test/suite/test_timestamp07.py index 5e25afe1675..4af4123913f 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp07.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp07.py @@ -41,9 +41,10 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): tablename2 = 'ts07_nots_logged' tablename3 = 'ts07_ts_logged' - key_format_values = [ - ('integer-row', dict(key_format='i')), - ('column', dict(key_format='r')), + format_values = [ + ('string-row', dict(key_format='i', value_format='S')), + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), ] types = [ @@ -63,20 +64,30 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): ('1000keys', dict(nkeys=1000)), ] - scenarios = make_scenarios(key_format_values, types, conncfg, nkeys) + scenarios = make_scenarios(format_values, types, conncfg, nkeys) # Binary values. - value = u'\u0001\u0002abcd\u0007\u0004' - value2 = u'\u0001\u0002dcba\u0007\u0004' - value3 = u'\u0001\u0002cdef\u0007\u0004' + def moreinit(self): + if self.value_format == '8t': + self.value = 2 + self.value2 = 4 + self.value3 = 6 + else: + self.value = u'\u0001\u0002abcd\u0007\u0004' + self.value2 = u'\u0001\u0002dcba\u0007\u0004' + self.value3 = u'\u0001\u0002cdef\u0007\u0004' # Check that a cursor (optionally started in a new transaction), sees the # expected value for a key - def check(self, session, txn_config, k, expected): + def check(self, session, txn_config, k, expected, flcs_expected): + # In FLCS the table extends under uncommitted writes and we expect to + # see zero rather than NOTFOUND. + if self.value_format == '8t' and flcs_expected is not None: + expected = flcs_expected if txn_config: session.begin_transaction(txn_config) c = session.open_cursor(self.uri + self.tablename, None) - if not expected: + if expected is None: c.set_key(k) self.assertEqual(c.search(), wiredtiger.WT_NOTFOUND) else: @@ -92,6 +103,13 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): c = session.open_cursor(self.uri + self.tablename, None) c2 = session.open_cursor(self.uri + self.tablename2, None) c3 = session.open_cursor(self.uri + self.tablename3, None) + + # In FLCS the values are bytes, which are numbers, but the tests below are via + # string inclusion rather than just equality of values. Not sure why that is, but + # I'm going to assume there's a reason for it and not change things. Compensate. + if self.value_format == '8t': + check_value = str(check_value) + count = 0 for k, v in c: if check_value in str(v): @@ -126,6 +144,13 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): c = session.open_cursor(self.uri + self.tablename, None) c2 = session.open_cursor(self.uri + self.tablename2, None) c3 = session.open_cursor(self.uri + self.tablename3, None) + + # In FLCS the values are bytes, which are numbers, but the tests below are via + # string inclusion rather than just equality of values. Not sure why that is, but + # I'm going to assume there's a reason for it and not change things. Compensate. + if self.value_format == '8t': + check_value = str(check_value) + # Count how many times the second value is present count = 0 for k, v in c: @@ -179,17 +204,19 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): uri = self.uri + self.tablename uri2 = self.uri + self.tablename2 uri3 = self.uri + self.tablename3 + self.moreinit() # # Open three tables: # 1. Table is not logged and uses timestamps. # 2. Table is logged and does not use timestamps. # 3. Table is logged and uses timestamps. # - self.session.create(uri, 'key_format={},value_format=S,log=(enabled=false)'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, format + ',log=(enabled=false)') c = self.session.open_cursor(uri) - self.session.create(uri2, 'key_format={},value_format=S'.format(self.key_format)) + self.session.create(uri2, format) c2 = self.session.open_cursor(uri2) - self.session.create(uri3, 'key_format={},value_format=S'.format(self.key_format)) + self.session.create(uri3, format) c3 = self.session.open_cursor(uri3) # print "tables created" @@ -211,9 +238,9 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): # timestamp. for k in orig_keys: self.check(self.session, 'read_timestamp=' + self.timestamp_str(k), - k, self.value) + k, self.value, None) self.check(self.session, 'read_timestamp=' + self.timestamp_str(k), - k + 1, None) + k + 1, None, None if k == self.nkeys else 0) # print "all values read, updating timestamps" diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp10.py b/src/third_party/wiredtiger/test/suite/test_timestamp10.py index be6dafc5c38..d8fb817a3a0 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp10.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp10.py @@ -45,9 +45,10 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): nentries = 10 table_cnt = 3 - key_format_values = [ - ('integer-row', dict(key_format='i')), - ('column', dict(key_format='r')), + format_values = [ + ('integer-row', dict(key_format='i', value_format='i')), + ('column', dict(key_format='r', value_format='i')), + ('column-fix', dict(key_format='r', value_format='8t')), ] types = [ ('all', dict(use_stable='false', run_wt=0)), @@ -60,7 +61,7 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): ('stable+wt', dict(use_stable='true', run_wt=1)), ('stable+wt2', dict(use_stable='true', run_wt=2)), ] - scenarios = make_scenarios(key_format_values, types) + scenarios = make_scenarios(format_values, types) def data_and_checkpoint(self): # @@ -68,7 +69,7 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): # Add data to each of them separately and checkpoint so that each one # has a different stable timestamp. # - basecfg = 'key_format={},value_format=i'.format(self.key_format) + basecfg = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(self.oplog_uri, basecfg) self.session.create(self.coll1_uri, basecfg + ',log=(enabled=false)') self.session.create(self.coll2_uri, basecfg + ',log=(enabled=false)') @@ -159,6 +160,11 @@ class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess): # be missing some. if self.use_stable == 'false' or i <= ts or table != self.table_cnt: self.assertEquals(curs[i], i) + elif self.value_format == '8t': + # For FLCS, expect the table to have extended under the lost values. + # We should see 0 and not the data that was written. + self.assertEqual(curs.search(), 0) + self.assertEqual(curs.get_value(), 0) else: self.assertEqual(curs.search(), wiredtiger.WT_NOTFOUND) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp11.py b/src/third_party/wiredtiger/test/suite/test_timestamp11.py index d5998bdc212..4d0af0562cf 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp11.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp11.py @@ -37,19 +37,34 @@ from wtscenario import make_scenarios class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' - key_format_values = [ - ('string-row', dict(key_format='S', usestrings=True)), - ('column', dict(key_format='r', usestrings=False)), + format_values = [ + ('string-row', dict(key_format='S', value_format='S')), + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test_timestamp_range(self): base = 'timestamp11' uri = 'file:' + base - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) - - key = 'key' if self.usestrings else 1 - key2 = 'key2' if self.usestrings else 2 + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, format) + + if self.key_format == 'r': + key = 1 + key2 = 2 + else: + key = 'key' + key2 = 'key2' + + if self.value_format == '8t': + value2 = 200 + value5 = 50 + valueNOTS = 111 + else: + value2 = 'value2' + value5 = 'value5' + valueNOTS = 'valueNOTS' # Test that mixed timestamp usage where some transactions use timestamps # and others don't behave in the expected way. @@ -59,8 +74,8 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() self.session.timestamp_transaction( 'commit_timestamp=' + self.timestamp_str(2)) - c[key] = 'value2' - c[key2] = 'value2' + c[key] = value2 + c[key2] = value2 self.session.commit_transaction() c.close() @@ -72,13 +87,13 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() self.session.timestamp_transaction( 'commit_timestamp=' + self.timestamp_str(5)) - c[key] = 'value5' + c[key] = value5 self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c[key2] = 'valueNOTS' + c[key2] = valueNOTS self.session.commit_transaction() c.close() @@ -95,15 +110,15 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri) self.session.begin_transaction() - self.assertEquals(c[key], 'value2') - self.assertEquals(c[key2], 'valueNOTS') + self.assertEquals(c[key], value2) + self.assertEquals(c[key2], valueNOTS) self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction('read_timestamp=' + stable_ts) - self.assertEquals(c[key], 'value2') - self.assertEquals(c[key2], 'valueNOTS') + self.assertEquals(c[key], value2) + self.assertEquals(c[key2], valueNOTS) self.session.commit_transaction() c.close() @@ -114,13 +129,13 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction() self.session.timestamp_transaction( 'commit_timestamp=' + self.timestamp_str(5)) - c[key2] = 'value5' + c[key2] = value5 self.session.commit_transaction() c.close() c = self.session.open_cursor(uri) self.session.begin_transaction() - c[key] = 'valueNOTS' + c[key] = valueNOTS self.session.commit_transaction() c.close() @@ -129,8 +144,8 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): # Without a timestamp. We should see the latest value for each. c = self.session.open_cursor(uri) self.session.begin_transaction() - self.assertEquals(c[key], 'valueNOTS') - self.assertEquals(c[key2], 'value5') + self.assertEquals(c[key], valueNOTS) + self.assertEquals(c[key2], value5) self.session.commit_transaction() c.close() @@ -138,8 +153,8 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): # value at timestamp 2. c = self.session.open_cursor(uri) self.session.begin_transaction('read_timestamp=' + stable_ts) - self.assertEquals(c[key], 'valueNOTS') - self.assertEquals(c[key2], 'valueNOTS') + self.assertEquals(c[key], valueNOTS) + self.assertEquals(c[key2], valueNOTS) self.session.commit_transaction() c.close() @@ -148,8 +163,8 @@ class test_timestamp11(wttest.WiredTigerTestCase, suite_subprocess): # we inserted at timestamp 5 after the non-timestamped insert. c = self.session.open_cursor(uri) self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5)) - self.assertEquals(c[key], 'valueNOTS') - self.assertEquals(c[key2], 'value5') + self.assertEquals(c[key], valueNOTS) + self.assertEquals(c[key2], value5) self.session.commit_transaction() c.close() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp12.py b/src/third_party/wiredtiger/test/suite/test_timestamp12.py index 5ccab50ce05..a4bab3a4e48 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp12.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp12.py @@ -38,16 +38,18 @@ class test_timestamp12(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' coll_uri = 'table:collection12' oplog_uri = 'table:oplog12' - key_format_values = [ - ('integer-row', dict(key_format='i')), - ('column', dict(key_format='r')), + + format_values = [ + ('integer-row', dict(key_format='i', value_format='i')), + ('column', dict(key_format='r', value_format='i')), + ('column-fix', dict(key_format='r', value_format='8t')), ] closecfg = [ ('dfl', dict(close_cfg='', all_expected=False)), ('use_stable', dict(close_cfg='use_timestamp=true', all_expected=False)), ('all_dirty', dict(close_cfg='use_timestamp=false', all_expected=True)), ] - scenarios = make_scenarios(key_format_values, closecfg) + scenarios = make_scenarios(format_values, closecfg) def verify_expected(self, op_exp, coll_exp): c_op = self.session.open_cursor(self.oplog_uri) @@ -71,7 +73,7 @@ class test_timestamp12(wttest.WiredTigerTestCase): # Add data to each of them separately and checkpoint so that each one # has a different stable timestamp. # - basecfg = 'key_format={},value_format=i'.format(self.key_format) + basecfg = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(self.oplog_uri, basecfg) self.session.create(self.coll_uri, basecfg + ',log=(enabled=false)') c_op = self.session.open_cursor(self.oplog_uri) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp13.py b/src/third_party/wiredtiger/test/suite/test_timestamp13.py index f65bdc21bcd..0a09864c22e 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp13.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp13.py @@ -41,6 +41,7 @@ class test_timestamp13(wttest.WiredTigerTestCase, suite_subprocess): scenarios = make_scenarios([ ('col', dict(extra_config=',key_format=r')), + ('col-fix', dict(extra_config=',key_format=r,value_format=8t')), ('lsm', dict(extra_config=',type=lsm')), ('row', dict(extra_config='')), ]) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp14.py b/src/third_party/wiredtiger/test/suite/test_timestamp14.py index 3b4c4bf40ae..213b309c528 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp14.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp14.py @@ -40,21 +40,23 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename session_config = 'isolation=snapshot' - key_format_values = [ - ('integer-row', dict(key_format='i')), - ('column', dict(key_format='r')), + format_values = [ + ('integer-row', dict(key_format='i', value_format='i')), + ('column', dict(key_format='r', value_format='i')), + ('column-fix', dict(key_format='r', value_format='8t')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test_all_durable_old(self): # This test was originally for testing the all_committed timestamp. # In the absence of prepared transactions, all_durable is identical to # all_committed so let's enforce the all_durable values instead. all_durable_uri = self.uri + '_all_durable' + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) session1 = self.setUpSessionOpen(self.conn) session2 = self.setUpSessionOpen(self.conn) - session1.create(all_durable_uri, 'key_format={},value_format=i'.format(self.key_format)) - session2.create(all_durable_uri, 'key_format={},value_format=i'.format(self.key_format)) + session1.create(all_durable_uri, format) + session2.create(all_durable_uri, format) # Scenario 0: No commit timestamp has ever been specified therefore # There is no all_durable timestamp and we will get an error @@ -138,8 +140,9 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): oldest_reader_uri = self.uri + '_oldest_reader_pinned' session1 = self.setUpSessionOpen(self.conn) session2 = self.setUpSessionOpen(self.conn) - session1.create(oldest_reader_uri, 'key_format={},value_format=i'.format(self.key_format)) - session2.create(oldest_reader_uri, 'key_format={},value_format=i'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + session1.create(oldest_reader_uri, format) + session2.create(oldest_reader_uri, format) # Nothing is reading so there is no oldest reader. self.assertRaisesException(wiredtiger.WiredTigerError, @@ -196,7 +199,8 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): def test_pinned_oldest(self): pinned_oldest_uri = self.uri + 'pinned_oldest' session1 = self.setUpSessionOpen(self.conn) - session1.create(pinned_oldest_uri, 'key_format={},value_format=i'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + session1.create(pinned_oldest_uri, format) # Confirm no oldest timestamp exists. self.assertRaisesException(wiredtiger.WiredTigerError, lambda: self.conn.query_timestamp('get=oldest')) @@ -247,7 +251,8 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): def test_all_durable(self): all_durable_uri = self.uri + '_all_durable' session1 = self.setUpSessionOpen(self.conn) - session1.create(all_durable_uri, 'key_format={},value_format=i'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + session1.create(all_durable_uri, format) # Since this is a non-prepared transaction, we'll be using the commit # timestamp when calculating all_durable since it's implied that they're @@ -335,8 +340,9 @@ class test_timestamp14(wttest.WiredTigerTestCase, suite_subprocess): all_uri = self.uri + 'pinned_oldest' session1 = self.setUpSessionOpen(self.conn) session2 = self.setUpSessionOpen(self.conn) - session1.create(all_uri, 'key_format={},value_format=i'.format(self.key_format)) - session2.create(all_uri, 'key_format={},value_format=i'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + session1.create(all_uri, format) + session2.create(all_uri, format) cur1 = session1.open_cursor(all_uri) cur2 = session2.open_cursor(all_uri) # Set up oldest timestamp. diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp17.py b/src/third_party/wiredtiger/test/suite/test_timestamp17.py index fe4ec606d23..af96e2e0c3f 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp17.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp17.py @@ -43,14 +43,16 @@ class test_timestamp17(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename session_config = 'isolation=snapshot' - key_format_values = [ - ('integer-row', dict(key_format='i')), - ('column', dict(key_format='r')), + format_values = [ + ('integer-row', dict(key_format='i', value_format='i')), + ('column', dict(key_format='r', value_format='i')), + ('column-fix', dict(key_format='r', value_format='8t')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test_inconsistent_timestamping(self): - self.session.create(self.uri, 'key_format={},value_format=i'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(self.uri, format) self.session.begin_transaction() cur1 = self.session.open_cursor(self.uri) cur1[1] = 1 @@ -70,10 +72,15 @@ class test_timestamp17(wttest.WiredTigerTestCase, suite_subprocess): self.session.commit_transaction('commit_timestamp=100') # Read before any updates and ensure we cannot find the key or value. + # (For FLCS we expect to read zeros since the table extends nontransactionally.) self.session.begin_transaction('read_timestamp=20') cur1.set_key(1) search_success = cur1.search() - self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) + if self.value_format == '8t': + self.assertEqual(search_success, 0) + self.assertEqual(cur1.get_value(), 0) + else: + self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) self.session.commit_transaction() # Read at 25 and we should see 1. @@ -95,24 +102,37 @@ class test_timestamp17(wttest.WiredTigerTestCase, suite_subprocess): self.assertEqual(2, value1) # Read at 100 and we should not find anything. + # (For FLCS, deleted values read as zero.) self.session.begin_transaction('read_timestamp=100') cur1.set_key(1) search_success = cur1.search() - self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) + if self.value_format == '8t': + self.assertEqual(search_success, 0) + self.assertEqual(cur1.get_value(), 0) + else: + self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) self.session.commit_transaction() # Read at 200 and we should still not find anything. self.session.begin_transaction('read_timestamp=200') cur1.set_key(1) search_success = cur1.search() - self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) + if self.value_format == '8t': + self.assertEqual(search_success, 0) + self.assertEqual(cur1.get_value(), 0) + else: + self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) self.session.commit_transaction() # Read at 300 for further validation. self.session.begin_transaction('read_timestamp=300') cur1.set_key(1) search_success = cur1.search() - self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) + if self.value_format == '8t': + self.assertEqual(search_success, 0) + self.assertEqual(cur1.get_value(), 0) + else: + self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) self.session.commit_transaction() # Move oldest timestamp forward and @@ -143,17 +163,26 @@ class test_timestamp17(wttest.WiredTigerTestCase, suite_subprocess): self.conn.set_timestamp('oldest_timestamp=100') # Read at 100 and we should not find anything. + # (Again, in FLCS deleted values read back as 0.) self.session.begin_transaction('read_timestamp=100') cur1.set_key(1) search_success = cur1.search() - self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) + if self.value_format == '8t': + self.assertEqual(search_success, 0) + self.assertEqual(cur1.get_value(), 0) + else: + self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) self.session.commit_transaction() # Read at 200 and we should not find anything. self.session.begin_transaction('read_timestamp=200') cur1.set_key(1) search_success = cur1.search() - self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) + if self.value_format == '8t': + self.assertEqual(search_success, 0) + self.assertEqual(cur1.get_value(), 0) + else: + self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) self.session.commit_transaction() # Move oldest timestamp to 200 to ensure history @@ -163,13 +192,21 @@ class test_timestamp17(wttest.WiredTigerTestCase, suite_subprocess): self.session.begin_transaction('read_timestamp=200') cur1.set_key(1) search_success = cur1.search() - self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) + if self.value_format == '8t': + self.assertEqual(search_success, 0) + self.assertEqual(cur1.get_value(), 0) + else: + self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) self.session.commit_transaction() self.session.begin_transaction('read_timestamp=250') cur1.set_key(1) search_success = cur1.search() - self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) + if self.value_format == '8t': + self.assertEqual(search_success, 0) + self.assertEqual(cur1.get_value(), 0) + else: + self.assertEqual(search_success, wiredtiger.WT_NOTFOUND) self.session.commit_transaction() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp18.py b/src/third_party/wiredtiger/test/suite/test_timestamp18.py index b41fc5ccb34..b6a92651ff6 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp18.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp18.py @@ -42,29 +42,37 @@ class test_timestamp18(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' - key_format_values = [ - ('string-row', dict(key_format='S', usestrings=True)), - ('column', dict(key_format='r', usestrings=False)), + format_values = [ + ('string-row', dict(key_format='S', value_format='S')), + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), ] non_ts_writes = [ ('insert', dict(delete=False)), ('delete', dict(delete=True)), ] - scenarios = make_scenarios(key_format_values, non_ts_writes) + scenarios = make_scenarios(format_values, non_ts_writes) def get_key(self, i): - return str(i) if self.usestrings else i + return str(i) if self.key_format == 'S' else i def test_ts_writes_with_non_ts_write(self): uri = 'table:test_timestamp18' - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, format) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) - value1 = 'a' * 500 - value2 = 'b' * 500 - value3 = 'c' * 500 - value4 = 'd' * 500 + if self.value_format == '8t': + value1 = 97 # 'a' + value2 = 98 # 'b' + value3 = 99 # 'c' + value4 = 100 # 'd' + else: + value1 = 'a' * 500 + value2 = 'b' * 500 + value3 = 'c' * 500 + value4 = 'd' * 500 # A series of timestamped writes on each key. for i in range(1, 10000): @@ -98,12 +106,16 @@ class test_timestamp18(wttest.WiredTigerTestCase): for ts in range(2, 4): self.session.begin_transaction('read_timestamp=' + self.timestamp_str(ts)) for i in range(1, 10000): - # The non-timestamped delete should cover all the previous writes and make them effectively - # invisible. + # The non-timestamped delete should cover all the previous writes and make them + # effectively invisible. (For FLCS, this means they read back as zero.) if i % 2 == 0: if self.delete: cursor.set_key(self.get_key(i)) - self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + if self.value_format == '8t': + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_value(), 0) + else: + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) else: self.assertEqual(cursor[self.get_key(i)], value4) # Otherwise, expect one of the timestamped writes. diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp19.py b/src/third_party/wiredtiger/test/suite/test_timestamp19.py index 17868e1b821..740fba4fdb2 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp19.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp19.py @@ -36,11 +36,12 @@ class test_timestamp19(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' - key_format_values = [ - ('integer-row', dict(key_format='i')), - ('column', dict(key_format='r')), + format_values = [ + ('integer-row', dict(key_format='i', value_format='S')), + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def updates(self, uri, value, ds, nrows, commit_ts): session = self.session @@ -53,7 +54,7 @@ class test_timestamp19(wttest.WiredTigerTestCase): def test_timestamp(self): uri = "table:test_timestamp19" - create_params = 'key_format={},value_format=S'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, create_params) ds = SimpleDataSet( @@ -61,9 +62,14 @@ class test_timestamp19(wttest.WiredTigerTestCase): ds.populate() nrows = 1000 - value_x = 'x' * 1000 - value_y = 'y' * 1000 - value_z = 'z' * 1000 + if self.value_format == '8t': + value_x = 120 # 'x' + value_y = 121 # 'y' + value_z = 122 # 'z' + else: + value_x = 'x' * 1000 + value_y = 'y' * 1000 + value_z = 'z' * 1000 # Set the oldest and stable timestamps to 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp20.py b/src/third_party/wiredtiger/test/suite/test_timestamp20.py index 7bfeb670004..b4b017d8560 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp20.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp20.py @@ -35,26 +35,35 @@ class test_timestamp20(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' - key_format_values = [ - ('string-row', dict(key_format='S', usestrings=True)), - ('column', dict(key_format='r', usestrings=False)), + format_values = [ + ('string-row', dict(key_format='S', value_format='S')), + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def get_key(self, i): - return str(i) if self.usestrings else i + return str(i) if self.key_format == 'S' else i def test_timestamp20_standard(self): uri = 'table:test_timestamp20' - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, format) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) - value1 = 'a' * 500 - value2 = 'b' * 500 - value3 = 'c' * 500 - value4 = 'd' * 500 - value5 = 'e' * 500 + if self.value_format == '8t': + value1 = 97 # 'a' + value2 = 98 # 'b' + value3 = 99 # 'c' + value4 = 100 # 'd' + value5 = 101 # 'e' + else: + value1 = 'a' * 500 + value2 = 'b' * 500 + value3 = 'c' * 500 + value4 = 'd' * 500 + value5 = 'e' * 500 for i in range(1, 10000): self.session.begin_transaction() @@ -99,8 +108,14 @@ class test_timestamp20(wttest.WiredTigerTestCase): # Corruptions to string types may go undetected since non-ASCII characters won't be included in # the conversion to a Python string. def test_timestamp20_modify(self): + # FLCS does not support modifies, so skip this test. + # Just return instead of using self.skipTest to avoid generating noise. + if self.value_format == '8t': + return + uri = 'table:test_timestamp20' - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, format) self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1)) cursor = self.session.open_cursor(uri) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp22.py b/src/third_party/wiredtiger/test/suite/test_timestamp22.py index c45eea97ac6..7081cdb4d46 100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp22.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp22.py @@ -48,11 +48,12 @@ class test_timestamp22(wttest.WiredTigerTestCase): SUCCESS = 'success' FAILURE = 'failure' - key_format_values = [ - ('integer-row', dict(key_format='i')), - ('column', dict(key_format='r')), + format_values = [ + ('integer-row', dict(key_format='i', value_format='S')), + ('column', dict(key_format='r', value_format='S')), + ('column-fix', dict(key_format='r', value_format='8t')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) # Control execution of an operation, looking for exceptions and error messages. # Usage: @@ -98,6 +99,8 @@ class test_timestamp22(wttest.WiredTigerTestCase): # Create a predictable value based on the iteration number and timestamp. def gen_value(self, iternum, ts): + if self.value_format == '8t': + return (iternum * 7 + ts * 13) % 255 return str(iternum) + '_' + str(ts) + '_' + 'x' * 1000 # Given a number representing an "approximate timestamp", generate a timestamp @@ -404,14 +407,15 @@ class test_timestamp22(wttest.WiredTigerTestCase): else: iterations = 1000 - create_params = 'key_format={},value_format=S'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(self.uri, create_params) self.set_global_timestamps(1, 1, -1, -1) # Create tables with no entries ds = SimpleDataSet( - self, self.uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') + self, self.uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') # We do a bunch of iterations, doing transactions, prepare, and global timestamp calls # with timestamps that are sometimes valid, sometimes not. We use the iteration number diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp23.py b/src/third_party/wiredtiger/test/suite/test_timestamp23.py index 60d97cf78fb..9c394d97570 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp23.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp23.py @@ -37,26 +37,33 @@ class test_timestamp23(wttest.WiredTigerTestCase): conn_config = '' session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test_timestamp(self): # Create a file that contains active history (content newer than the oldest timestamp). table_uri = 'table:timestamp23' ds = SimpleDataSet( - self, table_uri, 0, key_format=self.key_format, value_format='S', config='log=(enabled=false)') + self, table_uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() self.session.checkpoint() key = 5 - value_1 = 'a' * 500 - value_2 = 'b' * 500 - value_3 = 'c' * 500 + if self.value_format == '8t': + value_1 = 97 + value_2 = 98 + value_3 = 99 + else: + value_1 = 'a' * 500 + value_2 = 'b' * 500 + value_3 = 'c' * 500 # Pin oldest and stable to timestamp 1. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp24.py b/src/third_party/wiredtiger/test/suite/test_timestamp24.py index ec5814e301a..541280a5a0e 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp24.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp24.py @@ -38,12 +38,13 @@ class test_timestamp24(wttest.WiredTigerTestCase): conn_config = '' session_config = 'isolation=snapshot' - key_format_values = [ - ('column', dict(key_format='r')), - ('integer_row', dict(key_format='i')), + format_values = [ + ('column', dict(key_format='r', value_format='S')), + ('column_fix', dict(key_format='r', value_format='8t')), + ('integer_row', dict(key_format='i', value_format='S')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def evict(self, uri, session, key, value): evict_cursor = session.open_cursor(uri, None, "debug=(release_evict)") @@ -57,15 +58,22 @@ class test_timestamp24(wttest.WiredTigerTestCase): table_uri = 'table:timestamp24' ds = SimpleDataSet( - self, table_uri, 0, key_format=self.key_format, value_format='S', config='log=(enabled=false)') + self, table_uri, 0, key_format=self.key_format, value_format=self.value_format, + config='log=(enabled=false)') ds.populate() self.session.checkpoint() key = 5 - value_a = 'a' * 500 - value_b = 'b' * 500 - value_c = 'c' * 500 - value_d = 'd' * 500 + if self.value_format == '8t': + value_a = 97 + value_b = 98 + value_c = 99 + value_d = 100 + else: + value_a = 'a' * 500 + value_b = 'b' * 500 + value_c = 'c' * 500 + value_d = 'd' * 500 # Pin oldest and stable to timestamp 1. #self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) + diff --git a/src/third_party/wiredtiger/test/suite/test_txn14.py b/src/third_party/wiredtiger/test/suite/test_txn14.py index f3eaac6cd6b..12ea8182b61 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn14.py +++ b/src/third_party/wiredtiger/test/suite/test_txn14.py @@ -46,11 +46,18 @@ class test_txn14(wttest.WiredTigerTestCase, suite_subprocess): ('write', dict(sync='off')), ('sync', dict(sync='on')), ] - key_format_values = [ - ('integer-row', dict(key_format='i')), - ('column', dict(key_format='r')), + format_values = [ + ('integer-row', dict(key_format='i', value_format='i')), + ('column', dict(key_format='r', value_format='i')), + ('column-fix', dict(key_format='r', value_format='8t')), ] - scenarios = make_scenarios(sync_list, key_format_values) + scenarios = make_scenarios(sync_list, format_values) + + def mkvalue(self, i): + if self.value_format == '8t': + # Use 255 instead of 256 just for variety. + return i % 255 + return i def test_log_flush(self): # Here's the strategy: @@ -61,16 +68,16 @@ class test_txn14(wttest.WiredTigerTestCase, suite_subprocess): # - Make recovery run. # - Confirm flushed data is in the table. # - create_params = 'key_format={},value_format=i'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(self.t1, create_params) c = self.session.open_cursor(self.t1, None, None) for i in range(1, self.entries + 1): - c[i] = i + 1 + c[i] = self.mkvalue(i + 1) cfgarg='sync=%s' % self.sync self.pr('cfgarg ' + cfgarg) self.session.log_flush(cfgarg) for i in range(1, self.extra_entries + 1): - c[i+self.entries] = i + self.entries + 1 + c[i+self.entries] = self.mkvalue(i + self.entries + 1) c.close() self.session.log_flush(cfgarg) simulate_crash_restart(self, ".", "RESTART") @@ -78,7 +85,7 @@ class test_txn14(wttest.WiredTigerTestCase, suite_subprocess): i = 1 for key, value in c: self.assertEqual(i, key) - self.assertEqual(i+1, value) + self.assertEqual(self.mkvalue(i+1), value) i += 1 all = self.entries + self.extra_entries self.assertEqual(i, all + 1) diff --git a/src/third_party/wiredtiger/test/suite/test_txn15.py b/src/third_party/wiredtiger/test/suite/test_txn15.py index a266bf37582..c5663bf70a4 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn15.py +++ b/src/third_party/wiredtiger/test/suite/test_txn15.py @@ -47,9 +47,10 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): 'transaction_sync=(enabled=%s),' % self.conn_enable + \ 'transaction_sync=(method=%s),' % self.conn_method - key_format_values = [ - ('integer-row', dict(key_format='i')), - ('column', dict(key_format='r')), + format_values = [ + ('integer-row', dict(key_format='i', value_format='i')), + ('column', dict(key_format='r', value_format='i')), + ('column-fix', dict(key_format='r', value_format='8t')), ] conn_sync_enabled = [ ('en_off', dict(conn_enable='false')), @@ -74,9 +75,14 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): ('c_none', dict(commit_sync=None)), ('c_off', dict(commit_sync='sync=off')), ] - scenarios = make_scenarios(key_format_values, conn_sync_enabled, conn_sync_method, + scenarios = make_scenarios(format_values, conn_sync_enabled, conn_sync_method, begin_sync, commit_sync) + def mkvalue(self, i): + if self.value_format == '8t': + return i % 256 + return i + # Given the different configuration settings determine if this group # of settings would result in either a wait for write or sync. # Returns None, "write" or "sync". None means no waiting for either. @@ -114,7 +120,7 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): if self.begin_sync != None and self.commit_sync != None: return - create_params = 'key_format={},value_format=i'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(self.uri, create_params) stat_cursor = self.session.open_cursor('statistics:', None, None) @@ -131,7 +137,7 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(self.uri, None, None) self.session.begin_transaction(self.begin_sync) for i in range(1, self.entries + 1): - c[i] = i + 1 + c[i] = self.mkvalue(i + 1) self.session.commit_transaction(self.commit_sync) c.close() diff --git a/src/third_party/wiredtiger/test/suite/test_txn18.py b/src/third_party/wiredtiger/test/suite/test_txn18.py index e8bb10bdbcf..0c4b5d8e73b 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn18.py +++ b/src/third_party/wiredtiger/test/suite/test_txn18.py @@ -42,11 +42,17 @@ class test_txn18(wttest.WiredTigerTestCase, suite_subprocess): conn_recerror = conn_config + ',log=(recover=error)' conn_recon = conn_config + ',log=(recover=on)' - key_format_values = [ - ('integer-row', dict(key_format='i')), - ('column', dict(key_format='r')), + format_values = [ + ('integer-row', dict(key_format='i', value_format='i')), + ('column', dict(key_format='r', value_format='i')), + ('column-fix', dict(key_format='r', value_format='8t')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) + + def mkvalue(self, i): + if self.value_format == '8t': + return i % 256 + return i def simulate_crash(self, olddir, newdir): ''' Simulate a crash from olddir and restart in newdir. ''' @@ -77,7 +83,7 @@ class test_txn18(wttest.WiredTigerTestCase, suite_subprocess): # # If we aren't tracking file IDs properly, it's possible that # we'd end up apply the log records for t2 to table t1. - create_params = 'key_format={},value_format=i'.format(self.key_format) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(self.t1, create_params) # # Since we're logging, we need to flush out the meta-data file @@ -85,7 +91,7 @@ class test_txn18(wttest.WiredTigerTestCase, suite_subprocess): self.session.checkpoint() c = self.session.open_cursor(self.t1, None, None) for i in range(1, 10001): - c[i] = i + 1 + c[i] = self.mkvalue(i + 1) c.close() olddir = "." newdir = "RESTART" @@ -113,7 +119,7 @@ class test_txn18(wttest.WiredTigerTestCase, suite_subprocess): i = 1 for key, value in c: self.assertEqual(i, key) - self.assertEqual(i+1, value) + self.assertEqual(self.mkvalue(i+1), value) i += 1 self.assertEqual(i, 10001) c.close() diff --git a/src/third_party/wiredtiger/test/suite/test_txn20.py b/src/third_party/wiredtiger/test/suite/test_txn20.py index b6d0c8306f2..13b07a94e39 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn20.py +++ b/src/third_party/wiredtiger/test/suite/test_txn20.py @@ -34,23 +34,28 @@ import wttest from wtscenario import make_scenarios class test_txn20(wttest.WiredTigerTestCase): - uri = 'table:test_txn' - key_format_values = [ - ('string-row', dict(key_format='S', key='key')), - ('column', dict(key_format='r', key=12)), + + format_values = [ + ('string-row', dict(key_format='S', key='key', \ + value_format='S', old_value='value:old', new_value='value:new')), + ('column', dict(key_format='r', key=12, \ + value_format='S', old_value='value:old', new_value='value:new')), + ('column-fix', dict(key_format='r', key=12, \ + value_format='8t', old_value=89, new_value=167)), ] iso_types = [ ('isolation_read_uncommitted', dict(isolation='read-uncommitted')), ('isolation_read_committed', dict(isolation='read-committed')), ('isolation_snapshot', dict(isolation='snapshot')) ] - scenarios = make_scenarios(key_format_values, iso_types) + scenarios = make_scenarios(format_values, iso_types) old_value = 'value: old' new_value = 'value: new' def test_isolation_level(self): - self.session.create(self.uri, 'key_format={},value_format=S'.format(self.key_format)) + config = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(self.uri, config) cursor = self.session.open_cursor(self.uri, None) cursor[self.key] = self.old_value diff --git a/src/third_party/wiredtiger/test/suite/test_txn22.py b/src/third_party/wiredtiger/test/suite/test_txn22.py index 42da29540e4..cd1110ffbd0 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn22.py +++ b/src/third_party/wiredtiger/test/suite/test_txn22.py @@ -52,9 +52,13 @@ class test_txn22(wttest.WiredTigerTestCase, suite_subprocess): base_config = 'cache_size=1GB' conn_config = base_config - key_format_values = [ - ('integer-row', dict(key_format='i')), - ('column', dict(key_format='r')), + # Generate more rows for FLCS because otherwise it all fits on one page even with the + # smaller page size. + format_values = [ + ('integer-row', dict(key_format='i', value_format='S', extraconfig='', nrecords=1000)), + ('column', dict(key_format='r', value_format='S', extraconfig='', nrecords=1000)), + ('column-fix', dict(key_format='r', value_format='8t', extraconfig=',leaf_page_max=4096', + nrecords=10000)), ] # File to be corrupted @@ -80,11 +84,12 @@ class test_txn22(wttest.WiredTigerTestCase, suite_subprocess): "removal:WiredTiger.wt", ] - scenarios = make_scenarios(key_format_values, filename_scenarios) + scenarios = make_scenarios(format_values, filename_scenarios) uri = 'table:test_txn22' - nrecords = 1000 # records per table. def valuegen(self, i): + if self.value_format == '8t': + return i % 256 return str(i) + 'A' * 1024 # Insert a list of keys @@ -122,8 +127,8 @@ class test_txn22(wttest.WiredTigerTestCase, suite_subprocess): expect = list(range(1, self.nrecords + 1)) salvage_config = self.base_config + ',salvage=true' - create_params = 'key_format={},value_format=S'.format(self.key_format) - self.session.create(self.uri, create_params) + create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(self.uri, create_params + self.extraconfig) self.inserts(expect) # Simulate a crash by copying the contents of the directory diff --git a/src/third_party/wiredtiger/test/suite/test_txn23.py b/src/third_party/wiredtiger/test/suite/test_txn23.py index c0d420ca92b..1a1d77a9e50 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn23.py +++ b/src/third_party/wiredtiger/test/suite/test_txn23.py @@ -38,11 +38,13 @@ class test_txn23(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' conn_config = 'cache_size=5MB' - key_format_values = [ - ('integer-row', dict(key_format='i')), - ('column', dict(key_format='r')), + format_values = [ + ('integer-row', dict(key_format='i', value_format='S', extraconfig='')), + ('column', dict(key_format='r', value_format='S', extraconfig='')), + ('column-fix', dict(key_format='r', value_format='8t', + extraconfig='allocation_size=512,leaf_page_max=512')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def large_updates(self, uri, value, ds, nrows, commit_ts): # Update a large number of records. @@ -62,28 +64,42 @@ class test_txn23(wttest.WiredTigerTestCase): self.session.commit_transaction() def test_txn(self): - nrows = 2000 # Create a table. uri_1 = "table:txn23_1" ds_1 = SimpleDataSet( - self, uri_1, 0, key_format=self.key_format, value_format="S") + self, uri_1, 0, key_format=self.key_format, value_format=self.value_format, + config=self.extraconfig) ds_1.populate() # Create another table. uri_2 = "table:txn23_2" ds_2 = SimpleDataSet( - self, uri_2, 0, key_format=self.key_format, value_format="S") + self, uri_2, 0, key_format=self.key_format, value_format=self.value_format, + config=self.extraconfig) ds_2.populate() # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) - value_a = "aaaaa" * 100 - value_b = "bbbbb" * 100 - value_c = "ccccc" * 100 - value_d = "ddddd" * 100 + if self.value_format == '8t': + # Values are 1/500 the size, so in principle maybe we should use 500x as many rows. + # However, that takes a really long time, and to some extent we should also take the + # in-memory size of updates into account, so what I've done is pick a number of rows + # that makes it take about 2x the time of the VLCS and row-store versions. Hopefully + # that's enough memory usage to exercise the intended code paths. + nrows = 8000 + value_a = 97 + value_b = 98 + value_c = 99 + value_d = 100 + else: + nrows = 2000 + value_a = "aaaaa" * 100 + value_b = "bbbbb" * 100 + value_c = "ccccc" * 100 + value_d = "ddddd" * 100 # Perform several updates. self.large_updates(uri_1, value_d, ds_1, nrows, 20) diff --git a/src/third_party/wiredtiger/test/suite/test_txn24.py b/src/third_party/wiredtiger/test/suite/test_txn24.py index 11bb8a02f4e..4ecb540cfb7 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn24.py +++ b/src/third_party/wiredtiger/test/suite/test_txn24.py @@ -39,11 +39,12 @@ class test_txn24(wttest.WiredTigerTestCase): session_config = 'isolation=snapshot' - key_format_values = [ - ('integer-row', dict(key_format='i')), - ('column', dict(key_format='r')), + table_params_values = [ + ('integer-row', dict(key_format='i', value_format='S', extraconfig='')), + ('column', dict(key_format='r', value_format='S', extraconfig='')), + ('column-fix', dict(key_format='r', value_format='8t', extraconfig=',leaf_page_max=4096')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(table_params_values) def conn_config(self): # We want to either eliminate or keep the application thread role in eviction to minimum. @@ -56,12 +57,24 @@ class test_txn24(wttest.WiredTigerTestCase): # Create and populate a table. uri = "table:test_txn24" - table_params = 'key_format={},value_format=S'.format(self.key_format) - default_val = 'ABCD' * 60 - new_val = 'YYYY' * 60 - n_rows = 480000 - - self.session.create(uri, table_params) + table_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + + if self.value_format == '8t': + # Values are 1/240 the size, but as in-memory updates are considerably larger, we + # shouldn't just use 240x the number of rows. For now go with 3x, a number pulled from + # thin air that also makes it just about 3x slower than the VLCS case. It isn't clear + # whether it's really working as intended, and should maybe check deeper or use some + # kind of stats feedback to figure out how many rows to pump out instead of choosing + # in advance. + default_val = 45 + new_val = 101 + n_rows = 480000 * 3 + else: + default_val = 'ABCD' * 60 + new_val = 'YYYY' * 60 + n_rows = 480000 + + self.session.create(uri, table_params + self.extraconfig) cursor = self.session.open_cursor(uri, None) for i in range(1, n_rows + 1): cursor[i] = default_val @@ -78,31 +91,31 @@ class test_txn24(wttest.WiredTigerTestCase): # Start few sessions and transactions, make updates and try committing them. session2 = self.setUpSessionOpen(self.conn) cursor2 = session2.open_cursor(uri) - start_row = int(n_rows/4) - for i in range(0, 120000): + start_row = n_rows // 4 + for i in range(0, n_rows // 4): cursor2[start_row] = new_val start_row += 1 session3 = self.setUpSessionOpen(self.conn) cursor3 = session3.open_cursor(uri) - start_row = int(n_rows/2) - for i in range(0, 120000): + start_row = n_rows // 2 + for i in range(0, n_rows // 4): cursor3[start_row] = new_val start_row += 1 # At this point in time, we have made roughly 90% cache dirty. If we are not using - # snaphsots for eviction threads, the cache state will remain like this forever and we may + # snapshots for eviction threads, the cache state will remain like this forever and we may # never reach this part of code. We might get a rollback error by now or WT will panic with # cache stuck error. # - # Even if we dont get an error by now and if we try to insert new data at this point in + # Even if we don't get an error by now and if we try to insert new data at this point in # time, dirty cache usage will exceed 100% if eviction threads are not using snapshot # isolation. In that case, we will eventually get a rollback error for sure. session4 = self.setUpSessionOpen(self.conn) cursor4 = session4.open_cursor(uri) start_row = 2 - for i in range(0, 120000): + for i in range(0, n_rows // 4): cursor4[start_row] = new_val start_row += 1 diff --git a/src/third_party/wiredtiger/test/suite/test_txn25.py b/src/third_party/wiredtiger/test/suite/test_txn25.py index 6f1b25d4eec..b9930a0011f 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn25.py +++ b/src/third_party/wiredtiger/test/suite/test_txn25.py @@ -37,11 +37,12 @@ class test_txn25(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB,log=(enabled)' session_config = 'isolation=snapshot' - key_format_values = [ - ('string-row', dict(key_format='S', usestrings=True)), - ('column', dict(key_format='r', usestrings=False)), + format_values = [ + ('string-row', dict(key_format='S', usestrings=True, value_format='S')), + ('column', dict(key_format='r', usestrings=False, value_format='S')), + ('column-fix', dict(key_format='r', usestrings=False, value_format='8t')), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def getkey(self, i): if self.usestrings: @@ -51,14 +52,22 @@ class test_txn25(wttest.WiredTigerTestCase): def test_txn25(self): uri = 'file:test_txn25' - create_config = 'allocation_size=512,key_format={},value_format=S'.format(self.key_format) - self.session.create(uri, create_config) + create_config = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, 'allocation_size=512,' + create_config) # Populate the file and ensure that we start seeing some high transaction IDs in the system. nrows = 1000 - value1 = 'aaaaa' * 100 - value2 = 'bbbbb' * 100 - value3 = 'ccccc' * 100 + if self.value_format == '8t': + # Values are 1/500 the size, but for this we don't need to generate a lot of data, + # just a lot of transactions, so we can keep the same nrows. This will generate only + # one page, but that shouldn't affect the test criteria. + value1 = 97 + value2 = 98 + value3 = 99 + else: + value1 = 'aaaaa' * 100 + value2 = 'bbbbb' * 100 + value3 = 'ccccc' * 100 # Keep transaction ids around. session2 = self.conn.open_session() diff --git a/src/third_party/wiredtiger/test/suite/test_txn26.py b/src/third_party/wiredtiger/test/suite/test_txn26.py index e5b0a719867..c5edd0eb2f2 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn26.py +++ b/src/third_party/wiredtiger/test/suite/test_txn26.py @@ -41,23 +41,25 @@ class test_txn26(wttest.WiredTigerTestCase): conn_config = 'cache_size=50MB' session_config = 'isolation=snapshot' - key_format_values = [ - ('string-row', dict(key_format='S', key=str(0))), - ('column', dict(key_format='r', key=16)), + format_values = [ + ('string-row', dict(key_format='S', value_format='S', key=str(0))), + ('column', dict(key_format='r', value_format='S', key=16)), + ('column-fix', dict(key_format='r', value_format='8t', key=16)), ] - scenarios = make_scenarios(key_format_values) + scenarios = make_scenarios(format_values) def test_commit_larger_than_active_timestamp(self): if not wiredtiger.diagnostic_build(): self.skipTest('requires a diagnostic build') uri = 'table:test_txn26' - self.session.create(uri, 'key_format={},value_format=S'.format(self.key_format)) + config = 'key_format={},value_format={}'.format(self.key_format, self.value_format) + self.session.create(uri, config) cursor = self.session.open_cursor(uri) self.conn.set_timestamp( 'oldest_timestamp=' + self.timestamp_str(1) + ',stable_timestamp=' + self.timestamp_str(1)) - value = 'a' + value = 97 if self.value_format == '8t' else 'a' # Start a session with timestamp 10 session2 = self.conn.open_session(self.session_config) diff --git a/src/third_party/wiredtiger/test/suite/wtdataset.py b/src/third_party/wiredtiger/test/suite/wtdataset.py index dec62655c93..c4bd993ca80 100755 --- a/src/third_party/wiredtiger/test/suite/wtdataset.py +++ b/src/third_party/wiredtiger/test/suite/wtdataset.py @@ -97,6 +97,12 @@ class BaseDataSet(object): 0xac, 0xad, 0xae, 0xaf, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf) return value[i % len(value)] + elif value_format == '6t': + value = ( + 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x2a, 0x2b, + 0x2c, 0x2d, 0x2e, 0x2f, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, + 0x37, 0x38, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f) + return value[i % len(value)] else: raise AssertionError( 'value: object has unexpected format: ' + value_format) |