diff options
author | Don Anderson <dda@mongodb.com> | 2016-10-26 00:35:39 -0400 |
---|---|---|
committer | Michael Cahill <michael.cahill@mongodb.com> | 2016-10-26 15:35:39 +1100 |
commit | ef87391458eb9c4440f2f9dc1aa89b41363b6b9b (patch) | |
tree | 7d12b1466ea59910272c1f5fe2d5b48db052ca25 | |
parent | b6be49dd39979e55bf809111c71dce0389f6b728 (diff) | |
download | mongo-ef87391458eb9c4440f2f9dc1aa89b41363b6b9b.tar.gz |
WT-2947 replace test suite populate functions with *DataSet classes (#3083)
63 files changed, 981 insertions, 897 deletions
diff --git a/test/suite/helper.py b/test/suite/helper.py index 9f34b566b3c..d1f41f05e8b 100644 --- a/test/suite/helper.py +++ b/test/suite/helper.py @@ -30,6 +30,8 @@ import glob, os, shutil, string, subprocess import wiredtiger +from wtdataset import SimpleDataSet, SimpleIndexDataSet, ComplexDataSet + # python has a filecmp.cmp function, but different versions of python approach # file comparison differently. To make sure we get byte for byte comparison, # we define it here. @@ -121,188 +123,3 @@ def copy_wiredtiger_home(olddir, newdir, aligned=True): cmd_list = ['dd', inpf, outf, 'bs=300'] a = subprocess.Popen(cmd_list) a.wait() - -# create a simple_populate or complex_populate key -def key_populate(cursor, i): - key_format = cursor.key_format - if key_format == 'i' or key_format == 'r' or key_format == 'u': - return i - elif key_format == 'S': - return str('%015d' % i) - else: - raise AssertionError( - 'key_populate: object has unexpected format: ' + key_format) - -# create a simple_populate value -def value_populate(cursor, i): - value_format = cursor.value_format - if value_format == 'i' or value_format == 'r' or value_format == 'u': - return i - elif value_format == 'S': - return str(i) + ': abcdefghijklmnopqrstuvwxyz' - elif value_format == '8t': - value = ( - 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xaa, 0xab, - 0xac, 0xad, 0xae, 0xaf, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, - 0xb7, 0xb8, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf) - return value[i % len(value)] - else: - raise AssertionError( - 'value_populate: object has unexpected format: ' + value_format) - -# population of a simple object -# uri: object -# config: prefix of the session.create configuration string (defaults -# to string value formats) -# rows: entries to insert -def simple_populate(self, uri, config, rows): - self.pr('simple_populate: ' + uri + ' with ' + str(rows) + ' rows') - self.session.create(uri, 'value_format=S,' + config) - cursor = self.session.open_cursor(uri, None) - for i in range(1, rows + 1): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) - cursor.close() - -def simple_populate_check_cursor(self, cursor, rows): - i = 0 - for key,val in cursor: - i += 1 - self.assertEqual(key, key_populate(cursor, i)) - if cursor.value_format == '8t' and val == 0: # deleted - continue - self.assertEqual(val, value_populate(cursor, i)) - self.assertEqual(i, rows) - -def simple_populate_check(self, uri, rows): - self.pr('simple_populate_check: ' + uri) - cursor = self.session.open_cursor(uri, None) - simple_populate_check_cursor(self, cursor, rows) - cursor.close() - -# population of a simple object, with a single index -# uri: object -# config: prefix of the session.create configuration string (defaults -# to string value formats) -# rows: entries to insert -def simple_index_populate(self, uri, config, rows): - self.pr('simple_index_populate: ' + uri + ' with ' + str(rows) + ' rows') - self.session.create(uri, 'value_format=S,columns=(key0,value0),' + config) - indxname = 'index:' + uri.split(":")[1] - self.session.create(indxname + ':index1', 'columns=(value0,key0)') - cursor = self.session.open_cursor(uri, None) - for i in range(1, rows + 1): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) - cursor.close() - -def simple_index_populate_check_cursor(self, cursor, rows): - i = 0 - for key,val in cursor: - i += 1 - self.assertEqual(key, key_populate(cursor, i)) - if cursor.value_format == '8t' and val == 0: # deleted - continue - self.assertEqual(val, value_populate(cursor, i)) - self.assertEqual(i, rows) - -def simple_index_populate_check(self, uri, rows): - self.pr('simple_index_populate_check: ' + uri) - - # Check values in the main table. - cursor = self.session.open_cursor(uri, None) - simple_index_populate_check_cursor(self, cursor, rows) - - # Check values in the index. - indxname = 'index:' + uri.split(":")[1] - idxcursor = self.session.open_cursor(indxname + ':index1') - for i in range(1, rows + 1): - k = key_populate(cursor, i) - v = value_populate(cursor, i) - ik = (v,k) # The index key is columns=(v,k). - self.assertEqual(v, idxcursor[ik]) - idxcursor.close() - cursor.close() - -# Return the value stored in a complex object. -def complex_value_populate(cursor, i): - return [str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%26], - i, - str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%23], - str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%18]] - -# Return the number of column groups used -def complex_populate_colgroup_count(): - return 6 - -# Return the number of indices used -def complex_populate_index_count(): - return 6 - -# population of a complex object -# uri: object -# config: prefix of the session.create configuration string -# rows: entries to insert -def complex_populate(self, uri, config, rows): - complex_populate_type(self, uri, config, '', rows, '') -def complex_populate_cgconfig(self, uri, config, rows): - complex_populate_type(self, uri, config, config, rows, '') -def complex_populate_lsm(self, uri, config, rows): - complex_populate_type(self, uri, config, '', rows, 'type=lsm') -def complex_populate_cgconfig_lsm(self, uri, config, rows): - complex_populate_type(self, uri, config, config, rows, 'type=lsm') -def complex_populate_type(self, uri, config, cgconfig, rows, type): - self.session.create(uri, - config + ',value_format=SiSS,' + - 'columns=(record,column2,column3,column4,column5),' + - 'colgroups=(cgroup1,cgroup2,cgroup3,cgroup4,cgroup5,cgroup6)') - - cgname = 'colgroup:' + uri.split(":")[1] - cgcfg = ',' + cgconfig + ',' + type - self.session.create(cgname + ':cgroup1', 'columns=(column2)' + ',' + cgcfg) - self.session.create(cgname + ':cgroup2', 'columns=(column3)' + ',' + cgcfg) - self.session.create(cgname + ':cgroup3', 'columns=(column4)' + ',' + cgcfg) - self.session.create( - cgname + ':cgroup4', 'columns=(column2,column3)' + ',' + cgcfg) - self.session.create( - cgname + ':cgroup5', 'columns=(column3,column4)' + ',' + cgcfg) - self.session.create( - cgname + ':cgroup6', 'columns=(column2,column4,column5)' + ',' + cgcfg) - indxname = 'index:' + uri.split(":")[1] - self.session.create(indxname + ':indx1', 'columns=(column2)' + ',' + cgcfg) - self.session.create(indxname + ':indx2', 'columns=(column3)' + ',' + cgcfg) - self.session.create(indxname + ':indx3', 'columns=(column4)' + ',' + cgcfg) - self.session.create( - indxname + ':indx4', 'columns=(column2,column4)' + ',' + cgcfg) - cursor = self.session.open_cursor(uri, None) - for i in range(1, rows + 1): - cursor[key_populate(cursor, i)] = \ - tuple(complex_value_populate(cursor, i)) - cursor.close() - # add some indices after populating - self.session.create( - indxname + ':indx5', 'columns=(column3,column5)' + ',' + cgcfg) - self.session.create( - indxname + ':indx6', 'columns=(column3,column5,column4)' + ',' + cgcfg) - -def complex_populate_colgroup_name(self, uri, i): - return 'colgroup:' + uri.split(":")[1] + ':cgroup' + str(i + 1) - -def complex_populate_index_name(self, uri, i): - return 'index:' + uri.split(":")[1] + ':indx' + str(i + 1) - -def complex_populate_check_cursor(self, cursor, rows): - i = 0 - for key, s1, i2, s3, s4 in cursor: - i += 1 - self.assertEqual(key, key_populate(cursor, i)) - v = complex_value_populate(cursor, i) - self.assertEqual(s1, v[0]) - self.assertEqual(i2, v[1]) - self.assertEqual(s3, v[2]) - self.assertEqual(s4, v[3]) - self.assertEqual(i, rows) - -def complex_populate_check(self, uri, rows): - self.pr('complex_populate_check: ' + uri) - cursor = self.session.open_cursor(uri, None) - complex_populate_check_cursor(self, cursor, rows) - cursor.close() diff --git a/test/suite/test_backup01.py b/test/suite/test_backup01.py index 92d39514440..4e98b6d8e77 100644 --- a/test/suite/test_backup01.py +++ b/test/suite/test_backup01.py @@ -32,8 +32,8 @@ import shutil import string from suite_subprocess import suite_subprocess import wiredtiger, wttest -from helper import compare_files,\ - complex_populate, complex_populate_lsm, simple_populate +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet +from helper import compare_files # test_backup.py # Utilities: wt backup @@ -43,14 +43,14 @@ class test_backup(wttest.WiredTigerTestCase, suite_subprocess): pfx = 'test_backup' objs = [ - ( 'file:' + pfx + '.1', simple_populate, 0), - ( 'file:' + pfx + '.2', simple_populate, 0), - ('table:' + pfx + '.3', simple_populate, 0), - ('table:' + pfx + '.4', simple_populate, 0), - ('table:' + pfx + '.5', complex_populate, 0), - ('table:' + pfx + '.6', complex_populate, 0), - ('table:' + pfx + '.7', complex_populate_lsm, 1), - ('table:' + pfx + '.8', complex_populate_lsm, 1), + ( 'file:' + pfx + '.1', SimpleDataSet, 0), + ( 'file:' + pfx + '.2', SimpleDataSet, 0), + ('table:' + pfx + '.3', SimpleDataSet, 0), + ('table:' + pfx + '.4', SimpleDataSet, 0), + ('table:' + pfx + '.5', ComplexDataSet, 0), + ('table:' + pfx + '.6', ComplexDataSet, 0), + ('table:' + pfx + '.7', ComplexLSMDataSet, 1), + ('table:' + pfx + '.8', ComplexLSMDataSet, 1), ] # Populate a set of objects. @@ -59,7 +59,7 @@ class test_backup(wttest.WiredTigerTestCase, suite_subprocess): if i[2]: if skiplsm: continue - i[1](self, i[0], 'key_format=S', 100) + i[1](self, i[0], 100).populate() # Compare the original and backed-up files using the wt dump command. def compare(self, uri): diff --git a/test/suite/test_backup02.py b/test/suite/test_backup02.py index 398d55abd7a..d4089273be0 100644 --- a/test/suite/test_backup02.py +++ b/test/suite/test_backup02.py @@ -28,7 +28,6 @@ import Queue import threading, time, wiredtiger, wttest -from helper import key_populate, simple_populate from wtthread import backup_thread, checkpoint_thread, op_thread from wtscenario import make_scenarios diff --git a/test/suite/test_backup03.py b/test/suite/test_backup03.py index ba7ce60f519..73d05f0b0a1 100644 --- a/test/suite/test_backup03.py +++ b/test/suite/test_backup03.py @@ -27,11 +27,11 @@ # OTHER DEALINGS IN THE SOFTWARE. import glob, os, shutil, string +import wiredtiger, wttest +from helper import compare_files from suite_subprocess import suite_subprocess +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios -import wiredtiger, wttest -from helper import compare_files,\ - complex_populate, complex_populate_lsm, simple_populate # test_backup03.py # Utilities: wt backup @@ -50,10 +50,10 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): # to the backup to confirm the backup is correct. pfx = 'test_backup' objs = [ # Objects - ('table:' + pfx + '.1', simple_populate, 0), - ( 'lsm:' + pfx + '.2', simple_populate, 1), - ('table:' + pfx + '.3', complex_populate, 2), - ('table:' + pfx + '.4', complex_populate_lsm, 3), + ('table:' + pfx + '.1', SimpleDataSet, 0), + ( 'lsm:' + pfx + '.2', SimpleDataSet, 1), + ('table:' + pfx + '.3', ComplexDataSet, 2), + ('table:' + pfx + '.4', ComplexLSMDataSet, 3), ] list = [ ( 'backup_1', dict(big=0,list=[0])), # Target objects individually @@ -85,7 +85,7 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): rows = 200000 # Big object else: rows = 1000 # Small object - i[1](self, i[0], 'key_format=S', rows) + i[1](self, i[0], rows).populate() # Backup needs a checkpoint self.session.checkpoint(None) diff --git a/test/suite/test_backup04.py b/test/suite/test_backup04.py index dd4dbc1d60f..919649fed57 100644 --- a/test/suite/test_backup04.py +++ b/test/suite/test_backup04.py @@ -29,10 +29,11 @@ import Queue import threading, time, wiredtiger, wttest import glob, os, shutil +from helper import compare_files from suite_subprocess import suite_subprocess +from wtdataset import SimpleDataSet, simple_key from wtscenario import make_scenarios from wtthread import op_thread -from helper import compare_files, key_populate # test_backup04.py # Utilities: wt backup @@ -67,14 +68,14 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): self.pr('populate: ' + uri + ' with ' + str(rows) + ' rows') cursor = self.session.open_cursor(uri, None) for i in range(1, rows + 1): - cursor[key_populate(cursor, i)] = str(i) + ':' + 'a' * dsize + cursor[simple_key(cursor, i)] = str(i) + ':' + 'a' * dsize cursor.close() def update(self, uri, dsize, upd, rows): self.pr('update: ' + uri + ' with ' + str(rows) + ' rows') cursor = self.session.open_cursor(uri, None) for i in range(1, rows + 1): - cursor[key_populate(cursor, i)] = str(i) + ':' + upd * dsize + cursor[simple_key(cursor, i)] = str(i) + ':' + upd * dsize cursor.close() # Compare the original and backed-up files using the wt dump command. diff --git a/test/suite/test_bug004.py b/test/suite/test_bug004.py index 316d15d8080..464cc57e272 100644 --- a/test/suite/test_bug004.py +++ b/test/suite/test_bug004.py @@ -30,7 +30,7 @@ # Regression tests. import wiredtiger, wttest -from helper import key_populate, value_populate +from wtdataset import SimpleDataSet, simple_key, simple_value # Check to make sure we see the right versions of overflow keys and values # when they are deleted in reconciliation without having been instantiated @@ -50,8 +50,8 @@ class test_bug004(wttest.WiredTigerTestCase): self.session.create(self.uri, self.config) c1 = self.session.open_cursor(self.uri, None) for i in range(1, self.nentries): - c1[key_populate(c1, i) + 'abcdef' * 100] = \ - value_populate(c1, i) + 'abcdef' * 100 + c1[simple_key(c1, i) + 'abcdef' * 100] = \ + simple_value(c1, i) + 'abcdef' * 100 c1.close() # Verify the object, force it to disk, and verify the on-disk version. @@ -69,9 +69,9 @@ class test_bug004(wttest.WiredTigerTestCase): # currently do -- that's unlikely to change, but is a problem for the # test going forward.) c1 = self.session.open_cursor(self.uri, None) - c1.set_key(key_populate(c1, self.nentries - 5) + 'abcdef' * 100) + c1.set_key(simple_key(c1, self.nentries - 5) + 'abcdef' * 100) c2 = self.session.open_cursor(self.uri, None) - c2.set_key(key_populate(c2, self.nentries + 5) + 'abcdef' * 100) + c2.set_key(simple_key(c2, self.nentries + 5) + 'abcdef' * 100) self.session.truncate(None, c1, c2, None) c1.close() c2.close() @@ -81,14 +81,14 @@ class test_bug004(wttest.WiredTigerTestCase): # Use the snapshot cursor to retrieve the old key/value pairs c1 = tmp_session.open_cursor(self.uri, None) - c1.set_key(key_populate(c1, 1) + 'abcdef' * 100) + c1.set_key(simple_key(c1, 1) + 'abcdef' * 100) c1.search() for i in range(2, self.nentries): c1.next() self.assertEquals( - c1.get_key(), key_populate(c1, i) + 'abcdef' * 100) + c1.get_key(), simple_key(c1, i) + 'abcdef' * 100) self.assertEquals( - c1.get_value(), value_populate(c1, i) + 'abcdef' * 100) + c1.get_value(), simple_value(c1, i) + 'abcdef' * 100) if __name__ == '__main__': wttest.run() diff --git a/test/suite/test_bug005.py b/test/suite/test_bug005.py index 6437b329096..69df175ae67 100644 --- a/test/suite/test_bug005.py +++ b/test/suite/test_bug005.py @@ -30,7 +30,7 @@ # Regression tests. import wiredtiger, wttest -from helper import key_populate, value_populate +from wtdataset import SimpleDataSet, simple_key, simple_value # Check that verify works when the file has additional data after the last # checkpoint. @@ -43,7 +43,7 @@ class test_bug005(wttest.WiredTigerTestCase): self.session.create(self.uri, 'value_format=S,key_format=S') cursor = self.session.open_cursor(self.uri, None) for i in range(1, 1000): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) cursor.close() # Verify the object, force it to disk, and verify the on-disk version. diff --git a/test/suite/test_bug006.py b/test/suite/test_bug006.py index e01333f59de..c0f6055f720 100644 --- a/test/suite/test_bug006.py +++ b/test/suite/test_bug006.py @@ -30,7 +30,7 @@ # Regression tests. import wiredtiger, wttest -from helper import key_populate, value_populate +from wtdataset import SimpleDataSet, simple_key, simple_value from wtscenario import make_scenarios # Check that verify and salvage both raise exceptions if there is an open @@ -47,7 +47,7 @@ class test_bug006(wttest.WiredTigerTestCase): self.session.create(uri, 'value_format=S,key_format=S') cursor = self.session.open_cursor(uri, None) for i in range(1, 1000): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Table operations should fail, the cursor is open. self.assertRaises( diff --git a/test/suite/test_bug008.py b/test/suite/test_bug008.py index 8e787ae14d1..c54c92fc864 100644 --- a/test/suite/test_bug008.py +++ b/test/suite/test_bug008.py @@ -30,7 +30,7 @@ # Regression tests. import wiredtiger, wttest -from helper import simple_populate, key_populate, value_populate +from wtdataset import SimpleDataSet from wtscenario import make_scenarios # Test search/search-near operations, including invisible values and keys @@ -38,23 +38,25 @@ from wtscenario import make_scenarios class test_bug008(wttest.WiredTigerTestCase): uri = 'file:test_bug008' # This is a btree layer test. scenarios = make_scenarios([ - ('fix', dict(fmt='key_format=r,value_format=8t', empty=1, colvar=0)), - ('row', dict(fmt='key_format=S', empty=0, colvar=0)), - ('var', dict(fmt='key_format=r', empty=0, colvar=1)) + ('fix', dict(key_format='r', value_format='8t', empty=1, colvar=0)), + ('row', dict(key_format='S', value_format='S', empty=0, colvar=0)), + ('var', dict(key_format='r', value_format='S', empty=0, colvar=1)) ]) # Verify cursor search and search-near operations in an empty table. def test_search_empty(self): # Create the object and open a cursor. - self.session.create(self.uri, self.fmt) + ds = SimpleDataSet(self, self.uri, 0, key_format=self.key_format, + value_format=self.value_format) + ds.create() cursor = self.session.open_cursor(self.uri, None) # Search for a record past the end of the table, which should fail. - cursor.set_key(key_populate(cursor, 100)) + cursor.set_key(ds.key(100)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) # Search-near for a record past the end of the table, which should fail. - cursor.set_key(key_populate(cursor, 100)) + cursor.set_key(ds.key(100)) self.assertEqual(cursor.search_near(), wiredtiger.WT_NOTFOUND) # Verify cursor search and search-near operations at and past the end of @@ -62,35 +64,37 @@ class test_bug008(wttest.WiredTigerTestCase): def test_search_eot(self): # Populate the tree and reopen the connection, forcing it to disk # and moving the records to an on-page format. - simple_populate(self, self.uri, self.fmt, 100) + ds = SimpleDataSet(self, self.uri, 100, key_format=self.key_format, + value_format=self.value_format) + ds.populate() self.reopen_conn() # Open a cursor. cursor = self.session.open_cursor(self.uri, None) # Search for a record at the end of the table, which should succeed. - cursor.set_key(key_populate(cursor, 100)) + cursor.set_key(ds.key(100)) self.assertEqual(cursor.search(), 0) - self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) - self.assertEqual(cursor.get_value(), value_populate(cursor, 100)) + self.assertEqual(cursor.get_key(), ds.key(100)) + self.assertEqual(cursor.get_value(), ds.value(100)) # Search-near for a record at the end of the table, which should # succeed, returning the last record. - cursor.set_key(key_populate(cursor, 100)) + cursor.set_key(ds.key(100)) self.assertEqual(cursor.search_near(), 0) - self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) - self.assertEqual(cursor.get_value(), value_populate(cursor, 100)) + self.assertEqual(cursor.get_key(), ds.key(100)) + self.assertEqual(cursor.get_value(), ds.value(100)) # Search for a record past the end of the table, which should fail. - cursor.set_key(key_populate(cursor, 200)) + cursor.set_key(ds.key(200)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) # Search-near for a record past the end of the table, which should # succeed, returning the last record. - cursor.set_key(key_populate(cursor, 200)) + cursor.set_key(ds.key(200)) self.assertEqual(cursor.search_near(), -1) - self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) - self.assertEqual(cursor.get_value(), value_populate(cursor, 100)) + self.assertEqual(cursor.get_key(), ds.key(100)) + self.assertEqual(cursor.get_value(), ds.value(100)) # Verify cursor search-near operations before and after a set of # column-store duplicates. @@ -99,18 +103,20 @@ class test_bug008(wttest.WiredTigerTestCase): return # Populate the tree. - simple_populate(self, self.uri, self.fmt, 105) + ds = SimpleDataSet(self, self.uri, 105, key_format=self.key_format, + value_format=self.value_format) + ds.populate() # Set up deleted records before and after a set of duplicate records, # and make sure search/search-near returns the correct record. cursor = self.session.open_cursor(self.uri, None) for i in range(20, 100): - cursor[key_populate(cursor, i)] = '=== IDENTICAL VALUE ===' + cursor[ds.key(i)] = '=== IDENTICAL VALUE ===' for i in range(15, 25): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) self.assertEqual(cursor.remove(), 0) for i in range(95, 106): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) self.assertEqual(cursor.remove(), 0) cursor.close() @@ -123,26 +129,28 @@ class test_bug008(wttest.WiredTigerTestCase): # Search-near for a record in the deleted set before the duplicate set, # which should succeed, returning the first record in the duplicate set. - cursor.set_key(key_populate(cursor, 18)) + cursor.set_key(ds.key(18)) self.assertEqual(cursor.search_near(), 1) - self.assertEqual(cursor.get_key(), key_populate(cursor, 25)) + self.assertEqual(cursor.get_key(), ds.key(25)) # Search-near for a record in the deleted set after the duplicate set, # which should succeed, returning the last record in the duplicate set. - cursor.set_key(key_populate(cursor, 98)) + cursor.set_key(ds.key(98)) self.assertEqual(cursor.search_near(), -1) - self.assertEqual(cursor.get_key(), key_populate(cursor, 94)) + self.assertEqual(cursor.get_key(), ds.key(94)) # Verify cursor search and search-near operations on a file with a set of # on-page visible records, and a set of insert-list invisible records. def test_search_invisible_one(self): # Populate the tree. - simple_populate(self, self.uri, self.fmt, 100) + ds = SimpleDataSet(self, self.uri, 100, key_format=self.key_format, + value_format=self.value_format) + ds.populate() # Delete a range of records. for i in range(5, 10): cursor = self.session.open_cursor(self.uri, None) - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) self.assertEqual(cursor.remove(), 0) # Reopen the connection, forcing it to disk and moving the records to @@ -155,11 +163,11 @@ class test_bug008(wttest.WiredTigerTestCase): self.session.begin_transaction() cursor = self.session.open_cursor(self.uri, None) for i in range(5, 10): - cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000) + cursor[ds.key(i)] = ds.value(i + 1000) for i in range(30, 40): - cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000) + cursor[ds.key(i)] = ds.value(i + 1000) for i in range(100, 140): - cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000) + cursor[ds.key(i)] = ds.value(i + 1000) # Open a separate session and cursor. s = self.conn.open_session() @@ -168,7 +176,7 @@ class test_bug008(wttest.WiredTigerTestCase): # Search for an existing record in the deleted range, should not find # it. for i in range(5, 10): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) if self.empty: # Fixed-length column-store rows always exist. self.assertEqual(cursor.search(), 0) @@ -180,13 +188,13 @@ class test_bug008(wttest.WiredTigerTestCase): # Search for an existing record in the updated range, should see the # original value. for i in range(30, 40): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) self.assertEqual(cursor.search(), 0) - self.assertEqual(cursor.get_key(), key_populate(cursor, i)) + self.assertEqual(cursor.get_key(), ds.key(i)) # Search for a added record, should not find it. for i in range(120, 130): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) if self.empty: # Invisible updates to fixed-length column-store objects are # invisible to the reader, but the fact that they exist past @@ -203,7 +211,7 @@ class test_bug008(wttest.WiredTigerTestCase): # the next largest record. (This depends on the implementation behavior # which currently includes a bias to prefix search.) for i in range(5, 10): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) if self.empty: # Fixed-length column-store rows always exist. self.assertEqual(cursor.search_near(), 0) @@ -211,19 +219,19 @@ class test_bug008(wttest.WiredTigerTestCase): self.assertEqual(cursor.get_value(), 0) else: self.assertEqual(cursor.search_near(), 1) - self.assertEqual(cursor.get_key(), key_populate(cursor, 10)) + self.assertEqual(cursor.get_key(), ds.key(10)) # Search-near for an existing record in the updated range, should see # the original value. for i in range(30, 40): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) self.assertEqual(cursor.search_near(), 0) - self.assertEqual(cursor.get_key(), key_populate(cursor, i)) + self.assertEqual(cursor.get_key(), ds.key(i)) # Search-near for an added record, should find the previous largest # record. for i in range(120, 130): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) if self.empty: # Invisible updates to fixed-length column-store objects are # invisible to the reader, but the fact that they exist past @@ -234,7 +242,7 @@ class test_bug008(wttest.WiredTigerTestCase): self.assertEqual(cursor.get_value(), 0) else: self.assertEqual(cursor.search_near(), -1) - self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) + self.assertEqual(cursor.get_key(), ds.key(100)) # Verify cursor search and search-near operations on a file with a set of # on-page visible records, a set of insert-list visible records, and a set @@ -246,27 +254,29 @@ class test_bug008(wttest.WiredTigerTestCase): def test_search_invisible_two(self): # Populate the tree and reopen the connection, forcing it to disk # and moving the records to an on-page format. - simple_populate(self, self.uri, self.fmt, 100) + ds = SimpleDataSet(self, self.uri, 100, key_format=self.key_format, + value_format=self.value_format) + ds.populate() self.reopen_conn() # Add some additional visible records. cursor = self.session.open_cursor(self.uri, None) for i in range(100, 120): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[ds.key(i)] = ds.value(i) cursor.close() # Begin a transaction, and add some additional records. self.session.begin_transaction() cursor = self.session.open_cursor(self.uri, None) for i in range(120, 140): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[ds.key(i)] = ds.value(i) # Open a separate session and cursor. s = self.conn.open_session() cursor = s.open_cursor(self.uri, None) # Search for an invisible record. - cursor.set_key(key_populate(cursor, 130)) + cursor.set_key(ds.key(130)) if self.empty: # Invisible updates to fixed-length column-store objects are # invisible to the reader, but the fact that they exist past @@ -281,7 +291,7 @@ class test_bug008(wttest.WiredTigerTestCase): # Search-near for an invisible record, which should succeed, returning # the last visible record. - cursor.set_key(key_populate(cursor, 130)) + cursor.set_key(ds.key(130)) cursor.search_near() if self.empty: # Invisible updates to fixed-length column-store objects are @@ -294,8 +304,8 @@ class test_bug008(wttest.WiredTigerTestCase): else: # Otherwise, we should find the closest record for which we can see # the value. - self.assertEqual(cursor.get_key(), key_populate(cursor, 119)) - self.assertEqual(cursor.get_value(), value_populate(cursor, 119)) + self.assertEqual(cursor.get_key(), ds.key(119)) + self.assertEqual(cursor.get_value(), ds.value(119)) if __name__ == '__main__': wttest.run() diff --git a/test/suite/test_bug011.py b/test/suite/test_bug011.py index fceb7a22ddb..29bb08ec2e5 100644 --- a/test/suite/test_bug011.py +++ b/test/suite/test_bug011.py @@ -26,8 +26,8 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import helper, random, wiredtiger, wttest -from helper import simple_populate +import random, wiredtiger, wttest +from wtdataset import SimpleDataSet # test_bug011.py # Eviction working on more files than there are hazard pointers. @@ -46,11 +46,13 @@ class test_bug011(wttest.WiredTigerTestCase): def test_eviction(self): cursors = [] + datasets = [] for i in range(0, self.ntables): this_uri = 'table:%s-%03d' % (self.table_name, i) - simple_populate(self, this_uri, - 'key_format=S,allocation_size=1KB,leaf_page_max=1KB', - self.nrows) + ds = SimpleDataSet(self, this_uri, self.nrows, + config='allocation_size=1KB,leaf_page_max=1KB') + ds.populate() + datasets.append(ds) # Switch over to on-disk trees with multiple leaf pages self.reopen_conn() @@ -63,8 +65,7 @@ class test_bug011(wttest.WiredTigerTestCase): # Make use of the cache. for i in range(0, self.nops): for i in range(0, self.ntables): - cursors[i].set_key(helper.key_populate(cursors[i], - random.randint(0, self.nrows - 1))) + cursors[i].set_key(ds.key(random.randint(0, self.nrows - 1))) cursors[i].search() cursors[i].reset() diff --git a/test/suite/test_bug012.py b/test/suite/test_bug012.py index a28834f36d2..91f49d14b3f 100644 --- a/test/suite/test_bug012.py +++ b/test/suite/test_bug012.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import complex_populate +from wtdataset import ComplexDataSet # test_bug012.py class test_bug012(wttest.WiredTigerTestCase): @@ -71,10 +71,11 @@ class test_bug012(wttest.WiredTigerTestCase): # Test that we detect illegal extractors. # - # This test is a little fragile, we're depending on complex_populate to do + # This test is a little fragile, we're depending on ComplexDataSet to do # the heavy-lifting, so if that function changes, this could break. def test_illegal_extractor(self): - complex_populate(self, 'table:A', 'key_format=S', 10) + ds = ComplexDataSet(self, 'table:A', 10) + ds.populate() msg = '/unknown extractor/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.create('index:A:xyzzy', diff --git a/test/suite/test_bug014.py b/test/suite/test_bug014.py index c951cb45293..1dee933e839 100644 --- a/test/suite/test_bug014.py +++ b/test/suite/test_bug014.py @@ -27,7 +27,8 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import copy_wiredtiger_home, key_populate, simple_populate +from wtdataset import SimpleDataSet +from helper import copy_wiredtiger_home # test_bug014.py # JIRA WT-2115: fast-delete pages can be incorrectly lost due to a crash. @@ -35,8 +36,9 @@ class test_bug014(wttest.WiredTigerTestCase): def test_bug014(self): # Populate a table with 1000 keys on small pages. uri = 'table:test_bug014' - simple_populate(self, uri, - 'allocation_size=512,leaf_page_max=512,key_format=S', 1000) + ds = SimpleDataSet(self, uri, 1000, + config='allocation_size=512,leaf_page_max=512') + ds.populate() # Reopen it so we can fast-delete pages. self.reopen_conn() @@ -44,9 +46,9 @@ class test_bug014(wttest.WiredTigerTestCase): # Truncate a chunk of the key/value pairs inside a transaction. self.session.begin_transaction(None) start = self.session.open_cursor(uri, None) - start.set_key(key_populate(start, 250)) + start.set_key(ds.key(250)) end = self.session.open_cursor(uri, None) - end.set_key(key_populate(end, 500)) + end.set_key(ds.key(500)) self.session.truncate(None, start, end, None) start.close() end.close() @@ -66,7 +68,7 @@ class test_bug014(wttest.WiredTigerTestCase): # Confirm all of the records are there. for i in range(1, 1001): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) self.assertEqual(cursor.search(), 0) conn.close() diff --git a/test/suite/test_bug015.py b/test/suite/test_bug015.py index 7892f66759d..68cca49688f 100644 --- a/test/suite/test_bug015.py +++ b/test/suite/test_bug015.py @@ -27,7 +27,6 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import copy_wiredtiger_home, key_populate, simple_populate # test_bug015.py # JIRA WT-2162: index drop in a certain order triggers NULL pointer deref diff --git a/test/suite/test_bulk01.py b/test/suite/test_bulk01.py index addebe80647..8d5b6a04385 100644 --- a/test/suite/test_bulk01.py +++ b/test/suite/test_bulk01.py @@ -31,7 +31,7 @@ # import wiredtiger, wttest -from helper import key_populate, value_populate +from wtdataset import SimpleDataSet, simple_key, simple_value from wtscenario import make_scenarios # Smoke test bulk-load. @@ -61,7 +61,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk") for i in range(1, 1000): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Test a bulk-load triggers variable-length column-store RLE correctly. def test_bulk_load_var_rle(self): @@ -76,7 +76,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk") for i in range(1, 1000): - cursor[key_populate(cursor, i)] = value_populate(cursor, i/7) + cursor[simple_key(cursor, i)] = simple_value(cursor, i/7) # Test a bulk-load variable-length column-store append ignores any key. def test_bulk_load_var_append(self): @@ -88,13 +88,13 @@ class test_bulk_load(wttest.WiredTigerTestCase): 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk,append") for i in range(1, 1000): - cursor[key_populate(cursor, 37)] = value_populate(cursor, i) + cursor[simple_key(cursor, 37)] = simple_value(cursor, i) cursor.close() cursor = self.session.open_cursor(uri, None, None) for i in range(1, 1000): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(simple_key(cursor, i)) cursor.search() - self.assertEqual(cursor.get_value(), value_populate(cursor, i)) + self.assertEqual(cursor.get_value(), simple_value(cursor, i)) # Test that column-store bulk-load handles skipped records correctly. def test_bulk_load_col_delete(self): @@ -107,21 +107,21 @@ class test_bulk_load(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None, "bulk") for i in range(1, 1000): if i % 7 == 0: - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Ensure we create all the missing records. i = i + 1 - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) cursor.close() cursor = self.session.open_cursor(uri, None, None) # Verify all the records are there, in their proper state. for i in range(1, 1000): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(simple_key(cursor, i)) if i % 7 == 0: cursor.search() - self.assertEqual(cursor.get_value(), value_populate(cursor, i)) + self.assertEqual(cursor.get_value(), simple_value(cursor, i)) elif cursor.value_format == '8t': cursor.search() self.assertEqual(cursor.get_value(), 0) @@ -139,17 +139,17 @@ class test_bulk_load(wttest.WiredTigerTestCase): 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk") for i in range(1, 10): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # A big record -- if it's not efficient, we'll just hang. big = 18446744073709551606 - cursor[key_populate(cursor, big)] = value_populate(cursor, big) + cursor[simple_key(cursor, big)] = simple_value(cursor, big) cursor.close() cursor = self.session.open_cursor(uri, None, None) - cursor.set_key(key_populate(cursor, big)) + cursor.set_key(simple_key(cursor, big)) cursor.search() - self.assertEqual(cursor.get_value(), value_populate(cursor, big)) + self.assertEqual(cursor.get_value(), simple_value(cursor, big)) # Test that bulk-load out-of-order fails. def test_bulk_load_order_check(self): @@ -157,16 +157,16 @@ class test_bulk_load(wttest.WiredTigerTestCase): self.session.create(uri, 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk") - cursor[key_populate(cursor, 10)] = value_populate(cursor, 10) + cursor[simple_key(cursor, 10)] = simple_value(cursor, 10) for i in [1, 9, 10]: - cursor.set_key(key_populate(cursor, 1)) - cursor.set_value(value_populate(cursor, 1)) + cursor.set_key(simple_key(cursor, 1)) + cursor.set_value(simple_value(cursor, 1)) msg = '/than previously inserted key/' self.assertRaisesWithMessage( wiredtiger.WiredTigerError, lambda: cursor.insert(), msg) - cursor[key_populate(cursor, 11)] = value_populate(cursor, 11) + cursor[simple_key(cursor, 11)] = simple_value(cursor, 11) # Test that row-store bulk-load out-of-order can succeed. def test_bulk_load_row_order_nocheck(self): @@ -181,8 +181,8 @@ class test_bulk_load(wttest.WiredTigerTestCase): self.session.create(uri, 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk,skip_sort_check") - cursor[key_populate(cursor, 10)] = value_populate(cursor, 10) - cursor[key_populate(cursor, 1)] = value_populate(cursor, 1) + cursor[simple_key(cursor, 10)] = simple_value(cursor, 10) + cursor[simple_key(cursor, 1)] = simple_value(cursor, 1) if not wiredtiger.diagnostic_build(): self.skipTest('requires a diagnostic build') @@ -197,7 +197,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): uri = self.type + self.name self.session.create(uri, 'key_format=S,value_format=S') cursor = self.session.open_cursor(uri, None) - cursor[key_populate(cursor, 1)] = value_populate(cursor, 1) + cursor[simple_key(cursor, 1)] = simple_value(cursor, 1) # Close the insert cursor, else we'll get EBUSY. cursor.close() msg = '/bulk-load is only supported on newly created objects/' @@ -209,7 +209,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): uri = self.type + self.name self.session.create(uri, 'key_format=S,value_format=S') cursor = self.session.open_cursor(uri, None) - cursor[key_populate(cursor, 1)] = value_populate(cursor, 1) + cursor[simple_key(cursor, 1)] = simple_value(cursor, 1) # Don't close the insert cursor, we want EBUSY. self.assertRaises(wiredtiger.WiredTigerError, lambda: self.session.open_cursor(uri, None, "bulk")) diff --git a/test/suite/test_bulk02.py b/test/suite/test_bulk02.py index c959098179c..fb9240e91e7 100644 --- a/test/suite/test_bulk02.py +++ b/test/suite/test_bulk02.py @@ -30,10 +30,11 @@ # Bulk-load testing. import shutil, os -from helper import confirm_empty, key_populate, value_populate +import wiredtiger, wttest +from helper import confirm_empty from suite_subprocess import suite_subprocess +from wtdataset import SimpleDataSet, simple_key, simple_value from wtscenario import make_scenarios -import wiredtiger, wttest # test_bulkload_checkpoint # Test bulk-load with checkpoints. @@ -56,7 +57,7 @@ class test_bulkload_checkpoint(wttest.WiredTigerTestCase, suite_subprocess): self.session.create(self.uri, 'key_format=S,value_format=S') cursor = self.session.open_cursor(self.uri, None, 'bulk') for i in range(1, 10): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Checkpoint a few times (to test the drop code). for i in range(1, 5): @@ -110,7 +111,7 @@ class test_bulkload_backup(wttest.WiredTigerTestCase, suite_subprocess): self.session.create(self.uri, 'key_format=S,value_format=S') cursor = self.session.open_cursor(self.uri, None, 'bulk') for i in range(1, 10): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Test without a checkpoint, with an unnamed checkpoint, with a named # checkpoint. diff --git a/test/suite/test_checkpoint01.py b/test/suite/test_checkpoint01.py index 013cda96e4e..c0d004db78d 100644 --- a/test/suite/test_checkpoint01.py +++ b/test/suite/test_checkpoint01.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import key_populate, complex_populate_lsm, simple_populate +from wtdataset import SimpleDataSet, ComplexLSMDataSet from wtscenario import make_scenarios # test_checkpoint01.py @@ -145,7 +145,7 @@ class test_checkpoint_cursor(wttest.WiredTigerTestCase): # Check that you cannot open a checkpoint that doesn't exist. def test_checkpoint_dne(self): - simple_populate(self, self.uri, 'key_format=' + self.fmt, 100) + SimpleDataSet(self, self.uri, 100, key_format=self.fmt).populate() self.assertRaises(wiredtiger.WiredTigerError, lambda: self.session.open_cursor( self.uri, None, "checkpoint=checkpoint-1")) @@ -155,7 +155,7 @@ class test_checkpoint_cursor(wttest.WiredTigerTestCase): # Check that you can open checkpoints more than once. def test_checkpoint_multiple_open(self): - simple_populate(self, self.uri, 'key_format=' + self.fmt, 100) + SimpleDataSet(self, self.uri, 100, key_format=self.fmt).populate() self.session.checkpoint("name=checkpoint-1") c1 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-1") c2 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-1") @@ -172,7 +172,7 @@ class test_checkpoint_cursor(wttest.WiredTigerTestCase): # Check that you cannot drop a checkpoint if it's in use. def test_checkpoint_inuse(self): - simple_populate(self, self.uri, 'key_format=' + self.fmt, 100) + SimpleDataSet(self, self.uri, 100, key_format=self.fmt).populate() self.session.checkpoint("name=checkpoint-1") self.session.checkpoint("name=checkpoint-2") self.session.checkpoint("name=checkpoint-3") @@ -208,44 +208,49 @@ class test_checkpoint_target(wttest.WiredTigerTestCase): ('table', dict(uri='table:checkpoint',fmt='S')) ]) - def update(self, uri, value): + def update(self, uri, ds, value): cursor = self.session.open_cursor(uri, None, "overwrite") - cursor[key_populate(cursor, 10)] = value + cursor[ds.key(10)] = value cursor.close() - def check(self, uri, value): + def check(self, uri, ds, value): cursor = self.session.open_cursor(uri, None, "checkpoint=checkpoint-1") - self.assertEquals(cursor[key_populate(cursor, 10)], value) + self.assertEquals(cursor[ds.key(10)], value) cursor.close() def test_checkpoint_target(self): # Create 3 objects, change one record to an easily recognizable string. uri = self.uri + '1' - simple_populate(self, uri, 'key_format=' + self.fmt, 100) - self.update(uri, 'ORIGINAL') + ds1 = SimpleDataSet(self, uri, 100, key_format=self.fmt) + ds1.populate() + self.update(uri, ds1, 'ORIGINAL') + uri = self.uri + '2' - simple_populate(self, uri, 'key_format=' + self.fmt, 100) - self.update(uri, 'ORIGINAL') + ds2 = SimpleDataSet(self, uri, 100, key_format=self.fmt) + ds2.populate() + self.update(uri, ds2, 'ORIGINAL') + uri = self.uri + '3' - simple_populate(self, uri, 'key_format=' + self.fmt, 100) - self.update(uri, 'ORIGINAL') + ds3 = SimpleDataSet(self, uri, 100, key_format=self.fmt) + ds3.populate() + self.update(uri, ds3, 'ORIGINAL') # Checkpoint all three objects. self.session.checkpoint("name=checkpoint-1") # Update all 3 objects, then checkpoint two of the objects with the # same checkpoint name. - self.update(self.uri + '1', 'UPDATE') - self.update(self.uri + '2', 'UPDATE') - self.update(self.uri + '3', 'UPDATE') + self.update(self.uri + '1', ds1, 'UPDATE') + self.update(self.uri + '2', ds2, 'UPDATE') + self.update(self.uri + '3', ds3, 'UPDATE') target = 'target=("' + self.uri + '1"' + ',"' + self.uri + '2")' self.session.checkpoint("name=checkpoint-1," + target) # Confirm the checkpoint has the old value in objects that weren't # checkpointed, and the new value in objects that were checkpointed. - self.check(self.uri + '1', 'UPDATE') - self.check(self.uri + '2', 'UPDATE') - self.check(self.uri + '3', 'ORIGINAL') + self.check(self.uri + '1', ds1, 'UPDATE') + self.check(self.uri + '2', ds2, 'UPDATE') + self.check(self.uri + '3', ds3, 'ORIGINAL') # Check that you can't write checkpoint cursors. class test_checkpoint_cursor_update(wttest.WiredTigerTestCase): @@ -257,10 +262,11 @@ class test_checkpoint_cursor_update(wttest.WiredTigerTestCase): ]) def test_checkpoint_cursor_update(self): - simple_populate(self, self.uri, 'key_format=' + self.fmt, 100) + ds = SimpleDataSet(self, self.uri, 100, key_format=self.fmt) + ds.populate() self.session.checkpoint("name=ckpt") cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt") - cursor.set_key(key_populate(cursor, 10)) + cursor.set_key(ds.key(10)) cursor.set_value("XXX") msg = "/Unsupported cursor/" self.assertRaisesWithMessage(wiredtiger.WiredTigerError, @@ -284,12 +290,13 @@ class test_checkpoint_last(wttest.WiredTigerTestCase): # value. Repeat this action, we want to be sure the engine gets the # latest checkpoint information each time. uri = self.uri - simple_populate(self, uri, 'key_format=' + self.fmt, 100) + ds = SimpleDataSet(self, uri, 100, key_format=self.fmt) + ds.populate() for value in ('FIRST', 'SECOND', 'THIRD', 'FOURTH', 'FIFTH'): # Update the object. cursor = self.session.open_cursor(uri, None, "overwrite") - cursor[key_populate(cursor, 10)] = value + cursor[ds.key(10)] = value cursor.close() # Checkpoint the object. @@ -298,14 +305,15 @@ class test_checkpoint_last(wttest.WiredTigerTestCase): # Verify the "last" checkpoint sees the correct value. cursor = self.session.open_cursor( uri, None, "checkpoint=WiredTigerCheckpoint") - self.assertEquals(cursor[key_populate(cursor, 10)], value) + self.assertEquals(cursor[ds.key(10)], value) # Don't close the checkpoint cursor, we want it to remain open until # the test completes. # Check we can't use the reserved name as an application checkpoint name. class test_checkpoint_illegal_name(wttest.WiredTigerTestCase): def test_checkpoint_illegal_name(self): - simple_populate(self, "file:checkpoint", 'key_format=S', 100) + ds = SimpleDataSet(self, "file:checkpoint", 100, key_format='S') + ds.populate() msg = '/the checkpoint name.*is reserved/' for conf in ( 'name=WiredTigerCheckpoint', @@ -329,8 +337,8 @@ class test_checkpoint_illegal_name(wttest.WiredTigerTestCase): # Check we can't name checkpoints that include LSM tables. class test_checkpoint_lsm_name(wttest.WiredTigerTestCase): def test_checkpoint_lsm_name(self): - complex_populate_lsm(self, - "table:checkpoint", 'type=lsm,key_format=S', 1000) + ds = ComplexLSMDataSet(self, "table:checkpoint", 1000) + ds.populate() msg = '/object does not support named checkpoints/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.checkpoint("name=ckpt"), msg) diff --git a/test/suite/test_checkpoint02.py b/test/suite/test_checkpoint02.py index 0d166544472..b5d20fb73b1 100644 --- a/test/suite/test_checkpoint02.py +++ b/test/suite/test_checkpoint02.py @@ -28,7 +28,6 @@ import Queue import threading, time, wiredtiger, wttest -from helper import key_populate, simple_populate from wtthread import checkpoint_thread, op_thread from wtscenario import make_scenarios diff --git a/test/suite/test_colgap.py b/test/suite/test_colgap.py index dbe748ae58f..91df0fd6c1c 100644 --- a/test/suite/test_colgap.py +++ b/test/suite/test_colgap.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import simple_populate, key_populate, value_populate +from wtdataset import SimpleDataSet, simple_key, simple_value from wtscenario import make_scenarios # test_colgap.py @@ -61,14 +61,16 @@ class test_column_store_gap(wttest.WiredTigerTestCase): # namespace. If this runs in less-than-glacial time, it's working. def test_column_store_gap(self): uri = 'table:gap' - simple_populate(self, uri, 'key_format=r,value_format=S', 0) + # Initially just create tables. + ds = SimpleDataSet(self, uri, 0, key_format='r') + ds.populate() cursor = self.session.open_cursor(uri, None, None) self.nentries = 0 # Create a column-store table with large gaps in the name-space. v = [ 1000, 2000000000000, 30000000000000 ] for i in v: - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[ds.key(i)] = ds.value(i) self.nentries += 1 # In-memory cursor forward, backward. @@ -84,7 +86,9 @@ class test_column_store_gap(wttest.WiredTigerTestCase): def test_column_store_gap_traverse(self): uri = 'table:gap' - simple_populate(self, uri, 'key_format=r,value_format=S', 0) + # Initially just create tables. + ds = SimpleDataSet(self, uri, 0, key_format='r') + ds.populate() cursor = self.session.open_cursor(uri, None, None) self.nentries = 0 @@ -92,7 +96,7 @@ class test_column_store_gap(wttest.WiredTigerTestCase): # important, we just want some gaps. v = [ 1000, 1001, 2000, 2001] for i in v: - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[ds.key(i)] = ds.value(i) self.nentries += 1 # In-memory cursor forward, backward. @@ -111,7 +115,7 @@ class test_column_store_gap(wttest.WiredTigerTestCase): # so the traversal walks to them. v2 = [ 1500, 1501 ] for i in v2: - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[ds.key(i)] = ds.value(i) self.nentries += 1 # Tell the validation what to expect. @@ -168,7 +172,7 @@ class test_colmax(wttest.WiredTigerTestCase): # Optionaly make the big record the only record in the table. if not self.single: for i in range(1, 723): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Confirm searching past the end of the table works. if not self.bulk: @@ -176,7 +180,7 @@ class test_colmax(wttest.WiredTigerTestCase): self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) # Insert the big record. - cursor[key_populate(cursor, recno)] = value_populate(cursor, recno) + cursor[simple_key(cursor, recno)] = simple_value(cursor, recno) # Optionally flush to disk; re-open the cursor as necessary. if self.bulk or self.reopen: @@ -189,18 +193,18 @@ class test_colmax(wttest.WiredTigerTestCase): # Search for the large record. cursor.set_key(recno) self.assertEqual(cursor.search(), 0) - self.assertEqual(cursor.get_value(), value_populate(cursor, recno)) + self.assertEqual(cursor.get_value(), simple_value(cursor, recno)) # Update it. - cursor[key_populate(cursor, recno)] = value_populate(cursor, 37) + cursor[simple_key(cursor, recno)] = simple_value(cursor, 37) cursor.set_key(recno) self.assertEqual(cursor.search(), 0) - self.assertEqual(cursor.get_value(), value_populate(cursor, 37)) + self.assertEqual(cursor.get_value(), simple_value(cursor, 37)) # Remove it. cursor.set_key(recno) self.assertEqual(cursor.remove(), 0) - cursor.set_key(key_populate(cursor, recno)) + cursor.set_key(simple_key(cursor, recno)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) if __name__ == '__main__': diff --git a/test/suite/test_compact01.py b/test/suite/test_compact01.py index 8da1a0df4da..56ab6d39076 100644 --- a/test/suite/test_compact01.py +++ b/test/suite/test_compact01.py @@ -27,8 +27,8 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import complex_populate, simple_populate, key_populate from suite_subprocess import suite_subprocess +from wtdataset import SimpleDataSet, ComplexDataSet from wiredtiger import stat from wtscenario import make_scenarios @@ -39,14 +39,14 @@ class test_compact(wttest.WiredTigerTestCase, suite_subprocess): # Use a small page size because we want to create lots of pages. config = 'allocation_size=512,' +\ - 'leaf_page_max=512,value_format=S,key_format=S' + 'leaf_page_max=512,key_format=S' nentries = 50000 # The table is a complex object, give it roughly 5 pages per underlying # file. types = [ - ('file', dict(type='file:', pop=simple_populate, maxpages=5)), - ('table', dict(type='table:', pop=complex_populate, maxpages=50)) + ('file', dict(type='file:', dataset=SimpleDataSet, maxpages=5)), + ('table', dict(type='table:', dataset=ComplexDataSet, maxpages=50)) ] compact = [ ('method', dict(utility=0,reopen=0)), @@ -64,7 +64,8 @@ class test_compact(wttest.WiredTigerTestCase, suite_subprocess): def test_compact(self): # Populate an object uri = self.type + self.name - self.pop(self, uri, self.config, self.nentries - 1) + ds = self.dataset(self, uri, self.nentries - 1, config=self.config) + ds.populate() # Reopen the connection to force the object to disk. self.reopen_conn() @@ -76,9 +77,9 @@ class test_compact(wttest.WiredTigerTestCase, suite_subprocess): # Remove most of the object. c1 = self.session.open_cursor(uri, None) - c1.set_key(key_populate(c1, 5)) + c1.set_key(ds.key(5)) c2 = self.session.open_cursor(uri, None) - c2.set_key(key_populate(c2, self.nentries - 5)) + c2.set_key(ds.key(self.nentries - 5)) self.session.truncate(None, c1, c2, None) c1.close() c2.close() diff --git a/test/suite/test_cursor06.py b/test/suite/test_cursor06.py index 3a6240bc6c7..117e29b0605 100644 --- a/test/suite/test_cursor06.py +++ b/test/suite/test_cursor06.py @@ -27,8 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import key_populate, value_populate, simple_populate -from helper import complex_value_populate, complex_populate +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios # test_cursor06.py @@ -36,33 +35,26 @@ from wtscenario import make_scenarios class test_cursor06(wttest.WiredTigerTestCase): name = 'reconfigure' scenarios = make_scenarios([ - ('file-r', dict(type='file:', config='key_format=r', complex=0)), - ('file-S', dict(type='file:', config='key_format=S', complex=0)), - ('lsm-S', dict(type='lsm:', config='key_format=S', complex=0)), - ('table-r', - dict(type='table:', config='key_format=r', complex=0)), - ('table-S', - dict(type='table:', config='key_format=S', complex=0)), - ('table-r-complex', - dict(type='table:', config='key_format=r', complex=1)), - ('table-S-complex', - dict(type='table:', config='key_format=S', complex=1)), - ('table-S-complex-lsm', - dict(type='table:', config='key_format=S,type=lsm', complex=1)), + ('file-r', dict(type='file:', keyfmt='r', dataset=SimpleDataSet)), + ('file-S', dict(type='file:', keyfmt='S', dataset=SimpleDataSet)), + ('lsm-S', dict(type='lsm:', keyfmt='S', dataset=SimpleDataSet)), + ('table-r', dict(type='table:', keyfmt='r', dataset=SimpleDataSet)), + ('table-S', dict(type='table:', keyfmt='S', dataset=SimpleDataSet)), + ('table-r-complex', dict(type='table:', keyfmt='r', + dataset=ComplexDataSet)), + ('table-S-complex', dict(type='table:', keyfmt='S', + dataset=ComplexDataSet)), + ('table-S-complex-lsm', dict(type='table:', keyfmt='S', + dataset=ComplexLSMDataSet)), ]) def populate(self, uri): - if self.complex: - complex_populate(self, uri, self.config, 100) - else: - simple_populate(self, uri, self.config, 100) + self.ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + self.ds.populate() def set_kv(self, cursor): - cursor.set_key(key_populate(cursor, 10)) - if self.complex: - cursor.set_value(tuple(complex_value_populate(cursor, 10))) - else: - cursor.set_value(value_populate(cursor, 10)) + cursor.set_key(self.ds.key(10)) + cursor.set_value(self.ds.value(10)) def test_reconfigure_overwrite(self): uri = self.type + self.name diff --git a/test/suite/test_cursor09.py b/test/suite/test_cursor09.py index a05caea4f1f..9a1fc06b617 100644 --- a/test/suite/test_cursor09.py +++ b/test/suite/test_cursor09.py @@ -27,27 +27,24 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import key_populate, value_populate, simple_populate -from helper import complex_populate, complex_value_populate +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios # test_cursor09.py # JIRA WT-2217: insert resets key/value "set". class test_cursor09(wttest.WiredTigerTestCase): scenarios = make_scenarios([ - ('file-r', dict(type='file:', config='key_format=r', complex=0)), - ('file-S', dict(type='file:', config='key_format=S', complex=0)), - ('lsm-S', dict(type='lsm:', config='key_format=S', complex=0)), - ('table-r', - dict(type='table:', config='key_format=r', complex=0)), - ('table-S', - dict(type='table:', config='key_format=S', complex=0)), - ('table-r-complex', - dict(type='table:', config='key_format=r', complex=1)), - ('table-S-complex', - dict(type='table:', config='key_format=S', complex=1)), - ('table-S-complex-lsm', - dict(type='table:', config='key_format=S,type=lsm', complex=1)), + ('file-r', dict(type='file:', keyfmt='r', dataset=SimpleDataSet)), + ('file-S', dict(type='file:', keyfmt='S', dataset=SimpleDataSet)), + ('lsm-S', dict(type='lsm:', keyfmt='S', dataset=SimpleDataSet)), + ('table-r', dict(type='table:', keyfmt='r', dataset=SimpleDataSet)), + ('table-S', dict(type='table:', keyfmt='S', dataset=SimpleDataSet)), + ('table-r-complex', dict(type='table:', keyfmt='r', + dataset=ComplexDataSet)), + ('table-S-complex', dict(type='table:', keyfmt='S', + dataset=ComplexDataSet)), + ('table-S-complex-lsm', dict(type='table:', keyfmt='S', + dataset=ComplexLSMDataSet)), ]) # WT_CURSOR.insert doesn't leave the cursor positioned, verify any @@ -55,15 +52,11 @@ class test_cursor09(wttest.WiredTigerTestCase): def test_cursor09(self): uri = self.type + 'cursor09' - if self.complex: - complex_populate(self, uri, self.config, 100) - else: - simple_populate(self, uri, self.config, 100) + ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + ds.populate() cursor = self.session.open_cursor(uri, None, None) - cursor[key_populate(cursor, 10)] = \ - tuple(complex_value_populate(cursor, 10)) if self.complex \ - else value_populate(cursor, 10) + cursor[ds.key(10)] = ds.value(10) msg = '/requires key be set/' self.assertRaisesWithMessage( wiredtiger.WiredTigerError, cursor.search, msg) diff --git a/test/suite/test_cursor_compare.py b/test/suite/test_cursor_compare.py index 4610a2f30a9..c0feb1d4867 100644 --- a/test/suite/test_cursor_compare.py +++ b/test/suite/test_cursor_compare.py @@ -27,8 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest, exceptions -from helper import complex_populate, simple_populate, key_populate -from helper import complex_populate_index_name +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios # Test cursor comparisons. @@ -36,9 +35,9 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): name = 'test_compare' types = [ - ('file', dict(type='file:', config='')), - ('lsm', dict(type='table:', config=',type=lsm')), - ('table', dict(type='table:', config='')) + ('file', dict(type='file:', dataset=SimpleDataSet)), + ('lsm', dict(type='table:', dataset=ComplexLSMDataSet)), + ('table', dict(type='table:', dataset=ComplexDataSet)) ] keyfmt = [ ('integer', dict(keyfmt='i')), @@ -52,28 +51,20 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): uriX = self.type + 'compareX' # Build the object. + ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + dsX = self.dataset(self, uriX, 100, key_format=self.keyfmt) + ds.populate() + dsX.populate() if self.type == 'file:': - simple_populate( - self, uri, 'key_format=' + self.keyfmt + self.config, 100) - simple_populate( - self, uriX, 'key_format=' + self.keyfmt + self.config, 100) ix0_0 = None ix0_1 = None ix1_0 = None ixX_0 = None else: - complex_populate( - self, uri, 'key_format=' + self.keyfmt + self.config, 100) - complex_populate( - self, uriX, 'key_format=' + self.keyfmt + self.config, 100) - ix0_0 = self.session.open_cursor( - complex_populate_index_name(self, uri, 0), None) - ix0_1 = self.session.open_cursor( - complex_populate_index_name(self, uri, 0), None) - ix1_0 = self.session.open_cursor( - complex_populate_index_name(self, uri, 1), None) - ixX_0 = self.session.open_cursor( - complex_populate_index_name(self, uriX, 0), None) + ix0_0 = self.session.open_cursor(ds.index_name(0), None) + ix0_1 = self.session.open_cursor(ds.index_name(0), None) + ix1_0 = self.session.open_cursor(ds.index_name(1), None) + ixX_0 = self.session.open_cursor(dsX.index_name(0), None) ix0_0.next() ix0_1.next() ix1_0.next() @@ -90,17 +81,17 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): wiredtiger.WiredTigerError, lambda: c2.compare(c1), msg) # Test cursors before they're positioned. - c1.set_key(key_populate(c1, 10)) - c2.set_key(key_populate(c2, 20)) + c1.set_key(ds.key(10)) + c2.set_key(ds.key(20)) self.assertGreater(c2.compare(c1), 0) self.assertLess(c1.compare(c2), 0) - c2.set_key(key_populate(c2, 10)) + c2.set_key(ds.key(10)) self.assertEqual(c1.compare(c2), 0) self.assertEqual(c2.compare(c1), 0) # Confirm failure for different objects. cX = self.session.open_cursor(uriX, None) - cX.set_key(key_populate(cX, 10)) + cX.set_key(dsX.key(10)) msg = '/must reference the same object/' self.assertRaisesWithMessage( wiredtiger.WiredTigerError, lambda: cX.compare(c1), msg) @@ -125,20 +116,20 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): wiredtiger.WiredTigerError, lambda: ix0_0.compare(ix1_0), msg) # Test cursors after they're positioned (shouldn't matter for compare). - c1.set_key(key_populate(c1, 10)) + c1.set_key(ds.key(10)) self.assertEqual(c1.search(), 0) - c2.set_key(key_populate(c2, 20)) + c2.set_key(ds.key(20)) self.assertEqual(c2.search(), 0) self.assertGreater(c2.compare(c1), 0) self.assertLess(c1.compare(c2), 0) - c2.set_key(key_populate(c2, 10)) + c2.set_key(ds.key(10)) self.assertEqual(c2.search(), 0) self.assertEqual(c1.compare(c2), 0) self.assertEqual(c2.compare(c1), 0) # Confirm failure for different objects. cX = self.session.open_cursor(uriX, None) - cX.set_key(key_populate(cX, 10)) + cX.set_key(dsX.key(10)) self.assertEqual(cX.search(), 0) msg = '/must reference the same object/' self.assertRaisesWithMessage( @@ -149,28 +140,20 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): uriX = self.type + 'compareX' # Build the object. + ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + dsX = self.dataset(self, uriX, 100, key_format=self.keyfmt) + ds.populate() + dsX.populate() if self.type == 'file:': - simple_populate( - self, uri, 'key_format=' + self.keyfmt + self.config, 100) - simple_populate( - self, uriX, 'key_format=' + self.keyfmt + self.config, 100) ix0_0 = None ix0_1 = None ix1_0 = None ixX_0 = None else: - complex_populate( - self, uri, 'key_format=' + self.keyfmt + self.config, 100) - complex_populate( - self, uriX, 'key_format=' + self.keyfmt + self.config, 100) - ix0_0 = self.session.open_cursor( - complex_populate_index_name(self, uri, 0), None) - ix0_1 = self.session.open_cursor( - complex_populate_index_name(self, uri, 0), None) - ix1_0 = self.session.open_cursor( - complex_populate_index_name(self, uri, 1), None) - ixX_0 = self.session.open_cursor( - complex_populate_index_name(self, uriX, 0), None) + ix0_0 = self.session.open_cursor(ds.index_name(0), None) + ix0_1 = self.session.open_cursor(ds.index_name(0), None) + ix1_0 = self.session.open_cursor(ds.index_name(1), None) + ixX_0 = self.session.open_cursor(dsX.index_name(0), None) ix0_0.next() ix0_1.next() ix1_0.next() @@ -187,17 +170,17 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): wiredtiger.WiredTigerError, lambda: c2.equals(c1), msg) # Test cursors before they're positioned. - c1.set_key(key_populate(c1, 10)) - c2.set_key(key_populate(c2, 20)) + c1.set_key(ds.key(10)) + c2.set_key(ds.key(20)) self.assertFalse(c1.equals(c2)) self.assertFalse(c2.equals(c1)) - c2.set_key(key_populate(c2, 10)) + c2.set_key(ds.key(10)) self.assertTrue(c1.equals(c2)) self.assertTrue(c2.equals(c1)) # Confirm failure for different objects. cX = self.session.open_cursor(uriX, None) - cX.set_key(key_populate(cX, 10)) + cX.set_key(dsX.key(10)) msg = '/must reference the same object/' self.assertRaisesWithMessage( wiredtiger.WiredTigerError, lambda: cX.equals(c1), msg) @@ -222,20 +205,20 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): # Test cursors after they're positioned (internally, it's a different # search path if keys are positioned in the tree). - c1.set_key(key_populate(c1, 10)) + c1.set_key(ds.key(10)) self.assertEqual(c1.search(), 0) - c2.set_key(key_populate(c2, 20)) + c2.set_key(ds.key(20)) self.assertEqual(c2.search(), 0) self.assertFalse(c1.equals(c2)) self.assertFalse(c2.equals(c1)) - c2.set_key(key_populate(c2, 10)) + c2.set_key(ds.key(10)) self.assertEqual(c2.search(), 0) self.assertTrue(c1.equals(c2)) self.assertTrue(c2.equals(c1)) # Confirm failure for different objects. cX = self.session.open_cursor(uriX, None) - cX.set_key(key_populate(cX, 10)) + cX.set_key(dsX.key(10)) self.assertEqual(cX.search(), 0) msg = '/must reference the same object/' self.assertRaisesWithMessage( diff --git a/test/suite/test_cursor_pin.py b/test/suite/test_cursor_pin.py index b6119a93897..cb7045c7e41 100644 --- a/test/suite/test_cursor_pin.py +++ b/test/suite/test_cursor_pin.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import simple_populate, key_populate, value_populate +from wtdataset import SimpleDataSet from wtscenario import make_scenarios # test_cursor_pin.py @@ -36,7 +36,7 @@ from wtscenario import make_scenarios class test_cursor_pin(wttest.WiredTigerTestCase): uri = 'file:cursor_pin' nentries = 10000 - config = 'allocation_size=512,leaf_page_max=512,value_format=S,key_format=' + config = 'allocation_size=512,leaf_page_max=512' scenarios = make_scenarios([ ('recno', dict(keyfmt='r')), ('string', dict(keyfmt='S')), @@ -45,72 +45,75 @@ class test_cursor_pin(wttest.WiredTigerTestCase): # Create a multi-page file, confirm that a simple search to the local # page works, followed by a search to a different page. def test_smoke(self): - simple_populate(self, - self.uri, self.config + self.keyfmt, self.nentries) + ds = SimpleDataSet(self, self.uri, self.nentries, + config=self.config, key_format=self.keyfmt) + ds.populate() self.reopen_conn() c = self.session.open_cursor(self.uri, None) - c.set_key(key_populate(c, 100)) + c.set_key(ds.key(100)) self.assertEqual(c.search(), 0) - self.assertEqual(c.get_value(), value_populate(c, 100)) - c.set_key(key_populate(c, 101)) + self.assertEqual(c.get_value(), ds.value(100)) + c.set_key(ds.key(101)) self.assertEqual(c.search(), 0) - self.assertEqual(c.get_value(), value_populate(c, 101)) - c.set_key(key_populate(c, 9999)) + self.assertEqual(c.get_value(), ds.value(101)) + c.set_key(ds.key(9999)) self.assertEqual(c.search(), 0) - self.assertEqual(c.get_value(), value_populate(c, 9999)) + self.assertEqual(c.get_value(), ds.value(9999)) # Forward check. - def forward(self, c, max, notfound): + def forward(self, c, ds, max, notfound): for i in range(1, max + 1): - c.set_key(key_populate(c, i)) + c.set_key(ds.key(i)) if i in notfound: self.assertEqual(c.search(), wiredtiger.WT_NOTFOUND) else: self.assertEqual(c.search(), 0) - self.assertEqual(c.get_value(), value_populate(c, i)) + self.assertEqual(c.get_value(), ds.value(i)) # Backward check. - def backward(self, c, max, notfound): + def backward(self, c, ds, max, notfound): for i in range(max, 0, -1): - c.set_key(key_populate(c, i)) + c.set_key(ds.key(i)) if i in notfound: self.assertEqual(c.search(), wiredtiger.WT_NOTFOUND) else: self.assertEqual(c.search(), 0) - self.assertEqual(c.get_value(), value_populate(c, i)) + self.assertEqual(c.get_value(), ds.value(i)) # Create a multi-page file, search backward, forward to check page # boundaries. def test_basic(self): - simple_populate(self, - self.uri, self.config + self.keyfmt, self.nentries) + ds = SimpleDataSet(self, self.uri, self.nentries, + config=self.config, key_format=self.keyfmt) + ds.populate() self.reopen_conn() c = self.session.open_cursor(self.uri, None) - self.forward(c, self.nentries, []) - self.backward(c, self.nentries, []) + self.forward(c, ds, self.nentries, []) + self.backward(c, ds, self.nentries, []) # Create a multi-page file with a big chunk of missing space in the # middle (to exercise column-store searches). def test_missing(self): - simple_populate(self, - self.uri, self.config + self.keyfmt, self.nentries) + ds = SimpleDataSet(self, self.uri, self.nentries, + config=self.config, key_format=self.keyfmt) + ds.populate() c = self.session.open_cursor(self.uri, None) for i in range(self.nentries + 3000, self.nentries + 5001): - c[key_populate(c, i)] = value_populate(c, i) + c[ds.key(i)] = ds.value(i) self.reopen_conn() c = self.session.open_cursor(self.uri, None) - self.forward(c, self.nentries + 5000, + self.forward(c, ds, self.nentries + 5000, list(range(self.nentries + 1, self.nentries + 3000))) - self.backward(c, self.nentries + 5000, + self.backward(c, ds, self.nentries + 5000, list(range(self.nentries + 1, self.nentries + 3000))) # Insert into the empty space so we test searching inserted items. for i in range(self.nentries + 1000, self.nentries + 2001): - c[key_populate(c, i)] = value_populate(c, i) - self.forward(c, self.nentries + 5000, + c[ds.key(i)] = ds.value(i) + self.forward(c, ds, self.nentries + 5000, list(range(self.nentries + 1, self.nentries + 1000) +\ range(self.nentries + 2001, self.nentries + 3000))) - self.backward(c, self.nentries + 5000, + self.backward(c, ds, self.nentries + 5000, list(range(self.nentries + 1, self.nentries + 1000) +\ range(self.nentries + 2001, self.nentries + 3000))) diff --git a/test/suite/test_cursor_random.py b/test/suite/test_cursor_random.py index 205b9c3c9c5..3bda6dc9946 100644 --- a/test/suite/test_cursor_random.py +++ b/test/suite/test_cursor_random.py @@ -27,16 +27,15 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import complex_populate, simple_populate -from helper import key_populate, value_populate +from wtdataset import SimpleDataSet, ComplexDataSet, simple_key, simple_value from wtscenario import make_scenarios # test_cursor_random.py # Cursor next_random operations class test_cursor_random(wttest.WiredTigerTestCase): types = [ - ('file', dict(type='file:random')), - ('table', dict(type='table:random')) + ('file', dict(type='file:random', dataset=SimpleDataSet)), + ('table', dict(type='table:random', dataset=ComplexDataSet)) ] config = [ ('sample', dict(config='next_random=true,next_random_sample_size=35')), @@ -89,12 +88,9 @@ class test_cursor_random(wttest.WiredTigerTestCase): # where the values are in an insert list. def test_cursor_random_multiple_insert_records(self): uri = self.type - if uri.startswith('file:'): - simple_populate(self, uri, - 'allocation_size=512,leaf_page_max=512,key_format=S', 100) - else: - complex_populate(self, uri, - 'allocation_size=512,leaf_page_max=512,key_format=S', 100) + ds = self.dataset(self, uri, 100, + config='allocation_size=512,leaf_page_max=512') + ds.populate() # In a insert list, next_random always selects the middle key/value # pair, all we can do is confirm cursor.next works. @@ -105,12 +101,9 @@ class test_cursor_random(wttest.WiredTigerTestCase): # where the values are in a disk format page. def cursor_random_multiple_page_records(self, reopen): uri = self.type - if uri.startswith('file:'): - simple_populate(self, uri, - 'allocation_size=512,leaf_page_max=512,key_format=S', 10000) - else: - complex_populate(self, uri, - 'allocation_size=512,leaf_page_max=512,key_format=S', 10000) + ds = self.dataset(self, uri, 10000, + config='allocation_size=512,leaf_page_max=512') + ds.populate() # Optionally close the connection so everything is forced to disk, # insert lists are an entirely different path in the code. @@ -168,7 +161,7 @@ class test_cursor_random_invisible(wttest.WiredTigerTestCase): # Start a transaction. self.session.begin_transaction() for i in range(1, 100): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Open another session, the updates won't yet be visible, we shouldn't # find anything at all. @@ -182,19 +175,19 @@ class test_cursor_random_invisible(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None) # Insert a single leading record. - cursor[key_populate(cursor, 1)] = value_populate(cursor, 1) + cursor[simple_key(cursor, 1)] = simple_value(cursor, 1) # Start a transaction. self.session.begin_transaction() for i in range(2, 100): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Open another session, the updates won't yet be visible, we should # return the only possible record. s = self.conn.open_session() cursor = s.open_cursor(uri, None, self.config) self.assertEquals(cursor.next(), 0) - self.assertEqual(cursor.get_key(), key_populate(cursor, 1)) + self.assertEqual(cursor.get_key(), simple_key(cursor, 1)) def test_cursor_random_invisible_before(self): uri = self.type @@ -202,19 +195,19 @@ class test_cursor_random_invisible(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None) # Insert a single leading record. - cursor[key_populate(cursor, 99)] = value_populate(cursor, 99) + cursor[simple_key(cursor, 99)] = simple_value(cursor, 99) # Start a transaction. self.session.begin_transaction() for i in range(2, 100): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Open another session, the updates won't yet be visible, we should # return the only possible record. s = self.conn.open_session() cursor = s.open_cursor(uri, None, self.config) self.assertEquals(cursor.next(), 0) - self.assertEqual(cursor.get_key(), key_populate(cursor, 99)) + self.assertEqual(cursor.get_key(), simple_key(cursor, 99)) if __name__ == '__main__': wttest.run() diff --git a/test/suite/test_cursor_random02.py b/test/suite/test_cursor_random02.py index 195480d703b..d18d8efd94d 100644 --- a/test/suite/test_cursor_random02.py +++ b/test/suite/test_cursor_random02.py @@ -27,8 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import complex_populate, simple_populate -from helper import key_populate, value_populate +from wtdataset import SimpleDataSet from wtscenario import make_scenarios # test_cursor_random02.py @@ -56,12 +55,14 @@ class test_cursor_random02(wttest.WiredTigerTestCase): def test_cursor_random_reasonable_distribution(self): uri = self.type num_entries = self.records - config = 'key_format=S' if uri == 'table:random': - config = 'leaf_page_max=100MB,' + config + config = 'leaf_page_max=100MB' + else: + config = '' # Set the leaf-page-max value, otherwise the page might split. - simple_populate(self, uri, config, num_entries) + ds = SimpleDataSet(self, uri, num_entries, config=config) + ds.populate() # Setup an array to track which keys are seen visitedKeys = [0] * (num_entries + 1) # Setup a counter to see when we find a sequential key diff --git a/test/suite/test_drop.py b/test/suite/test_drop.py index a3e80214295..e241c05aa68 100644 --- a/test/suite/test_drop.py +++ b/test/suite/test_drop.py @@ -28,8 +28,8 @@ import os, time import wiredtiger, wttest -from helper import confirm_does_not_exist, complex_populate, \ - complex_populate_index_name, simple_populate +from helper import confirm_does_not_exist +from wtdataset import SimpleDataSet, ComplexDataSet from wtscenario import make_scenarios # test_drop.py @@ -45,9 +45,10 @@ class test_drop(wttest.WiredTigerTestCase): ]) # Populate an object, remove it and confirm it no longer exists. - def drop(self, populate, with_cursor, reopen, drop_index): + def drop(self, dataset, with_cursor, reopen, drop_index): uri = self.uri + self.name - populate(self, uri, 'key_format=S' + self.extra_config, 10) + ds = dataset(self, uri, 10, config=self.extra_config) + ds.populate() # Open cursors should cause failure. if with_cursor: @@ -60,7 +61,7 @@ class test_drop(wttest.WiredTigerTestCase): self.reopen_conn() if drop_index: - drop_uri = complex_populate_index_name(self, uri, 0) + drop_uri = ds.index_name(0) else: drop_uri = uri self.session.drop(drop_uri, None) @@ -73,7 +74,7 @@ class test_drop(wttest.WiredTigerTestCase): # case has no indices. for with_cursor in [False, True]: for reopen in [False, True]: - self.drop(simple_populate, with_cursor, reopen, False) + self.drop(SimpleDataSet, with_cursor, reopen, False) # A complex, multi-file table object. # Try all test combinations. @@ -81,7 +82,7 @@ class test_drop(wttest.WiredTigerTestCase): for with_cursor in [False, True]: for reopen in [False, True]: for drop_index in [False, True]: - self.drop(complex_populate, with_cursor, + self.drop(ComplexDataSet, with_cursor, reopen, drop_index) # Test drop of a non-existent object: force succeeds, without force fails. diff --git a/test/suite/test_drop02.py b/test/suite/test_drop02.py index 677ba3866b2..017aa64e312 100644 --- a/test/suite/test_drop02.py +++ b/test/suite/test_drop02.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import simple_populate +from wtdataset import SimpleDataSet # test_drop02.py # Test dropping an LSM tree on first open. There was a bug where this @@ -38,7 +38,8 @@ class test_drop02(wttest.WiredTigerTestCase): # Populate an object, remove it and confirm it no longer exists. def test_drop(self): uri = 'lsm:' + self.name - simple_populate(self, uri, 'key_format=S', 100000) + ds = SimpleDataSet(self, uri, 100000) + ds.populate() self.reopen_conn() self.session.drop(uri, None) diff --git a/test/suite/test_dump.py b/test/suite/test_dump.py index 280d5870359..f6a83c32489 100644 --- a/test/suite/test_dump.py +++ b/test/suite/test_dump.py @@ -28,19 +28,18 @@ import os, shutil import wiredtiger, wttest -from helper import \ - complex_populate, complex_populate_check, \ - simple_populate, simple_populate_check, \ - simple_index_populate, simple_index_populate_check + from suite_subprocess import suite_subprocess from wtscenario import make_scenarios +from wtdataset import SimpleDataSet, SimpleIndexDataSet, SimpleLSMDataSet, \ + ComplexDataSet, ComplexLSMDataSet # test_dump.py # Utilities: wt dump # Test the dump utility (I'm not testing the dump cursors, that's what the # utility uses underneath). class test_dump(wttest.WiredTigerTestCase, suite_subprocess): - dir='dump.dir' # Backup directory name + dir = 'dump.dir' # Backup directory name name = 'test_dump' name2 = 'test_dumpb' @@ -56,30 +55,20 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): ('string', dict(keyfmt='S')) ] types = [ - ('file', dict(uri='file:', config='', lsm=False, - populate=simple_populate, - populate_check=simple_populate_check)), - ('lsm', dict(uri='lsm:', config='', lsm=True, - populate=simple_populate, - populate_check=simple_populate_check)), - ('table-simple', dict(uri='table:', config='', lsm=False, - populate=simple_populate, - populate_check=simple_populate_check)), - ('table-index', dict(uri='table:', config='', lsm=False, - populate=simple_index_populate, - populate_check=simple_index_populate_check)), - ('table-simple-lsm', dict(uri='table:', config='type=lsm', lsm=True, - populate=simple_populate, - populate_check=simple_populate_check)), - ('table-complex', dict(uri='table:', config='', lsm=False, - populate=complex_populate, - populate_check=complex_populate_check)), - ('table-complex-lsm', dict(uri='table:', config='type=lsm', lsm=True, - populate=complex_populate, - populate_check=complex_populate_check)) + ('file', dict(uri='file:', dataset=SimpleDataSet)), + ('lsm', dict(uri='lsm:', dataset=SimpleDataSet)), + ('table-simple', dict(uri='table:', dataset=SimpleDataSet)), + ('table-index', dict(uri='table:', dataset=SimpleIndexDataSet)), + ('table-simple-lsm', dict(uri='table:', dataset=SimpleLSMDataSet)), + ('table-complex', dict(uri='table:', dataset=ComplexDataSet)), + ('table-complex-lsm', dict(uri='table:', dataset=ComplexLSMDataSet)), ] scenarios = make_scenarios(types, keyfmt, dumpfmt) + def skip(self): + return (self.dataset.is_lsm() or self.uri == 'lsm:') and \ + self.keyfmt == 'r' + # Extract the values lines from the dump output. def value_lines(self, fname): # mode: @@ -108,14 +97,14 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): # Dump, re-load and do a content comparison. def test_dump(self): # LSM and column-store isn't a valid combination. - if self.lsm and self.keyfmt == 'r': + if self.skip(): return # Create the object. uri = self.uri + self.name uri2 = self.uri + self.name2 - self.populate(self, uri, - self.config + ',key_format=' + self.keyfmt, self.nentries) + pop = self.dataset(self, uri, self.nentries, key_format=self.keyfmt) + pop.populate() # Dump the object. os.mkdir(self.dir) @@ -136,14 +125,14 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): # Check the object's contents self.reopen_conn(self.dir) - self.populate_check(self, uri, self.nentries) + pop.check() # Re-load the object again in the original directory. self.reopen_conn('.') self.runWt(['-h', self.dir, 'load', '-f', 'dump.out']) # Check the contents, they shouldn't have changed. - self.populate_check(self, uri, self.nentries) + pop.check() # Re-load the object again, but confirm -n (no overwrite) fails. self.runWt(['-h', self.dir, 'load', '-n', '-f', 'dump.out'], @@ -151,7 +140,7 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): self.check_non_empty_file('errfile.out') # If there are indices, dump one of them and check the output. - if self.populate == complex_populate: + if self.dataset == ComplexDataSet: indexuri = 'index:' + self.name + ':indx1' hexopt = ['-x'] if self.hex == 1 else [] self.runWt(['-h', self.dir, 'dump'] + hexopt + [indexuri], @@ -166,7 +155,8 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): # Check the contents in the new table. self.reopen_conn(self.dir) - self.populate_check(self, uri2, self.nentries) + pop = self.dataset(self, uri2, self.nentries, key_format=self.keyfmt) + pop.check() if __name__ == '__main__': wttest.run() diff --git a/test/suite/test_dupc.py b/test/suite/test_dupc.py index 12b18f1ba79..c0cf6acc75e 100644 --- a/test/suite/test_dupc.py +++ b/test/suite/test_dupc.py @@ -32,7 +32,7 @@ import os, time import wiredtiger, wttest -from helper import complex_populate, key_populate, simple_populate +from wtdataset import SimpleDataSet, ComplexDataSet from wtscenario import make_scenarios # Test session.open_cursor with cursor duplication. @@ -40,8 +40,6 @@ class test_duplicate_cursor(wttest.WiredTigerTestCase): name = 'test_dupc' nentries = 1000 - config = 'key_format=' - scenarios = make_scenarios([ ('file-r', dict(uri='file:', fmt='r')), ('file-S', dict(uri='file:', fmt='S')), @@ -51,7 +49,7 @@ class test_duplicate_cursor(wttest.WiredTigerTestCase): # Iterate through an object, duplicate the cursor and checking that it # matches the original and is set to the same record. - def iterate(self, uri): + def iterate(self, uri, ds): cursor = self.session.open_cursor(uri, None, None) next = 0 while True: @@ -59,10 +57,10 @@ class test_duplicate_cursor(wttest.WiredTigerTestCase): if nextret != 0: break next += 1 - self.assertEqual(cursor.get_key(), key_populate(cursor, next)) + self.assertEqual(cursor.get_key(), ds.key(next)) dupc = self.session.open_cursor(None, cursor, None) self.assertEqual(cursor.compare(dupc), 0) - self.assertEqual(dupc.get_key(), key_populate(dupc, next)) + self.assertEqual(dupc.get_key(), ds.key(next)) cursor.close() cursor = dupc self.assertEqual(next, self.nentries) @@ -73,14 +71,16 @@ class test_duplicate_cursor(wttest.WiredTigerTestCase): uri = self.uri + self.name # A simple, one-file file or table object. - simple_populate(self, uri, self.config + self.fmt, self.nentries) - self.iterate(uri) + ds = SimpleDataSet(self, uri, self.nentries, key_format=self.fmt) + ds.populate() + self.iterate(uri, ds) self.session.drop(uri, None) # A complex, multi-file table object. if self.uri == "table:": - complex_populate(self, uri, self.config + self.fmt, self.nentries) - self.iterate(uri) + ds = ComplexDataSet(self, uri, self.nentries, key_format=self.fmt) + ds.populate() + self.iterate(uri, ds) self.session.drop(uri, None) if __name__ == '__main__': diff --git a/test/suite/test_empty.py b/test/suite/test_empty.py index 9fe88107412..578bec618c9 100644 --- a/test/suite/test_empty.py +++ b/test/suite/test_empty.py @@ -28,7 +28,7 @@ import os import wiredtiger, wttest -from helper import key_populate +from wtdataset import simple_key from wtscenario import make_scenarios # test_empty.py @@ -64,7 +64,7 @@ class test_empty(wttest.WiredTigerTestCase): # Add a few records to the object and remove them. cursor = self.session.open_cursor(uri, None, None) for i in range(1,5): - key = key_populate(cursor, i) + key = simple_key(cursor, i) cursor[key] = "XXX" del cursor[key] diff --git a/test/suite/test_inmem01.py b/test/suite/test_inmem01.py index c6ae7ff6c4b..1af43bbd9d9 100644 --- a/test/suite/test_inmem01.py +++ b/test/suite/test_inmem01.py @@ -28,8 +28,7 @@ import wiredtiger, wttest from time import sleep -from helper import simple_populate, simple_populate_check -from helper import key_populate, value_populate +from wtdataset import SimpleDataSet from wtscenario import make_scenarios # test_inmem01.py @@ -41,54 +40,60 @@ class test_inmem01(wttest.WiredTigerTestCase): table_config = ',memory_page_max=32k,leaf_page_max=4k' scenarios = make_scenarios([ - ('col', dict(fmt='key_format=r,value_format=S')), - ('fix', dict(fmt='key_format=r,value_format=8t')), - ('row', dict(fmt='key_format=S,value_format=S')) + ('col', dict(keyfmt='r', valuefmt='S')), + ('fix', dict(keyfmt='r', valuefmt='8t')), + ('row', dict(keyfmt='S', valuefmt='S')), ]) # Smoke-test in-memory configurations, add a small amount of data and # ensure it's visible. def test_insert(self): - config = self.fmt + self.table_config - simple_populate(self, self.uri, config, 1000) - simple_populate_check(self, self.uri, 1000) + ds = SimpleDataSet(self, self.uri, 1000, key_format=self.keyfmt, + value_format=self.valuefmt, config=self.table_config) + ds.populate() + ds.check() # Add more data than fits into the configured cache and verify it fails. def test_insert_over_capacity(self): - config = self.fmt + self.table_config msg = '/WT_CACHE_FULL.*/' + ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt, + value_format=self.valuefmt, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, - lambda:simple_populate(self, self.uri, config, 10000000), msg) + lambda:ds.populate(), msg) # Figure out the last key we successfully inserted, and check all # previous inserts are still there. cursor = self.session.open_cursor(self.uri, None) cursor.prev() last_key = int(cursor.get_key()) - simple_populate_check(self, self.uri, last_key) + ds = SimpleDataSet(self, self.uri, last_key, key_format=self.keyfmt, + value_format=self.valuefmt, config=self.table_config) + ds.check() # Fill the cache with data, remove some data, ensure more data can be # inserted (after a reasonable amount of time for space to be reclaimed). def test_insert_over_delete(self): - config = self.fmt + self.table_config msg = '/WT_CACHE_FULL.*/' + ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt, + value_format=self.valuefmt, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, - lambda:simple_populate(self, self.uri, config, 10000000), msg) + lambda:ds.populate(), msg) # Now that the database contains as much data as will fit into # the configured cache, verify removes succeed. cursor = self.session.open_cursor(self.uri, None) for i in range(1, 100): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) cursor.remove() # Run queries after adding, removing and re-inserting data. # Try out keeping a cursor open while adding new data. def test_insert_over_delete_replace(self): - config = self.fmt + self.table_config msg = '/WT_CACHE_FULL.*/' + ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt, + value_format=self.valuefmt, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, - lambda:simple_populate(self, self.uri, config, 10000000), msg) + lambda:ds.populate(), msg) cursor = self.session.open_cursor(self.uri, None) cursor.prev() @@ -98,7 +103,7 @@ class test_inmem01(wttest.WiredTigerTestCase): # the configured cache, verify removes succeed. cursor = self.session.open_cursor(self.uri, None) for i in range(1, last_key / 4, 1): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) cursor.remove() cursor.reset() @@ -106,7 +111,7 @@ class test_inmem01(wttest.WiredTigerTestCase): inserted = False for i in range(1, 1000): try: - cursor[key_populate(cursor, 1)] = value_populate(cursor, 1) + cursor[ds.key(1)] = ds.value(1) except wiredtiger.WiredTigerError: cursor.reset() sleep(1) diff --git a/test/suite/test_inmem02.py b/test/suite/test_inmem02.py index 9eb8330b2a3..b5e07fea967 100644 --- a/test/suite/test_inmem02.py +++ b/test/suite/test_inmem02.py @@ -28,9 +28,7 @@ import wiredtiger, wttest from time import sleep -from helper import simple_populate, simple_populate_check -from helper import key_populate, value_populate -from wtscenario import make_scenarios +from wtdataset import SimpleDataSet # test_inmem02.py # Test in-memory with ignore-cache-size setting. @@ -38,7 +36,7 @@ class test_inmem02(wttest.WiredTigerTestCase): uri = 'table:inmem02' conn_config = \ 'cache_size=3MB,file_manager=(close_idle_time=0),in_memory=true' - table_config = 'key_format=S,value_format=S,memory_page_max=32k,leaf_page_max=4k' + table_config = 'memory_page_max=32k,leaf_page_max=4k' # Add more data than fits into the configured cache and verify it fails. def test_insert_over_allowed(self): @@ -50,9 +48,9 @@ class test_inmem02(wttest.WiredTigerTestCase): # Populate a table with enough data to fill the cache. msg = '/WT_CACHE_FULL.*/' + ds = SimpleDataSet(self, self.uri, 10000000, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, - lambda:simple_populate( - self, self.uri, self.table_config, 10000000), msg) + lambda:ds.populate(), msg) # Add some content to the new table cursor = self.session.open_cursor(self.uri + '_over', None) diff --git a/test/suite/test_jsondump01.py b/test/suite/test_jsondump01.py index dc8027c2115..c7fa9cdf397 100644 --- a/test/suite/test_jsondump01.py +++ b/test/suite/test_jsondump01.py @@ -28,11 +28,9 @@ import os, json import wiredtiger, wttest -from helper import \ - complex_populate, complex_populate_check, complex_populate_check_cursor,\ - simple_populate, simple_populate_check, simple_populate_check_cursor, \ - simple_index_populate, simple_index_populate_check, \ - simple_index_populate_check_cursor, compare_files +from wtdataset import SimpleDataSet, SimpleLSMDataSet, SimpleIndexDataSet, \ + ComplexDataSet, ComplexLSMDataSet +from helper import compare_files from suite_subprocess import suite_subprocess from wtscenario import make_scenarios @@ -40,7 +38,8 @@ from wtscenario import make_scenarios # It emulates a WT cursor well enough for the *_check_cursor methods. # They just need an iterable object. class FakeCursor: - def __init__(self, keyfmt, valuefmt, rows): + def __init__(self, uri, keyfmt, valuefmt, rows): + self.uri = uri self.key_format = keyfmt self.value_format = valuefmt self.rows = rows @@ -79,47 +78,30 @@ class test_jsondump01(wttest.WiredTigerTestCase, suite_subprocess): ('string', dict(keyfmt='S')) ] types = [ - ('file', dict(uri='file:', config='', lsm=False, - populate=simple_populate, - populate_check=simple_populate_check, - populate_check_cursor=simple_populate_check_cursor)), - ('lsm', dict(uri='lsm:', config='', lsm=True, - populate=simple_populate, - populate_check=simple_populate_check, - populate_check_cursor=simple_populate_check_cursor)), - ('table-simple', dict(uri='table:', config='', lsm=False, - populate=simple_populate, - populate_check=simple_populate_check, - populate_check_cursor=simple_populate_check_cursor)), - ('table-index', dict(uri='table:', config='', lsm=False, - populate=simple_index_populate, - populate_check=simple_index_populate_check, - populate_check_cursor=simple_index_populate_check_cursor)), - ('table-simple-lsm', dict(uri='table:', config='type=lsm', lsm=True, - populate=simple_populate, - populate_check=simple_populate_check, - populate_check_cursor=simple_populate_check_cursor)), - ('table-complex', dict(uri='table:', config='', lsm=False, - populate=complex_populate, - populate_check=complex_populate_check, - populate_check_cursor=complex_populate_check_cursor)), - ('table-complex-lsm', dict(uri='table:', config='type=lsm', lsm=True, - populate=complex_populate, - populate_check=complex_populate_check, - populate_check_cursor=complex_populate_check_cursor)) + ('file', dict(uri='file:', dataset=SimpleDataSet)), + ('lsm', dict(uri='lsm:', dataset=SimpleDataSet)), + ('table-simple', dict(uri='table:', dataset=SimpleDataSet)), + ('table-index', dict(uri='table:', dataset=SimpleIndexDataSet)), + ('table-simple-lsm', dict(uri='table:', dataset=SimpleLSMDataSet)), + ('table-complex', dict(uri='table:', dataset=ComplexDataSet)), + ('table-complex-lsm', dict(uri='table:', dataset=ComplexLSMDataSet)) ] scenarios = make_scenarios(types, keyfmt) + def skip(self): + return (self.dataset.is_lsm() or self.uri == 'lsm:') and \ + self.keyfmt == 'r' + # Dump using util, re-load using python's JSON, and do a content comparison. def test_jsondump_util(self): # LSM and column-store isn't a valid combination. - if self.lsm and self.keyfmt == 'r': + if self.skip(): return # Create the object. uri = self.uri + self.name - self.populate(self, uri, self.config + ',key_format=' + self.keyfmt, - self.nentries) + ds = self.dataset(self, uri, self.nentries, key_format=self.keyfmt) + ds.populate() # Dump the object. self.runWt(['dump', '-j', uri], outfilename='jsondump.out') @@ -141,20 +123,21 @@ class test_jsondump01(wttest.WiredTigerTestCase, suite_subprocess): # check the contents of the data we read. # we only use a wt cursor to get the key_format/value_format. cursor = self.session.open_cursor(uri, None) - fake = FakeCursor(cursor.key_format, cursor.value_format, data) + fake = FakeCursor(uri, cursor.key_format, cursor.value_format, data) cursor.close() - self.populate_check_cursor(self, fake, self.nentries) + ds.check_cursor(fake) # Dump using util, re-load using python's JSON, and do a content comparison. def test_jsonload_util(self): # LSM and column-store isn't a valid combination. - if self.lsm and self.keyfmt == 'r': + if self.skip(): return # Create the object. uri = self.uri + self.name uri2 = self.uri + self.name2 - self.populate(self, uri, 'key_format=' + self.keyfmt, self.nentries) + ds = self.dataset(self, uri, self.nentries, key_format=self.keyfmt) + ds.populate() # Dump the object. self.runWt(['dump', '-j', uri], outfilename='jsondump.out') @@ -165,7 +148,9 @@ class test_jsondump01(wttest.WiredTigerTestCase, suite_subprocess): self.runWt(loadcmd) # Check the contents of the data we read. - self.populate_check(self, uri2, self.nentries) + # We use the dataset only for checking. + ds2 = self.dataset(self, uri2, self.nentries, key_format=self.keyfmt) + ds2.check() # Reload into the original uri, and dump into another file. self.session.drop(uri, None) @@ -175,7 +160,7 @@ class test_jsondump01(wttest.WiredTigerTestCase, suite_subprocess): # Compare the two outputs, and check the content again. compare_files(self, 'jsondump.out', 'jsondump2.out') - self.populate_check(self, uri, self.nentries) + ds.check() if __name__ == '__main__': wttest.run() diff --git a/test/suite/test_lsm01.py b/test/suite/test_lsm01.py index f6cee20e896..b44df4bae14 100644 --- a/test/suite/test_lsm01.py +++ b/test/suite/test_lsm01.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wtscenario, wttest -from helper import simple_populate +from wtdataset import SimpleDataSet # test_lsm01.py # Test LSM tree configuration options. @@ -77,7 +77,7 @@ class test_lsm01(wttest.WiredTigerTestCase): args += ')' # Close the LSM configuration option group self.verbose(3, 'Test LSM with config: ' + args + ' count: ' + str(self.nrecs)) - simple_populate(self, self.uri, args, self.nrecs) + SimpleDataSet(self, self.uri, self.nrecs).populate() # TODO: Adding an explicit drop here can cause deadlocks, if a merge # is still happening. See issue #349. diff --git a/test/suite/test_lsm02.py b/test/suite/test_lsm02.py index 85067665017..e9628139a97 100644 --- a/test/suite/test_lsm02.py +++ b/test/suite/test_lsm02.py @@ -27,7 +27,6 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wtscenario, wttest -from helper import simple_populate # test_lsm02.py # Test LSM schema level operations diff --git a/test/suite/test_lsm03.py b/test/suite/test_lsm03.py index 448d864c646..d916db415da 100644 --- a/test/suite/test_lsm03.py +++ b/test/suite/test_lsm03.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wtscenario, wttest -from helper import simple_populate +from wtdataset import SimpleDataSet # test_lsm03.py # Check to make sure that LSM schema operations don't get EBUSY when @@ -37,13 +37,14 @@ class test_lsm03(wttest.WiredTigerTestCase): # Use small pages so we generate some internal layout # Setup LSM so multiple chunks are present - config = 'key_format=S,allocation_size=512,internal_page_max=512' + \ + config = 'allocation_size=512,internal_page_max=512' + \ ',leaf_page_max=1k,lsm=(chunk_size=512k,merge_min=10)' # Populate an object then drop it. def test_lsm_drop_active(self): uri = 'lsm:' + self.name - simple_populate(self, uri, self.config, 10000) + ds = SimpleDataSet(self, uri, 10000, config=self.config) + ds.populate() # Force to disk self.reopen_conn() @@ -55,6 +56,7 @@ class test_lsm03(wttest.WiredTigerTestCase): cursor.close() # Add enough records that a merge should be running - simple_populate(self, uri, self.config, 50000) + ds = SimpleDataSet(self, uri, 50000, config=self.config) + ds.populate() # The drop should succeed even when LSM work units are active self.session.drop(uri) diff --git a/test/suite/test_nsnap01.py b/test/suite/test_nsnap01.py index 7e8951750f8..4d5555277fe 100644 --- a/test/suite/test_nsnap01.py +++ b/test/suite/test_nsnap01.py @@ -30,7 +30,7 @@ # Named snapshots: basic API from suite_subprocess import suite_subprocess -from helper import simple_populate +from wtdataset import SimpleDataSet import wiredtiger, wttest class test_nsnap01(wttest.WiredTigerTestCase, suite_subprocess): @@ -53,7 +53,7 @@ class test_nsnap01(wttest.WiredTigerTestCase, suite_subprocess): def test_named_snapshots(self): # Populate a table end = start = 0 - simple_populate(self, self.uri, 'key_format=i', 0) + SimpleDataSet(self, self.uri, 0, key_format='i').populate() # Now run a workload: # every iteration: diff --git a/test/suite/test_nsnap02.py b/test/suite/test_nsnap02.py index 510c9d421ef..ed1c96ebe50 100644 --- a/test/suite/test_nsnap02.py +++ b/test/suite/test_nsnap02.py @@ -30,7 +30,7 @@ # Named snapshots: Combinations of dropping snapshots from suite_subprocess import suite_subprocess -from helper import simple_populate +from wtdataset import SimpleDataSet import wiredtiger, wttest class test_nsnap02(wttest.WiredTigerTestCase, suite_subprocess): @@ -63,7 +63,7 @@ class test_nsnap02(wttest.WiredTigerTestCase, suite_subprocess): def create_snapshots(self): # Populate a table end = start = 0 - simple_populate(self, self.uri, 'key_format=i', 0) + SimpleDataSet(self, self.uri, 0, key_format='i').populate() # Create a set of snapshots # Each snapshot has a bunch of new data diff --git a/test/suite/test_nsnap03.py b/test/suite/test_nsnap03.py index 3986c0c1a0a..6964fb914c3 100644 --- a/test/suite/test_nsnap03.py +++ b/test/suite/test_nsnap03.py @@ -30,7 +30,7 @@ # Named snapshots: Access and create from multiple sessions from suite_subprocess import suite_subprocess -from helper import simple_populate +from wtdataset import SimpleDataSet import wiredtiger, wttest class test_nsnap03(wttest.WiredTigerTestCase, suite_subprocess): @@ -55,7 +55,7 @@ class test_nsnap03(wttest.WiredTigerTestCase, suite_subprocess): def test_named_snapshots(self): # Populate a table end = start = 0 - simple_populate(self, self.uri, 'key_format=i', 0) + SimpleDataSet(self, self.uri, 0, key_format='i').populate() # Now run a workload: # every iteration: diff --git a/test/suite/test_nsnap04.py b/test/suite/test_nsnap04.py index f9ef26b5600..60901dd2ee3 100644 --- a/test/suite/test_nsnap04.py +++ b/test/suite/test_nsnap04.py @@ -29,9 +29,9 @@ # test_nsnap04.py # Named snapshots: Create snapshot from running transaction -from suite_subprocess import suite_subprocess -from helper import simple_populate import wiredtiger, wttest +from suite_subprocess import suite_subprocess +from wtdataset import SimpleDataSet class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_nsnap04' @@ -52,7 +52,7 @@ class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess): def test_named_snapshots(self): # Populate a table end = start = 0 - simple_populate(self, self.uri, 'key_format=i', 0) + SimpleDataSet(self, self.uri, 0, key_format='i').populate() snapshots = [] c = self.session.open_cursor(self.uri) diff --git a/test/suite/test_overwrite.py b/test/suite/test_overwrite.py index b7af1612243..4739abaa578 100644 --- a/test/suite/test_overwrite.py +++ b/test/suite/test_overwrite.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from helper import key_populate, simple_populate +from wtdataset import SimpleDataSet from wtscenario import make_scenarios # test_overwrite.py @@ -35,22 +35,23 @@ from wtscenario import make_scenarios class test_overwrite(wttest.WiredTigerTestCase): name = 'overwrite' scenarios = make_scenarios([ - ('file-r', dict(type='file:',keyfmt='r')), - ('file-S', dict(type='file:',keyfmt='S')), - ('lsm-S', dict(type='lsm:',keyfmt='S')), - ('table-r', dict(type='table:',keyfmt='r')), - ('table-S', dict(type='table:',keyfmt='S')), + ('file-r', dict(type='file:', keyfmt='r', dataset=SimpleDataSet)), + ('file-S', dict(type='file:', keyfmt='S', dataset=SimpleDataSet)), + ('lsm-S', dict(type='lsm:', keyfmt='S', dataset=SimpleDataSet)), + ('table-r', dict(type='table:', keyfmt='r', dataset=SimpleDataSet)), + ('table-S', dict(type='table:', keyfmt='S', dataset=SimpleDataSet)), ]) # Confirm a cursor configured with/without overwrite correctly handles # non-existent records during insert, remove and update operations. def test_overwrite_insert(self): uri = self.type + self.name - simple_populate(self, uri, 'key_format=' + self.keyfmt, 100) + ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + ds.populate() # Insert of an existing record with overwrite off fails. cursor = self.session.open_cursor(uri, None, "overwrite=false") - cursor.set_key(key_populate(cursor, 5)) + cursor.set_key(ds.key(5)) cursor.set_value('XXXXXXXXXX') self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.insert()) @@ -60,78 +61,80 @@ class test_overwrite(wttest.WiredTigerTestCase): # failure modes are for non-existent records, and you cannot duplicate # cursor pointing to non-existent records. cursor = self.session.open_cursor(uri, None, "overwrite=false") - cursor.set_key(key_populate(cursor, 5)) + cursor.set_key(ds.key(5)) dupc = self.session.open_cursor(None, cursor, "overwrite=true") dupc.set_value('XXXXXXXXXX') self.assertEquals(dupc.insert(), 0) # Insert of an existing record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) - cursor.set_key(key_populate(cursor, 6)) + cursor.set_key(ds.key(6)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.insert(), 0) # Insert of a non-existent record with overwrite off succeeds. cursor = self.session.open_cursor(uri, None, "overwrite=false") - cursor.set_key(key_populate(cursor, 200)) + cursor.set_key(ds.key(200)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.insert(), 0) # Insert of a non-existent record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) - cursor.set_key(key_populate(cursor, 201)) + cursor.set_key(ds.key(201)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.insert(), 0) def test_overwrite_remove(self): uri = self.type + self.name - simple_populate(self, uri, 'key_format=' + self.keyfmt, 100) + ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + ds.populate() # Remove of an existing record with overwrite off succeeds. cursor = self.session.open_cursor(uri, None, "overwrite=false") - cursor.set_key(key_populate(cursor, 5)) + cursor.set_key(ds.key(5)) self.assertEquals(cursor.remove(), 0) # Remove of an existing record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) - cursor.set_key(key_populate(cursor, 6)) + cursor.set_key(ds.key(6)) self.assertEquals(cursor.remove(), 0) # Remove of a non-existent record with overwrite off fails. cursor = self.session.open_cursor(uri, None, "overwrite=false") - cursor.set_key(key_populate(cursor, 200)) + cursor.set_key(ds.key(200)) self.assertEquals(cursor.remove(), wiredtiger.WT_NOTFOUND) # Remove of a non-existent record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) - cursor.set_key(key_populate(cursor, 201)) + cursor.set_key(ds.key(201)) self.assertEquals(cursor.remove(), 0) def test_overwrite_update(self): uri = self.type + self.name - simple_populate(self, uri, 'key_format=' + self.keyfmt, 100) + ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + ds.populate() # Update of an existing record with overwrite off succeeds. cursor = self.session.open_cursor(uri, None, "overwrite=false") - cursor.set_key(key_populate(cursor, 5)) + cursor.set_key(ds.key(5)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.update(), 0) # Update of an existing record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) - cursor.set_key(key_populate(cursor, 6)) + cursor.set_key(ds.key(6)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.update(), 0) # Update of a non-existent record with overwrite off fails. cursor = self.session.open_cursor(uri, None, "overwrite=false") - cursor.set_key(key_populate(cursor, 200)) + cursor.set_key(ds.key(200)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.update(), wiredtiger.WT_NOTFOUND) # Update of a non-existent record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) - cursor.set_key(key_populate(cursor, 201)) + cursor.set_key(ds.key(201)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.update(), 0) diff --git a/test/suite/test_readonly03.py b/test/suite/test_readonly03.py index a5607fcfe80..f30c591ca59 100644 --- a/test/suite/test_readonly03.py +++ b/test/suite/test_readonly03.py @@ -31,9 +31,9 @@ # all return ENOTSUP. # -from helper import simple_populate -from suite_subprocess import suite_subprocess import os, sys, wiredtiger, wttest +from suite_subprocess import suite_subprocess +from wtdataset import SimpleDataSet class test_readonly03(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:test_readonly03' @@ -61,7 +61,8 @@ class test_readonly03(wttest.WiredTigerTestCase, suite_subprocess): create_params = 'key_format=i,value_format=i' entries = 10 # Create a database and a table. - simple_populate(self, self.uri, create_params, entries) + SimpleDataSet(self, self.uri, entries, key_format='i', + value_format='i').populate() # # Now close and reopen. Note that the connection function diff --git a/test/suite/test_rebalance.py b/test/suite/test_rebalance.py index acb73d82a13..2d160bafec0 100644 --- a/test/suite/test_rebalance.py +++ b/test/suite/test_rebalance.py @@ -28,7 +28,7 @@ import os, time import wiredtiger, wttest -from helper import complex_populate, simple_populate +from wtdataset import SimpleDataSet, ComplexDataSet from wtscenario import make_scenarios # test_rebalance.py @@ -38,7 +38,7 @@ class test_rebalance(wttest.WiredTigerTestCase): # Use small pages so we generate some internal layout # Setup LSM so multiple chunks are present - config = 'key_format=S,allocation_size=512,internal_page_max=512' + \ + config = 'allocation_size=512,internal_page_max=512' + \ ',leaf_page_max=1k,lsm=(chunk_size=512k,merge_min=10)' scenarios = make_scenarios([ @@ -48,9 +48,10 @@ class test_rebalance(wttest.WiredTigerTestCase): ]) # Populate an object, then rebalance it. - def rebalance(self, populate, with_cursor): + def rebalance(self, dataset, with_cursor): uri = self.uri + self.name - populate(self, uri, self.config, 10000) + ds = dataset(self, uri, 10000, config=self.config) + ds.populate() # Force to disk, we don't rebalance in-memory objects. self.reopen_conn() @@ -68,13 +69,13 @@ class test_rebalance(wttest.WiredTigerTestCase): # Test rebalance of an object. def test_rebalance(self): # Simple file or table object. - self.rebalance(simple_populate, False) - self.rebalance(simple_populate, True) + self.rebalance(SimpleDataSet, False) + self.rebalance(SimpleDataSet, True) # A complex, multi-file table object. if self.uri == "table:": - self.rebalance(complex_populate, False) - self.rebalance(complex_populate, True) + self.rebalance(ComplexDataSet, False) + self.rebalance(ComplexDataSet, True) if __name__ == '__main__': wttest.run() diff --git a/test/suite/test_reconfig01.py b/test/suite/test_reconfig01.py index fb3fb7edac6..e76becac76a 100644 --- a/test/suite/test_reconfig01.py +++ b/test/suite/test_reconfig01.py @@ -28,7 +28,7 @@ import time import wiredtiger, wttest -from helper import simple_populate +from wtdataset import SimpleDataSet # test_reconfig01.py # Smoke-test the connection reconfiguration operations. @@ -70,7 +70,7 @@ class test_reconfig01(wttest.WiredTigerTestCase): # Take all the defaults. uri = "lsm:test_reconfig" nrecs = 10 - simple_populate(self, uri, 'key_format=S', nrecs) + SimpleDataSet(self, uri, nrecs).populate() # Sleep to make sure all threads are started. time.sleep(2) # Now that an LSM tree exists, reconfigure LSM manager threads. @@ -78,7 +78,7 @@ class test_reconfig01(wttest.WiredTigerTestCase): self.conn.reconfigure("lsm_manager=(worker_thread_max=10)") # Generate some work nrecs = 20 - simple_populate(self, uri, 'key_format=S', nrecs) + SimpleDataSet(self, uri, nrecs).populate() # Now reconfigure fewer threads. self.conn.reconfigure("lsm_manager=(worker_thread_max=3)") diff --git a/test/suite/test_reconfig02.py b/test/suite/test_reconfig02.py index 9d9ac220aa7..36a78a1805f 100644 --- a/test/suite/test_reconfig02.py +++ b/test/suite/test_reconfig02.py @@ -28,7 +28,6 @@ import fnmatch, os, time import wiredtiger, wttest -from helper import simple_populate # test_reconfig02.py # Smoke-test the connection reconfiguration operations. diff --git a/test/suite/test_reconfig03.py b/test/suite/test_reconfig03.py index e9d39ea5a76..0019bf4814e 100644 --- a/test/suite/test_reconfig03.py +++ b/test/suite/test_reconfig03.py @@ -28,7 +28,7 @@ import fnmatch, os, time import wiredtiger, wttest -from helper import simple_populate +from wtdataset import SimpleDataSet # test_reconfig03.py # Test the connection reconfiguration operations used in the MongoDB @@ -41,16 +41,16 @@ class test_reconfig03(wttest.WiredTigerTestCase): # can run after we've made modifications. def test_reconfig03_mdb(self): entries = 10000 - simple_populate(self, self.uri, 'key_format=S', entries) + SimpleDataSet(self, self.uri, entries).populate() time.sleep(1) self.conn.reconfigure("eviction_target=81") - simple_populate(self, self.uri, 'key_format=S', entries * 2) + SimpleDataSet(self, self.uri, entries * 2).populate() time.sleep(1) self.conn.reconfigure("cache_size=81M") - simple_populate(self, self.uri, 'key_format=S', entries * 3) + SimpleDataSet(self, self.uri, entries * 3).populate() time.sleep(1) self.conn.reconfigure("eviction_dirty_target=82") - simple_populate(self, self.uri, 'key_format=S', entries * 4) + SimpleDataSet(self, self.uri, entries * 4).populate() time.sleep(1) self.conn.reconfigure("shared_cache=(chunk=11MB, name=bar, reserve=12MB, size=1G)") diff --git a/test/suite/test_rename.py b/test/suite/test_rename.py index 1979bbb802a..4e3af8e13e0 100644 --- a/test/suite/test_rename.py +++ b/test/suite/test_rename.py @@ -28,9 +28,8 @@ import os, time import wiredtiger, wttest -from helper import confirm_does_not_exist,\ - complex_populate, complex_populate_check,\ - simple_populate, simple_populate_check +from helper import confirm_does_not_exist +from wtdataset import SimpleDataSet, ComplexDataSet from wtscenario import make_scenarios # test_rename.py @@ -46,10 +45,13 @@ class test_rename(wttest.WiredTigerTestCase): # Populate and object, and rename it a couple of times, confirming the # old name doesn't exist and the new name has the right contents. - def rename(self, populate, check, with_cursor): + def rename(self, dataset, with_cursor): uri1 = self.uri + self.name1 uri2 = self.uri + self.name2 - populate(self, uri1, 'key_format=S', 10) + # Only populate uri1, we keep ds2 for checking. + ds1 = dataset(self, uri1, 10) + ds2 = dataset(self, uri2, 10) + ds1.populate() # Open cursors should cause failure. if with_cursor: @@ -60,24 +62,24 @@ class test_rename(wttest.WiredTigerTestCase): self.session.rename(uri1, uri2, None) confirm_does_not_exist(self, uri1) - check(self, uri2, 10) + ds2.check() self.session.rename(uri2, uri1, None) confirm_does_not_exist(self, uri2) - check(self, uri1, 10) + ds1.check() self.session.drop(uri1) # Test rename of an object. def test_rename(self): # Simple, one-file file or table object. - self.rename(simple_populate, simple_populate_check, False) - self.rename(simple_populate, simple_populate_check, True) + self.rename(SimpleDataSet, False) + self.rename(SimpleDataSet, True) # A complex, multi-file table object. if self.uri == "table:": - self.rename(complex_populate, complex_populate_check, False) - self.rename(complex_populate, complex_populate_check, True) + self.rename(ComplexDataSet, False) + self.rename(ComplexDataSet, True) def test_rename_dne(self): uri1 = self.uri + self.name1 diff --git a/test/suite/test_shared_cache01.py b/test/suite/test_shared_cache01.py index 896a1e569ef..70560a625ee 100644 --- a/test/suite/test_shared_cache01.py +++ b/test/suite/test_shared_cache01.py @@ -31,7 +31,6 @@ import os import shutil import wiredtiger, wttest from wttest import unittest -from helper import key_populate, simple_populate # test_shared_cache01.py # Checkpoint tests diff --git a/test/suite/test_shared_cache02.py b/test/suite/test_shared_cache02.py index 4afccbe59f7..7cde6c86695 100644 --- a/test/suite/test_shared_cache02.py +++ b/test/suite/test_shared_cache02.py @@ -31,7 +31,6 @@ import os import shutil import wiredtiger, wttest from wttest import unittest -from helper import key_populate, simple_populate # test_shared_cache02.py # Shared cache tests diff --git a/test/suite/test_split.py b/test/suite/test_split.py index 28bf6bc59b0..411778f21ae 100644 --- a/test/suite/test_split.py +++ b/test/suite/test_split.py @@ -32,9 +32,6 @@ import wiredtiger, wttest from wiredtiger import stat -from helper import confirm_empty,\ - key_populate, value_populate, simple_populate,\ - complex_populate, complex_value_populate # Test splits class test_split(wttest.WiredTigerTestCase): diff --git a/test/suite/test_stat01.py b/test/suite/test_stat01.py index 1ad51ee9882..2b04a3cbcd5 100644 --- a/test/suite/test_stat01.py +++ b/test/suite/test_stat01.py @@ -28,7 +28,7 @@ import helper, wiredtiger, wttest from wiredtiger import stat -from helper import key_populate, simple_populate +from wtdataset import SimpleDataSet, simple_key from wtscenario import make_scenarios # test_stat01.py @@ -89,8 +89,8 @@ class test_stat01(wttest.WiredTigerTestCase): # Test simple connection statistics. def test_basic_conn_stats(self): # Build an object and force some writes. - config = self.config + ',key_format=' + self.keyfmt - simple_populate(self, self.uri, config, 1000) + SimpleDataSet(self, self.uri, 1000, + config=self.config, key_format=self.keyfmt).populate() self.session.checkpoint(None) # See that we can get a specific stat value by its key and verify its @@ -113,7 +113,7 @@ class test_stat01(wttest.WiredTigerTestCase): value = "" for i in range(1, self.nentries): value = value + 1000 * "a" - cursor[key_populate(cursor, i)] = value + cursor[simple_key(cursor, i)] = value cursor.close() # Force the object to disk, otherwise we can't check the overflow count. @@ -140,9 +140,10 @@ class test_stat01(wttest.WiredTigerTestCase): # Test simple per-checkpoint statistics. def test_checkpoint_stats(self): + ds = SimpleDataSet(self, self.uri, self.nentries, + config=self.config, key_format=self.keyfmt) for name in ('first', 'second', 'third'): - config = self.config + ',key_format=' + self.keyfmt - helper.simple_populate(self, self.uri, config, self.nentries) + ds.populate() self.session.checkpoint('name=' + name) cursor = self.session.open_cursor( 'statistics:' + self.uri, None, 'checkpoint=' + name) diff --git a/test/suite/test_stat02.py b/test/suite/test_stat02.py index 047d2c74499..cecda7f1ddc 100644 --- a/test/suite/test_stat02.py +++ b/test/suite/test_stat02.py @@ -28,22 +28,21 @@ import itertools, wiredtiger, wttest from suite_subprocess import suite_subprocess +from wtdataset import SimpleDataSet, SimpleLSMDataSet, ComplexDataSet, \ + ComplexLSMDataSet from wtscenario import make_scenarios from wiredtiger import stat -from helper import complex_populate, complex_populate_lsm, simple_populate # test_stat02.py # Statistics cursor configurations. class test_stat_cursor_config(wttest.WiredTigerTestCase): pfx = 'test_stat_cursor_config' uri = [ - ('file', dict(uri='file:' + pfx, pop=simple_populate, cfg='')), - ('table', dict(uri='table:' + pfx, pop=simple_populate, cfg='')), - ('table-lsm', - dict(uri='table:' + pfx, pop=simple_populate, cfg=',type=lsm')), - ('complex', dict(uri='table:' + pfx, pop=complex_populate, cfg='')), - ('complex-lsm', - dict(uri='table:' + pfx, pop=complex_populate_lsm, cfg='')) + ('file', dict(uri='file:' + pfx, dataset=SimpleDataSet)), + ('table', dict(uri='table:' + pfx, dataset=SimpleDataSet)), + ('table-lsm', dict(uri='table:' + pfx, dataset=SimpleLSMDataSet)), + ('complex', dict(uri='table:' + pfx, dataset=ComplexDataSet)), + ('complex-lsm', dict(uri='table:' + pfx, dataset=ComplexLSMDataSet)) ] data_config = [ ('none', dict(data_config='none', ok=[])), @@ -66,7 +65,7 @@ class test_stat_cursor_config(wttest.WiredTigerTestCase): # For each database/cursor configuration, confirm the right combinations # succeed or fail. def test_stat_cursor_config(self): - self.pop(self, self.uri, 'key_format=S' + self.cfg, 100) + self.dataset(self, self.uri, 100).populate() config = 'statistics=(' if self.cursor_config != 'empty': config = config + self.cursor_config @@ -85,7 +84,7 @@ class test_stat_cursor_conn_clear(wttest.WiredTigerTestCase): def test_stat_cursor_conn_clear(self): uri = 'table:' + self.pfx - complex_populate(self, uri, 'key_format=S', 100) + ComplexDataSet(self, uri, 100).populate() # cursor_insert should clear # cache_bytes_dirty should not clear @@ -103,17 +102,17 @@ class test_stat_cursor_dsrc_clear(wttest.WiredTigerTestCase): pfx = 'test_stat_cursor_dsrc_clear' uri = [ - ('dsrc_clear_1', dict(uri='file:' + pfx, pop=simple_populate)), - ('dsrc_clear_2', dict(uri='table:' + pfx, pop=simple_populate)), - ('dsrc_clear_3', dict(uri='table:' + pfx, pop=complex_populate)), - ('dsrc_clear_4', dict(uri='table:' + pfx, pop=complex_populate_lsm)) + ('dsrc_clear_1', dict(uri='file:' + pfx, dataset=SimpleDataSet)), + ('dsrc_clear_2', dict(uri='table:' + pfx, dataset=SimpleDataSet)), + ('dsrc_clear_3', dict(uri='table:' + pfx, dataset=ComplexDataSet)), + ('dsrc_clear_4', dict(uri='table:' + pfx, dataset=ComplexLSMDataSet)) ] scenarios = make_scenarios(uri) conn_config = 'statistics=(all)' def test_stat_cursor_dsrc_clear(self): - self.pop(self, self.uri, 'key_format=S', 100) + self.dataset(self, self.uri, 100).populate() # cursor_insert should clear # @@ -132,17 +131,17 @@ class test_stat_cursor_fast(wttest.WiredTigerTestCase): pfx = 'test_stat_cursor_fast' uri = [ - ('fast_1', dict(uri='file:' + pfx, pop=simple_populate)), - ('fast_2', dict(uri='table:' + pfx, pop=simple_populate)), - ('fast_3', dict(uri='table:' + pfx, pop=complex_populate)), - ('fast_4', dict(uri='table:' + pfx, pop=complex_populate_lsm)) + ('fast_1', dict(uri='file:' + pfx, dataset=SimpleDataSet)), + ('fast_2', dict(uri='table:' + pfx, dataset=SimpleDataSet)), + ('fast_3', dict(uri='table:' + pfx, dataset=ComplexDataSet)), + ('fast_4', dict(uri='table:' + pfx, dataset=ComplexLSMDataSet)) ] scenarios = make_scenarios(uri) conn_config = 'statistics=(all)' def test_stat_cursor_fast(self): - self.pop(self, self.uri, 'key_format=S', 100) + self.dataset(self, self.uri, 100).populate() # A "fast" cursor shouldn't see the underlying btree statistics. # Check "fast" first, otherwise we get a copy of the statistics @@ -174,17 +173,17 @@ class test_stat_cursor_dsrc_error(wttest.WiredTigerTestCase): pfx = 'test_stat_cursor_dsrc_error' uri = [ - ('dsrc_error_1', dict(uri='file:' + pfx, pop=simple_populate)), - ('dsrc_error_2', dict(uri='table:' + pfx, pop=simple_populate)), - ('dsrc_error_3', dict(uri='table:' + pfx, pop=complex_populate)), - ('dsrc_error_4', dict(uri='table:' + pfx, pop=complex_populate_lsm)) + ('dsrc_error_1', dict(uri='file:' + pfx, dataset=SimpleDataSet)), + ('dsrc_error_2', dict(uri='table:' + pfx, dataset=SimpleDataSet)), + ('dsrc_error_3', dict(uri='table:' + pfx, dataset=ComplexDataSet)), + ('dsrc_error_4', dict(uri='table:' + pfx, dataset=ComplexLSMDataSet)) ] scenarios = make_scenarios(uri) conn_config = 'statistics=(all)' def test_stat_cursor_dsrc_error(self): - self.pop(self, self.uri, 'key_format=S', 100) + self.dataset(self, self.uri, 100).populate() args = ['all', 'fast'] for i in list(itertools.permutations(args, 2)): config = 'statistics=(' + i[0] + ',' + i[1] + ')' @@ -200,7 +199,7 @@ class test_stat_cursor_dsrc_cache_walk(wttest.WiredTigerTestCase): conn_config = 'statistics=(none)' def test_stat_cursor_dsrc_cache_walk(self): - simple_populate(self, self.uri, 'key_format=S', 100) + SimpleDataSet(self, self.uri, 100).populate() # Ensure that it's an error to get cache_walk stats if none is set msg = '/doesn\'t match the database statistics/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, diff --git a/test/suite/test_stat03.py b/test/suite/test_stat03.py index 1570ac4c9d8..7e5cf46ef13 100644 --- a/test/suite/test_stat03.py +++ b/test/suite/test_stat03.py @@ -30,10 +30,7 @@ import itertools, wiredtiger, wttest from suite_subprocess import suite_subprocess from wiredtiger import stat -from helper import complex_populate, complex_populate_lsm, simple_populate -from helper import key_populate, complex_value_populate, value_populate -from helper import complex_populate_colgroup_count, complex_populate_index_count -from helper import complex_populate_colgroup_name, complex_populate_index_name +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios # test_stat03.py @@ -41,14 +38,11 @@ from wtscenario import make_scenarios class test_stat_cursor_reset(wttest.WiredTigerTestCase): pfx = 'test_stat_cursor_reset' uri = [ - ('file-simple', - dict(uri='file:' + pfx, pop=simple_populate)), - ('table-simple', - dict(uri='table:' + pfx, pop=simple_populate)), - ('table-complex', - dict(uri='table:' + pfx, pop=complex_populate)), - ('table-complex-lsm', - dict(uri='table:' + pfx, pop=complex_populate_lsm)), + ('file-simple', dict(uri='file:' + pfx, dataset=SimpleDataSet)), + ('table-simple', dict(uri='table:' + pfx, dataset=SimpleDataSet)), + ('table-complex', dict(uri='table:' + pfx, dataset=ComplexDataSet)), + ('table-complex-lsm', dict(uri='table:' + pfx, + dataset=ComplexLSMDataSet)) ] scenarios = make_scenarios(uri) @@ -59,25 +53,23 @@ class test_stat_cursor_reset(wttest.WiredTigerTestCase): 'statistics:' + uri, None, 'statistics=(all)') def test_stat_cursor_reset(self): + n = 100 + ds = self.dataset(self, self.uri, n) + ds.populate() + # The number of btree_entries reported is influenced by the # number of column groups and indices. Each insert will have # a multiplied effect. - if self.pop == simple_populate: + if self.dataset == SimpleDataSet: multiplier = 1 # no declared colgroup is like one big colgroup else: - multiplier = complex_populate_colgroup_count() + \ - complex_populate_index_count() - n = 100 - self.pop(self, self.uri, 'key_format=S', n) + multiplier = ds.colgroup_count() + ds.index_count() statc = self.stat_cursor(self.uri) self.assertEqual(statc[stat.dsrc.btree_entries][2], n * multiplier) c = self.session.open_cursor(self.uri) - c.set_key(key_populate(c, 200)) - if self.pop == simple_populate: - c.set_value(value_populate(c, 200)) - else: - c.set_value(tuple(complex_value_populate(c, 200))) + c.set_key(ds.key(200)) + c.set_value(ds.value(200)) c.insert() # Test that cursor reset re-loads the values. @@ -88,14 +80,12 @@ class test_stat_cursor_reset(wttest.WiredTigerTestCase): # For applications with indices and/or column groups, verify # that there is a way to count the base number of entries. - if self.pop != simple_populate: + if self.dataset != SimpleDataSet: statc.close() - statc = self.stat_cursor( - complex_populate_index_name(self, self.uri, 0)) + statc = self.stat_cursor(ds.index_name(0)) self.assertEqual(statc[stat.dsrc.btree_entries][2], n) statc.close() - statc = self.stat_cursor( - complex_populate_colgroup_name(self, self.uri, 0)) + statc = self.stat_cursor(ds.colgroup_name(0)) self.assertEqual(statc[stat.dsrc.btree_entries][2], n) statc.close() diff --git a/test/suite/test_stat05.py b/test/suite/test_stat05.py index 62562f78ed6..ef4d65e85e4 100644 --- a/test/suite/test_stat05.py +++ b/test/suite/test_stat05.py @@ -30,8 +30,7 @@ import itertools, wiredtiger, wttest from suite_subprocess import suite_subprocess from wtscenario import make_scenarios from wiredtiger import stat -from helper import complex_populate, complex_populate_lsm, simple_populate -from helper import complex_value_populate, key_populate, value_populate +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet # test_stat05.py # Statistics cursor using size only @@ -40,17 +39,17 @@ class test_stat_cursor_config(wttest.WiredTigerTestCase): conn_config = 'statistics=(fast)' uri = [ - ('file', dict(uri='file:' + pfx, pop=simple_populate, cfg='')), - ('table', dict(uri='table:' + pfx, pop=simple_populate, cfg='')), - ('inmem', dict(uri='table:' + pfx, pop=simple_populate, cfg='', + ('file', dict(uri='file:' + pfx, dataset=SimpleDataSet, cfg='')), + ('table', dict(uri='table:' + pfx, dataset=SimpleDataSet, cfg='')), + ('inmem', dict(uri='table:' + pfx, dataset=SimpleDataSet, cfg='', conn_config = 'in_memory,statistics=(fast)')), - ('table-lsm', dict(uri='table:' + pfx, pop=simple_populate, - cfg=',type=lsm,lsm=(chunk_size=1MB,merge_min=2)', + ('table-lsm', dict(uri='table:' + pfx, dataset=SimpleDataSet, + cfg='lsm=(chunk_size=1MB,merge_min=2)', conn_config = 'statistics=(fast),eviction_dirty_target=99,eviction_dirty_trigger=99')), - ('complex', dict(uri='table:' + pfx, pop=complex_populate, cfg='')), + ('complex', dict(uri='table:' + pfx, dataset=ComplexDataSet, cfg='')), ('complex-lsm', - dict(uri='table:' + pfx, pop=complex_populate_lsm, - cfg=',lsm=(chunk_size=1MB,merge_min=2)', + dict(uri='table:' + pfx, dataset=ComplexLSMDataSet, + cfg='lsm=(chunk_size=1MB,merge_min=2)', conn_config = 'statistics=(fast),eviction_dirty_target=99,eviction_dirty_trigger=99')), ] @@ -68,17 +67,14 @@ class test_stat_cursor_config(wttest.WiredTigerTestCase): # the cursor open succeeds. Insert enough data that LSM tables to need to # switch and merge. def test_stat_cursor_size(self): - self.pop(self, self.uri, 'key_format=S' + self.cfg, 100) + ds = self.dataset(self, self.uri, 100, config=self.cfg) + ds.populate() self.openAndWalkStatCursor() cursor = self.session.open_cursor(self.uri, None) for i in range(100, 40000 + 1): if i % 100 == 0: self.openAndWalkStatCursor() - if self.pop == simple_populate: - cursor[key_populate(cursor, i)] = value_populate(cursor, i) - else: - cursor[key_populate(cursor, i)] = \ - tuple(complex_value_populate(cursor, i)) + cursor[ds.key(i)] = ds.value(i) cursor.close() self.openAndWalkStatCursor() diff --git a/test/suite/test_truncate01.py b/test/suite/test_truncate01.py index 37cd1f12c79..2319eeddbef 100644 --- a/test/suite/test_truncate01.py +++ b/test/suite/test_truncate01.py @@ -31,9 +31,8 @@ # import wiredtiger, wttest -from helper import confirm_empty,\ - key_populate, value_populate, simple_populate,\ - complex_populate, complex_value_populate +from helper import confirm_empty +from wtdataset import SimpleDataSet, ComplexDataSet from wtscenario import make_scenarios # Test truncation arguments. @@ -49,7 +48,7 @@ class test_truncate_arguments(wttest.WiredTigerTestCase): # either cursor specified, expect errors. def test_truncate_bad_args(self): uri = self.type + self.name - simple_populate(self, uri, 'key_format=S', 100) + SimpleDataSet(self, uri, 100).populate() msg = '/either a URI or start/stop cursors/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.truncate(None, None, None, None), msg) @@ -64,11 +63,12 @@ class test_truncate_arguments(wttest.WiredTigerTestCase): uri = self.type + self.name msg = '/requires key be set/' - simple_populate(self, uri, 'key_format=S', 100) + ds = SimpleDataSet(self, uri, 100) + ds.populate() c1 = self.session.open_cursor(uri, None) c2 = self.session.open_cursor(uri, None) - c2.set_key(key_populate(c2, 10)) + c2.set_key(ds.key(10)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.truncate(None, c1, c2, None), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, @@ -89,13 +89,13 @@ class test_truncate_uri(wttest.WiredTigerTestCase): uri = self.type + self.name # A simple, one-file file or table object. - simple_populate(self, uri, 'key_format=S', 100) + SimpleDataSet(self, uri, 100).populate() self.session.truncate(uri, None, None, None) confirm_empty(self, uri) self.session.drop(uri, None) if self.type == "table:": - complex_populate(self, uri, 'key_format=S', 100) + ComplexDataSet(self, uri, 100).populate() self.session.truncate(uri, None, None, None) confirm_empty(self, uri) self.session.drop(uri, None) @@ -118,16 +118,17 @@ class test_truncate_cursor_order(wttest.WiredTigerTestCase): # Test an illegal order, then confirm that equal cursors works. def test_truncate_cursor_order(self): uri = self.type + self.name - simple_populate(self, uri, 'key_format=' + self.keyfmt, 100) + ds = SimpleDataSet(self, uri, 100, key_format=self.keyfmt) + ds.populate() c1 = self.session.open_cursor(uri, None) c2 = self.session.open_cursor(uri, None) - c1.set_key(key_populate(c1, 20)) - c2.set_key(key_populate(c2, 10)) + c1.set_key(ds.key(20)) + c2.set_key(ds.key(10)) msg = '/the start cursor position is after the stop cursor position/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.truncate(None, c1, c2, None), msg) - c2.set_key(key_populate(c2, 20)) + c2.set_key(ds.key(20)) self.session.truncate(None, c1, c2, None) # Test truncation of cursors past the end of the object. @@ -150,22 +151,24 @@ class test_truncate_cursor_end(wttest.WiredTigerTestCase): uri = self.type + self.name # A simple, one-file file or table object. - simple_populate(self, uri, 'key_format=' + self.keyfmt, 100) + ds = SimpleDataSet(self, uri, 100, key_format=self.keyfmt) + ds.populate() c1 = self.session.open_cursor(uri, None) - c1.set_key(key_populate(c1, 1000)) + c1.set_key(ds.key(1000)) c2 = self.session.open_cursor(uri, None) - c2.set_key(key_populate(c2, 2000)) + c2.set_key(ds.key(2000)) self.session.truncate(None, c1, c2, None) self.assertEquals(c1.close(), 0) self.assertEquals(c2.close(), 0) self.session.drop(uri) if self.type == "table:": - complex_populate(self, uri, 'key_format=' + self.keyfmt, 100) + ds = ComplexDataSet(self, uri, 100, key_format=self.keyfmt) + ds.populate() c1 = self.session.open_cursor(uri, None) - c1.set_key(key_populate(c1, 1000)) + c1.set_key(ds.key(1000)) c2 = self.session.open_cursor(uri, None) - c2.set_key(key_populate(c2, 2000)) + c2.set_key(ds.key(2000)) self.session.truncate(None, c1, c2, None) self.assertEquals(c1.close(), 0) self.assertEquals(c2.close(), 0) @@ -179,13 +182,12 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): # The underlying table routines don't easily support 8t value types, limit # those tests to file objects. types = [ - ('file', dict(type='file:',\ - config='allocation_size=512,leaf_page_max=512,key_format=')), - ('file8t', dict(type='file:',\ - config='allocation_size=512,\ - leaf_page_max=512,value_format=8t,key_format=')), - ('table', dict(type='table:',\ - config='allocation_size=512,leaf_page_max=512,key_format=')), + ('file', dict(type='file:', valuefmt='S', + config='allocation_size=512,leaf_page_max=512')), + ('file8t', dict(type='file:', valuefmt='8t', + config='allocation_size=512,leaf_page_max=512')), + ('table', dict(type='table:', valuefmt='S', + config='allocation_size=512,leaf_page_max=512')), ] keyfmt = [ ('integer', dict(keyfmt='i')), @@ -204,18 +206,18 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): scenarios = make_scenarios(types, keyfmt, size, reopen) # Set a cursor key. - def cursorKey(self, uri, key): + def cursorKey(self, ds, uri, key): if key == -1: return None cursor = self.session.open_cursor(uri, None) - cursor.set_key(key_populate(cursor, key)) + cursor.set_key(ds.key(key)) return cursor # Truncate a range using cursors, and check the results. - def truncateRangeAndCheck(self, uri, begin, end, expected): + def truncateRangeAndCheck(self, ds, uri, begin, end, expected): self.pr('truncateRangeAndCheck: ' + str(begin) + ',' + str(end)) - cur1 = self.cursorKey(uri, begin) - cur2 = self.cursorKey(uri, end) + cur1 = self.cursorKey(ds, uri, begin) + cur2 = self.cursorKey(ds, uri, end) self.session.truncate(None, cur1, cur2, None) if not cur1: begin = 1 @@ -234,7 +236,7 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): # Check the expected values against the object. cursor = self.session.open_cursor(uri, None) for i in range(begin, end + 1): - expected[key_populate(cursor, i)] = [0] + expected[ds.key(i)] = [0] for k, v in expected.iteritems(): cursor.set_key(k) if v == [0] and \ @@ -306,6 +308,10 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): (self.skip, self.skip) # middle to same middle ] + # Using this data set to compare only, it doesn't create or populate. + ds = SimpleDataSet(self, uri, 0, key_format=self.keyfmt, + value_format=self.valuefmt, config=self.config) + # Build the layout we're going to test total = self.nentries for begin_skipped,begin_insert,end_skipped,end_insert in layout: @@ -332,15 +338,16 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): # Create the object. self.session.create( - uri, 'value_format=S,' + self.config + self.keyfmt) + uri, self.config + ',key_format=' + self.keyfmt + + ',value_format=' + self.valuefmt) # Insert the records that aren't skipped or inserted. start = begin_skipped + begin_insert stop = self.nentries - (end_skipped + end_insert) cursor = self.session.open_cursor(uri, None) for i in range(start + 1, stop + 1): - k = key_populate(cursor, i) - v = value_populate(cursor, i) + k = ds.key(i) + v = ds.value(i) cursor[k] = v expected[k] = [v] cursor.close() @@ -355,33 +362,33 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): start = 0 for i in range(0, begin_skipped): start += 1 - k = key_populate(cursor, start) + k = ds.key(start) expected[k] = [0] # Optionally insert leading records. for i in range(0, begin_insert): start += 1 - k = key_populate(cursor, start) - v = value_populate(cursor, start) + k = ds.key(start) + v = ds.value(start) cursor[k] = v expected[k] = [v] # Optionally insert trailing skipped records. for i in range(0, end_skipped): stop += 1 - k = key_populate(cursor, stop) + k = ds.key(stop) expected[k] = [0] # Optionally insert trailing records. for i in range(0, end_insert): stop += 1 - k = key_populate(cursor, stop) - v = value_populate(cursor, stop) + k = ds.key(stop) + v = ds.value(stop) cursor[k] = v expected[k] = [v] cursor.close() - self.truncateRangeAndCheck(uri, begin, end, expected) + self.truncateRangeAndCheck(ds, uri, begin, end, expected) self.session.drop(uri, None) # Test truncation of complex tables using cursors. We can't do the kind of @@ -423,16 +430,16 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): ''' # Create the object. - complex_populate( - self, uri, self.config + self.keyfmt, self.nentries) + ds = ComplexDataSet(self, uri, self.nentries, + config=self.config, key_format=self.keyfmt) + ds.populate() # Build a dictionary of what the object should look like for # later comparison cursor = self.session.open_cursor(uri, None) expected = {} for i in range(1, self.nentries + 1): - expected[key_populate(cursor, i)] = \ - complex_value_populate(cursor, i) + expected[ds.key(i)] = ds.comparable_value(i) cursor.close() # Optionally close and re-open the object to get a disk image @@ -440,7 +447,7 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): if self.reopen: self.reopen_conn() - self.truncateRangeAndCheck(uri, begin, end, expected) + self.truncateRangeAndCheck(ds, uri, begin, end, expected) self.session.drop(uri, None) if __name__ == '__main__': diff --git a/test/suite/test_truncate02.py b/test/suite/test_truncate02.py index 7184cd67242..73fed362354 100644 --- a/test/suite/test_truncate02.py +++ b/test/suite/test_truncate02.py @@ -31,7 +31,7 @@ # import wiredtiger, wttest -from helper import key_populate, value_populate, simple_populate +from wtdataset import SimpleDataSet from wtscenario import make_scenarios # test_truncate_fast_delete @@ -46,8 +46,7 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase): # of individual pages in the file. types = [ ('file', dict(type='file:', config=\ - 'allocation_size=512,leaf_page_max=512,' +\ - 'value_format=S,key_format=')), + 'allocation_size=512,leaf_page_max=512')), ] # This is all about testing the btree layer, not the schema layer, test @@ -123,15 +122,17 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase): ''' # Create the object. - simple_populate(self, uri, self.config + self.keyfmt, self.nentries) + ds = SimpleDataSet(self, uri, self.nentries, + config=self.config, key_format=self.keyfmt) + ds.populate() # Optionally add a few overflow records so we block fast delete on # those pages. if self.overflow: cursor = self.session.open_cursor(uri, None, 'overwrite=false') for i in range(1, self.nentries, 3123): - cursor.set_key(key_populate(cursor, i)) - cursor.set_value(value_populate(cursor, i)) + cursor.set_key(ds.key(i)) + cursor.set_value(ds.value(i)) cursor.update() cursor.close() @@ -143,21 +144,21 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None, 'overwrite=false') if self.readbefore: for i in range(1, self.nentries, 737): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) cursor.search() if self.writebefore: for i in range(1, self.nentries, 988): - cursor.set_key(key_populate(cursor, i)) - cursor.set_value(value_populate(cursor, i + 100)) + cursor.set_key(ds.key(i)) + cursor.set_value(ds.value(i + 100)) cursor.update() cursor.close() # Begin a transaction, and truncate a big range of rows. self.session.begin_transaction(None) start = self.session.open_cursor(uri, None) - start.set_key(key_populate(start, 10)) + start.set_key(ds.key(10)) end = self.session.open_cursor(uri, None) - end.set_key(key_populate(end, self.nentries - 10)) + end.set_key(ds.key(self.nentries - 10)) self.session.truncate(None, start, end, None) start.close() end.close() @@ -167,12 +168,12 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None, 'overwrite=false') if self.readafter: for i in range(1, self.nentries, 1123): - cursor.set_key(key_populate(cursor, i)) + cursor.set_key(ds.key(i)) cursor.search() if self.writeafter: for i in range(1, self.nentries, 621): - cursor.set_key(key_populate(cursor, i)) - cursor.set_value(value_populate(cursor, i + 100)) + cursor.set_key(ds.key(i)) + cursor.set_value(ds.value(i + 100)) cursor.update() cursor.close() diff --git a/test/suite/test_truncate03.py b/test/suite/test_truncate03.py index 49044ac5690..2b4628950b3 100644 --- a/test/suite/test_truncate03.py +++ b/test/suite/test_truncate03.py @@ -30,7 +30,7 @@ # session level operations on tables import wiredtiger, wttest -from helper import key_populate, simple_populate, value_populate +from wtdataset import SimpleDataSet # A standalone test case that exercises address-deleted cells. class test_truncate_address_deleted(wttest.WiredTigerTestCase): @@ -39,15 +39,15 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase): # Use a small page size and lots of keys because we want to create lots # of individual pages in the file. nentries = 10000 - config = 'allocation_size=512,' +\ - 'leaf_page_max=512,value_format=S,key_format=S' + config = 'allocation_size=512,leaf_page_max=512' # address_deleted routine: # Create an object that has a bunch of address-deleted cells on disk. # Recover the object, and turn the address-deleted cells into free pages. def address_deleted(self): # Create the object, force it to disk, and verify the object. - simple_populate(self, self.uri, self.config, self.nentries) + ds = SimpleDataSet(self, self.uri, self.nentries, config=self.config) + ds.populate() self.reopen_conn() self.session.verify(self.uri) @@ -59,9 +59,9 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase): # Truncate a big range of rows; the leaf pages aren't in memory, so # leaf page references will be deleted without being read. start = self.session.open_cursor(self.uri, None) - start.set_key(key_populate(start, 10)) + start.set_key(ds.key(10)) end = self.session.open_cursor(self.uri, None) - end.set_key(key_populate(end, self.nentries - 10)) + end.set_key(ds.key(self.nentries - 10)) self.session.truncate(None, start, end, None) self.assertEqual(start.close(), 0) self.assertEqual(end.close(), 0) @@ -78,7 +78,7 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase): # we get a good look at all the internal pages and the address-deleted # cells. cursor = self.session.open_cursor(self.uri, None) - cursor.set_key(key_populate(cursor, 5)) + cursor.set_key(ds.key(5)) cursor.set_value("changed value") self.assertEqual(cursor.update(), 0) cursor.reset() @@ -88,12 +88,13 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase): # Checkpoint, freeing the pages. self.session.checkpoint() + return ds # Test object creation, recovery, and conversion of address-deleted cells # into free pages. def test_truncate_address_deleted_free(self): # Create the object on disk. - self.address_deleted() + ds = self.address_deleted() # That's all just verify that worked. self.session.verify(self.uri) @@ -103,7 +104,7 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase): # empty pages by a reader after the underlying leaf pages are removed. def test_truncate_address_deleted_empty_page(self): # Create the object on disk. - self.address_deleted() + ds = self.address_deleted() # Open a cursor and update pages in the middle of the range, forcing # creation of empty pages (once the underlying leaf page is freed, we @@ -111,11 +112,11 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase): # the value as well as write the page and get it back. cursor = self.session.open_cursor(self.uri, None) for i in range(3000, 7000, 137): - k = key_populate(cursor, i) + k = ds.key(i) v = 'changed value: ' + str(i) cursor[k] = v for i in range(3000, 7000, 137): - k = key_populate(cursor, i) + k = ds.key(i) v = 'changed value: ' + str(i) cursor.set_key(k) self.assertEqual(cursor.search(), 0) @@ -128,7 +129,7 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri, None) for i in range(3000, 7000, 137): - k = key_populate(cursor, i) + k = ds.key(i) v = 'changed value: ' + str(i) cursor.set_key(k) self.assertEqual(cursor.search(), 0) diff --git a/test/suite/test_txn06.py b/test/suite/test_txn06.py index e4636e40e2e..2bff97f6aac 100644 --- a/test/suite/test_txn06.py +++ b/test/suite/test_txn06.py @@ -30,7 +30,7 @@ # Transactions: test long-running snapshots from suite_subprocess import suite_subprocess -from helper import simple_populate +from wtdataset import SimpleDataSet import wiredtiger, wttest class test_txn06(wttest.WiredTigerTestCase, suite_subprocess): @@ -47,7 +47,7 @@ class test_txn06(wttest.WiredTigerTestCase, suite_subprocess): def test_long_running(self): # Populate a table - simple_populate(self, self.source_uri, 'key_format=S', self.nrows) + SimpleDataSet(self, self.source_uri, self.nrows).populate() # Now scan the table and copy the rows into a new table c_src = self.session.create(self.uri, "key_format=S") diff --git a/test/suite/test_txn11.py b/test/suite/test_txn11.py index dfb02799434..147bf3a76c0 100644 --- a/test/suite/test_txn11.py +++ b/test/suite/test_txn11.py @@ -31,7 +31,7 @@ import fnmatch, os, time from suite_subprocess import suite_subprocess -from helper import simple_populate +from wtdataset import SimpleDataSet import wttest class test_txn11(wttest.WiredTigerTestCase, suite_subprocess): @@ -64,7 +64,7 @@ class test_txn11(wttest.WiredTigerTestCase, suite_subprocess): def test_ops(self): # Populate a table - simple_populate(self, self.source_uri, 'key_format=S', self.nrows) + SimpleDataSet(self, self.source_uri, self.nrows).populate() # Run forced checkpoints self.run_checkpoints() diff --git a/test/suite/test_upgrade.py b/test/suite/test_upgrade.py index 1c2e3f6cda8..4eb6a9e6817 100644 --- a/test/suite/test_upgrade.py +++ b/test/suite/test_upgrade.py @@ -28,7 +28,7 @@ import os, time import wiredtiger, wttest -from helper import complex_populate, simple_populate +from wtdataset import SimpleDataSet, ComplexDataSet from wtscenario import make_scenarios # test_upgrade.py @@ -42,9 +42,9 @@ class test_upgrade(wttest.WiredTigerTestCase): ]) # Populate an object, then upgrade it. - def upgrade(self, populate, with_cursor): + def upgrade(self, dataset, with_cursor): uri = self.uri + self.name - populate(self, uri, 'key_format=S', 10) + dataset(self, uri, 10).populate() # Open cursors should cause failure. if with_cursor: @@ -59,13 +59,13 @@ class test_upgrade(wttest.WiredTigerTestCase): # Test upgrade of an object. def test_upgrade(self): # Simple file or table object. - self.upgrade(simple_populate, False) - self.upgrade(simple_populate, True) + self.upgrade(SimpleDataSet, False) + self.upgrade(SimpleDataSet, True) # A complex, multi-file table object. if self.uri == "table:": - self.upgrade(complex_populate, False) - self.upgrade(complex_populate, True) + self.upgrade(ComplexDataSet, False) + self.upgrade(ComplexDataSet, True) if __name__ == '__main__': wttest.run() diff --git a/test/suite/test_util02.py b/test/suite/test_util02.py index 13f52d6aeea..7aa24605ed1 100644 --- a/test/suite/test_util02.py +++ b/test/suite/test_util02.py @@ -29,8 +29,8 @@ import string, os import wiredtiger, wttest from suite_subprocess import suite_subprocess +from wtdataset import ComplexDataSet from wtscenario import make_scenarios -from helper import complex_populate # test_util02.py # Utilities: wt load @@ -169,7 +169,7 @@ class test_load_commandline(wttest.WiredTigerTestCase, suite_subprocess): def load_commandline(self, args, fail): errfile= "errfile" - complex_populate(self, self.uri, "key_format=S,value_format=S", 20) + ComplexDataSet(self, self.uri, 20).populate() self.runWt(["dump", self.uri], outfilename="dump.out") loadargs = ["load", "-f", "dump.out"] + args self.runWt(loadargs, errfilename=errfile, failure=fail) diff --git a/test/suite/test_util13.py b/test/suite/test_util13.py index 9804dc700ba..7890d4fdb1b 100644 --- a/test/suite/test_util13.py +++ b/test/suite/test_util13.py @@ -28,11 +28,9 @@ import os, re, string from suite_subprocess import suite_subprocess -import itertools, wiredtiger, wttest +from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet +import wiredtiger, wttest -from helper import complex_populate_cgconfig, complex_populate_cgconfig_lsm -from helper import simple_populate -from helper import complex_populate_check, simple_populate_check from wtscenario import make_scenarios # test_util13.py @@ -52,25 +50,23 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): # Select table configuration settings that are not the default. # types = [ - ('file-simple', dict(uri='file:' + pfx, pop=simple_populate, - populate_check=simple_populate_check, - table_config='prefix_compression_min=3', cfg='')), - ('lsm-simple', dict(uri='lsm:' + pfx, pop=simple_populate, - populate_check=simple_populate_check, - table_config='lsm=(bloom_bit_count=29)', - cfg='bloom_bit_count=29')), - ('table-simple', dict(uri='table:' + pfx, pop=simple_populate, - populate_check=simple_populate_check, - table_config='split_pct=50', cfg='')), + ('file-simple', dict(uri='file:' + pfx, dataset=SimpleDataSet, + table_config='prefix_compression_min=3', cfg='', + cg_config='')), + ('lsm-simple', dict(uri='lsm:' + pfx, dataset=SimpleDataSet, + table_config='lsm=(bloom_bit_count=29)', cfg='bloom_bit_count=29', + cg_config='')), + ('table-simple', dict(uri='table:' + pfx, dataset=SimpleDataSet, + table_config='split_pct=50', cfg='', + cg_config='')), ('table-complex', - dict(uri='table:' + pfx, pop=complex_populate_cgconfig, - populate_check=complex_populate_check, - table_config='allocation_size=512B', cfg='')), + dict(uri='table:' + pfx, dataset=ComplexDataSet, + table_config='allocation_size=512B', cfg='', + cg_config='allocation_size=512B')), ('table-complex-lsm', - dict(uri='table:' + pfx, pop=complex_populate_cgconfig_lsm, - populate_check=complex_populate_check, - table_config='lsm=(merge_max=5)', - cfg='merge_max=5')), + dict(uri='table:' + pfx, dataset=ComplexLSMDataSet, + table_config='lsm=(merge_max=5)', cfg='merge_max=5', + cg_config='lsm=(merge_max=5)')) ] scenarios = make_scenarios(types) @@ -83,7 +79,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): #print "compare_config Expected config " #print expected_cfg cfg_orig = actual_cfg - if self.pop != simple_populate: + if self.dataset != SimpleDataSet: # # If we have a complex config, strip out the colgroups and # columns from the config. Doing so allows us to keep the @@ -127,7 +123,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): break return True - def load_recheck(self, expect_subset, dump_out): + def load_recheck(self, ds, expect_subset, dump_out): newdump = "newdump.out" os.mkdir(self.dir) self.runWt(['-h', self.dir, 'load', '-f', dump_out]) @@ -135,7 +131,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): conn = self.wiredtiger_open(self.dir) session = conn.open_session() cursor = session.open_cursor(self.uri, None, None) - self.populate_check + ds.check() conn.close() dumpargs = ["-h"] dumpargs.append(self.dir) @@ -150,8 +146,9 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): # The number of btree_entries reported is influenced by the # number of column groups and indices. Each insert will have # a multiplied effect. - self.pop(self, self.uri, - 'key_format=S,value_format=S,' + self.table_config, self.nentries) + ds = self.dataset(self, self.uri, self.nentries, + config=self.table_config, cgconfig=self.cg_config) + ds.populate() ver = wiredtiger.wiredtiger_version() verstring = str(ver[1]) + '.' + str(ver[2]) + '.' + str(ver[3]) @@ -166,7 +163,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): expectout.write('Header\n') expectout.write(self.uri + '\n') # Check the config on the colgroup itself for complex tables. - if self.pop != simple_populate: + if self.dataset != SimpleDataSet: expectout.write('key_format=S\n') expectout.write('colgroup:' + self.pfx + ':cgroup1\n') if self.cfg == '': @@ -182,7 +179,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): self.runWt(dumpargs, outfilename=outfile) self.assertTrue(self.compare_files(expectfile, outfile)) - self.assertTrue(self.load_recheck(expectfile, outfile)) + self.assertTrue(self.load_recheck(ds, expectfile, outfile)) if __name__ == '__main__': wttest.run() diff --git a/test/suite/wtdataset.py b/test/suite/wtdataset.py new file mode 100644 index 00000000000..cb77735510c --- /dev/null +++ b/test/suite/wtdataset.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +class BaseDataSet(object): + """ + BaseDataSet is an abstract base class for other *DataSet classes. + An object of this type should not be created directly. These classes + represent test data sets that can be used to populate tables and + to check the contents of existing tables. + """ + def __init__(self, testcase, uri, rows, **kwargs): + self.testcase = testcase + self.uri = uri + self.rows = rows + self.key_format = kwargs.get('key_format', 'S') + self.value_format = kwargs.get('value_format', 'S') + self.config = kwargs.get('config', '') + + def create(self): + self.testcase.session.create(self.uri, 'key_format=' + self.key_format + + ',value_format=' + self.value_format + + ',' + self.config) + + def fill(self): + c = self.testcase.session.open_cursor(self.uri, None) + for i in range(1, self.rows + 1): + c[self.key(i)] = self.value(i) + c.close() + + def postfill(self): + pass + + @classmethod + def is_lsm(cls): + return False + + def populate(self): + self.testcase.pr('populate: ' + self.uri + ' with ' + + str(self.rows) + ' rows') + self.create() + self.fill() + self.postfill() + + # Create a key for a Simple or Complex data set. + @staticmethod + def key_by_format(i, key_format): + if key_format == 'i' or key_format == 'r' or key_format == 'u': + return i + elif key_format == 'S': + return str('%015d' % i) + else: + raise AssertionError( + 'key: object has unexpected format: ' + key_format) + + # Create a value for a Simple data set. + @staticmethod + def value_by_format(i, value_format): + if value_format == 'i' or value_format == 'r' or value_format == 'u': + return i + elif value_format == 'S': + return str(i) + ': abcdefghijklmnopqrstuvwxyz' + elif value_format == '8t': + value = ( + 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xaa, 0xab, + 0xac, 0xad, 0xae, 0xaf, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, + 0xb7, 0xb8, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf) + return value[i % len(value)] + else: + raise AssertionError( + 'value: object has unexpected format: ' + + value_format) + + # Create a key for this data set. Simple and Complex data sets have + # the same key space. + def key(self, i): + return BaseDataSet.key_by_format(i, self.key_format) + + def check(self): + self.testcase.pr('check: ' + self.uri) + cursor = self.testcase.session.open_cursor(self.uri, None) + self.check_cursor(cursor) + cursor.close() + +class SimpleDataSet(BaseDataSet): + """ + SimpleDataSet creates a table with a single key and value that is + populated with predefined data, up to the requested number of rows. + key_format and value_format may be set in the constructor to + override the simple string defaults. + """ + def __init__(self, testcase, uri, rows, **kwargs): + super(SimpleDataSet, self).__init__(testcase, uri, rows, **kwargs) + + def value(self, i): + return BaseDataSet.value_by_format(i, self.value_format) + + def check_cursor(self, cursor): + i = 0 + for key, val in cursor: + i += 1 + self.testcase.assertEqual(key, self.key(i)) + if cursor.value_format == '8t' and val == 0: # deleted + continue + self.testcase.assertEqual(val, self.value(i)) + self.testcase.assertEqual(i, self.rows) + +class SimpleLSMDataSet(SimpleDataSet): + """ + SimpleLSMDataSet is identical to SimpleDataSet, but using LSM files + via the type=lsm configuration. + """ + def __init__(self, testcase, uri, rows, **kwargs): + kwargs['config'] = kwargs.get('config', '') + ',type=lsm' + super(SimpleLSMDataSet, self).__init__( + testcase, uri, rows, **kwargs) + + @classmethod + def is_lsm(cls): + return True + +class SimpleIndexDataSet(SimpleDataSet): + """ + SimpleIndexDataSet is identical to SimpleDataSet, adding one index + that maps the value to the key. + """ + def __init__(self, testcase, uri, rows, **kwargs): + self.indexname = 'index:' + uri.split(":")[1] + ':index1' + self.origconfig = kwargs.get('config', '') + kwargs['config'] = self.origconfig + ',columns=(key0,value0)' + super(SimpleIndexDataSet, self).__init__( + testcase, uri, rows, **kwargs) + + def create(self): + super(SimpleIndexDataSet, self).create() + self.testcase.session.create(self.indexname, 'columns=(value0,key0),' + + self.origconfig) + + def check(self): + BaseDataSet.check(self) + + # Check values in the index. + idxcursor = self.testcase.session.open_cursor(self.indexname) + for i in range(1, self.rows + 1): + k = self.key(i) + v = self.value(i) + ik = (v, k) # The index key is columns=(v,k). + self.testcase.assertEqual(v, idxcursor[ik]) + idxcursor.close() + +class SimpleIndexLSMDataSet(SimpleIndexDataSet): + """ + SimpleIndexLSMDataSet is identical to SimpleIndexDataSet, but + using LSM files. + """ + def __init__(self, testcase, uri, rows, **kwargs): + kwargs['config'] = kwargs.get('config', '') + ',type=lsm' + super(SimpleIndexLSMDataSet, self).__init__( + testcase, uri, rows, **kwargs) + + @classmethod + def is_lsm(cls): + return True + +class ComplexDataSet(BaseDataSet): + """ + ComplexDataSet populates a table with a mixed set of indices + and column groups. Some indices are created before the + table is populated, some after. + """ + def __init__(self, testcase, uri, rows, **kwargs): + self.indexlist = [ + ['indx1', 'column2'], + ['indx2', 'column3'], + ['indx3', 'column4'], + ['indx4', 'column2,column4'], + ['indx5', 'column3,column5'], + ['indx6', 'column3,column5,column4']] + self.cglist = [ + ['cgroup1', 'column2'], + ['cgroup2', 'column3'], + ['cgroup3', 'column4'], + ['cgroup4', 'column2,column3'], + ['cgroup5', 'column3,column4'], + ['cgroup6', 'column2,column4,column5']] + self.cgconfig = kwargs.pop('cgconfig', '') + config = kwargs.get('config', '') + config += ',columns=(record,column2,column3,column4,column5),' + \ + 'colgroups=(cgroup1,cgroup2,cgroup3,cgroup4,cgroup5,cgroup6)' + kwargs['config'] = config + kwargs['value_format'] = 'SiSS' + super(ComplexDataSet, self).__init__(testcase, uri, rows, **kwargs) + + def create(self): + config = 'key_format=' + self.key_format + \ + ',value_format=' + self.value_format + ',' + self.config + session = self.testcase.session + ##self.testcase.tty('URI=' + self.uri + 'CONFIG=' + config) + session.create(self.uri, config) + tablepart = self.uri.split(":")[1] + ':' + for cg in self.cglist: + session.create('colgroup:' + tablepart + cg[0], + ',columns=(' + cg[1] + '),' + self.cgconfig) + for index in self.indexlist[0:4]: + session.create('index:' + tablepart + index[0], + ',columns=(' + index[1] + '),' + self.config) + + def postfill(self): + # add some indices after filling the table + tablepart = self.uri.split(":")[1] + ':' + session = self.testcase.session + for index in self.indexlist[4:]: + session.create('index:' + tablepart + index[0], + ',columns=(' + index[1] + ')') + + def colgroup_count(self): + return len(self.cglist) + + def colgroup_name(self, i): + return 'colgroup:' + self.uri.split(":")[1] + ':' + self.cglist[i][0] + + def index_count(self): + return len(self.indexlist) + + def index_name(self, i): + return 'index:' + self.uri.split(":")[1] + ':' + self.indexlist[i][0] + + # A value suitable for checking the value returned by a cursor, as + # cursor.get_value() returns a list. + def comparable_value(self, i): + return [str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%26], + i, + str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%23], + str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%18]] + + # A value suitable for assigning to a cursor, as + # cursor.set_value() expects a tuple when there it is used with + # a single argument and the value is composite. + def value(self, i): + return tuple(self.comparable_value(i)) + + def check_cursor(self, cursor): + i = 0 + for key, s1, i2, s3, s4 in cursor: + i += 1 + self.testcase.assertEqual(key, self.key(i)) + v = self.value(i) + self.testcase.assertEqual(s1, v[0]) + self.testcase.assertEqual(i2, v[1]) + self.testcase.assertEqual(s3, v[2]) + self.testcase.assertEqual(s4, v[3]) + self.testcase.assertEqual(i, self.rows) + +class ComplexLSMDataSet(ComplexDataSet): + """ + ComplexLSMDataSet is identical to ComplexDataSet, but using LSM files. + """ + def __init__(self, testcase, uri, rows, **kwargs): + kwargs['cgconfig'] = kwargs.get('cgconfig', '') + ',type=lsm' + super(ComplexLSMDataSet, self).__init__( + testcase, uri, rows, **kwargs) + + @classmethod + def is_lsm(cls): + return True + +# create a key based on a cursor as a shortcut to creating a SimpleDataSet +def simple_key(cursor, i): + return BaseDataSet.key_by_format(i, cursor.key_format) + +# create a value based on a cursor as a shortcut to creating a SimpleDataSet +def simple_value(cursor, i): + return BaseDataSet.value_by_format(i, cursor.value_format) + +# create a key based on a cursor as a shortcut to creating a ComplexDataSet +def complex_key(cursor, i): + return BaseDataSet.key_by_format(i, cursor.key_format) |