summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDon Anderson <dda@mongodb.com>2016-10-26 00:35:39 -0400
committerMichael Cahill <michael.cahill@mongodb.com>2016-10-26 15:35:39 +1100
commitef87391458eb9c4440f2f9dc1aa89b41363b6b9b (patch)
tree7d12b1466ea59910272c1f5fe2d5b48db052ca25
parentb6be49dd39979e55bf809111c71dce0389f6b728 (diff)
downloadmongo-ef87391458eb9c4440f2f9dc1aa89b41363b6b9b.tar.gz
WT-2947 replace test suite populate functions with *DataSet classes (#3083)
-rw-r--r--test/suite/helper.py187
-rw-r--r--test/suite/test_backup01.py22
-rw-r--r--test/suite/test_backup02.py1
-rw-r--r--test/suite/test_backup03.py16
-rw-r--r--test/suite/test_backup04.py7
-rw-r--r--test/suite/test_bug004.py16
-rw-r--r--test/suite/test_bug005.py4
-rw-r--r--test/suite/test_bug006.py4
-rw-r--r--test/suite/test_bug008.py106
-rw-r--r--test/suite/test_bug011.py15
-rw-r--r--test/suite/test_bug012.py7
-rw-r--r--test/suite/test_bug014.py14
-rw-r--r--test/suite/test_bug015.py1
-rw-r--r--test/suite/test_bulk01.py44
-rw-r--r--test/suite/test_bulk02.py9
-rw-r--r--test/suite/test_checkpoint01.py64
-rw-r--r--test/suite/test_checkpoint02.py1
-rw-r--r--test/suite/test_colgap.py28
-rw-r--r--test/suite/test_compact01.py15
-rw-r--r--test/suite/test_cursor06.py40
-rw-r--r--test/suite/test_cursor09.py37
-rw-r--r--test/suite/test_cursor_compare.py89
-rw-r--r--test/suite/test_cursor_pin.py59
-rw-r--r--test/suite/test_cursor_random.py39
-rw-r--r--test/suite/test_cursor_random02.py11
-rw-r--r--test/suite/test_drop.py15
-rw-r--r--test/suite/test_drop02.py5
-rw-r--r--test/suite/test_dump.py56
-rw-r--r--test/suite/test_dupc.py20
-rw-r--r--test/suite/test_empty.py4
-rw-r--r--test/suite/test_inmem01.py41
-rw-r--r--test/suite/test_inmem02.py10
-rw-r--r--test/suite/test_jsondump01.py71
-rw-r--r--test/suite/test_lsm01.py4
-rw-r--r--test/suite/test_lsm02.py1
-rw-r--r--test/suite/test_lsm03.py10
-rw-r--r--test/suite/test_nsnap01.py4
-rw-r--r--test/suite/test_nsnap02.py4
-rw-r--r--test/suite/test_nsnap03.py4
-rw-r--r--test/suite/test_nsnap04.py6
-rw-r--r--test/suite/test_overwrite.py47
-rw-r--r--test/suite/test_readonly03.py7
-rw-r--r--test/suite/test_rebalance.py17
-rw-r--r--test/suite/test_reconfig01.py6
-rw-r--r--test/suite/test_reconfig02.py1
-rw-r--r--test/suite/test_reconfig03.py10
-rw-r--r--test/suite/test_rename.py24
-rw-r--r--test/suite/test_shared_cache01.py1
-rw-r--r--test/suite/test_shared_cache02.py1
-rw-r--r--test/suite/test_split.py3
-rw-r--r--test/suite/test_stat01.py13
-rw-r--r--test/suite/test_stat02.py51
-rw-r--r--test/suite/test_stat03.py44
-rw-r--r--test/suite/test_stat05.py28
-rw-r--r--test/suite/test_truncate01.py99
-rw-r--r--test/suite/test_truncate02.py29
-rw-r--r--test/suite/test_truncate03.py25
-rw-r--r--test/suite/test_txn06.py4
-rw-r--r--test/suite/test_txn11.py4
-rw-r--r--test/suite/test_upgrade.py14
-rw-r--r--test/suite/test_util02.py4
-rw-r--r--test/suite/test_util13.py53
-rw-r--r--test/suite/wtdataset.py302
63 files changed, 981 insertions, 897 deletions
diff --git a/test/suite/helper.py b/test/suite/helper.py
index 9f34b566b3c..d1f41f05e8b 100644
--- a/test/suite/helper.py
+++ b/test/suite/helper.py
@@ -30,6 +30,8 @@
import glob, os, shutil, string, subprocess
import wiredtiger
+from wtdataset import SimpleDataSet, SimpleIndexDataSet, ComplexDataSet
+
# python has a filecmp.cmp function, but different versions of python approach
# file comparison differently. To make sure we get byte for byte comparison,
# we define it here.
@@ -121,188 +123,3 @@ def copy_wiredtiger_home(olddir, newdir, aligned=True):
cmd_list = ['dd', inpf, outf, 'bs=300']
a = subprocess.Popen(cmd_list)
a.wait()
-
-# create a simple_populate or complex_populate key
-def key_populate(cursor, i):
- key_format = cursor.key_format
- if key_format == 'i' or key_format == 'r' or key_format == 'u':
- return i
- elif key_format == 'S':
- return str('%015d' % i)
- else:
- raise AssertionError(
- 'key_populate: object has unexpected format: ' + key_format)
-
-# create a simple_populate value
-def value_populate(cursor, i):
- value_format = cursor.value_format
- if value_format == 'i' or value_format == 'r' or value_format == 'u':
- return i
- elif value_format == 'S':
- return str(i) + ': abcdefghijklmnopqrstuvwxyz'
- elif value_format == '8t':
- value = (
- 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xaa, 0xab,
- 0xac, 0xad, 0xae, 0xaf, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6,
- 0xb7, 0xb8, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf)
- return value[i % len(value)]
- else:
- raise AssertionError(
- 'value_populate: object has unexpected format: ' + value_format)
-
-# population of a simple object
-# uri: object
-# config: prefix of the session.create configuration string (defaults
-# to string value formats)
-# rows: entries to insert
-def simple_populate(self, uri, config, rows):
- self.pr('simple_populate: ' + uri + ' with ' + str(rows) + ' rows')
- self.session.create(uri, 'value_format=S,' + config)
- cursor = self.session.open_cursor(uri, None)
- for i in range(1, rows + 1):
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
- cursor.close()
-
-def simple_populate_check_cursor(self, cursor, rows):
- i = 0
- for key,val in cursor:
- i += 1
- self.assertEqual(key, key_populate(cursor, i))
- if cursor.value_format == '8t' and val == 0: # deleted
- continue
- self.assertEqual(val, value_populate(cursor, i))
- self.assertEqual(i, rows)
-
-def simple_populate_check(self, uri, rows):
- self.pr('simple_populate_check: ' + uri)
- cursor = self.session.open_cursor(uri, None)
- simple_populate_check_cursor(self, cursor, rows)
- cursor.close()
-
-# population of a simple object, with a single index
-# uri: object
-# config: prefix of the session.create configuration string (defaults
-# to string value formats)
-# rows: entries to insert
-def simple_index_populate(self, uri, config, rows):
- self.pr('simple_index_populate: ' + uri + ' with ' + str(rows) + ' rows')
- self.session.create(uri, 'value_format=S,columns=(key0,value0),' + config)
- indxname = 'index:' + uri.split(":")[1]
- self.session.create(indxname + ':index1', 'columns=(value0,key0)')
- cursor = self.session.open_cursor(uri, None)
- for i in range(1, rows + 1):
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
- cursor.close()
-
-def simple_index_populate_check_cursor(self, cursor, rows):
- i = 0
- for key,val in cursor:
- i += 1
- self.assertEqual(key, key_populate(cursor, i))
- if cursor.value_format == '8t' and val == 0: # deleted
- continue
- self.assertEqual(val, value_populate(cursor, i))
- self.assertEqual(i, rows)
-
-def simple_index_populate_check(self, uri, rows):
- self.pr('simple_index_populate_check: ' + uri)
-
- # Check values in the main table.
- cursor = self.session.open_cursor(uri, None)
- simple_index_populate_check_cursor(self, cursor, rows)
-
- # Check values in the index.
- indxname = 'index:' + uri.split(":")[1]
- idxcursor = self.session.open_cursor(indxname + ':index1')
- for i in range(1, rows + 1):
- k = key_populate(cursor, i)
- v = value_populate(cursor, i)
- ik = (v,k) # The index key is columns=(v,k).
- self.assertEqual(v, idxcursor[ik])
- idxcursor.close()
- cursor.close()
-
-# Return the value stored in a complex object.
-def complex_value_populate(cursor, i):
- return [str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%26],
- i,
- str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%23],
- str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%18]]
-
-# Return the number of column groups used
-def complex_populate_colgroup_count():
- return 6
-
-# Return the number of indices used
-def complex_populate_index_count():
- return 6
-
-# population of a complex object
-# uri: object
-# config: prefix of the session.create configuration string
-# rows: entries to insert
-def complex_populate(self, uri, config, rows):
- complex_populate_type(self, uri, config, '', rows, '')
-def complex_populate_cgconfig(self, uri, config, rows):
- complex_populate_type(self, uri, config, config, rows, '')
-def complex_populate_lsm(self, uri, config, rows):
- complex_populate_type(self, uri, config, '', rows, 'type=lsm')
-def complex_populate_cgconfig_lsm(self, uri, config, rows):
- complex_populate_type(self, uri, config, config, rows, 'type=lsm')
-def complex_populate_type(self, uri, config, cgconfig, rows, type):
- self.session.create(uri,
- config + ',value_format=SiSS,' +
- 'columns=(record,column2,column3,column4,column5),' +
- 'colgroups=(cgroup1,cgroup2,cgroup3,cgroup4,cgroup5,cgroup6)')
-
- cgname = 'colgroup:' + uri.split(":")[1]
- cgcfg = ',' + cgconfig + ',' + type
- self.session.create(cgname + ':cgroup1', 'columns=(column2)' + ',' + cgcfg)
- self.session.create(cgname + ':cgroup2', 'columns=(column3)' + ',' + cgcfg)
- self.session.create(cgname + ':cgroup3', 'columns=(column4)' + ',' + cgcfg)
- self.session.create(
- cgname + ':cgroup4', 'columns=(column2,column3)' + ',' + cgcfg)
- self.session.create(
- cgname + ':cgroup5', 'columns=(column3,column4)' + ',' + cgcfg)
- self.session.create(
- cgname + ':cgroup6', 'columns=(column2,column4,column5)' + ',' + cgcfg)
- indxname = 'index:' + uri.split(":")[1]
- self.session.create(indxname + ':indx1', 'columns=(column2)' + ',' + cgcfg)
- self.session.create(indxname + ':indx2', 'columns=(column3)' + ',' + cgcfg)
- self.session.create(indxname + ':indx3', 'columns=(column4)' + ',' + cgcfg)
- self.session.create(
- indxname + ':indx4', 'columns=(column2,column4)' + ',' + cgcfg)
- cursor = self.session.open_cursor(uri, None)
- for i in range(1, rows + 1):
- cursor[key_populate(cursor, i)] = \
- tuple(complex_value_populate(cursor, i))
- cursor.close()
- # add some indices after populating
- self.session.create(
- indxname + ':indx5', 'columns=(column3,column5)' + ',' + cgcfg)
- self.session.create(
- indxname + ':indx6', 'columns=(column3,column5,column4)' + ',' + cgcfg)
-
-def complex_populate_colgroup_name(self, uri, i):
- return 'colgroup:' + uri.split(":")[1] + ':cgroup' + str(i + 1)
-
-def complex_populate_index_name(self, uri, i):
- return 'index:' + uri.split(":")[1] + ':indx' + str(i + 1)
-
-def complex_populate_check_cursor(self, cursor, rows):
- i = 0
- for key, s1, i2, s3, s4 in cursor:
- i += 1
- self.assertEqual(key, key_populate(cursor, i))
- v = complex_value_populate(cursor, i)
- self.assertEqual(s1, v[0])
- self.assertEqual(i2, v[1])
- self.assertEqual(s3, v[2])
- self.assertEqual(s4, v[3])
- self.assertEqual(i, rows)
-
-def complex_populate_check(self, uri, rows):
- self.pr('complex_populate_check: ' + uri)
- cursor = self.session.open_cursor(uri, None)
- complex_populate_check_cursor(self, cursor, rows)
- cursor.close()
diff --git a/test/suite/test_backup01.py b/test/suite/test_backup01.py
index 92d39514440..4e98b6d8e77 100644
--- a/test/suite/test_backup01.py
+++ b/test/suite/test_backup01.py
@@ -32,8 +32,8 @@ import shutil
import string
from suite_subprocess import suite_subprocess
import wiredtiger, wttest
-from helper import compare_files,\
- complex_populate, complex_populate_lsm, simple_populate
+from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet
+from helper import compare_files
# test_backup.py
# Utilities: wt backup
@@ -43,14 +43,14 @@ class test_backup(wttest.WiredTigerTestCase, suite_subprocess):
pfx = 'test_backup'
objs = [
- ( 'file:' + pfx + '.1', simple_populate, 0),
- ( 'file:' + pfx + '.2', simple_populate, 0),
- ('table:' + pfx + '.3', simple_populate, 0),
- ('table:' + pfx + '.4', simple_populate, 0),
- ('table:' + pfx + '.5', complex_populate, 0),
- ('table:' + pfx + '.6', complex_populate, 0),
- ('table:' + pfx + '.7', complex_populate_lsm, 1),
- ('table:' + pfx + '.8', complex_populate_lsm, 1),
+ ( 'file:' + pfx + '.1', SimpleDataSet, 0),
+ ( 'file:' + pfx + '.2', SimpleDataSet, 0),
+ ('table:' + pfx + '.3', SimpleDataSet, 0),
+ ('table:' + pfx + '.4', SimpleDataSet, 0),
+ ('table:' + pfx + '.5', ComplexDataSet, 0),
+ ('table:' + pfx + '.6', ComplexDataSet, 0),
+ ('table:' + pfx + '.7', ComplexLSMDataSet, 1),
+ ('table:' + pfx + '.8', ComplexLSMDataSet, 1),
]
# Populate a set of objects.
@@ -59,7 +59,7 @@ class test_backup(wttest.WiredTigerTestCase, suite_subprocess):
if i[2]:
if skiplsm:
continue
- i[1](self, i[0], 'key_format=S', 100)
+ i[1](self, i[0], 100).populate()
# Compare the original and backed-up files using the wt dump command.
def compare(self, uri):
diff --git a/test/suite/test_backup02.py b/test/suite/test_backup02.py
index 398d55abd7a..d4089273be0 100644
--- a/test/suite/test_backup02.py
+++ b/test/suite/test_backup02.py
@@ -28,7 +28,6 @@
import Queue
import threading, time, wiredtiger, wttest
-from helper import key_populate, simple_populate
from wtthread import backup_thread, checkpoint_thread, op_thread
from wtscenario import make_scenarios
diff --git a/test/suite/test_backup03.py b/test/suite/test_backup03.py
index ba7ce60f519..73d05f0b0a1 100644
--- a/test/suite/test_backup03.py
+++ b/test/suite/test_backup03.py
@@ -27,11 +27,11 @@
# OTHER DEALINGS IN THE SOFTWARE.
import glob, os, shutil, string
+import wiredtiger, wttest
+from helper import compare_files
from suite_subprocess import suite_subprocess
+from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet
from wtscenario import make_scenarios
-import wiredtiger, wttest
-from helper import compare_files,\
- complex_populate, complex_populate_lsm, simple_populate
# test_backup03.py
# Utilities: wt backup
@@ -50,10 +50,10 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess):
# to the backup to confirm the backup is correct.
pfx = 'test_backup'
objs = [ # Objects
- ('table:' + pfx + '.1', simple_populate, 0),
- ( 'lsm:' + pfx + '.2', simple_populate, 1),
- ('table:' + pfx + '.3', complex_populate, 2),
- ('table:' + pfx + '.4', complex_populate_lsm, 3),
+ ('table:' + pfx + '.1', SimpleDataSet, 0),
+ ( 'lsm:' + pfx + '.2', SimpleDataSet, 1),
+ ('table:' + pfx + '.3', ComplexDataSet, 2),
+ ('table:' + pfx + '.4', ComplexLSMDataSet, 3),
]
list = [
( 'backup_1', dict(big=0,list=[0])), # Target objects individually
@@ -85,7 +85,7 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess):
rows = 200000 # Big object
else:
rows = 1000 # Small object
- i[1](self, i[0], 'key_format=S', rows)
+ i[1](self, i[0], rows).populate()
# Backup needs a checkpoint
self.session.checkpoint(None)
diff --git a/test/suite/test_backup04.py b/test/suite/test_backup04.py
index dd4dbc1d60f..919649fed57 100644
--- a/test/suite/test_backup04.py
+++ b/test/suite/test_backup04.py
@@ -29,10 +29,11 @@
import Queue
import threading, time, wiredtiger, wttest
import glob, os, shutil
+from helper import compare_files
from suite_subprocess import suite_subprocess
+from wtdataset import SimpleDataSet, simple_key
from wtscenario import make_scenarios
from wtthread import op_thread
-from helper import compare_files, key_populate
# test_backup04.py
# Utilities: wt backup
@@ -67,14 +68,14 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess):
self.pr('populate: ' + uri + ' with ' + str(rows) + ' rows')
cursor = self.session.open_cursor(uri, None)
for i in range(1, rows + 1):
- cursor[key_populate(cursor, i)] = str(i) + ':' + 'a' * dsize
+ cursor[simple_key(cursor, i)] = str(i) + ':' + 'a' * dsize
cursor.close()
def update(self, uri, dsize, upd, rows):
self.pr('update: ' + uri + ' with ' + str(rows) + ' rows')
cursor = self.session.open_cursor(uri, None)
for i in range(1, rows + 1):
- cursor[key_populate(cursor, i)] = str(i) + ':' + upd * dsize
+ cursor[simple_key(cursor, i)] = str(i) + ':' + upd * dsize
cursor.close()
# Compare the original and backed-up files using the wt dump command.
diff --git a/test/suite/test_bug004.py b/test/suite/test_bug004.py
index 316d15d8080..464cc57e272 100644
--- a/test/suite/test_bug004.py
+++ b/test/suite/test_bug004.py
@@ -30,7 +30,7 @@
# Regression tests.
import wiredtiger, wttest
-from helper import key_populate, value_populate
+from wtdataset import SimpleDataSet, simple_key, simple_value
# Check to make sure we see the right versions of overflow keys and values
# when they are deleted in reconciliation without having been instantiated
@@ -50,8 +50,8 @@ class test_bug004(wttest.WiredTigerTestCase):
self.session.create(self.uri, self.config)
c1 = self.session.open_cursor(self.uri, None)
for i in range(1, self.nentries):
- c1[key_populate(c1, i) + 'abcdef' * 100] = \
- value_populate(c1, i) + 'abcdef' * 100
+ c1[simple_key(c1, i) + 'abcdef' * 100] = \
+ simple_value(c1, i) + 'abcdef' * 100
c1.close()
# Verify the object, force it to disk, and verify the on-disk version.
@@ -69,9 +69,9 @@ class test_bug004(wttest.WiredTigerTestCase):
# currently do -- that's unlikely to change, but is a problem for the
# test going forward.)
c1 = self.session.open_cursor(self.uri, None)
- c1.set_key(key_populate(c1, self.nentries - 5) + 'abcdef' * 100)
+ c1.set_key(simple_key(c1, self.nentries - 5) + 'abcdef' * 100)
c2 = self.session.open_cursor(self.uri, None)
- c2.set_key(key_populate(c2, self.nentries + 5) + 'abcdef' * 100)
+ c2.set_key(simple_key(c2, self.nentries + 5) + 'abcdef' * 100)
self.session.truncate(None, c1, c2, None)
c1.close()
c2.close()
@@ -81,14 +81,14 @@ class test_bug004(wttest.WiredTigerTestCase):
# Use the snapshot cursor to retrieve the old key/value pairs
c1 = tmp_session.open_cursor(self.uri, None)
- c1.set_key(key_populate(c1, 1) + 'abcdef' * 100)
+ c1.set_key(simple_key(c1, 1) + 'abcdef' * 100)
c1.search()
for i in range(2, self.nentries):
c1.next()
self.assertEquals(
- c1.get_key(), key_populate(c1, i) + 'abcdef' * 100)
+ c1.get_key(), simple_key(c1, i) + 'abcdef' * 100)
self.assertEquals(
- c1.get_value(), value_populate(c1, i) + 'abcdef' * 100)
+ c1.get_value(), simple_value(c1, i) + 'abcdef' * 100)
if __name__ == '__main__':
wttest.run()
diff --git a/test/suite/test_bug005.py b/test/suite/test_bug005.py
index 6437b329096..69df175ae67 100644
--- a/test/suite/test_bug005.py
+++ b/test/suite/test_bug005.py
@@ -30,7 +30,7 @@
# Regression tests.
import wiredtiger, wttest
-from helper import key_populate, value_populate
+from wtdataset import SimpleDataSet, simple_key, simple_value
# Check that verify works when the file has additional data after the last
# checkpoint.
@@ -43,7 +43,7 @@ class test_bug005(wttest.WiredTigerTestCase):
self.session.create(self.uri, 'value_format=S,key_format=S')
cursor = self.session.open_cursor(self.uri, None)
for i in range(1, 1000):
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+ cursor[simple_key(cursor, i)] = simple_value(cursor, i)
cursor.close()
# Verify the object, force it to disk, and verify the on-disk version.
diff --git a/test/suite/test_bug006.py b/test/suite/test_bug006.py
index e01333f59de..c0f6055f720 100644
--- a/test/suite/test_bug006.py
+++ b/test/suite/test_bug006.py
@@ -30,7 +30,7 @@
# Regression tests.
import wiredtiger, wttest
-from helper import key_populate, value_populate
+from wtdataset import SimpleDataSet, simple_key, simple_value
from wtscenario import make_scenarios
# Check that verify and salvage both raise exceptions if there is an open
@@ -47,7 +47,7 @@ class test_bug006(wttest.WiredTigerTestCase):
self.session.create(uri, 'value_format=S,key_format=S')
cursor = self.session.open_cursor(uri, None)
for i in range(1, 1000):
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+ cursor[simple_key(cursor, i)] = simple_value(cursor, i)
# Table operations should fail, the cursor is open.
self.assertRaises(
diff --git a/test/suite/test_bug008.py b/test/suite/test_bug008.py
index 8e787ae14d1..c54c92fc864 100644
--- a/test/suite/test_bug008.py
+++ b/test/suite/test_bug008.py
@@ -30,7 +30,7 @@
# Regression tests.
import wiredtiger, wttest
-from helper import simple_populate, key_populate, value_populate
+from wtdataset import SimpleDataSet
from wtscenario import make_scenarios
# Test search/search-near operations, including invisible values and keys
@@ -38,23 +38,25 @@ from wtscenario import make_scenarios
class test_bug008(wttest.WiredTigerTestCase):
uri = 'file:test_bug008' # This is a btree layer test.
scenarios = make_scenarios([
- ('fix', dict(fmt='key_format=r,value_format=8t', empty=1, colvar=0)),
- ('row', dict(fmt='key_format=S', empty=0, colvar=0)),
- ('var', dict(fmt='key_format=r', empty=0, colvar=1))
+ ('fix', dict(key_format='r', value_format='8t', empty=1, colvar=0)),
+ ('row', dict(key_format='S', value_format='S', empty=0, colvar=0)),
+ ('var', dict(key_format='r', value_format='S', empty=0, colvar=1))
])
# Verify cursor search and search-near operations in an empty table.
def test_search_empty(self):
# Create the object and open a cursor.
- self.session.create(self.uri, self.fmt)
+ ds = SimpleDataSet(self, self.uri, 0, key_format=self.key_format,
+ value_format=self.value_format)
+ ds.create()
cursor = self.session.open_cursor(self.uri, None)
# Search for a record past the end of the table, which should fail.
- cursor.set_key(key_populate(cursor, 100))
+ cursor.set_key(ds.key(100))
self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND)
# Search-near for a record past the end of the table, which should fail.
- cursor.set_key(key_populate(cursor, 100))
+ cursor.set_key(ds.key(100))
self.assertEqual(cursor.search_near(), wiredtiger.WT_NOTFOUND)
# Verify cursor search and search-near operations at and past the end of
@@ -62,35 +64,37 @@ class test_bug008(wttest.WiredTigerTestCase):
def test_search_eot(self):
# Populate the tree and reopen the connection, forcing it to disk
# and moving the records to an on-page format.
- simple_populate(self, self.uri, self.fmt, 100)
+ ds = SimpleDataSet(self, self.uri, 100, key_format=self.key_format,
+ value_format=self.value_format)
+ ds.populate()
self.reopen_conn()
# Open a cursor.
cursor = self.session.open_cursor(self.uri, None)
# Search for a record at the end of the table, which should succeed.
- cursor.set_key(key_populate(cursor, 100))
+ cursor.set_key(ds.key(100))
self.assertEqual(cursor.search(), 0)
- self.assertEqual(cursor.get_key(), key_populate(cursor, 100))
- self.assertEqual(cursor.get_value(), value_populate(cursor, 100))
+ self.assertEqual(cursor.get_key(), ds.key(100))
+ self.assertEqual(cursor.get_value(), ds.value(100))
# Search-near for a record at the end of the table, which should
# succeed, returning the last record.
- cursor.set_key(key_populate(cursor, 100))
+ cursor.set_key(ds.key(100))
self.assertEqual(cursor.search_near(), 0)
- self.assertEqual(cursor.get_key(), key_populate(cursor, 100))
- self.assertEqual(cursor.get_value(), value_populate(cursor, 100))
+ self.assertEqual(cursor.get_key(), ds.key(100))
+ self.assertEqual(cursor.get_value(), ds.value(100))
# Search for a record past the end of the table, which should fail.
- cursor.set_key(key_populate(cursor, 200))
+ cursor.set_key(ds.key(200))
self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND)
# Search-near for a record past the end of the table, which should
# succeed, returning the last record.
- cursor.set_key(key_populate(cursor, 200))
+ cursor.set_key(ds.key(200))
self.assertEqual(cursor.search_near(), -1)
- self.assertEqual(cursor.get_key(), key_populate(cursor, 100))
- self.assertEqual(cursor.get_value(), value_populate(cursor, 100))
+ self.assertEqual(cursor.get_key(), ds.key(100))
+ self.assertEqual(cursor.get_value(), ds.value(100))
# Verify cursor search-near operations before and after a set of
# column-store duplicates.
@@ -99,18 +103,20 @@ class test_bug008(wttest.WiredTigerTestCase):
return
# Populate the tree.
- simple_populate(self, self.uri, self.fmt, 105)
+ ds = SimpleDataSet(self, self.uri, 105, key_format=self.key_format,
+ value_format=self.value_format)
+ ds.populate()
# Set up deleted records before and after a set of duplicate records,
# and make sure search/search-near returns the correct record.
cursor = self.session.open_cursor(self.uri, None)
for i in range(20, 100):
- cursor[key_populate(cursor, i)] = '=== IDENTICAL VALUE ==='
+ cursor[ds.key(i)] = '=== IDENTICAL VALUE ==='
for i in range(15, 25):
- cursor.set_key(key_populate(cursor, i))
+ cursor.set_key(ds.key(i))
self.assertEqual(cursor.remove(), 0)
for i in range(95, 106):
- cursor.set_key(key_populate(cursor, i))
+ cursor.set_key(ds.key(i))
self.assertEqual(cursor.remove(), 0)
cursor.close()
@@ -123,26 +129,28 @@ class test_bug008(wttest.WiredTigerTestCase):
# Search-near for a record in the deleted set before the duplicate set,
# which should succeed, returning the first record in the duplicate set.
- cursor.set_key(key_populate(cursor, 18))
+ cursor.set_key(ds.key(18))
self.assertEqual(cursor.search_near(), 1)
- self.assertEqual(cursor.get_key(), key_populate(cursor, 25))
+ self.assertEqual(cursor.get_key(), ds.key(25))
# Search-near for a record in the deleted set after the duplicate set,
# which should succeed, returning the last record in the duplicate set.
- cursor.set_key(key_populate(cursor, 98))
+ cursor.set_key(ds.key(98))
self.assertEqual(cursor.search_near(), -1)
- self.assertEqual(cursor.get_key(), key_populate(cursor, 94))
+ self.assertEqual(cursor.get_key(), ds.key(94))
# Verify cursor search and search-near operations on a file with a set of
# on-page visible records, and a set of insert-list invisible records.
def test_search_invisible_one(self):
# Populate the tree.
- simple_populate(self, self.uri, self.fmt, 100)
+ ds = SimpleDataSet(self, self.uri, 100, key_format=self.key_format,
+ value_format=self.value_format)
+ ds.populate()
# Delete a range of records.
for i in range(5, 10):
cursor = self.session.open_cursor(self.uri, None)
- cursor.set_key(key_populate(cursor, i))
+ cursor.set_key(ds.key(i))
self.assertEqual(cursor.remove(), 0)
# Reopen the connection, forcing it to disk and moving the records to
@@ -155,11 +163,11 @@ class test_bug008(wttest.WiredTigerTestCase):
self.session.begin_transaction()
cursor = self.session.open_cursor(self.uri, None)
for i in range(5, 10):
- cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000)
+ cursor[ds.key(i)] = ds.value(i + 1000)
for i in range(30, 40):
- cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000)
+ cursor[ds.key(i)] = ds.value(i + 1000)
for i in range(100, 140):
- cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000)
+ cursor[ds.key(i)] = ds.value(i + 1000)
# Open a separate session and cursor.
s = self.conn.open_session()
@@ -168,7 +176,7 @@ class test_bug008(wttest.WiredTigerTestCase):
# Search for an existing record in the deleted range, should not find
# it.
for i in range(5, 10):
- cursor.set_key(key_populate(cursor, i))
+ cursor.set_key(ds.key(i))
if self.empty:
# Fixed-length column-store rows always exist.
self.assertEqual(cursor.search(), 0)
@@ -180,13 +188,13 @@ class test_bug008(wttest.WiredTigerTestCase):
# Search for an existing record in the updated range, should see the
# original value.
for i in range(30, 40):
- cursor.set_key(key_populate(cursor, i))
+ cursor.set_key(ds.key(i))
self.assertEqual(cursor.search(), 0)
- self.assertEqual(cursor.get_key(), key_populate(cursor, i))
+ self.assertEqual(cursor.get_key(), ds.key(i))
# Search for a added record, should not find it.
for i in range(120, 130):
- cursor.set_key(key_populate(cursor, i))
+ cursor.set_key(ds.key(i))
if self.empty:
# Invisible updates to fixed-length column-store objects are
# invisible to the reader, but the fact that they exist past
@@ -203,7 +211,7 @@ class test_bug008(wttest.WiredTigerTestCase):
# the next largest record. (This depends on the implementation behavior
# which currently includes a bias to prefix search.)
for i in range(5, 10):
- cursor.set_key(key_populate(cursor, i))
+ cursor.set_key(ds.key(i))
if self.empty:
# Fixed-length column-store rows always exist.
self.assertEqual(cursor.search_near(), 0)
@@ -211,19 +219,19 @@ class test_bug008(wttest.WiredTigerTestCase):
self.assertEqual(cursor.get_value(), 0)
else:
self.assertEqual(cursor.search_near(), 1)
- self.assertEqual(cursor.get_key(), key_populate(cursor, 10))
+ self.assertEqual(cursor.get_key(), ds.key(10))
# Search-near for an existing record in the updated range, should see
# the original value.
for i in range(30, 40):
- cursor.set_key(key_populate(cursor, i))
+ cursor.set_key(ds.key(i))
self.assertEqual(cursor.search_near(), 0)
- self.assertEqual(cursor.get_key(), key_populate(cursor, i))
+ self.assertEqual(cursor.get_key(), ds.key(i))
# Search-near for an added record, should find the previous largest
# record.
for i in range(120, 130):
- cursor.set_key(key_populate(cursor, i))
+ cursor.set_key(ds.key(i))
if self.empty:
# Invisible updates to fixed-length column-store objects are
# invisible to the reader, but the fact that they exist past
@@ -234,7 +242,7 @@ class test_bug008(wttest.WiredTigerTestCase):
self.assertEqual(cursor.get_value(), 0)
else:
self.assertEqual(cursor.search_near(), -1)
- self.assertEqual(cursor.get_key(), key_populate(cursor, 100))
+ self.assertEqual(cursor.get_key(), ds.key(100))
# Verify cursor search and search-near operations on a file with a set of
# on-page visible records, a set of insert-list visible records, and a set
@@ -246,27 +254,29 @@ class test_bug008(wttest.WiredTigerTestCase):
def test_search_invisible_two(self):
# Populate the tree and reopen the connection, forcing it to disk
# and moving the records to an on-page format.
- simple_populate(self, self.uri, self.fmt, 100)
+ ds = SimpleDataSet(self, self.uri, 100, key_format=self.key_format,
+ value_format=self.value_format)
+ ds.populate()
self.reopen_conn()
# Add some additional visible records.
cursor = self.session.open_cursor(self.uri, None)
for i in range(100, 120):
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+ cursor[ds.key(i)] = ds.value(i)
cursor.close()
# Begin a transaction, and add some additional records.
self.session.begin_transaction()
cursor = self.session.open_cursor(self.uri, None)
for i in range(120, 140):
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+ cursor[ds.key(i)] = ds.value(i)
# Open a separate session and cursor.
s = self.conn.open_session()
cursor = s.open_cursor(self.uri, None)
# Search for an invisible record.
- cursor.set_key(key_populate(cursor, 130))
+ cursor.set_key(ds.key(130))
if self.empty:
# Invisible updates to fixed-length column-store objects are
# invisible to the reader, but the fact that they exist past
@@ -281,7 +291,7 @@ class test_bug008(wttest.WiredTigerTestCase):
# Search-near for an invisible record, which should succeed, returning
# the last visible record.
- cursor.set_key(key_populate(cursor, 130))
+ cursor.set_key(ds.key(130))
cursor.search_near()
if self.empty:
# Invisible updates to fixed-length column-store objects are
@@ -294,8 +304,8 @@ class test_bug008(wttest.WiredTigerTestCase):
else:
# Otherwise, we should find the closest record for which we can see
# the value.
- self.assertEqual(cursor.get_key(), key_populate(cursor, 119))
- self.assertEqual(cursor.get_value(), value_populate(cursor, 119))
+ self.assertEqual(cursor.get_key(), ds.key(119))
+ self.assertEqual(cursor.get_value(), ds.value(119))
if __name__ == '__main__':
wttest.run()
diff --git a/test/suite/test_bug011.py b/test/suite/test_bug011.py
index fceb7a22ddb..29bb08ec2e5 100644
--- a/test/suite/test_bug011.py
+++ b/test/suite/test_bug011.py
@@ -26,8 +26,8 @@
# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
-import helper, random, wiredtiger, wttest
-from helper import simple_populate
+import random, wiredtiger, wttest
+from wtdataset import SimpleDataSet
# test_bug011.py
# Eviction working on more files than there are hazard pointers.
@@ -46,11 +46,13 @@ class test_bug011(wttest.WiredTigerTestCase):
def test_eviction(self):
cursors = []
+ datasets = []
for i in range(0, self.ntables):
this_uri = 'table:%s-%03d' % (self.table_name, i)
- simple_populate(self, this_uri,
- 'key_format=S,allocation_size=1KB,leaf_page_max=1KB',
- self.nrows)
+ ds = SimpleDataSet(self, this_uri, self.nrows,
+ config='allocation_size=1KB,leaf_page_max=1KB')
+ ds.populate()
+ datasets.append(ds)
# Switch over to on-disk trees with multiple leaf pages
self.reopen_conn()
@@ -63,8 +65,7 @@ class test_bug011(wttest.WiredTigerTestCase):
# Make use of the cache.
for i in range(0, self.nops):
for i in range(0, self.ntables):
- cursors[i].set_key(helper.key_populate(cursors[i],
- random.randint(0, self.nrows - 1)))
+ cursors[i].set_key(ds.key(random.randint(0, self.nrows - 1)))
cursors[i].search()
cursors[i].reset()
diff --git a/test/suite/test_bug012.py b/test/suite/test_bug012.py
index a28834f36d2..91f49d14b3f 100644
--- a/test/suite/test_bug012.py
+++ b/test/suite/test_bug012.py
@@ -27,7 +27,7 @@
# OTHER DEALINGS IN THE SOFTWARE.
import wiredtiger, wttest
-from helper import complex_populate
+from wtdataset import ComplexDataSet
# test_bug012.py
class test_bug012(wttest.WiredTigerTestCase):
@@ -71,10 +71,11 @@ class test_bug012(wttest.WiredTigerTestCase):
# Test that we detect illegal extractors.
#
- # This test is a little fragile, we're depending on complex_populate to do
+ # This test is a little fragile, we're depending on ComplexDataSet to do
# the heavy-lifting, so if that function changes, this could break.
def test_illegal_extractor(self):
- complex_populate(self, 'table:A', 'key_format=S', 10)
+ ds = ComplexDataSet(self, 'table:A', 10)
+ ds.populate()
msg = '/unknown extractor/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda:
self.session.create('index:A:xyzzy',
diff --git a/test/suite/test_bug014.py b/test/suite/test_bug014.py
index c951cb45293..1dee933e839 100644
--- a/test/suite/test_bug014.py
+++ b/test/suite/test_bug014.py
@@ -27,7 +27,8 @@
# OTHER DEALINGS IN THE SOFTWARE.
import wiredtiger, wttest
-from helper import copy_wiredtiger_home, key_populate, simple_populate
+from wtdataset import SimpleDataSet
+from helper import copy_wiredtiger_home
# test_bug014.py
# JIRA WT-2115: fast-delete pages can be incorrectly lost due to a crash.
@@ -35,8 +36,9 @@ class test_bug014(wttest.WiredTigerTestCase):
def test_bug014(self):
# Populate a table with 1000 keys on small pages.
uri = 'table:test_bug014'
- simple_populate(self, uri,
- 'allocation_size=512,leaf_page_max=512,key_format=S', 1000)
+ ds = SimpleDataSet(self, uri, 1000,
+ config='allocation_size=512,leaf_page_max=512')
+ ds.populate()
# Reopen it so we can fast-delete pages.
self.reopen_conn()
@@ -44,9 +46,9 @@ class test_bug014(wttest.WiredTigerTestCase):
# Truncate a chunk of the key/value pairs inside a transaction.
self.session.begin_transaction(None)
start = self.session.open_cursor(uri, None)
- start.set_key(key_populate(start, 250))
+ start.set_key(ds.key(250))
end = self.session.open_cursor(uri, None)
- end.set_key(key_populate(end, 500))
+ end.set_key(ds.key(500))
self.session.truncate(None, start, end, None)
start.close()
end.close()
@@ -66,7 +68,7 @@ class test_bug014(wttest.WiredTigerTestCase):
# Confirm all of the records are there.
for i in range(1, 1001):
- cursor.set_key(key_populate(cursor, i))
+ cursor.set_key(ds.key(i))
self.assertEqual(cursor.search(), 0)
conn.close()
diff --git a/test/suite/test_bug015.py b/test/suite/test_bug015.py
index 7892f66759d..68cca49688f 100644
--- a/test/suite/test_bug015.py
+++ b/test/suite/test_bug015.py
@@ -27,7 +27,6 @@
# OTHER DEALINGS IN THE SOFTWARE.
import wiredtiger, wttest
-from helper import copy_wiredtiger_home, key_populate, simple_populate
# test_bug015.py
# JIRA WT-2162: index drop in a certain order triggers NULL pointer deref
diff --git a/test/suite/test_bulk01.py b/test/suite/test_bulk01.py
index addebe80647..8d5b6a04385 100644
--- a/test/suite/test_bulk01.py
+++ b/test/suite/test_bulk01.py
@@ -31,7 +31,7 @@
#
import wiredtiger, wttest
-from helper import key_populate, value_populate
+from wtdataset import SimpleDataSet, simple_key, simple_value
from wtscenario import make_scenarios
# Smoke test bulk-load.
@@ -61,7 +61,7 @@ class test_bulk_load(wttest.WiredTigerTestCase):
'key_format=' + self.keyfmt + ',value_format=' + self.valfmt)
cursor = self.session.open_cursor(uri, None, "bulk")
for i in range(1, 1000):
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+ cursor[simple_key(cursor, i)] = simple_value(cursor, i)
# Test a bulk-load triggers variable-length column-store RLE correctly.
def test_bulk_load_var_rle(self):
@@ -76,7 +76,7 @@ class test_bulk_load(wttest.WiredTigerTestCase):
'key_format=' + self.keyfmt + ',value_format=' + self.valfmt)
cursor = self.session.open_cursor(uri, None, "bulk")
for i in range(1, 1000):
- cursor[key_populate(cursor, i)] = value_populate(cursor, i/7)
+ cursor[simple_key(cursor, i)] = simple_value(cursor, i/7)
# Test a bulk-load variable-length column-store append ignores any key.
def test_bulk_load_var_append(self):
@@ -88,13 +88,13 @@ class test_bulk_load(wttest.WiredTigerTestCase):
'key_format=' + self.keyfmt + ',value_format=' + self.valfmt)
cursor = self.session.open_cursor(uri, None, "bulk,append")
for i in range(1, 1000):
- cursor[key_populate(cursor, 37)] = value_populate(cursor, i)
+ cursor[simple_key(cursor, 37)] = simple_value(cursor, i)
cursor.close()
cursor = self.session.open_cursor(uri, None, None)
for i in range(1, 1000):
- cursor.set_key(key_populate(cursor, i))
+ cursor.set_key(simple_key(cursor, i))
cursor.search()
- self.assertEqual(cursor.get_value(), value_populate(cursor, i))
+ self.assertEqual(cursor.get_value(), simple_value(cursor, i))
# Test that column-store bulk-load handles skipped records correctly.
def test_bulk_load_col_delete(self):
@@ -107,21 +107,21 @@ class test_bulk_load(wttest.WiredTigerTestCase):
cursor = self.session.open_cursor(uri, None, "bulk")
for i in range(1, 1000):
if i % 7 == 0:
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+ cursor[simple_key(cursor, i)] = simple_value(cursor, i)
# Ensure we create all the missing records.
i = i + 1
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+ cursor[simple_key(cursor, i)] = simple_value(cursor, i)
cursor.close()
cursor = self.session.open_cursor(uri, None, None)
# Verify all the records are there, in their proper state.
for i in range(1, 1000):
- cursor.set_key(key_populate(cursor, i))
+ cursor.set_key(simple_key(cursor, i))
if i % 7 == 0:
cursor.search()
- self.assertEqual(cursor.get_value(), value_populate(cursor, i))
+ self.assertEqual(cursor.get_value(), simple_value(cursor, i))
elif cursor.value_format == '8t':
cursor.search()
self.assertEqual(cursor.get_value(), 0)
@@ -139,17 +139,17 @@ class test_bulk_load(wttest.WiredTigerTestCase):
'key_format=' + self.keyfmt + ',value_format=' + self.valfmt)
cursor = self.session.open_cursor(uri, None, "bulk")
for i in range(1, 10):
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+ cursor[simple_key(cursor, i)] = simple_value(cursor, i)
# A big record -- if it's not efficient, we'll just hang.
big = 18446744073709551606
- cursor[key_populate(cursor, big)] = value_populate(cursor, big)
+ cursor[simple_key(cursor, big)] = simple_value(cursor, big)
cursor.close()
cursor = self.session.open_cursor(uri, None, None)
- cursor.set_key(key_populate(cursor, big))
+ cursor.set_key(simple_key(cursor, big))
cursor.search()
- self.assertEqual(cursor.get_value(), value_populate(cursor, big))
+ self.assertEqual(cursor.get_value(), simple_value(cursor, big))
# Test that bulk-load out-of-order fails.
def test_bulk_load_order_check(self):
@@ -157,16 +157,16 @@ class test_bulk_load(wttest.WiredTigerTestCase):
self.session.create(uri,
'key_format=' + self.keyfmt + ',value_format=' + self.valfmt)
cursor = self.session.open_cursor(uri, None, "bulk")
- cursor[key_populate(cursor, 10)] = value_populate(cursor, 10)
+ cursor[simple_key(cursor, 10)] = simple_value(cursor, 10)
for i in [1, 9, 10]:
- cursor.set_key(key_populate(cursor, 1))
- cursor.set_value(value_populate(cursor, 1))
+ cursor.set_key(simple_key(cursor, 1))
+ cursor.set_value(simple_value(cursor, 1))
msg = '/than previously inserted key/'
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError, lambda: cursor.insert(), msg)
- cursor[key_populate(cursor, 11)] = value_populate(cursor, 11)
+ cursor[simple_key(cursor, 11)] = simple_value(cursor, 11)
# Test that row-store bulk-load out-of-order can succeed.
def test_bulk_load_row_order_nocheck(self):
@@ -181,8 +181,8 @@ class test_bulk_load(wttest.WiredTigerTestCase):
self.session.create(uri,
'key_format=' + self.keyfmt + ',value_format=' + self.valfmt)
cursor = self.session.open_cursor(uri, None, "bulk,skip_sort_check")
- cursor[key_populate(cursor, 10)] = value_populate(cursor, 10)
- cursor[key_populate(cursor, 1)] = value_populate(cursor, 1)
+ cursor[simple_key(cursor, 10)] = simple_value(cursor, 10)
+ cursor[simple_key(cursor, 1)] = simple_value(cursor, 1)
if not wiredtiger.diagnostic_build():
self.skipTest('requires a diagnostic build')
@@ -197,7 +197,7 @@ class test_bulk_load(wttest.WiredTigerTestCase):
uri = self.type + self.name
self.session.create(uri, 'key_format=S,value_format=S')
cursor = self.session.open_cursor(uri, None)
- cursor[key_populate(cursor, 1)] = value_populate(cursor, 1)
+ cursor[simple_key(cursor, 1)] = simple_value(cursor, 1)
# Close the insert cursor, else we'll get EBUSY.
cursor.close()
msg = '/bulk-load is only supported on newly created objects/'
@@ -209,7 +209,7 @@ class test_bulk_load(wttest.WiredTigerTestCase):
uri = self.type + self.name
self.session.create(uri, 'key_format=S,value_format=S')
cursor = self.session.open_cursor(uri, None)
- cursor[key_populate(cursor, 1)] = value_populate(cursor, 1)
+ cursor[simple_key(cursor, 1)] = simple_value(cursor, 1)
# Don't close the insert cursor, we want EBUSY.
self.assertRaises(wiredtiger.WiredTigerError,
lambda: self.session.open_cursor(uri, None, "bulk"))
diff --git a/test/suite/test_bulk02.py b/test/suite/test_bulk02.py
index c959098179c..fb9240e91e7 100644
--- a/test/suite/test_bulk02.py
+++ b/test/suite/test_bulk02.py
@@ -30,10 +30,11 @@
# Bulk-load testing.
import shutil, os
-from helper import confirm_empty, key_populate, value_populate
+import wiredtiger, wttest
+from helper import confirm_empty
from suite_subprocess import suite_subprocess
+from wtdataset import SimpleDataSet, simple_key, simple_value
from wtscenario import make_scenarios
-import wiredtiger, wttest
# test_bulkload_checkpoint
# Test bulk-load with checkpoints.
@@ -56,7 +57,7 @@ class test_bulkload_checkpoint(wttest.WiredTigerTestCase, suite_subprocess):
self.session.create(self.uri, 'key_format=S,value_format=S')
cursor = self.session.open_cursor(self.uri, None, 'bulk')
for i in range(1, 10):
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+ cursor[simple_key(cursor, i)] = simple_value(cursor, i)
# Checkpoint a few times (to test the drop code).
for i in range(1, 5):
@@ -110,7 +111,7 @@ class test_bulkload_backup(wttest.WiredTigerTestCase, suite_subprocess):
self.session.create(self.uri, 'key_format=S,value_format=S')
cursor = self.session.open_cursor(self.uri, None, 'bulk')
for i in range(1, 10):
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+ cursor[simple_key(cursor, i)] = simple_value(cursor, i)
# Test without a checkpoint, with an unnamed checkpoint, with a named
# checkpoint.
diff --git a/test/suite/test_checkpoint01.py b/test/suite/test_checkpoint01.py
index 013cda96e4e..c0d004db78d 100644
--- a/test/suite/test_checkpoint01.py
+++ b/test/suite/test_checkpoint01.py
@@ -27,7 +27,7 @@
# OTHER DEALINGS IN THE SOFTWARE.
import wiredtiger, wttest
-from helper import key_populate, complex_populate_lsm, simple_populate
+from wtdataset import SimpleDataSet, ComplexLSMDataSet
from wtscenario import make_scenarios
# test_checkpoint01.py
@@ -145,7 +145,7 @@ class test_checkpoint_cursor(wttest.WiredTigerTestCase):
# Check that you cannot open a checkpoint that doesn't exist.
def test_checkpoint_dne(self):
- simple_populate(self, self.uri, 'key_format=' + self.fmt, 100)
+ SimpleDataSet(self, self.uri, 100, key_format=self.fmt).populate()
self.assertRaises(wiredtiger.WiredTigerError,
lambda: self.session.open_cursor(
self.uri, None, "checkpoint=checkpoint-1"))
@@ -155,7 +155,7 @@ class test_checkpoint_cursor(wttest.WiredTigerTestCase):
# Check that you can open checkpoints more than once.
def test_checkpoint_multiple_open(self):
- simple_populate(self, self.uri, 'key_format=' + self.fmt, 100)
+ SimpleDataSet(self, self.uri, 100, key_format=self.fmt).populate()
self.session.checkpoint("name=checkpoint-1")
c1 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-1")
c2 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-1")
@@ -172,7 +172,7 @@ class test_checkpoint_cursor(wttest.WiredTigerTestCase):
# Check that you cannot drop a checkpoint if it's in use.
def test_checkpoint_inuse(self):
- simple_populate(self, self.uri, 'key_format=' + self.fmt, 100)
+ SimpleDataSet(self, self.uri, 100, key_format=self.fmt).populate()
self.session.checkpoint("name=checkpoint-1")
self.session.checkpoint("name=checkpoint-2")
self.session.checkpoint("name=checkpoint-3")
@@ -208,44 +208,49 @@ class test_checkpoint_target(wttest.WiredTigerTestCase):
('table', dict(uri='table:checkpoint',fmt='S'))
])
- def update(self, uri, value):
+ def update(self, uri, ds, value):
cursor = self.session.open_cursor(uri, None, "overwrite")
- cursor[key_populate(cursor, 10)] = value
+ cursor[ds.key(10)] = value
cursor.close()
- def check(self, uri, value):
+ def check(self, uri, ds, value):
cursor = self.session.open_cursor(uri, None, "checkpoint=checkpoint-1")
- self.assertEquals(cursor[key_populate(cursor, 10)], value)
+ self.assertEquals(cursor[ds.key(10)], value)
cursor.close()
def test_checkpoint_target(self):
# Create 3 objects, change one record to an easily recognizable string.
uri = self.uri + '1'
- simple_populate(self, uri, 'key_format=' + self.fmt, 100)
- self.update(uri, 'ORIGINAL')
+ ds1 = SimpleDataSet(self, uri, 100, key_format=self.fmt)
+ ds1.populate()
+ self.update(uri, ds1, 'ORIGINAL')
+
uri = self.uri + '2'
- simple_populate(self, uri, 'key_format=' + self.fmt, 100)
- self.update(uri, 'ORIGINAL')
+ ds2 = SimpleDataSet(self, uri, 100, key_format=self.fmt)
+ ds2.populate()
+ self.update(uri, ds2, 'ORIGINAL')
+
uri = self.uri + '3'
- simple_populate(self, uri, 'key_format=' + self.fmt, 100)
- self.update(uri, 'ORIGINAL')
+ ds3 = SimpleDataSet(self, uri, 100, key_format=self.fmt)
+ ds3.populate()
+ self.update(uri, ds3, 'ORIGINAL')
# Checkpoint all three objects.
self.session.checkpoint("name=checkpoint-1")
# Update all 3 objects, then checkpoint two of the objects with the
# same checkpoint name.
- self.update(self.uri + '1', 'UPDATE')
- self.update(self.uri + '2', 'UPDATE')
- self.update(self.uri + '3', 'UPDATE')
+ self.update(self.uri + '1', ds1, 'UPDATE')
+ self.update(self.uri + '2', ds2, 'UPDATE')
+ self.update(self.uri + '3', ds3, 'UPDATE')
target = 'target=("' + self.uri + '1"' + ',"' + self.uri + '2")'
self.session.checkpoint("name=checkpoint-1," + target)
# Confirm the checkpoint has the old value in objects that weren't
# checkpointed, and the new value in objects that were checkpointed.
- self.check(self.uri + '1', 'UPDATE')
- self.check(self.uri + '2', 'UPDATE')
- self.check(self.uri + '3', 'ORIGINAL')
+ self.check(self.uri + '1', ds1, 'UPDATE')
+ self.check(self.uri + '2', ds2, 'UPDATE')
+ self.check(self.uri + '3', ds3, 'ORIGINAL')
# Check that you can't write checkpoint cursors.
class test_checkpoint_cursor_update(wttest.WiredTigerTestCase):
@@ -257,10 +262,11 @@ class test_checkpoint_cursor_update(wttest.WiredTigerTestCase):
])
def test_checkpoint_cursor_update(self):
- simple_populate(self, self.uri, 'key_format=' + self.fmt, 100)
+ ds = SimpleDataSet(self, self.uri, 100, key_format=self.fmt)
+ ds.populate()
self.session.checkpoint("name=ckpt")
cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt")
- cursor.set_key(key_populate(cursor, 10))
+ cursor.set_key(ds.key(10))
cursor.set_value("XXX")
msg = "/Unsupported cursor/"
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
@@ -284,12 +290,13 @@ class test_checkpoint_last(wttest.WiredTigerTestCase):
# value. Repeat this action, we want to be sure the engine gets the
# latest checkpoint information each time.
uri = self.uri
- simple_populate(self, uri, 'key_format=' + self.fmt, 100)
+ ds = SimpleDataSet(self, uri, 100, key_format=self.fmt)
+ ds.populate()
for value in ('FIRST', 'SECOND', 'THIRD', 'FOURTH', 'FIFTH'):
# Update the object.
cursor = self.session.open_cursor(uri, None, "overwrite")
- cursor[key_populate(cursor, 10)] = value
+ cursor[ds.key(10)] = value
cursor.close()
# Checkpoint the object.
@@ -298,14 +305,15 @@ class test_checkpoint_last(wttest.WiredTigerTestCase):
# Verify the "last" checkpoint sees the correct value.
cursor = self.session.open_cursor(
uri, None, "checkpoint=WiredTigerCheckpoint")
- self.assertEquals(cursor[key_populate(cursor, 10)], value)
+ self.assertEquals(cursor[ds.key(10)], value)
# Don't close the checkpoint cursor, we want it to remain open until
# the test completes.
# Check we can't use the reserved name as an application checkpoint name.
class test_checkpoint_illegal_name(wttest.WiredTigerTestCase):
def test_checkpoint_illegal_name(self):
- simple_populate(self, "file:checkpoint", 'key_format=S', 100)
+ ds = SimpleDataSet(self, "file:checkpoint", 100, key_format='S')
+ ds.populate()
msg = '/the checkpoint name.*is reserved/'
for conf in (
'name=WiredTigerCheckpoint',
@@ -329,8 +337,8 @@ class test_checkpoint_illegal_name(wttest.WiredTigerTestCase):
# Check we can't name checkpoints that include LSM tables.
class test_checkpoint_lsm_name(wttest.WiredTigerTestCase):
def test_checkpoint_lsm_name(self):
- complex_populate_lsm(self,
- "table:checkpoint", 'type=lsm,key_format=S', 1000)
+ ds = ComplexLSMDataSet(self, "table:checkpoint", 1000)
+ ds.populate()
msg = '/object does not support named checkpoints/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: self.session.checkpoint("name=ckpt"), msg)
diff --git a/test/suite/test_checkpoint02.py b/test/suite/test_checkpoint02.py
index 0d166544472..b5d20fb73b1 100644
--- a/test/suite/test_checkpoint02.py
+++ b/test/suite/test_checkpoint02.py
@@ -28,7 +28,6 @@
import Queue
import threading, time, wiredtiger, wttest
-from helper import key_populate, simple_populate
from wtthread import checkpoint_thread, op_thread
from wtscenario import make_scenarios
diff --git a/test/suite/test_colgap.py b/test/suite/test_colgap.py
index dbe748ae58f..91df0fd6c1c 100644
--- a/test/suite/test_colgap.py
+++ b/test/suite/test_colgap.py
@@ -27,7 +27,7 @@
# OTHER DEALINGS IN THE SOFTWARE.
import wiredtiger, wttest
-from helper import simple_populate, key_populate, value_populate
+from wtdataset import SimpleDataSet, simple_key, simple_value
from wtscenario import make_scenarios
# test_colgap.py
@@ -61,14 +61,16 @@ class test_column_store_gap(wttest.WiredTigerTestCase):
# namespace. If this runs in less-than-glacial time, it's working.
def test_column_store_gap(self):
uri = 'table:gap'
- simple_populate(self, uri, 'key_format=r,value_format=S', 0)
+ # Initially just create tables.
+ ds = SimpleDataSet(self, uri, 0, key_format='r')
+ ds.populate()
cursor = self.session.open_cursor(uri, None, None)
self.nentries = 0
# Create a column-store table with large gaps in the name-space.
v = [ 1000, 2000000000000, 30000000000000 ]
for i in v:
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+ cursor[ds.key(i)] = ds.value(i)
self.nentries += 1
# In-memory cursor forward, backward.
@@ -84,7 +86,9 @@ class test_column_store_gap(wttest.WiredTigerTestCase):
def test_column_store_gap_traverse(self):
uri = 'table:gap'
- simple_populate(self, uri, 'key_format=r,value_format=S', 0)
+ # Initially just create tables.
+ ds = SimpleDataSet(self, uri, 0, key_format='r')
+ ds.populate()
cursor = self.session.open_cursor(uri, None, None)
self.nentries = 0
@@ -92,7 +96,7 @@ class test_column_store_gap(wttest.WiredTigerTestCase):
# important, we just want some gaps.
v = [ 1000, 1001, 2000, 2001]
for i in v:
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+ cursor[ds.key(i)] = ds.value(i)
self.nentries += 1
# In-memory cursor forward, backward.
@@ -111,7 +115,7 @@ class test_column_store_gap(wttest.WiredTigerTestCase):
# so the traversal walks to them.
v2 = [ 1500, 1501 ]
for i in v2:
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+ cursor[ds.key(i)] = ds.value(i)
self.nentries += 1
# Tell the validation what to expect.
@@ -168,7 +172,7 @@ class test_colmax(wttest.WiredTigerTestCase):
# Optionaly make the big record the only record in the table.
if not self.single:
for i in range(1, 723):
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+ cursor[simple_key(cursor, i)] = simple_value(cursor, i)
# Confirm searching past the end of the table works.
if not self.bulk:
@@ -176,7 +180,7 @@ class test_colmax(wttest.WiredTigerTestCase):
self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND)
# Insert the big record.
- cursor[key_populate(cursor, recno)] = value_populate(cursor, recno)
+ cursor[simple_key(cursor, recno)] = simple_value(cursor, recno)
# Optionally flush to disk; re-open the cursor as necessary.
if self.bulk or self.reopen:
@@ -189,18 +193,18 @@ class test_colmax(wttest.WiredTigerTestCase):
# Search for the large record.
cursor.set_key(recno)
self.assertEqual(cursor.search(), 0)
- self.assertEqual(cursor.get_value(), value_populate(cursor, recno))
+ self.assertEqual(cursor.get_value(), simple_value(cursor, recno))
# Update it.
- cursor[key_populate(cursor, recno)] = value_populate(cursor, 37)
+ cursor[simple_key(cursor, recno)] = simple_value(cursor, 37)
cursor.set_key(recno)
self.assertEqual(cursor.search(), 0)
- self.assertEqual(cursor.get_value(), value_populate(cursor, 37))
+ self.assertEqual(cursor.get_value(), simple_value(cursor, 37))
# Remove it.
cursor.set_key(recno)
self.assertEqual(cursor.remove(), 0)
- cursor.set_key(key_populate(cursor, recno))
+ cursor.set_key(simple_key(cursor, recno))
self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND)
if __name__ == '__main__':
diff --git a/test/suite/test_compact01.py b/test/suite/test_compact01.py
index 8da1a0df4da..56ab6d39076 100644
--- a/test/suite/test_compact01.py
+++ b/test/suite/test_compact01.py
@@ -27,8 +27,8 @@
# OTHER DEALINGS IN THE SOFTWARE.
import wiredtiger, wttest
-from helper import complex_populate, simple_populate, key_populate
from suite_subprocess import suite_subprocess
+from wtdataset import SimpleDataSet, ComplexDataSet
from wiredtiger import stat
from wtscenario import make_scenarios
@@ -39,14 +39,14 @@ class test_compact(wttest.WiredTigerTestCase, suite_subprocess):
# Use a small page size because we want to create lots of pages.
config = 'allocation_size=512,' +\
- 'leaf_page_max=512,value_format=S,key_format=S'
+ 'leaf_page_max=512,key_format=S'
nentries = 50000
# The table is a complex object, give it roughly 5 pages per underlying
# file.
types = [
- ('file', dict(type='file:', pop=simple_populate, maxpages=5)),
- ('table', dict(type='table:', pop=complex_populate, maxpages=50))
+ ('file', dict(type='file:', dataset=SimpleDataSet, maxpages=5)),
+ ('table', dict(type='table:', dataset=ComplexDataSet, maxpages=50))
]
compact = [
('method', dict(utility=0,reopen=0)),
@@ -64,7 +64,8 @@ class test_compact(wttest.WiredTigerTestCase, suite_subprocess):
def test_compact(self):
# Populate an object
uri = self.type + self.name
- self.pop(self, uri, self.config, self.nentries - 1)
+ ds = self.dataset(self, uri, self.nentries - 1, config=self.config)
+ ds.populate()
# Reopen the connection to force the object to disk.
self.reopen_conn()
@@ -76,9 +77,9 @@ class test_compact(wttest.WiredTigerTestCase, suite_subprocess):
# Remove most of the object.
c1 = self.session.open_cursor(uri, None)
- c1.set_key(key_populate(c1, 5))
+ c1.set_key(ds.key(5))
c2 = self.session.open_cursor(uri, None)
- c2.set_key(key_populate(c2, self.nentries - 5))
+ c2.set_key(ds.key(self.nentries - 5))
self.session.truncate(None, c1, c2, None)
c1.close()
c2.close()
diff --git a/test/suite/test_cursor06.py b/test/suite/test_cursor06.py
index 3a6240bc6c7..117e29b0605 100644
--- a/test/suite/test_cursor06.py
+++ b/test/suite/test_cursor06.py
@@ -27,8 +27,7 @@
# OTHER DEALINGS IN THE SOFTWARE.
import wiredtiger, wttest
-from helper import key_populate, value_populate, simple_populate
-from helper import complex_value_populate, complex_populate
+from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet
from wtscenario import make_scenarios
# test_cursor06.py
@@ -36,33 +35,26 @@ from wtscenario import make_scenarios
class test_cursor06(wttest.WiredTigerTestCase):
name = 'reconfigure'
scenarios = make_scenarios([
- ('file-r', dict(type='file:', config='key_format=r', complex=0)),
- ('file-S', dict(type='file:', config='key_format=S', complex=0)),
- ('lsm-S', dict(type='lsm:', config='key_format=S', complex=0)),
- ('table-r',
- dict(type='table:', config='key_format=r', complex=0)),
- ('table-S',
- dict(type='table:', config='key_format=S', complex=0)),
- ('table-r-complex',
- dict(type='table:', config='key_format=r', complex=1)),
- ('table-S-complex',
- dict(type='table:', config='key_format=S', complex=1)),
- ('table-S-complex-lsm',
- dict(type='table:', config='key_format=S,type=lsm', complex=1)),
+ ('file-r', dict(type='file:', keyfmt='r', dataset=SimpleDataSet)),
+ ('file-S', dict(type='file:', keyfmt='S', dataset=SimpleDataSet)),
+ ('lsm-S', dict(type='lsm:', keyfmt='S', dataset=SimpleDataSet)),
+ ('table-r', dict(type='table:', keyfmt='r', dataset=SimpleDataSet)),
+ ('table-S', dict(type='table:', keyfmt='S', dataset=SimpleDataSet)),
+ ('table-r-complex', dict(type='table:', keyfmt='r',
+ dataset=ComplexDataSet)),
+ ('table-S-complex', dict(type='table:', keyfmt='S',
+ dataset=ComplexDataSet)),
+ ('table-S-complex-lsm', dict(type='table:', keyfmt='S',
+ dataset=ComplexLSMDataSet)),
])
def populate(self, uri):
- if self.complex:
- complex_populate(self, uri, self.config, 100)
- else:
- simple_populate(self, uri, self.config, 100)
+ self.ds = self.dataset(self, uri, 100, key_format=self.keyfmt)
+ self.ds.populate()
def set_kv(self, cursor):
- cursor.set_key(key_populate(cursor, 10))
- if self.complex:
- cursor.set_value(tuple(complex_value_populate(cursor, 10)))
- else:
- cursor.set_value(value_populate(cursor, 10))
+ cursor.set_key(self.ds.key(10))
+ cursor.set_value(self.ds.value(10))
def test_reconfigure_overwrite(self):
uri = self.type + self.name
diff --git a/test/suite/test_cursor09.py b/test/suite/test_cursor09.py
index a05caea4f1f..9a1fc06b617 100644
--- a/test/suite/test_cursor09.py
+++ b/test/suite/test_cursor09.py
@@ -27,27 +27,24 @@
# OTHER DEALINGS IN THE SOFTWARE.
import wiredtiger, wttest
-from helper import key_populate, value_populate, simple_populate
-from helper import complex_populate, complex_value_populate
+from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet
from wtscenario import make_scenarios
# test_cursor09.py
# JIRA WT-2217: insert resets key/value "set".
class test_cursor09(wttest.WiredTigerTestCase):
scenarios = make_scenarios([
- ('file-r', dict(type='file:', config='key_format=r', complex=0)),
- ('file-S', dict(type='file:', config='key_format=S', complex=0)),
- ('lsm-S', dict(type='lsm:', config='key_format=S', complex=0)),
- ('table-r',
- dict(type='table:', config='key_format=r', complex=0)),
- ('table-S',
- dict(type='table:', config='key_format=S', complex=0)),
- ('table-r-complex',
- dict(type='table:', config='key_format=r', complex=1)),
- ('table-S-complex',
- dict(type='table:', config='key_format=S', complex=1)),
- ('table-S-complex-lsm',
- dict(type='table:', config='key_format=S,type=lsm', complex=1)),
+ ('file-r', dict(type='file:', keyfmt='r', dataset=SimpleDataSet)),
+ ('file-S', dict(type='file:', keyfmt='S', dataset=SimpleDataSet)),
+ ('lsm-S', dict(type='lsm:', keyfmt='S', dataset=SimpleDataSet)),
+ ('table-r', dict(type='table:', keyfmt='r', dataset=SimpleDataSet)),
+ ('table-S', dict(type='table:', keyfmt='S', dataset=SimpleDataSet)),
+ ('table-r-complex', dict(type='table:', keyfmt='r',
+ dataset=ComplexDataSet)),
+ ('table-S-complex', dict(type='table:', keyfmt='S',
+ dataset=ComplexDataSet)),
+ ('table-S-complex-lsm', dict(type='table:', keyfmt='S',
+ dataset=ComplexLSMDataSet)),
])
# WT_CURSOR.insert doesn't leave the cursor positioned, verify any
@@ -55,15 +52,11 @@ class test_cursor09(wttest.WiredTigerTestCase):
def test_cursor09(self):
uri = self.type + 'cursor09'
- if self.complex:
- complex_populate(self, uri, self.config, 100)
- else:
- simple_populate(self, uri, self.config, 100)
+ ds = self.dataset(self, uri, 100, key_format=self.keyfmt)
+ ds.populate()
cursor = self.session.open_cursor(uri, None, None)
- cursor[key_populate(cursor, 10)] = \
- tuple(complex_value_populate(cursor, 10)) if self.complex \
- else value_populate(cursor, 10)
+ cursor[ds.key(10)] = ds.value(10)
msg = '/requires key be set/'
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError, cursor.search, msg)
diff --git a/test/suite/test_cursor_compare.py b/test/suite/test_cursor_compare.py
index 4610a2f30a9..c0feb1d4867 100644
--- a/test/suite/test_cursor_compare.py
+++ b/test/suite/test_cursor_compare.py
@@ -27,8 +27,7 @@
# OTHER DEALINGS IN THE SOFTWARE.
import wiredtiger, wttest, exceptions
-from helper import complex_populate, simple_populate, key_populate
-from helper import complex_populate_index_name
+from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet
from wtscenario import make_scenarios
# Test cursor comparisons.
@@ -36,9 +35,9 @@ class test_cursor_comparison(wttest.WiredTigerTestCase):
name = 'test_compare'
types = [
- ('file', dict(type='file:', config='')),
- ('lsm', dict(type='table:', config=',type=lsm')),
- ('table', dict(type='table:', config=''))
+ ('file', dict(type='file:', dataset=SimpleDataSet)),
+ ('lsm', dict(type='table:', dataset=ComplexLSMDataSet)),
+ ('table', dict(type='table:', dataset=ComplexDataSet))
]
keyfmt = [
('integer', dict(keyfmt='i')),
@@ -52,28 +51,20 @@ class test_cursor_comparison(wttest.WiredTigerTestCase):
uriX = self.type + 'compareX'
# Build the object.
+ ds = self.dataset(self, uri, 100, key_format=self.keyfmt)
+ dsX = self.dataset(self, uriX, 100, key_format=self.keyfmt)
+ ds.populate()
+ dsX.populate()
if self.type == 'file:':
- simple_populate(
- self, uri, 'key_format=' + self.keyfmt + self.config, 100)
- simple_populate(
- self, uriX, 'key_format=' + self.keyfmt + self.config, 100)
ix0_0 = None
ix0_1 = None
ix1_0 = None
ixX_0 = None
else:
- complex_populate(
- self, uri, 'key_format=' + self.keyfmt + self.config, 100)
- complex_populate(
- self, uriX, 'key_format=' + self.keyfmt + self.config, 100)
- ix0_0 = self.session.open_cursor(
- complex_populate_index_name(self, uri, 0), None)
- ix0_1 = self.session.open_cursor(
- complex_populate_index_name(self, uri, 0), None)
- ix1_0 = self.session.open_cursor(
- complex_populate_index_name(self, uri, 1), None)
- ixX_0 = self.session.open_cursor(
- complex_populate_index_name(self, uriX, 0), None)
+ ix0_0 = self.session.open_cursor(ds.index_name(0), None)
+ ix0_1 = self.session.open_cursor(ds.index_name(0), None)
+ ix1_0 = self.session.open_cursor(ds.index_name(1), None)
+ ixX_0 = self.session.open_cursor(dsX.index_name(0), None)
ix0_0.next()
ix0_1.next()
ix1_0.next()
@@ -90,17 +81,17 @@ class test_cursor_comparison(wttest.WiredTigerTestCase):
wiredtiger.WiredTigerError, lambda: c2.compare(c1), msg)
# Test cursors before they're positioned.
- c1.set_key(key_populate(c1, 10))
- c2.set_key(key_populate(c2, 20))
+ c1.set_key(ds.key(10))
+ c2.set_key(ds.key(20))
self.assertGreater(c2.compare(c1), 0)
self.assertLess(c1.compare(c2), 0)
- c2.set_key(key_populate(c2, 10))
+ c2.set_key(ds.key(10))
self.assertEqual(c1.compare(c2), 0)
self.assertEqual(c2.compare(c1), 0)
# Confirm failure for different objects.
cX = self.session.open_cursor(uriX, None)
- cX.set_key(key_populate(cX, 10))
+ cX.set_key(dsX.key(10))
msg = '/must reference the same object/'
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError, lambda: cX.compare(c1), msg)
@@ -125,20 +116,20 @@ class test_cursor_comparison(wttest.WiredTigerTestCase):
wiredtiger.WiredTigerError, lambda: ix0_0.compare(ix1_0), msg)
# Test cursors after they're positioned (shouldn't matter for compare).
- c1.set_key(key_populate(c1, 10))
+ c1.set_key(ds.key(10))
self.assertEqual(c1.search(), 0)
- c2.set_key(key_populate(c2, 20))
+ c2.set_key(ds.key(20))
self.assertEqual(c2.search(), 0)
self.assertGreater(c2.compare(c1), 0)
self.assertLess(c1.compare(c2), 0)
- c2.set_key(key_populate(c2, 10))
+ c2.set_key(ds.key(10))
self.assertEqual(c2.search(), 0)
self.assertEqual(c1.compare(c2), 0)
self.assertEqual(c2.compare(c1), 0)
# Confirm failure for different objects.
cX = self.session.open_cursor(uriX, None)
- cX.set_key(key_populate(cX, 10))
+ cX.set_key(dsX.key(10))
self.assertEqual(cX.search(), 0)
msg = '/must reference the same object/'
self.assertRaisesWithMessage(
@@ -149,28 +140,20 @@ class test_cursor_comparison(wttest.WiredTigerTestCase):
uriX = self.type + 'compareX'
# Build the object.
+ ds = self.dataset(self, uri, 100, key_format=self.keyfmt)
+ dsX = self.dataset(self, uriX, 100, key_format=self.keyfmt)
+ ds.populate()
+ dsX.populate()
if self.type == 'file:':
- simple_populate(
- self, uri, 'key_format=' + self.keyfmt + self.config, 100)
- simple_populate(
- self, uriX, 'key_format=' + self.keyfmt + self.config, 100)
ix0_0 = None
ix0_1 = None
ix1_0 = None
ixX_0 = None
else:
- complex_populate(
- self, uri, 'key_format=' + self.keyfmt + self.config, 100)
- complex_populate(
- self, uriX, 'key_format=' + self.keyfmt + self.config, 100)
- ix0_0 = self.session.open_cursor(
- complex_populate_index_name(self, uri, 0), None)
- ix0_1 = self.session.open_cursor(
- complex_populate_index_name(self, uri, 0), None)
- ix1_0 = self.session.open_cursor(
- complex_populate_index_name(self, uri, 1), None)
- ixX_0 = self.session.open_cursor(
- complex_populate_index_name(self, uriX, 0), None)
+ ix0_0 = self.session.open_cursor(ds.index_name(0), None)
+ ix0_1 = self.session.open_cursor(ds.index_name(0), None)
+ ix1_0 = self.session.open_cursor(ds.index_name(1), None)
+ ixX_0 = self.session.open_cursor(dsX.index_name(0), None)
ix0_0.next()
ix0_1.next()
ix1_0.next()
@@ -187,17 +170,17 @@ class test_cursor_comparison(wttest.WiredTigerTestCase):
wiredtiger.WiredTigerError, lambda: c2.equals(c1), msg)
# Test cursors before they're positioned.
- c1.set_key(key_populate(c1, 10))
- c2.set_key(key_populate(c2, 20))
+ c1.set_key(ds.key(10))
+ c2.set_key(ds.key(20))
self.assertFalse(c1.equals(c2))
self.assertFalse(c2.equals(c1))
- c2.set_key(key_populate(c2, 10))
+ c2.set_key(ds.key(10))
self.assertTrue(c1.equals(c2))
self.assertTrue(c2.equals(c1))
# Confirm failure for different objects.
cX = self.session.open_cursor(uriX, None)
- cX.set_key(key_populate(cX, 10))
+ cX.set_key(dsX.key(10))
msg = '/must reference the same object/'
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError, lambda: cX.equals(c1), msg)
@@ -222,20 +205,20 @@ class test_cursor_comparison(wttest.WiredTigerTestCase):
# Test cursors after they're positioned (internally, it's a different
# search path if keys are positioned in the tree).
- c1.set_key(key_populate(c1, 10))
+ c1.set_key(ds.key(10))
self.assertEqual(c1.search(), 0)
- c2.set_key(key_populate(c2, 20))
+ c2.set_key(ds.key(20))
self.assertEqual(c2.search(), 0)
self.assertFalse(c1.equals(c2))
self.assertFalse(c2.equals(c1))
- c2.set_key(key_populate(c2, 10))
+ c2.set_key(ds.key(10))
self.assertEqual(c2.search(), 0)
self.assertTrue(c1.equals(c2))
self.assertTrue(c2.equals(c1))
# Confirm failure for different objects.
cX = self.session.open_cursor(uriX, None)
- cX.set_key(key_populate(cX, 10))
+ cX.set_key(dsX.key(10))
self.assertEqual(cX.search(), 0)
msg = '/must reference the same object/'
self.assertRaisesWithMessage(
diff --git a/test/suite/test_cursor_pin.py b/test/suite/test_cursor_pin.py
index b6119a93897..cb7045c7e41 100644
--- a/test/suite/test_cursor_pin.py
+++ b/test/suite/test_cursor_pin.py
@@ -27,7 +27,7 @@
# OTHER DEALINGS IN THE SOFTWARE.
import wiredtiger, wttest
-from helper import simple_populate, key_populate, value_populate
+from wtdataset import SimpleDataSet
from wtscenario import make_scenarios
# test_cursor_pin.py
@@ -36,7 +36,7 @@ from wtscenario import make_scenarios
class test_cursor_pin(wttest.WiredTigerTestCase):
uri = 'file:cursor_pin'
nentries = 10000
- config = 'allocation_size=512,leaf_page_max=512,value_format=S,key_format='
+ config = 'allocation_size=512,leaf_page_max=512'
scenarios = make_scenarios([
('recno', dict(keyfmt='r')),
('string', dict(keyfmt='S')),
@@ -45,72 +45,75 @@ class test_cursor_pin(wttest.WiredTigerTestCase):
# Create a multi-page file, confirm that a simple search to the local
# page works, followed by a search to a different page.
def test_smoke(self):
- simple_populate(self,
- self.uri, self.config + self.keyfmt, self.nentries)
+ ds = SimpleDataSet(self, self.uri, self.nentries,
+ config=self.config, key_format=self.keyfmt)
+ ds.populate()
self.reopen_conn()
c = self.session.open_cursor(self.uri, None)
- c.set_key(key_populate(c, 100))
+ c.set_key(ds.key(100))
self.assertEqual(c.search(), 0)
- self.assertEqual(c.get_value(), value_populate(c, 100))
- c.set_key(key_populate(c, 101))
+ self.assertEqual(c.get_value(), ds.value(100))
+ c.set_key(ds.key(101))
self.assertEqual(c.search(), 0)
- self.assertEqual(c.get_value(), value_populate(c, 101))
- c.set_key(key_populate(c, 9999))
+ self.assertEqual(c.get_value(), ds.value(101))
+ c.set_key(ds.key(9999))
self.assertEqual(c.search(), 0)
- self.assertEqual(c.get_value(), value_populate(c, 9999))
+ self.assertEqual(c.get_value(), ds.value(9999))
# Forward check.
- def forward(self, c, max, notfound):
+ def forward(self, c, ds, max, notfound):
for i in range(1, max + 1):
- c.set_key(key_populate(c, i))
+ c.set_key(ds.key(i))
if i in notfound:
self.assertEqual(c.search(), wiredtiger.WT_NOTFOUND)
else:
self.assertEqual(c.search(), 0)
- self.assertEqual(c.get_value(), value_populate(c, i))
+ self.assertEqual(c.get_value(), ds.value(i))
# Backward check.
- def backward(self, c, max, notfound):
+ def backward(self, c, ds, max, notfound):
for i in range(max, 0, -1):
- c.set_key(key_populate(c, i))
+ c.set_key(ds.key(i))
if i in notfound:
self.assertEqual(c.search(), wiredtiger.WT_NOTFOUND)
else:
self.assertEqual(c.search(), 0)
- self.assertEqual(c.get_value(), value_populate(c, i))
+ self.assertEqual(c.get_value(), ds.value(i))
# Create a multi-page file, search backward, forward to check page
# boundaries.
def test_basic(self):
- simple_populate(self,
- self.uri, self.config + self.keyfmt, self.nentries)
+ ds = SimpleDataSet(self, self.uri, self.nentries,
+ config=self.config, key_format=self.keyfmt)
+ ds.populate()
self.reopen_conn()
c = self.session.open_cursor(self.uri, None)
- self.forward(c, self.nentries, [])
- self.backward(c, self.nentries, [])
+ self.forward(c, ds, self.nentries, [])
+ self.backward(c, ds, self.nentries, [])
# Create a multi-page file with a big chunk of missing space in the
# middle (to exercise column-store searches).
def test_missing(self):
- simple_populate(self,
- self.uri, self.config + self.keyfmt, self.nentries)
+ ds = SimpleDataSet(self, self.uri, self.nentries,
+ config=self.config, key_format=self.keyfmt)
+ ds.populate()
c = self.session.open_cursor(self.uri, None)
for i in range(self.nentries + 3000, self.nentries + 5001):
- c[key_populate(c, i)] = value_populate(c, i)
+ c[ds.key(i)] = ds.value(i)
self.reopen_conn()
c = self.session.open_cursor(self.uri, None)
- self.forward(c, self.nentries + 5000,
+ self.forward(c, ds, self.nentries + 5000,
list(range(self.nentries + 1, self.nentries + 3000)))
- self.backward(c, self.nentries + 5000,
+ self.backward(c, ds, self.nentries + 5000,
list(range(self.nentries + 1, self.nentries + 3000)))
# Insert into the empty space so we test searching inserted items.
for i in range(self.nentries + 1000, self.nentries + 2001):
- c[key_populate(c, i)] = value_populate(c, i)
- self.forward(c, self.nentries + 5000,
+ c[ds.key(i)] = ds.value(i)
+ self.forward(c, ds, self.nentries + 5000,
list(range(self.nentries + 1, self.nentries + 1000) +\
range(self.nentries + 2001, self.nentries + 3000)))
- self.backward(c, self.nentries + 5000,
+ self.backward(c, ds, self.nentries + 5000,
list(range(self.nentries + 1, self.nentries + 1000) +\
range(self.nentries + 2001, self.nentries + 3000)))
diff --git a/test/suite/test_cursor_random.py b/test/suite/test_cursor_random.py
index 205b9c3c9c5..3bda6dc9946 100644
--- a/test/suite/test_cursor_random.py
+++ b/test/suite/test_cursor_random.py
@@ -27,16 +27,15 @@
# OTHER DEALINGS IN THE SOFTWARE.
import wiredtiger, wttest
-from helper import complex_populate, simple_populate
-from helper import key_populate, value_populate
+from wtdataset import SimpleDataSet, ComplexDataSet, simple_key, simple_value
from wtscenario import make_scenarios
# test_cursor_random.py
# Cursor next_random operations
class test_cursor_random(wttest.WiredTigerTestCase):
types = [
- ('file', dict(type='file:random')),
- ('table', dict(type='table:random'))
+ ('file', dict(type='file:random', dataset=SimpleDataSet)),
+ ('table', dict(type='table:random', dataset=ComplexDataSet))
]
config = [
('sample', dict(config='next_random=true,next_random_sample_size=35')),
@@ -89,12 +88,9 @@ class test_cursor_random(wttest.WiredTigerTestCase):
# where the values are in an insert list.
def test_cursor_random_multiple_insert_records(self):
uri = self.type
- if uri.startswith('file:'):
- simple_populate(self, uri,
- 'allocation_size=512,leaf_page_max=512,key_format=S', 100)
- else:
- complex_populate(self, uri,
- 'allocation_size=512,leaf_page_max=512,key_format=S', 100)
+ ds = self.dataset(self, uri, 100,
+ config='allocation_size=512,leaf_page_max=512')
+ ds.populate()
# In a insert list, next_random always selects the middle key/value
# pair, all we can do is confirm cursor.next works.
@@ -105,12 +101,9 @@ class test_cursor_random(wttest.WiredTigerTestCase):
# where the values are in a disk format page.
def cursor_random_multiple_page_records(self, reopen):
uri = self.type
- if uri.startswith('file:'):
- simple_populate(self, uri,
- 'allocation_size=512,leaf_page_max=512,key_format=S', 10000)
- else:
- complex_populate(self, uri,
- 'allocation_size=512,leaf_page_max=512,key_format=S', 10000)
+ ds = self.dataset(self, uri, 10000,
+ config='allocation_size=512,leaf_page_max=512')
+ ds.populate()
# Optionally close the connection so everything is forced to disk,
# insert lists are an entirely different path in the code.
@@ -168,7 +161,7 @@ class test_cursor_random_invisible(wttest.WiredTigerTestCase):
# Start a transaction.
self.session.begin_transaction()
for i in range(1, 100):
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+ cursor[simple_key(cursor, i)] = simple_value(cursor, i)
# Open another session, the updates won't yet be visible, we shouldn't
# find anything at all.
@@ -182,19 +175,19 @@ class test_cursor_random_invisible(wttest.WiredTigerTestCase):
cursor = self.session.open_cursor(uri, None)
# Insert a single leading record.
- cursor[key_populate(cursor, 1)] = value_populate(cursor, 1)
+ cursor[simple_key(cursor, 1)] = simple_value(cursor, 1)
# Start a transaction.
self.session.begin_transaction()
for i in range(2, 100):
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+ cursor[simple_key(cursor, i)] = simple_value(cursor, i)
# Open another session, the updates won't yet be visible, we should
# return the only possible record.
s = self.conn.open_session()
cursor = s.open_cursor(uri, None, self.config)
self.assertEquals(cursor.next(), 0)
- self.assertEqual(cursor.get_key(), key_populate(cursor, 1))
+ self.assertEqual(cursor.get_key(), simple_key(cursor, 1))
def test_cursor_random_invisible_before(self):
uri = self.type
@@ -202,19 +195,19 @@ class test_cursor_random_invisible(wttest.WiredTigerTestCase):
cursor = self.session.open_cursor(uri, None)
# Insert a single leading record.
- cursor[key_populate(cursor, 99)] = value_populate(cursor, 99)
+ cursor[simple_key(cursor, 99)] = simple_value(cursor, 99)
# Start a transaction.
self.session.begin_transaction()
for i in range(2, 100):
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+ cursor[simple_key(cursor, i)] = simple_value(cursor, i)
# Open another session, the updates won't yet be visible, we should
# return the only possible record.
s = self.conn.open_session()
cursor = s.open_cursor(uri, None, self.config)
self.assertEquals(cursor.next(), 0)
- self.assertEqual(cursor.get_key(), key_populate(cursor, 99))
+ self.assertEqual(cursor.get_key(), simple_key(cursor, 99))
if __name__ == '__main__':
wttest.run()
diff --git a/test/suite/test_cursor_random02.py b/test/suite/test_cursor_random02.py
index 195480d703b..d18d8efd94d 100644
--- a/test/suite/test_cursor_random02.py
+++ b/test/suite/test_cursor_random02.py
@@ -27,8 +27,7 @@
# OTHER DEALINGS IN THE SOFTWARE.
import wiredtiger, wttest
-from helper import complex_populate, simple_populate
-from helper import key_populate, value_populate
+from wtdataset import SimpleDataSet
from wtscenario import make_scenarios
# test_cursor_random02.py
@@ -56,12 +55,14 @@ class test_cursor_random02(wttest.WiredTigerTestCase):
def test_cursor_random_reasonable_distribution(self):
uri = self.type
num_entries = self.records
- config = 'key_format=S'
if uri == 'table:random':
- config = 'leaf_page_max=100MB,' + config
+ config = 'leaf_page_max=100MB'
+ else:
+ config = ''
# Set the leaf-page-max value, otherwise the page might split.
- simple_populate(self, uri, config, num_entries)
+ ds = SimpleDataSet(self, uri, num_entries, config=config)
+ ds.populate()
# Setup an array to track which keys are seen
visitedKeys = [0] * (num_entries + 1)
# Setup a counter to see when we find a sequential key
diff --git a/test/suite/test_drop.py b/test/suite/test_drop.py
index a3e80214295..e241c05aa68 100644
--- a/test/suite/test_drop.py
+++ b/test/suite/test_drop.py
@@ -28,8 +28,8 @@
import os, time
import wiredtiger, wttest
-from helper import confirm_does_not_exist, complex_populate, \
- complex_populate_index_name, simple_populate
+from helper import confirm_does_not_exist
+from wtdataset import SimpleDataSet, ComplexDataSet
from wtscenario import make_scenarios
# test_drop.py
@@ -45,9 +45,10 @@ class test_drop(wttest.WiredTigerTestCase):
])
# Populate an object, remove it and confirm it no longer exists.
- def drop(self, populate, with_cursor, reopen, drop_index):
+ def drop(self, dataset, with_cursor, reopen, drop_index):
uri = self.uri + self.name
- populate(self, uri, 'key_format=S' + self.extra_config, 10)
+ ds = dataset(self, uri, 10, config=self.extra_config)
+ ds.populate()
# Open cursors should cause failure.
if with_cursor:
@@ -60,7 +61,7 @@ class test_drop(wttest.WiredTigerTestCase):
self.reopen_conn()
if drop_index:
- drop_uri = complex_populate_index_name(self, uri, 0)
+ drop_uri = ds.index_name(0)
else:
drop_uri = uri
self.session.drop(drop_uri, None)
@@ -73,7 +74,7 @@ class test_drop(wttest.WiredTigerTestCase):
# case has no indices.
for with_cursor in [False, True]:
for reopen in [False, True]:
- self.drop(simple_populate, with_cursor, reopen, False)
+ self.drop(SimpleDataSet, with_cursor, reopen, False)
# A complex, multi-file table object.
# Try all test combinations.
@@ -81,7 +82,7 @@ class test_drop(wttest.WiredTigerTestCase):
for with_cursor in [False, True]:
for reopen in [False, True]:
for drop_index in [False, True]:
- self.drop(complex_populate, with_cursor,
+ self.drop(ComplexDataSet, with_cursor,
reopen, drop_index)
# Test drop of a non-existent object: force succeeds, without force fails.
diff --git a/test/suite/test_drop02.py b/test/suite/test_drop02.py
index 677ba3866b2..017aa64e312 100644
--- a/test/suite/test_drop02.py
+++ b/test/suite/test_drop02.py
@@ -27,7 +27,7 @@
# OTHER DEALINGS IN THE SOFTWARE.
import wiredtiger, wttest
-from helper import simple_populate
+from wtdataset import SimpleDataSet
# test_drop02.py
# Test dropping an LSM tree on first open. There was a bug where this
@@ -38,7 +38,8 @@ class test_drop02(wttest.WiredTigerTestCase):
# Populate an object, remove it and confirm it no longer exists.
def test_drop(self):
uri = 'lsm:' + self.name
- simple_populate(self, uri, 'key_format=S', 100000)
+ ds = SimpleDataSet(self, uri, 100000)
+ ds.populate()
self.reopen_conn()
self.session.drop(uri, None)
diff --git a/test/suite/test_dump.py b/test/suite/test_dump.py
index 280d5870359..f6a83c32489 100644
--- a/test/suite/test_dump.py
+++ b/test/suite/test_dump.py
@@ -28,19 +28,18 @@
import os, shutil
import wiredtiger, wttest
-from helper import \
- complex_populate, complex_populate_check, \
- simple_populate, simple_populate_check, \
- simple_index_populate, simple_index_populate_check
+
from suite_subprocess import suite_subprocess
from wtscenario import make_scenarios
+from wtdataset import SimpleDataSet, SimpleIndexDataSet, SimpleLSMDataSet, \
+ ComplexDataSet, ComplexLSMDataSet
# test_dump.py
# Utilities: wt dump
# Test the dump utility (I'm not testing the dump cursors, that's what the
# utility uses underneath).
class test_dump(wttest.WiredTigerTestCase, suite_subprocess):
- dir='dump.dir' # Backup directory name
+ dir = 'dump.dir' # Backup directory name
name = 'test_dump'
name2 = 'test_dumpb'
@@ -56,30 +55,20 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess):
('string', dict(keyfmt='S'))
]
types = [
- ('file', dict(uri='file:', config='', lsm=False,
- populate=simple_populate,
- populate_check=simple_populate_check)),
- ('lsm', dict(uri='lsm:', config='', lsm=True,
- populate=simple_populate,
- populate_check=simple_populate_check)),
- ('table-simple', dict(uri='table:', config='', lsm=False,
- populate=simple_populate,
- populate_check=simple_populate_check)),
- ('table-index', dict(uri='table:', config='', lsm=False,
- populate=simple_index_populate,
- populate_check=simple_index_populate_check)),
- ('table-simple-lsm', dict(uri='table:', config='type=lsm', lsm=True,
- populate=simple_populate,
- populate_check=simple_populate_check)),
- ('table-complex', dict(uri='table:', config='', lsm=False,
- populate=complex_populate,
- populate_check=complex_populate_check)),
- ('table-complex-lsm', dict(uri='table:', config='type=lsm', lsm=True,
- populate=complex_populate,
- populate_check=complex_populate_check))
+ ('file', dict(uri='file:', dataset=SimpleDataSet)),
+ ('lsm', dict(uri='lsm:', dataset=SimpleDataSet)),
+ ('table-simple', dict(uri='table:', dataset=SimpleDataSet)),
+ ('table-index', dict(uri='table:', dataset=SimpleIndexDataSet)),
+ ('table-simple-lsm', dict(uri='table:', dataset=SimpleLSMDataSet)),
+ ('table-complex', dict(uri='table:', dataset=ComplexDataSet)),
+ ('table-complex-lsm', dict(uri='table:', dataset=ComplexLSMDataSet)),
]
scenarios = make_scenarios(types, keyfmt, dumpfmt)
+ def skip(self):
+ return (self.dataset.is_lsm() or self.uri == 'lsm:') and \
+ self.keyfmt == 'r'
+
# Extract the values lines from the dump output.
def value_lines(self, fname):
# mode:
@@ -108,14 +97,14 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess):
# Dump, re-load and do a content comparison.
def test_dump(self):
# LSM and column-store isn't a valid combination.
- if self.lsm and self.keyfmt == 'r':
+ if self.skip():
return
# Create the object.
uri = self.uri + self.name
uri2 = self.uri + self.name2
- self.populate(self, uri,
- self.config + ',key_format=' + self.keyfmt, self.nentries)
+ pop = self.dataset(self, uri, self.nentries, key_format=self.keyfmt)
+ pop.populate()
# Dump the object.
os.mkdir(self.dir)
@@ -136,14 +125,14 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess):
# Check the object's contents
self.reopen_conn(self.dir)
- self.populate_check(self, uri, self.nentries)
+ pop.check()
# Re-load the object again in the original directory.
self.reopen_conn('.')
self.runWt(['-h', self.dir, 'load', '-f', 'dump.out'])
# Check the contents, they shouldn't have changed.
- self.populate_check(self, uri, self.nentries)
+ pop.check()
# Re-load the object again, but confirm -n (no overwrite) fails.
self.runWt(['-h', self.dir, 'load', '-n', '-f', 'dump.out'],
@@ -151,7 +140,7 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess):
self.check_non_empty_file('errfile.out')
# If there are indices, dump one of them and check the output.
- if self.populate == complex_populate:
+ if self.dataset == ComplexDataSet:
indexuri = 'index:' + self.name + ':indx1'
hexopt = ['-x'] if self.hex == 1 else []
self.runWt(['-h', self.dir, 'dump'] + hexopt + [indexuri],
@@ -166,7 +155,8 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess):
# Check the contents in the new table.
self.reopen_conn(self.dir)
- self.populate_check(self, uri2, self.nentries)
+ pop = self.dataset(self, uri2, self.nentries, key_format=self.keyfmt)
+ pop.check()
if __name__ == '__main__':
wttest.run()
diff --git a/test/suite/test_dupc.py b/test/suite/test_dupc.py
index 12b18f1ba79..c0cf6acc75e 100644
--- a/test/suite/test_dupc.py
+++ b/test/suite/test_dupc.py
@@ -32,7 +32,7 @@
import os, time
import wiredtiger, wttest
-from helper import complex_populate, key_populate, simple_populate
+from wtdataset import SimpleDataSet, ComplexDataSet
from wtscenario import make_scenarios
# Test session.open_cursor with cursor duplication.
@@ -40,8 +40,6 @@ class test_duplicate_cursor(wttest.WiredTigerTestCase):
name = 'test_dupc'
nentries = 1000
- config = 'key_format='
-
scenarios = make_scenarios([
('file-r', dict(uri='file:', fmt='r')),
('file-S', dict(uri='file:', fmt='S')),
@@ -51,7 +49,7 @@ class test_duplicate_cursor(wttest.WiredTigerTestCase):
# Iterate through an object, duplicate the cursor and checking that it
# matches the original and is set to the same record.
- def iterate(self, uri):
+ def iterate(self, uri, ds):
cursor = self.session.open_cursor(uri, None, None)
next = 0
while True:
@@ -59,10 +57,10 @@ class test_duplicate_cursor(wttest.WiredTigerTestCase):
if nextret != 0:
break
next += 1
- self.assertEqual(cursor.get_key(), key_populate(cursor, next))
+ self.assertEqual(cursor.get_key(), ds.key(next))
dupc = self.session.open_cursor(None, cursor, None)
self.assertEqual(cursor.compare(dupc), 0)
- self.assertEqual(dupc.get_key(), key_populate(dupc, next))
+ self.assertEqual(dupc.get_key(), ds.key(next))
cursor.close()
cursor = dupc
self.assertEqual(next, self.nentries)
@@ -73,14 +71,16 @@ class test_duplicate_cursor(wttest.WiredTigerTestCase):
uri = self.uri + self.name
# A simple, one-file file or table object.
- simple_populate(self, uri, self.config + self.fmt, self.nentries)
- self.iterate(uri)
+ ds = SimpleDataSet(self, uri, self.nentries, key_format=self.fmt)
+ ds.populate()
+ self.iterate(uri, ds)
self.session.drop(uri, None)
# A complex, multi-file table object.
if self.uri == "table:":
- complex_populate(self, uri, self.config + self.fmt, self.nentries)
- self.iterate(uri)
+ ds = ComplexDataSet(self, uri, self.nentries, key_format=self.fmt)
+ ds.populate()
+ self.iterate(uri, ds)
self.session.drop(uri, None)
if __name__ == '__main__':
diff --git a/test/suite/test_empty.py b/test/suite/test_empty.py
index 9fe88107412..578bec618c9 100644
--- a/test/suite/test_empty.py
+++ b/test/suite/test_empty.py
@@ -28,7 +28,7 @@
import os
import wiredtiger, wttest
-from helper import key_populate
+from wtdataset import simple_key
from wtscenario import make_scenarios
# test_empty.py
@@ -64,7 +64,7 @@ class test_empty(wttest.WiredTigerTestCase):
# Add a few records to the object and remove them.
cursor = self.session.open_cursor(uri, None, None)
for i in range(1,5):
- key = key_populate(cursor, i)
+ key = simple_key(cursor, i)
cursor[key] = "XXX"
del cursor[key]
diff --git a/test/suite/test_inmem01.py b/test/suite/test_inmem01.py
index c6ae7ff6c4b..1af43bbd9d9 100644
--- a/test/suite/test_inmem01.py
+++ b/test/suite/test_inmem01.py
@@ -28,8 +28,7 @@
import wiredtiger, wttest
from time import sleep
-from helper import simple_populate, simple_populate_check
-from helper import key_populate, value_populate
+from wtdataset import SimpleDataSet
from wtscenario import make_scenarios
# test_inmem01.py
@@ -41,54 +40,60 @@ class test_inmem01(wttest.WiredTigerTestCase):
table_config = ',memory_page_max=32k,leaf_page_max=4k'
scenarios = make_scenarios([
- ('col', dict(fmt='key_format=r,value_format=S')),
- ('fix', dict(fmt='key_format=r,value_format=8t')),
- ('row', dict(fmt='key_format=S,value_format=S'))
+ ('col', dict(keyfmt='r', valuefmt='S')),
+ ('fix', dict(keyfmt='r', valuefmt='8t')),
+ ('row', dict(keyfmt='S', valuefmt='S')),
])
# Smoke-test in-memory configurations, add a small amount of data and
# ensure it's visible.
def test_insert(self):
- config = self.fmt + self.table_config
- simple_populate(self, self.uri, config, 1000)
- simple_populate_check(self, self.uri, 1000)
+ ds = SimpleDataSet(self, self.uri, 1000, key_format=self.keyfmt,
+ value_format=self.valuefmt, config=self.table_config)
+ ds.populate()
+ ds.check()
# Add more data than fits into the configured cache and verify it fails.
def test_insert_over_capacity(self):
- config = self.fmt + self.table_config
msg = '/WT_CACHE_FULL.*/'
+ ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt,
+ value_format=self.valuefmt, config=self.table_config)
self.assertRaisesHavingMessage(wiredtiger.WiredTigerError,
- lambda:simple_populate(self, self.uri, config, 10000000), msg)
+ lambda:ds.populate(), msg)
# Figure out the last key we successfully inserted, and check all
# previous inserts are still there.
cursor = self.session.open_cursor(self.uri, None)
cursor.prev()
last_key = int(cursor.get_key())
- simple_populate_check(self, self.uri, last_key)
+ ds = SimpleDataSet(self, self.uri, last_key, key_format=self.keyfmt,
+ value_format=self.valuefmt, config=self.table_config)
+ ds.check()
# Fill the cache with data, remove some data, ensure more data can be
# inserted (after a reasonable amount of time for space to be reclaimed).
def test_insert_over_delete(self):
- config = self.fmt + self.table_config
msg = '/WT_CACHE_FULL.*/'
+ ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt,
+ value_format=self.valuefmt, config=self.table_config)
self.assertRaisesHavingMessage(wiredtiger.WiredTigerError,
- lambda:simple_populate(self, self.uri, config, 10000000), msg)
+ lambda:ds.populate(), msg)
# Now that the database contains as much data as will fit into
# the configured cache, verify removes succeed.
cursor = self.session.open_cursor(self.uri, None)
for i in range(1, 100):
- cursor.set_key(key_populate(cursor, i))
+ cursor.set_key(ds.key(i))
cursor.remove()
# Run queries after adding, removing and re-inserting data.
# Try out keeping a cursor open while adding new data.
def test_insert_over_delete_replace(self):
- config = self.fmt + self.table_config
msg = '/WT_CACHE_FULL.*/'
+ ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt,
+ value_format=self.valuefmt, config=self.table_config)
self.assertRaisesHavingMessage(wiredtiger.WiredTigerError,
- lambda:simple_populate(self, self.uri, config, 10000000), msg)
+ lambda:ds.populate(), msg)
cursor = self.session.open_cursor(self.uri, None)
cursor.prev()
@@ -98,7 +103,7 @@ class test_inmem01(wttest.WiredTigerTestCase):
# the configured cache, verify removes succeed.
cursor = self.session.open_cursor(self.uri, None)
for i in range(1, last_key / 4, 1):
- cursor.set_key(key_populate(cursor, i))
+ cursor.set_key(ds.key(i))
cursor.remove()
cursor.reset()
@@ -106,7 +111,7 @@ class test_inmem01(wttest.WiredTigerTestCase):
inserted = False
for i in range(1, 1000):
try:
- cursor[key_populate(cursor, 1)] = value_populate(cursor, 1)
+ cursor[ds.key(1)] = ds.value(1)
except wiredtiger.WiredTigerError:
cursor.reset()
sleep(1)
diff --git a/test/suite/test_inmem02.py b/test/suite/test_inmem02.py
index 9eb8330b2a3..b5e07fea967 100644
--- a/test/suite/test_inmem02.py
+++ b/test/suite/test_inmem02.py
@@ -28,9 +28,7 @@
import wiredtiger, wttest
from time import sleep
-from helper import simple_populate, simple_populate_check
-from helper import key_populate, value_populate
-from wtscenario import make_scenarios
+from wtdataset import SimpleDataSet
# test_inmem02.py
# Test in-memory with ignore-cache-size setting.
@@ -38,7 +36,7 @@ class test_inmem02(wttest.WiredTigerTestCase):
uri = 'table:inmem02'
conn_config = \
'cache_size=3MB,file_manager=(close_idle_time=0),in_memory=true'
- table_config = 'key_format=S,value_format=S,memory_page_max=32k,leaf_page_max=4k'
+ table_config = 'memory_page_max=32k,leaf_page_max=4k'
# Add more data than fits into the configured cache and verify it fails.
def test_insert_over_allowed(self):
@@ -50,9 +48,9 @@ class test_inmem02(wttest.WiredTigerTestCase):
# Populate a table with enough data to fill the cache.
msg = '/WT_CACHE_FULL.*/'
+ ds = SimpleDataSet(self, self.uri, 10000000, config=self.table_config)
self.assertRaisesHavingMessage(wiredtiger.WiredTigerError,
- lambda:simple_populate(
- self, self.uri, self.table_config, 10000000), msg)
+ lambda:ds.populate(), msg)
# Add some content to the new table
cursor = self.session.open_cursor(self.uri + '_over', None)
diff --git a/test/suite/test_jsondump01.py b/test/suite/test_jsondump01.py
index dc8027c2115..c7fa9cdf397 100644
--- a/test/suite/test_jsondump01.py
+++ b/test/suite/test_jsondump01.py
@@ -28,11 +28,9 @@
import os, json
import wiredtiger, wttest
-from helper import \
- complex_populate, complex_populate_check, complex_populate_check_cursor,\
- simple_populate, simple_populate_check, simple_populate_check_cursor, \
- simple_index_populate, simple_index_populate_check, \
- simple_index_populate_check_cursor, compare_files
+from wtdataset import SimpleDataSet, SimpleLSMDataSet, SimpleIndexDataSet, \
+ ComplexDataSet, ComplexLSMDataSet
+from helper import compare_files
from suite_subprocess import suite_subprocess
from wtscenario import make_scenarios
@@ -40,7 +38,8 @@ from wtscenario import make_scenarios
# It emulates a WT cursor well enough for the *_check_cursor methods.
# They just need an iterable object.
class FakeCursor:
- def __init__(self, keyfmt, valuefmt, rows):
+ def __init__(self, uri, keyfmt, valuefmt, rows):
+ self.uri = uri
self.key_format = keyfmt
self.value_format = valuefmt
self.rows = rows
@@ -79,47 +78,30 @@ class test_jsondump01(wttest.WiredTigerTestCase, suite_subprocess):
('string', dict(keyfmt='S'))
]
types = [
- ('file', dict(uri='file:', config='', lsm=False,
- populate=simple_populate,
- populate_check=simple_populate_check,
- populate_check_cursor=simple_populate_check_cursor)),
- ('lsm', dict(uri='lsm:', config='', lsm=True,
- populate=simple_populate,
- populate_check=simple_populate_check,
- populate_check_cursor=simple_populate_check_cursor)),
- ('table-simple', dict(uri='table:', config='', lsm=False,
- populate=simple_populate,
- populate_check=simple_populate_check,
- populate_check_cursor=simple_populate_check_cursor)),
- ('table-index', dict(uri='table:', config='', lsm=False,
- populate=simple_index_populate,
- populate_check=simple_index_populate_check,
- populate_check_cursor=simple_index_populate_check_cursor)),
- ('table-simple-lsm', dict(uri='table:', config='type=lsm', lsm=True,
- populate=simple_populate,
- populate_check=simple_populate_check,
- populate_check_cursor=simple_populate_check_cursor)),
- ('table-complex', dict(uri='table:', config='', lsm=False,
- populate=complex_populate,
- populate_check=complex_populate_check,
- populate_check_cursor=complex_populate_check_cursor)),
- ('table-complex-lsm', dict(uri='table:', config='type=lsm', lsm=True,
- populate=complex_populate,
- populate_check=complex_populate_check,
- populate_check_cursor=complex_populate_check_cursor))
+ ('file', dict(uri='file:', dataset=SimpleDataSet)),
+ ('lsm', dict(uri='lsm:', dataset=SimpleDataSet)),
+ ('table-simple', dict(uri='table:', dataset=SimpleDataSet)),
+ ('table-index', dict(uri='table:', dataset=SimpleIndexDataSet)),
+ ('table-simple-lsm', dict(uri='table:', dataset=SimpleLSMDataSet)),
+ ('table-complex', dict(uri='table:', dataset=ComplexDataSet)),
+ ('table-complex-lsm', dict(uri='table:', dataset=ComplexLSMDataSet))
]
scenarios = make_scenarios(types, keyfmt)
+ def skip(self):
+ return (self.dataset.is_lsm() or self.uri == 'lsm:') and \
+ self.keyfmt == 'r'
+
# Dump using util, re-load using python's JSON, and do a content comparison.
def test_jsondump_util(self):
# LSM and column-store isn't a valid combination.
- if self.lsm and self.keyfmt == 'r':
+ if self.skip():
return
# Create the object.
uri = self.uri + self.name
- self.populate(self, uri, self.config + ',key_format=' + self.keyfmt,
- self.nentries)
+ ds = self.dataset(self, uri, self.nentries, key_format=self.keyfmt)
+ ds.populate()
# Dump the object.
self.runWt(['dump', '-j', uri], outfilename='jsondump.out')
@@ -141,20 +123,21 @@ class test_jsondump01(wttest.WiredTigerTestCase, suite_subprocess):
# check the contents of the data we read.
# we only use a wt cursor to get the key_format/value_format.
cursor = self.session.open_cursor(uri, None)
- fake = FakeCursor(cursor.key_format, cursor.value_format, data)
+ fake = FakeCursor(uri, cursor.key_format, cursor.value_format, data)
cursor.close()
- self.populate_check_cursor(self, fake, self.nentries)
+ ds.check_cursor(fake)
# Dump using util, re-load using python's JSON, and do a content comparison.
def test_jsonload_util(self):
# LSM and column-store isn't a valid combination.
- if self.lsm and self.keyfmt == 'r':
+ if self.skip():
return
# Create the object.
uri = self.uri + self.name
uri2 = self.uri + self.name2
- self.populate(self, uri, 'key_format=' + self.keyfmt, self.nentries)
+ ds = self.dataset(self, uri, self.nentries, key_format=self.keyfmt)
+ ds.populate()
# Dump the object.
self.runWt(['dump', '-j', uri], outfilename='jsondump.out')
@@ -165,7 +148,9 @@ class test_jsondump01(wttest.WiredTigerTestCase, suite_subprocess):
self.runWt(loadcmd)
# Check the contents of the data we read.
- self.populate_check(self, uri2, self.nentries)
+ # We use the dataset only for checking.
+ ds2 = self.dataset(self, uri2, self.nentries, key_format=self.keyfmt)
+ ds2.check()
# Reload into the original uri, and dump into another file.
self.session.drop(uri, None)
@@ -175,7 +160,7 @@ class test_jsondump01(wttest.WiredTigerTestCase, suite_subprocess):
# Compare the two outputs, and check the content again.
compare_files(self, 'jsondump.out', 'jsondump2.out')
- self.populate_check(self, uri, self.nentries)
+ ds.check()
if __name__ == '__main__':
wttest.run()
diff --git a/test/suite/test_lsm01.py b/test/suite/test_lsm01.py
index f6cee20e896..b44df4bae14 100644
--- a/test/suite/test_lsm01.py
+++ b/test/suite/test_lsm01.py
@@ -27,7 +27,7 @@
# OTHER DEALINGS IN THE SOFTWARE.
import wiredtiger, wtscenario, wttest
-from helper import simple_populate
+from wtdataset import SimpleDataSet
# test_lsm01.py
# Test LSM tree configuration options.
@@ -77,7 +77,7 @@ class test_lsm01(wttest.WiredTigerTestCase):
args += ')' # Close the LSM configuration option group
self.verbose(3,
'Test LSM with config: ' + args + ' count: ' + str(self.nrecs))
- simple_populate(self, self.uri, args, self.nrecs)
+ SimpleDataSet(self, self.uri, self.nrecs).populate()
# TODO: Adding an explicit drop here can cause deadlocks, if a merge
# is still happening. See issue #349.
diff --git a/test/suite/test_lsm02.py b/test/suite/test_lsm02.py
index 85067665017..e9628139a97 100644
--- a/test/suite/test_lsm02.py
+++ b/test/suite/test_lsm02.py
@@ -27,7 +27,6 @@
# OTHER DEALINGS IN THE SOFTWARE.
import wiredtiger, wtscenario, wttest
-from helper import simple_populate
# test_lsm02.py
# Test LSM schema level operations
diff --git a/test/suite/test_lsm03.py b/test/suite/test_lsm03.py
index 448d864c646..d916db415da 100644
--- a/test/suite/test_lsm03.py
+++ b/test/suite/test_lsm03.py
@@ -27,7 +27,7 @@
# OTHER DEALINGS IN THE SOFTWARE.
import wiredtiger, wtscenario, wttest
-from helper import simple_populate
+from wtdataset import SimpleDataSet
# test_lsm03.py
# Check to make sure that LSM schema operations don't get EBUSY when
@@ -37,13 +37,14 @@ class test_lsm03(wttest.WiredTigerTestCase):
# Use small pages so we generate some internal layout
# Setup LSM so multiple chunks are present
- config = 'key_format=S,allocation_size=512,internal_page_max=512' + \
+ config = 'allocation_size=512,internal_page_max=512' + \
',leaf_page_max=1k,lsm=(chunk_size=512k,merge_min=10)'
# Populate an object then drop it.
def test_lsm_drop_active(self):
uri = 'lsm:' + self.name
- simple_populate(self, uri, self.config, 10000)
+ ds = SimpleDataSet(self, uri, 10000, config=self.config)
+ ds.populate()
# Force to disk
self.reopen_conn()
@@ -55,6 +56,7 @@ class test_lsm03(wttest.WiredTigerTestCase):
cursor.close()
# Add enough records that a merge should be running
- simple_populate(self, uri, self.config, 50000)
+ ds = SimpleDataSet(self, uri, 50000, config=self.config)
+ ds.populate()
# The drop should succeed even when LSM work units are active
self.session.drop(uri)
diff --git a/test/suite/test_nsnap01.py b/test/suite/test_nsnap01.py
index 7e8951750f8..4d5555277fe 100644
--- a/test/suite/test_nsnap01.py
+++ b/test/suite/test_nsnap01.py
@@ -30,7 +30,7 @@
# Named snapshots: basic API
from suite_subprocess import suite_subprocess
-from helper import simple_populate
+from wtdataset import SimpleDataSet
import wiredtiger, wttest
class test_nsnap01(wttest.WiredTigerTestCase, suite_subprocess):
@@ -53,7 +53,7 @@ class test_nsnap01(wttest.WiredTigerTestCase, suite_subprocess):
def test_named_snapshots(self):
# Populate a table
end = start = 0
- simple_populate(self, self.uri, 'key_format=i', 0)
+ SimpleDataSet(self, self.uri, 0, key_format='i').populate()
# Now run a workload:
# every iteration:
diff --git a/test/suite/test_nsnap02.py b/test/suite/test_nsnap02.py
index 510c9d421ef..ed1c96ebe50 100644
--- a/test/suite/test_nsnap02.py
+++ b/test/suite/test_nsnap02.py
@@ -30,7 +30,7 @@
# Named snapshots: Combinations of dropping snapshots
from suite_subprocess import suite_subprocess
-from helper import simple_populate
+from wtdataset import SimpleDataSet
import wiredtiger, wttest
class test_nsnap02(wttest.WiredTigerTestCase, suite_subprocess):
@@ -63,7 +63,7 @@ class test_nsnap02(wttest.WiredTigerTestCase, suite_subprocess):
def create_snapshots(self):
# Populate a table
end = start = 0
- simple_populate(self, self.uri, 'key_format=i', 0)
+ SimpleDataSet(self, self.uri, 0, key_format='i').populate()
# Create a set of snapshots
# Each snapshot has a bunch of new data
diff --git a/test/suite/test_nsnap03.py b/test/suite/test_nsnap03.py
index 3986c0c1a0a..6964fb914c3 100644
--- a/test/suite/test_nsnap03.py
+++ b/test/suite/test_nsnap03.py
@@ -30,7 +30,7 @@
# Named snapshots: Access and create from multiple sessions
from suite_subprocess import suite_subprocess
-from helper import simple_populate
+from wtdataset import SimpleDataSet
import wiredtiger, wttest
class test_nsnap03(wttest.WiredTigerTestCase, suite_subprocess):
@@ -55,7 +55,7 @@ class test_nsnap03(wttest.WiredTigerTestCase, suite_subprocess):
def test_named_snapshots(self):
# Populate a table
end = start = 0
- simple_populate(self, self.uri, 'key_format=i', 0)
+ SimpleDataSet(self, self.uri, 0, key_format='i').populate()
# Now run a workload:
# every iteration:
diff --git a/test/suite/test_nsnap04.py b/test/suite/test_nsnap04.py
index f9ef26b5600..60901dd2ee3 100644
--- a/test/suite/test_nsnap04.py
+++ b/test/suite/test_nsnap04.py
@@ -29,9 +29,9 @@
# test_nsnap04.py
# Named snapshots: Create snapshot from running transaction
-from suite_subprocess import suite_subprocess
-from helper import simple_populate
import wiredtiger, wttest
+from suite_subprocess import suite_subprocess
+from wtdataset import SimpleDataSet
class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess):
tablename = 'test_nsnap04'
@@ -52,7 +52,7 @@ class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess):
def test_named_snapshots(self):
# Populate a table
end = start = 0
- simple_populate(self, self.uri, 'key_format=i', 0)
+ SimpleDataSet(self, self.uri, 0, key_format='i').populate()
snapshots = []
c = self.session.open_cursor(self.uri)
diff --git a/test/suite/test_overwrite.py b/test/suite/test_overwrite.py
index b7af1612243..4739abaa578 100644
--- a/test/suite/test_overwrite.py
+++ b/test/suite/test_overwrite.py
@@ -27,7 +27,7 @@
# OTHER DEALINGS IN THE SOFTWARE.
import wiredtiger, wttest
-from helper import key_populate, simple_populate
+from wtdataset import SimpleDataSet
from wtscenario import make_scenarios
# test_overwrite.py
@@ -35,22 +35,23 @@ from wtscenario import make_scenarios
class test_overwrite(wttest.WiredTigerTestCase):
name = 'overwrite'
scenarios = make_scenarios([
- ('file-r', dict(type='file:',keyfmt='r')),
- ('file-S', dict(type='file:',keyfmt='S')),
- ('lsm-S', dict(type='lsm:',keyfmt='S')),
- ('table-r', dict(type='table:',keyfmt='r')),
- ('table-S', dict(type='table:',keyfmt='S')),
+ ('file-r', dict(type='file:', keyfmt='r', dataset=SimpleDataSet)),
+ ('file-S', dict(type='file:', keyfmt='S', dataset=SimpleDataSet)),
+ ('lsm-S', dict(type='lsm:', keyfmt='S', dataset=SimpleDataSet)),
+ ('table-r', dict(type='table:', keyfmt='r', dataset=SimpleDataSet)),
+ ('table-S', dict(type='table:', keyfmt='S', dataset=SimpleDataSet)),
])
# Confirm a cursor configured with/without overwrite correctly handles
# non-existent records during insert, remove and update operations.
def test_overwrite_insert(self):
uri = self.type + self.name
- simple_populate(self, uri, 'key_format=' + self.keyfmt, 100)
+ ds = self.dataset(self, uri, 100, key_format=self.keyfmt)
+ ds.populate()
# Insert of an existing record with overwrite off fails.
cursor = self.session.open_cursor(uri, None, "overwrite=false")
- cursor.set_key(key_populate(cursor, 5))
+ cursor.set_key(ds.key(5))
cursor.set_value('XXXXXXXXXX')
self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.insert())
@@ -60,78 +61,80 @@ class test_overwrite(wttest.WiredTigerTestCase):
# failure modes are for non-existent records, and you cannot duplicate
# cursor pointing to non-existent records.
cursor = self.session.open_cursor(uri, None, "overwrite=false")
- cursor.set_key(key_populate(cursor, 5))
+ cursor.set_key(ds.key(5))
dupc = self.session.open_cursor(None, cursor, "overwrite=true")
dupc.set_value('XXXXXXXXXX')
self.assertEquals(dupc.insert(), 0)
# Insert of an existing record with overwrite on succeeds.
cursor = self.session.open_cursor(uri, None)
- cursor.set_key(key_populate(cursor, 6))
+ cursor.set_key(ds.key(6))
cursor.set_value('XXXXXXXXXX')
self.assertEquals(cursor.insert(), 0)
# Insert of a non-existent record with overwrite off succeeds.
cursor = self.session.open_cursor(uri, None, "overwrite=false")
- cursor.set_key(key_populate(cursor, 200))
+ cursor.set_key(ds.key(200))
cursor.set_value('XXXXXXXXXX')
self.assertEquals(cursor.insert(), 0)
# Insert of a non-existent record with overwrite on succeeds.
cursor = self.session.open_cursor(uri, None)
- cursor.set_key(key_populate(cursor, 201))
+ cursor.set_key(ds.key(201))
cursor.set_value('XXXXXXXXXX')
self.assertEquals(cursor.insert(), 0)
def test_overwrite_remove(self):
uri = self.type + self.name
- simple_populate(self, uri, 'key_format=' + self.keyfmt, 100)
+ ds = self.dataset(self, uri, 100, key_format=self.keyfmt)
+ ds.populate()
# Remove of an existing record with overwrite off succeeds.
cursor = self.session.open_cursor(uri, None, "overwrite=false")
- cursor.set_key(key_populate(cursor, 5))
+ cursor.set_key(ds.key(5))
self.assertEquals(cursor.remove(), 0)
# Remove of an existing record with overwrite on succeeds.
cursor = self.session.open_cursor(uri, None)
- cursor.set_key(key_populate(cursor, 6))
+ cursor.set_key(ds.key(6))
self.assertEquals(cursor.remove(), 0)
# Remove of a non-existent record with overwrite off fails.
cursor = self.session.open_cursor(uri, None, "overwrite=false")
- cursor.set_key(key_populate(cursor, 200))
+ cursor.set_key(ds.key(200))
self.assertEquals(cursor.remove(), wiredtiger.WT_NOTFOUND)
# Remove of a non-existent record with overwrite on succeeds.
cursor = self.session.open_cursor(uri, None)
- cursor.set_key(key_populate(cursor, 201))
+ cursor.set_key(ds.key(201))
self.assertEquals(cursor.remove(), 0)
def test_overwrite_update(self):
uri = self.type + self.name
- simple_populate(self, uri, 'key_format=' + self.keyfmt, 100)
+ ds = self.dataset(self, uri, 100, key_format=self.keyfmt)
+ ds.populate()
# Update of an existing record with overwrite off succeeds.
cursor = self.session.open_cursor(uri, None, "overwrite=false")
- cursor.set_key(key_populate(cursor, 5))
+ cursor.set_key(ds.key(5))
cursor.set_value('XXXXXXXXXX')
self.assertEquals(cursor.update(), 0)
# Update of an existing record with overwrite on succeeds.
cursor = self.session.open_cursor(uri, None)
- cursor.set_key(key_populate(cursor, 6))
+ cursor.set_key(ds.key(6))
cursor.set_value('XXXXXXXXXX')
self.assertEquals(cursor.update(), 0)
# Update of a non-existent record with overwrite off fails.
cursor = self.session.open_cursor(uri, None, "overwrite=false")
- cursor.set_key(key_populate(cursor, 200))
+ cursor.set_key(ds.key(200))
cursor.set_value('XXXXXXXXXX')
self.assertEquals(cursor.update(), wiredtiger.WT_NOTFOUND)
# Update of a non-existent record with overwrite on succeeds.
cursor = self.session.open_cursor(uri, None)
- cursor.set_key(key_populate(cursor, 201))
+ cursor.set_key(ds.key(201))
cursor.set_value('XXXXXXXXXX')
self.assertEquals(cursor.update(), 0)
diff --git a/test/suite/test_readonly03.py b/test/suite/test_readonly03.py
index a5607fcfe80..f30c591ca59 100644
--- a/test/suite/test_readonly03.py
+++ b/test/suite/test_readonly03.py
@@ -31,9 +31,9 @@
# all return ENOTSUP.
#
-from helper import simple_populate
-from suite_subprocess import suite_subprocess
import os, sys, wiredtiger, wttest
+from suite_subprocess import suite_subprocess
+from wtdataset import SimpleDataSet
class test_readonly03(wttest.WiredTigerTestCase, suite_subprocess):
uri = 'table:test_readonly03'
@@ -61,7 +61,8 @@ class test_readonly03(wttest.WiredTigerTestCase, suite_subprocess):
create_params = 'key_format=i,value_format=i'
entries = 10
# Create a database and a table.
- simple_populate(self, self.uri, create_params, entries)
+ SimpleDataSet(self, self.uri, entries, key_format='i',
+ value_format='i').populate()
#
# Now close and reopen. Note that the connection function
diff --git a/test/suite/test_rebalance.py b/test/suite/test_rebalance.py
index acb73d82a13..2d160bafec0 100644
--- a/test/suite/test_rebalance.py
+++ b/test/suite/test_rebalance.py
@@ -28,7 +28,7 @@
import os, time
import wiredtiger, wttest
-from helper import complex_populate, simple_populate
+from wtdataset import SimpleDataSet, ComplexDataSet
from wtscenario import make_scenarios
# test_rebalance.py
@@ -38,7 +38,7 @@ class test_rebalance(wttest.WiredTigerTestCase):
# Use small pages so we generate some internal layout
# Setup LSM so multiple chunks are present
- config = 'key_format=S,allocation_size=512,internal_page_max=512' + \
+ config = 'allocation_size=512,internal_page_max=512' + \
',leaf_page_max=1k,lsm=(chunk_size=512k,merge_min=10)'
scenarios = make_scenarios([
@@ -48,9 +48,10 @@ class test_rebalance(wttest.WiredTigerTestCase):
])
# Populate an object, then rebalance it.
- def rebalance(self, populate, with_cursor):
+ def rebalance(self, dataset, with_cursor):
uri = self.uri + self.name
- populate(self, uri, self.config, 10000)
+ ds = dataset(self, uri, 10000, config=self.config)
+ ds.populate()
# Force to disk, we don't rebalance in-memory objects.
self.reopen_conn()
@@ -68,13 +69,13 @@ class test_rebalance(wttest.WiredTigerTestCase):
# Test rebalance of an object.
def test_rebalance(self):
# Simple file or table object.
- self.rebalance(simple_populate, False)
- self.rebalance(simple_populate, True)
+ self.rebalance(SimpleDataSet, False)
+ self.rebalance(SimpleDataSet, True)
# A complex, multi-file table object.
if self.uri == "table:":
- self.rebalance(complex_populate, False)
- self.rebalance(complex_populate, True)
+ self.rebalance(ComplexDataSet, False)
+ self.rebalance(ComplexDataSet, True)
if __name__ == '__main__':
wttest.run()
diff --git a/test/suite/test_reconfig01.py b/test/suite/test_reconfig01.py
index fb3fb7edac6..e76becac76a 100644
--- a/test/suite/test_reconfig01.py
+++ b/test/suite/test_reconfig01.py
@@ -28,7 +28,7 @@
import time
import wiredtiger, wttest
-from helper import simple_populate
+from wtdataset import SimpleDataSet
# test_reconfig01.py
# Smoke-test the connection reconfiguration operations.
@@ -70,7 +70,7 @@ class test_reconfig01(wttest.WiredTigerTestCase):
# Take all the defaults.
uri = "lsm:test_reconfig"
nrecs = 10
- simple_populate(self, uri, 'key_format=S', nrecs)
+ SimpleDataSet(self, uri, nrecs).populate()
# Sleep to make sure all threads are started.
time.sleep(2)
# Now that an LSM tree exists, reconfigure LSM manager threads.
@@ -78,7 +78,7 @@ class test_reconfig01(wttest.WiredTigerTestCase):
self.conn.reconfigure("lsm_manager=(worker_thread_max=10)")
# Generate some work
nrecs = 20
- simple_populate(self, uri, 'key_format=S', nrecs)
+ SimpleDataSet(self, uri, nrecs).populate()
# Now reconfigure fewer threads.
self.conn.reconfigure("lsm_manager=(worker_thread_max=3)")
diff --git a/test/suite/test_reconfig02.py b/test/suite/test_reconfig02.py
index 9d9ac220aa7..36a78a1805f 100644
--- a/test/suite/test_reconfig02.py
+++ b/test/suite/test_reconfig02.py
@@ -28,7 +28,6 @@
import fnmatch, os, time
import wiredtiger, wttest
-from helper import simple_populate
# test_reconfig02.py
# Smoke-test the connection reconfiguration operations.
diff --git a/test/suite/test_reconfig03.py b/test/suite/test_reconfig03.py
index e9d39ea5a76..0019bf4814e 100644
--- a/test/suite/test_reconfig03.py
+++ b/test/suite/test_reconfig03.py
@@ -28,7 +28,7 @@
import fnmatch, os, time
import wiredtiger, wttest
-from helper import simple_populate
+from wtdataset import SimpleDataSet
# test_reconfig03.py
# Test the connection reconfiguration operations used in the MongoDB
@@ -41,16 +41,16 @@ class test_reconfig03(wttest.WiredTigerTestCase):
# can run after we've made modifications.
def test_reconfig03_mdb(self):
entries = 10000
- simple_populate(self, self.uri, 'key_format=S', entries)
+ SimpleDataSet(self, self.uri, entries).populate()
time.sleep(1)
self.conn.reconfigure("eviction_target=81")
- simple_populate(self, self.uri, 'key_format=S', entries * 2)
+ SimpleDataSet(self, self.uri, entries * 2).populate()
time.sleep(1)
self.conn.reconfigure("cache_size=81M")
- simple_populate(self, self.uri, 'key_format=S', entries * 3)
+ SimpleDataSet(self, self.uri, entries * 3).populate()
time.sleep(1)
self.conn.reconfigure("eviction_dirty_target=82")
- simple_populate(self, self.uri, 'key_format=S', entries * 4)
+ SimpleDataSet(self, self.uri, entries * 4).populate()
time.sleep(1)
self.conn.reconfigure("shared_cache=(chunk=11MB, name=bar, reserve=12MB, size=1G)")
diff --git a/test/suite/test_rename.py b/test/suite/test_rename.py
index 1979bbb802a..4e3af8e13e0 100644
--- a/test/suite/test_rename.py
+++ b/test/suite/test_rename.py
@@ -28,9 +28,8 @@
import os, time
import wiredtiger, wttest
-from helper import confirm_does_not_exist,\
- complex_populate, complex_populate_check,\
- simple_populate, simple_populate_check
+from helper import confirm_does_not_exist
+from wtdataset import SimpleDataSet, ComplexDataSet
from wtscenario import make_scenarios
# test_rename.py
@@ -46,10 +45,13 @@ class test_rename(wttest.WiredTigerTestCase):
# Populate and object, and rename it a couple of times, confirming the
# old name doesn't exist and the new name has the right contents.
- def rename(self, populate, check, with_cursor):
+ def rename(self, dataset, with_cursor):
uri1 = self.uri + self.name1
uri2 = self.uri + self.name2
- populate(self, uri1, 'key_format=S', 10)
+ # Only populate uri1, we keep ds2 for checking.
+ ds1 = dataset(self, uri1, 10)
+ ds2 = dataset(self, uri2, 10)
+ ds1.populate()
# Open cursors should cause failure.
if with_cursor:
@@ -60,24 +62,24 @@ class test_rename(wttest.WiredTigerTestCase):
self.session.rename(uri1, uri2, None)
confirm_does_not_exist(self, uri1)
- check(self, uri2, 10)
+ ds2.check()
self.session.rename(uri2, uri1, None)
confirm_does_not_exist(self, uri2)
- check(self, uri1, 10)
+ ds1.check()
self.session.drop(uri1)
# Test rename of an object.
def test_rename(self):
# Simple, one-file file or table object.
- self.rename(simple_populate, simple_populate_check, False)
- self.rename(simple_populate, simple_populate_check, True)
+ self.rename(SimpleDataSet, False)
+ self.rename(SimpleDataSet, True)
# A complex, multi-file table object.
if self.uri == "table:":
- self.rename(complex_populate, complex_populate_check, False)
- self.rename(complex_populate, complex_populate_check, True)
+ self.rename(ComplexDataSet, False)
+ self.rename(ComplexDataSet, True)
def test_rename_dne(self):
uri1 = self.uri + self.name1
diff --git a/test/suite/test_shared_cache01.py b/test/suite/test_shared_cache01.py
index 896a1e569ef..70560a625ee 100644
--- a/test/suite/test_shared_cache01.py
+++ b/test/suite/test_shared_cache01.py
@@ -31,7 +31,6 @@ import os
import shutil
import wiredtiger, wttest
from wttest import unittest
-from helper import key_populate, simple_populate
# test_shared_cache01.py
# Checkpoint tests
diff --git a/test/suite/test_shared_cache02.py b/test/suite/test_shared_cache02.py
index 4afccbe59f7..7cde6c86695 100644
--- a/test/suite/test_shared_cache02.py
+++ b/test/suite/test_shared_cache02.py
@@ -31,7 +31,6 @@ import os
import shutil
import wiredtiger, wttest
from wttest import unittest
-from helper import key_populate, simple_populate
# test_shared_cache02.py
# Shared cache tests
diff --git a/test/suite/test_split.py b/test/suite/test_split.py
index 28bf6bc59b0..411778f21ae 100644
--- a/test/suite/test_split.py
+++ b/test/suite/test_split.py
@@ -32,9 +32,6 @@
import wiredtiger, wttest
from wiredtiger import stat
-from helper import confirm_empty,\
- key_populate, value_populate, simple_populate,\
- complex_populate, complex_value_populate
# Test splits
class test_split(wttest.WiredTigerTestCase):
diff --git a/test/suite/test_stat01.py b/test/suite/test_stat01.py
index 1ad51ee9882..2b04a3cbcd5 100644
--- a/test/suite/test_stat01.py
+++ b/test/suite/test_stat01.py
@@ -28,7 +28,7 @@
import helper, wiredtiger, wttest
from wiredtiger import stat
-from helper import key_populate, simple_populate
+from wtdataset import SimpleDataSet, simple_key
from wtscenario import make_scenarios
# test_stat01.py
@@ -89,8 +89,8 @@ class test_stat01(wttest.WiredTigerTestCase):
# Test simple connection statistics.
def test_basic_conn_stats(self):
# Build an object and force some writes.
- config = self.config + ',key_format=' + self.keyfmt
- simple_populate(self, self.uri, config, 1000)
+ SimpleDataSet(self, self.uri, 1000,
+ config=self.config, key_format=self.keyfmt).populate()
self.session.checkpoint(None)
# See that we can get a specific stat value by its key and verify its
@@ -113,7 +113,7 @@ class test_stat01(wttest.WiredTigerTestCase):
value = ""
for i in range(1, self.nentries):
value = value + 1000 * "a"
- cursor[key_populate(cursor, i)] = value
+ cursor[simple_key(cursor, i)] = value
cursor.close()
# Force the object to disk, otherwise we can't check the overflow count.
@@ -140,9 +140,10 @@ class test_stat01(wttest.WiredTigerTestCase):
# Test simple per-checkpoint statistics.
def test_checkpoint_stats(self):
+ ds = SimpleDataSet(self, self.uri, self.nentries,
+ config=self.config, key_format=self.keyfmt)
for name in ('first', 'second', 'third'):
- config = self.config + ',key_format=' + self.keyfmt
- helper.simple_populate(self, self.uri, config, self.nentries)
+ ds.populate()
self.session.checkpoint('name=' + name)
cursor = self.session.open_cursor(
'statistics:' + self.uri, None, 'checkpoint=' + name)
diff --git a/test/suite/test_stat02.py b/test/suite/test_stat02.py
index 047d2c74499..cecda7f1ddc 100644
--- a/test/suite/test_stat02.py
+++ b/test/suite/test_stat02.py
@@ -28,22 +28,21 @@
import itertools, wiredtiger, wttest
from suite_subprocess import suite_subprocess
+from wtdataset import SimpleDataSet, SimpleLSMDataSet, ComplexDataSet, \
+ ComplexLSMDataSet
from wtscenario import make_scenarios
from wiredtiger import stat
-from helper import complex_populate, complex_populate_lsm, simple_populate
# test_stat02.py
# Statistics cursor configurations.
class test_stat_cursor_config(wttest.WiredTigerTestCase):
pfx = 'test_stat_cursor_config'
uri = [
- ('file', dict(uri='file:' + pfx, pop=simple_populate, cfg='')),
- ('table', dict(uri='table:' + pfx, pop=simple_populate, cfg='')),
- ('table-lsm',
- dict(uri='table:' + pfx, pop=simple_populate, cfg=',type=lsm')),
- ('complex', dict(uri='table:' + pfx, pop=complex_populate, cfg='')),
- ('complex-lsm',
- dict(uri='table:' + pfx, pop=complex_populate_lsm, cfg=''))
+ ('file', dict(uri='file:' + pfx, dataset=SimpleDataSet)),
+ ('table', dict(uri='table:' + pfx, dataset=SimpleDataSet)),
+ ('table-lsm', dict(uri='table:' + pfx, dataset=SimpleLSMDataSet)),
+ ('complex', dict(uri='table:' + pfx, dataset=ComplexDataSet)),
+ ('complex-lsm', dict(uri='table:' + pfx, dataset=ComplexLSMDataSet))
]
data_config = [
('none', dict(data_config='none', ok=[])),
@@ -66,7 +65,7 @@ class test_stat_cursor_config(wttest.WiredTigerTestCase):
# For each database/cursor configuration, confirm the right combinations
# succeed or fail.
def test_stat_cursor_config(self):
- self.pop(self, self.uri, 'key_format=S' + self.cfg, 100)
+ self.dataset(self, self.uri, 100).populate()
config = 'statistics=('
if self.cursor_config != 'empty':
config = config + self.cursor_config
@@ -85,7 +84,7 @@ class test_stat_cursor_conn_clear(wttest.WiredTigerTestCase):
def test_stat_cursor_conn_clear(self):
uri = 'table:' + self.pfx
- complex_populate(self, uri, 'key_format=S', 100)
+ ComplexDataSet(self, uri, 100).populate()
# cursor_insert should clear
# cache_bytes_dirty should not clear
@@ -103,17 +102,17 @@ class test_stat_cursor_dsrc_clear(wttest.WiredTigerTestCase):
pfx = 'test_stat_cursor_dsrc_clear'
uri = [
- ('dsrc_clear_1', dict(uri='file:' + pfx, pop=simple_populate)),
- ('dsrc_clear_2', dict(uri='table:' + pfx, pop=simple_populate)),
- ('dsrc_clear_3', dict(uri='table:' + pfx, pop=complex_populate)),
- ('dsrc_clear_4', dict(uri='table:' + pfx, pop=complex_populate_lsm))
+ ('dsrc_clear_1', dict(uri='file:' + pfx, dataset=SimpleDataSet)),
+ ('dsrc_clear_2', dict(uri='table:' + pfx, dataset=SimpleDataSet)),
+ ('dsrc_clear_3', dict(uri='table:' + pfx, dataset=ComplexDataSet)),
+ ('dsrc_clear_4', dict(uri='table:' + pfx, dataset=ComplexLSMDataSet))
]
scenarios = make_scenarios(uri)
conn_config = 'statistics=(all)'
def test_stat_cursor_dsrc_clear(self):
- self.pop(self, self.uri, 'key_format=S', 100)
+ self.dataset(self, self.uri, 100).populate()
# cursor_insert should clear
#
@@ -132,17 +131,17 @@ class test_stat_cursor_fast(wttest.WiredTigerTestCase):
pfx = 'test_stat_cursor_fast'
uri = [
- ('fast_1', dict(uri='file:' + pfx, pop=simple_populate)),
- ('fast_2', dict(uri='table:' + pfx, pop=simple_populate)),
- ('fast_3', dict(uri='table:' + pfx, pop=complex_populate)),
- ('fast_4', dict(uri='table:' + pfx, pop=complex_populate_lsm))
+ ('fast_1', dict(uri='file:' + pfx, dataset=SimpleDataSet)),
+ ('fast_2', dict(uri='table:' + pfx, dataset=SimpleDataSet)),
+ ('fast_3', dict(uri='table:' + pfx, dataset=ComplexDataSet)),
+ ('fast_4', dict(uri='table:' + pfx, dataset=ComplexLSMDataSet))
]
scenarios = make_scenarios(uri)
conn_config = 'statistics=(all)'
def test_stat_cursor_fast(self):
- self.pop(self, self.uri, 'key_format=S', 100)
+ self.dataset(self, self.uri, 100).populate()
# A "fast" cursor shouldn't see the underlying btree statistics.
# Check "fast" first, otherwise we get a copy of the statistics
@@ -174,17 +173,17 @@ class test_stat_cursor_dsrc_error(wttest.WiredTigerTestCase):
pfx = 'test_stat_cursor_dsrc_error'
uri = [
- ('dsrc_error_1', dict(uri='file:' + pfx, pop=simple_populate)),
- ('dsrc_error_2', dict(uri='table:' + pfx, pop=simple_populate)),
- ('dsrc_error_3', dict(uri='table:' + pfx, pop=complex_populate)),
- ('dsrc_error_4', dict(uri='table:' + pfx, pop=complex_populate_lsm))
+ ('dsrc_error_1', dict(uri='file:' + pfx, dataset=SimpleDataSet)),
+ ('dsrc_error_2', dict(uri='table:' + pfx, dataset=SimpleDataSet)),
+ ('dsrc_error_3', dict(uri='table:' + pfx, dataset=ComplexDataSet)),
+ ('dsrc_error_4', dict(uri='table:' + pfx, dataset=ComplexLSMDataSet))
]
scenarios = make_scenarios(uri)
conn_config = 'statistics=(all)'
def test_stat_cursor_dsrc_error(self):
- self.pop(self, self.uri, 'key_format=S', 100)
+ self.dataset(self, self.uri, 100).populate()
args = ['all', 'fast']
for i in list(itertools.permutations(args, 2)):
config = 'statistics=(' + i[0] + ',' + i[1] + ')'
@@ -200,7 +199,7 @@ class test_stat_cursor_dsrc_cache_walk(wttest.WiredTigerTestCase):
conn_config = 'statistics=(none)'
def test_stat_cursor_dsrc_cache_walk(self):
- simple_populate(self, self.uri, 'key_format=S', 100)
+ SimpleDataSet(self, self.uri, 100).populate()
# Ensure that it's an error to get cache_walk stats if none is set
msg = '/doesn\'t match the database statistics/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
diff --git a/test/suite/test_stat03.py b/test/suite/test_stat03.py
index 1570ac4c9d8..7e5cf46ef13 100644
--- a/test/suite/test_stat03.py
+++ b/test/suite/test_stat03.py
@@ -30,10 +30,7 @@ import itertools, wiredtiger, wttest
from suite_subprocess import suite_subprocess
from wiredtiger import stat
-from helper import complex_populate, complex_populate_lsm, simple_populate
-from helper import key_populate, complex_value_populate, value_populate
-from helper import complex_populate_colgroup_count, complex_populate_index_count
-from helper import complex_populate_colgroup_name, complex_populate_index_name
+from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet
from wtscenario import make_scenarios
# test_stat03.py
@@ -41,14 +38,11 @@ from wtscenario import make_scenarios
class test_stat_cursor_reset(wttest.WiredTigerTestCase):
pfx = 'test_stat_cursor_reset'
uri = [
- ('file-simple',
- dict(uri='file:' + pfx, pop=simple_populate)),
- ('table-simple',
- dict(uri='table:' + pfx, pop=simple_populate)),
- ('table-complex',
- dict(uri='table:' + pfx, pop=complex_populate)),
- ('table-complex-lsm',
- dict(uri='table:' + pfx, pop=complex_populate_lsm)),
+ ('file-simple', dict(uri='file:' + pfx, dataset=SimpleDataSet)),
+ ('table-simple', dict(uri='table:' + pfx, dataset=SimpleDataSet)),
+ ('table-complex', dict(uri='table:' + pfx, dataset=ComplexDataSet)),
+ ('table-complex-lsm', dict(uri='table:' + pfx,
+ dataset=ComplexLSMDataSet))
]
scenarios = make_scenarios(uri)
@@ -59,25 +53,23 @@ class test_stat_cursor_reset(wttest.WiredTigerTestCase):
'statistics:' + uri, None, 'statistics=(all)')
def test_stat_cursor_reset(self):
+ n = 100
+ ds = self.dataset(self, self.uri, n)
+ ds.populate()
+
# The number of btree_entries reported is influenced by the
# number of column groups and indices. Each insert will have
# a multiplied effect.
- if self.pop == simple_populate:
+ if self.dataset == SimpleDataSet:
multiplier = 1 # no declared colgroup is like one big colgroup
else:
- multiplier = complex_populate_colgroup_count() + \
- complex_populate_index_count()
- n = 100
- self.pop(self, self.uri, 'key_format=S', n)
+ multiplier = ds.colgroup_count() + ds.index_count()
statc = self.stat_cursor(self.uri)
self.assertEqual(statc[stat.dsrc.btree_entries][2], n * multiplier)
c = self.session.open_cursor(self.uri)
- c.set_key(key_populate(c, 200))
- if self.pop == simple_populate:
- c.set_value(value_populate(c, 200))
- else:
- c.set_value(tuple(complex_value_populate(c, 200)))
+ c.set_key(ds.key(200))
+ c.set_value(ds.value(200))
c.insert()
# Test that cursor reset re-loads the values.
@@ -88,14 +80,12 @@ class test_stat_cursor_reset(wttest.WiredTigerTestCase):
# For applications with indices and/or column groups, verify
# that there is a way to count the base number of entries.
- if self.pop != simple_populate:
+ if self.dataset != SimpleDataSet:
statc.close()
- statc = self.stat_cursor(
- complex_populate_index_name(self, self.uri, 0))
+ statc = self.stat_cursor(ds.index_name(0))
self.assertEqual(statc[stat.dsrc.btree_entries][2], n)
statc.close()
- statc = self.stat_cursor(
- complex_populate_colgroup_name(self, self.uri, 0))
+ statc = self.stat_cursor(ds.colgroup_name(0))
self.assertEqual(statc[stat.dsrc.btree_entries][2], n)
statc.close()
diff --git a/test/suite/test_stat05.py b/test/suite/test_stat05.py
index 62562f78ed6..ef4d65e85e4 100644
--- a/test/suite/test_stat05.py
+++ b/test/suite/test_stat05.py
@@ -30,8 +30,7 @@ import itertools, wiredtiger, wttest
from suite_subprocess import suite_subprocess
from wtscenario import make_scenarios
from wiredtiger import stat
-from helper import complex_populate, complex_populate_lsm, simple_populate
-from helper import complex_value_populate, key_populate, value_populate
+from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet
# test_stat05.py
# Statistics cursor using size only
@@ -40,17 +39,17 @@ class test_stat_cursor_config(wttest.WiredTigerTestCase):
conn_config = 'statistics=(fast)'
uri = [
- ('file', dict(uri='file:' + pfx, pop=simple_populate, cfg='')),
- ('table', dict(uri='table:' + pfx, pop=simple_populate, cfg='')),
- ('inmem', dict(uri='table:' + pfx, pop=simple_populate, cfg='',
+ ('file', dict(uri='file:' + pfx, dataset=SimpleDataSet, cfg='')),
+ ('table', dict(uri='table:' + pfx, dataset=SimpleDataSet, cfg='')),
+ ('inmem', dict(uri='table:' + pfx, dataset=SimpleDataSet, cfg='',
conn_config = 'in_memory,statistics=(fast)')),
- ('table-lsm', dict(uri='table:' + pfx, pop=simple_populate,
- cfg=',type=lsm,lsm=(chunk_size=1MB,merge_min=2)',
+ ('table-lsm', dict(uri='table:' + pfx, dataset=SimpleDataSet,
+ cfg='lsm=(chunk_size=1MB,merge_min=2)',
conn_config = 'statistics=(fast),eviction_dirty_target=99,eviction_dirty_trigger=99')),
- ('complex', dict(uri='table:' + pfx, pop=complex_populate, cfg='')),
+ ('complex', dict(uri='table:' + pfx, dataset=ComplexDataSet, cfg='')),
('complex-lsm',
- dict(uri='table:' + pfx, pop=complex_populate_lsm,
- cfg=',lsm=(chunk_size=1MB,merge_min=2)',
+ dict(uri='table:' + pfx, dataset=ComplexLSMDataSet,
+ cfg='lsm=(chunk_size=1MB,merge_min=2)',
conn_config = 'statistics=(fast),eviction_dirty_target=99,eviction_dirty_trigger=99')),
]
@@ -68,17 +67,14 @@ class test_stat_cursor_config(wttest.WiredTigerTestCase):
# the cursor open succeeds. Insert enough data that LSM tables to need to
# switch and merge.
def test_stat_cursor_size(self):
- self.pop(self, self.uri, 'key_format=S' + self.cfg, 100)
+ ds = self.dataset(self, self.uri, 100, config=self.cfg)
+ ds.populate()
self.openAndWalkStatCursor()
cursor = self.session.open_cursor(self.uri, None)
for i in range(100, 40000 + 1):
if i % 100 == 0:
self.openAndWalkStatCursor()
- if self.pop == simple_populate:
- cursor[key_populate(cursor, i)] = value_populate(cursor, i)
- else:
- cursor[key_populate(cursor, i)] = \
- tuple(complex_value_populate(cursor, i))
+ cursor[ds.key(i)] = ds.value(i)
cursor.close()
self.openAndWalkStatCursor()
diff --git a/test/suite/test_truncate01.py b/test/suite/test_truncate01.py
index 37cd1f12c79..2319eeddbef 100644
--- a/test/suite/test_truncate01.py
+++ b/test/suite/test_truncate01.py
@@ -31,9 +31,8 @@
#
import wiredtiger, wttest
-from helper import confirm_empty,\
- key_populate, value_populate, simple_populate,\
- complex_populate, complex_value_populate
+from helper import confirm_empty
+from wtdataset import SimpleDataSet, ComplexDataSet
from wtscenario import make_scenarios
# Test truncation arguments.
@@ -49,7 +48,7 @@ class test_truncate_arguments(wttest.WiredTigerTestCase):
# either cursor specified, expect errors.
def test_truncate_bad_args(self):
uri = self.type + self.name
- simple_populate(self, uri, 'key_format=S', 100)
+ SimpleDataSet(self, uri, 100).populate()
msg = '/either a URI or start/stop cursors/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: self.session.truncate(None, None, None, None), msg)
@@ -64,11 +63,12 @@ class test_truncate_arguments(wttest.WiredTigerTestCase):
uri = self.type + self.name
msg = '/requires key be set/'
- simple_populate(self, uri, 'key_format=S', 100)
+ ds = SimpleDataSet(self, uri, 100)
+ ds.populate()
c1 = self.session.open_cursor(uri, None)
c2 = self.session.open_cursor(uri, None)
- c2.set_key(key_populate(c2, 10))
+ c2.set_key(ds.key(10))
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: self.session.truncate(None, c1, c2, None), msg)
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
@@ -89,13 +89,13 @@ class test_truncate_uri(wttest.WiredTigerTestCase):
uri = self.type + self.name
# A simple, one-file file or table object.
- simple_populate(self, uri, 'key_format=S', 100)
+ SimpleDataSet(self, uri, 100).populate()
self.session.truncate(uri, None, None, None)
confirm_empty(self, uri)
self.session.drop(uri, None)
if self.type == "table:":
- complex_populate(self, uri, 'key_format=S', 100)
+ ComplexDataSet(self, uri, 100).populate()
self.session.truncate(uri, None, None, None)
confirm_empty(self, uri)
self.session.drop(uri, None)
@@ -118,16 +118,17 @@ class test_truncate_cursor_order(wttest.WiredTigerTestCase):
# Test an illegal order, then confirm that equal cursors works.
def test_truncate_cursor_order(self):
uri = self.type + self.name
- simple_populate(self, uri, 'key_format=' + self.keyfmt, 100)
+ ds = SimpleDataSet(self, uri, 100, key_format=self.keyfmt)
+ ds.populate()
c1 = self.session.open_cursor(uri, None)
c2 = self.session.open_cursor(uri, None)
- c1.set_key(key_populate(c1, 20))
- c2.set_key(key_populate(c2, 10))
+ c1.set_key(ds.key(20))
+ c2.set_key(ds.key(10))
msg = '/the start cursor position is after the stop cursor position/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: self.session.truncate(None, c1, c2, None), msg)
- c2.set_key(key_populate(c2, 20))
+ c2.set_key(ds.key(20))
self.session.truncate(None, c1, c2, None)
# Test truncation of cursors past the end of the object.
@@ -150,22 +151,24 @@ class test_truncate_cursor_end(wttest.WiredTigerTestCase):
uri = self.type + self.name
# A simple, one-file file or table object.
- simple_populate(self, uri, 'key_format=' + self.keyfmt, 100)
+ ds = SimpleDataSet(self, uri, 100, key_format=self.keyfmt)
+ ds.populate()
c1 = self.session.open_cursor(uri, None)
- c1.set_key(key_populate(c1, 1000))
+ c1.set_key(ds.key(1000))
c2 = self.session.open_cursor(uri, None)
- c2.set_key(key_populate(c2, 2000))
+ c2.set_key(ds.key(2000))
self.session.truncate(None, c1, c2, None)
self.assertEquals(c1.close(), 0)
self.assertEquals(c2.close(), 0)
self.session.drop(uri)
if self.type == "table:":
- complex_populate(self, uri, 'key_format=' + self.keyfmt, 100)
+ ds = ComplexDataSet(self, uri, 100, key_format=self.keyfmt)
+ ds.populate()
c1 = self.session.open_cursor(uri, None)
- c1.set_key(key_populate(c1, 1000))
+ c1.set_key(ds.key(1000))
c2 = self.session.open_cursor(uri, None)
- c2.set_key(key_populate(c2, 2000))
+ c2.set_key(ds.key(2000))
self.session.truncate(None, c1, c2, None)
self.assertEquals(c1.close(), 0)
self.assertEquals(c2.close(), 0)
@@ -179,13 +182,12 @@ class test_truncate_cursor(wttest.WiredTigerTestCase):
# The underlying table routines don't easily support 8t value types, limit
# those tests to file objects.
types = [
- ('file', dict(type='file:',\
- config='allocation_size=512,leaf_page_max=512,key_format=')),
- ('file8t', dict(type='file:',\
- config='allocation_size=512,\
- leaf_page_max=512,value_format=8t,key_format=')),
- ('table', dict(type='table:',\
- config='allocation_size=512,leaf_page_max=512,key_format=')),
+ ('file', dict(type='file:', valuefmt='S',
+ config='allocation_size=512,leaf_page_max=512')),
+ ('file8t', dict(type='file:', valuefmt='8t',
+ config='allocation_size=512,leaf_page_max=512')),
+ ('table', dict(type='table:', valuefmt='S',
+ config='allocation_size=512,leaf_page_max=512')),
]
keyfmt = [
('integer', dict(keyfmt='i')),
@@ -204,18 +206,18 @@ class test_truncate_cursor(wttest.WiredTigerTestCase):
scenarios = make_scenarios(types, keyfmt, size, reopen)
# Set a cursor key.
- def cursorKey(self, uri, key):
+ def cursorKey(self, ds, uri, key):
if key == -1:
return None
cursor = self.session.open_cursor(uri, None)
- cursor.set_key(key_populate(cursor, key))
+ cursor.set_key(ds.key(key))
return cursor
# Truncate a range using cursors, and check the results.
- def truncateRangeAndCheck(self, uri, begin, end, expected):
+ def truncateRangeAndCheck(self, ds, uri, begin, end, expected):
self.pr('truncateRangeAndCheck: ' + str(begin) + ',' + str(end))
- cur1 = self.cursorKey(uri, begin)
- cur2 = self.cursorKey(uri, end)
+ cur1 = self.cursorKey(ds, uri, begin)
+ cur2 = self.cursorKey(ds, uri, end)
self.session.truncate(None, cur1, cur2, None)
if not cur1:
begin = 1
@@ -234,7 +236,7 @@ class test_truncate_cursor(wttest.WiredTigerTestCase):
# Check the expected values against the object.
cursor = self.session.open_cursor(uri, None)
for i in range(begin, end + 1):
- expected[key_populate(cursor, i)] = [0]
+ expected[ds.key(i)] = [0]
for k, v in expected.iteritems():
cursor.set_key(k)
if v == [0] and \
@@ -306,6 +308,10 @@ class test_truncate_cursor(wttest.WiredTigerTestCase):
(self.skip, self.skip) # middle to same middle
]
+ # Using this data set to compare only, it doesn't create or populate.
+ ds = SimpleDataSet(self, uri, 0, key_format=self.keyfmt,
+ value_format=self.valuefmt, config=self.config)
+
# Build the layout we're going to test
total = self.nentries
for begin_skipped,begin_insert,end_skipped,end_insert in layout:
@@ -332,15 +338,16 @@ class test_truncate_cursor(wttest.WiredTigerTestCase):
# Create the object.
self.session.create(
- uri, 'value_format=S,' + self.config + self.keyfmt)
+ uri, self.config + ',key_format=' + self.keyfmt +
+ ',value_format=' + self.valuefmt)
# Insert the records that aren't skipped or inserted.
start = begin_skipped + begin_insert
stop = self.nentries - (end_skipped + end_insert)
cursor = self.session.open_cursor(uri, None)
for i in range(start + 1, stop + 1):
- k = key_populate(cursor, i)
- v = value_populate(cursor, i)
+ k = ds.key(i)
+ v = ds.value(i)
cursor[k] = v
expected[k] = [v]
cursor.close()
@@ -355,33 +362,33 @@ class test_truncate_cursor(wttest.WiredTigerTestCase):
start = 0
for i in range(0, begin_skipped):
start += 1
- k = key_populate(cursor, start)
+ k = ds.key(start)
expected[k] = [0]
# Optionally insert leading records.
for i in range(0, begin_insert):
start += 1
- k = key_populate(cursor, start)
- v = value_populate(cursor, start)
+ k = ds.key(start)
+ v = ds.value(start)
cursor[k] = v
expected[k] = [v]
# Optionally insert trailing skipped records.
for i in range(0, end_skipped):
stop += 1
- k = key_populate(cursor, stop)
+ k = ds.key(stop)
expected[k] = [0]
# Optionally insert trailing records.
for i in range(0, end_insert):
stop += 1
- k = key_populate(cursor, stop)
- v = value_populate(cursor, stop)
+ k = ds.key(stop)
+ v = ds.value(stop)
cursor[k] = v
expected[k] = [v]
cursor.close()
- self.truncateRangeAndCheck(uri, begin, end, expected)
+ self.truncateRangeAndCheck(ds, uri, begin, end, expected)
self.session.drop(uri, None)
# Test truncation of complex tables using cursors. We can't do the kind of
@@ -423,16 +430,16 @@ class test_truncate_cursor(wttest.WiredTigerTestCase):
'''
# Create the object.
- complex_populate(
- self, uri, self.config + self.keyfmt, self.nentries)
+ ds = ComplexDataSet(self, uri, self.nentries,
+ config=self.config, key_format=self.keyfmt)
+ ds.populate()
# Build a dictionary of what the object should look like for
# later comparison
cursor = self.session.open_cursor(uri, None)
expected = {}
for i in range(1, self.nentries + 1):
- expected[key_populate(cursor, i)] = \
- complex_value_populate(cursor, i)
+ expected[ds.key(i)] = ds.comparable_value(i)
cursor.close()
# Optionally close and re-open the object to get a disk image
@@ -440,7 +447,7 @@ class test_truncate_cursor(wttest.WiredTigerTestCase):
if self.reopen:
self.reopen_conn()
- self.truncateRangeAndCheck(uri, begin, end, expected)
+ self.truncateRangeAndCheck(ds, uri, begin, end, expected)
self.session.drop(uri, None)
if __name__ == '__main__':
diff --git a/test/suite/test_truncate02.py b/test/suite/test_truncate02.py
index 7184cd67242..73fed362354 100644
--- a/test/suite/test_truncate02.py
+++ b/test/suite/test_truncate02.py
@@ -31,7 +31,7 @@
#
import wiredtiger, wttest
-from helper import key_populate, value_populate, simple_populate
+from wtdataset import SimpleDataSet
from wtscenario import make_scenarios
# test_truncate_fast_delete
@@ -46,8 +46,7 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase):
# of individual pages in the file.
types = [
('file', dict(type='file:', config=\
- 'allocation_size=512,leaf_page_max=512,' +\
- 'value_format=S,key_format=')),
+ 'allocation_size=512,leaf_page_max=512')),
]
# This is all about testing the btree layer, not the schema layer, test
@@ -123,15 +122,17 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase):
'''
# Create the object.
- simple_populate(self, uri, self.config + self.keyfmt, self.nentries)
+ ds = SimpleDataSet(self, uri, self.nentries,
+ config=self.config, key_format=self.keyfmt)
+ ds.populate()
# Optionally add a few overflow records so we block fast delete on
# those pages.
if self.overflow:
cursor = self.session.open_cursor(uri, None, 'overwrite=false')
for i in range(1, self.nentries, 3123):
- cursor.set_key(key_populate(cursor, i))
- cursor.set_value(value_populate(cursor, i))
+ cursor.set_key(ds.key(i))
+ cursor.set_value(ds.value(i))
cursor.update()
cursor.close()
@@ -143,21 +144,21 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase):
cursor = self.session.open_cursor(uri, None, 'overwrite=false')
if self.readbefore:
for i in range(1, self.nentries, 737):
- cursor.set_key(key_populate(cursor, i))
+ cursor.set_key(ds.key(i))
cursor.search()
if self.writebefore:
for i in range(1, self.nentries, 988):
- cursor.set_key(key_populate(cursor, i))
- cursor.set_value(value_populate(cursor, i + 100))
+ cursor.set_key(ds.key(i))
+ cursor.set_value(ds.value(i + 100))
cursor.update()
cursor.close()
# Begin a transaction, and truncate a big range of rows.
self.session.begin_transaction(None)
start = self.session.open_cursor(uri, None)
- start.set_key(key_populate(start, 10))
+ start.set_key(ds.key(10))
end = self.session.open_cursor(uri, None)
- end.set_key(key_populate(end, self.nentries - 10))
+ end.set_key(ds.key(self.nentries - 10))
self.session.truncate(None, start, end, None)
start.close()
end.close()
@@ -167,12 +168,12 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase):
cursor = self.session.open_cursor(uri, None, 'overwrite=false')
if self.readafter:
for i in range(1, self.nentries, 1123):
- cursor.set_key(key_populate(cursor, i))
+ cursor.set_key(ds.key(i))
cursor.search()
if self.writeafter:
for i in range(1, self.nentries, 621):
- cursor.set_key(key_populate(cursor, i))
- cursor.set_value(value_populate(cursor, i + 100))
+ cursor.set_key(ds.key(i))
+ cursor.set_value(ds.value(i + 100))
cursor.update()
cursor.close()
diff --git a/test/suite/test_truncate03.py b/test/suite/test_truncate03.py
index 49044ac5690..2b4628950b3 100644
--- a/test/suite/test_truncate03.py
+++ b/test/suite/test_truncate03.py
@@ -30,7 +30,7 @@
# session level operations on tables
import wiredtiger, wttest
-from helper import key_populate, simple_populate, value_populate
+from wtdataset import SimpleDataSet
# A standalone test case that exercises address-deleted cells.
class test_truncate_address_deleted(wttest.WiredTigerTestCase):
@@ -39,15 +39,15 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase):
# Use a small page size and lots of keys because we want to create lots
# of individual pages in the file.
nentries = 10000
- config = 'allocation_size=512,' +\
- 'leaf_page_max=512,value_format=S,key_format=S'
+ config = 'allocation_size=512,leaf_page_max=512'
# address_deleted routine:
# Create an object that has a bunch of address-deleted cells on disk.
# Recover the object, and turn the address-deleted cells into free pages.
def address_deleted(self):
# Create the object, force it to disk, and verify the object.
- simple_populate(self, self.uri, self.config, self.nentries)
+ ds = SimpleDataSet(self, self.uri, self.nentries, config=self.config)
+ ds.populate()
self.reopen_conn()
self.session.verify(self.uri)
@@ -59,9 +59,9 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase):
# Truncate a big range of rows; the leaf pages aren't in memory, so
# leaf page references will be deleted without being read.
start = self.session.open_cursor(self.uri, None)
- start.set_key(key_populate(start, 10))
+ start.set_key(ds.key(10))
end = self.session.open_cursor(self.uri, None)
- end.set_key(key_populate(end, self.nentries - 10))
+ end.set_key(ds.key(self.nentries - 10))
self.session.truncate(None, start, end, None)
self.assertEqual(start.close(), 0)
self.assertEqual(end.close(), 0)
@@ -78,7 +78,7 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase):
# we get a good look at all the internal pages and the address-deleted
# cells.
cursor = self.session.open_cursor(self.uri, None)
- cursor.set_key(key_populate(cursor, 5))
+ cursor.set_key(ds.key(5))
cursor.set_value("changed value")
self.assertEqual(cursor.update(), 0)
cursor.reset()
@@ -88,12 +88,13 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase):
# Checkpoint, freeing the pages.
self.session.checkpoint()
+ return ds
# Test object creation, recovery, and conversion of address-deleted cells
# into free pages.
def test_truncate_address_deleted_free(self):
# Create the object on disk.
- self.address_deleted()
+ ds = self.address_deleted()
# That's all just verify that worked.
self.session.verify(self.uri)
@@ -103,7 +104,7 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase):
# empty pages by a reader after the underlying leaf pages are removed.
def test_truncate_address_deleted_empty_page(self):
# Create the object on disk.
- self.address_deleted()
+ ds = self.address_deleted()
# Open a cursor and update pages in the middle of the range, forcing
# creation of empty pages (once the underlying leaf page is freed, we
@@ -111,11 +112,11 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase):
# the value as well as write the page and get it back.
cursor = self.session.open_cursor(self.uri, None)
for i in range(3000, 7000, 137):
- k = key_populate(cursor, i)
+ k = ds.key(i)
v = 'changed value: ' + str(i)
cursor[k] = v
for i in range(3000, 7000, 137):
- k = key_populate(cursor, i)
+ k = ds.key(i)
v = 'changed value: ' + str(i)
cursor.set_key(k)
self.assertEqual(cursor.search(), 0)
@@ -128,7 +129,7 @@ class test_truncate_address_deleted(wttest.WiredTigerTestCase):
cursor = self.session.open_cursor(self.uri, None)
for i in range(3000, 7000, 137):
- k = key_populate(cursor, i)
+ k = ds.key(i)
v = 'changed value: ' + str(i)
cursor.set_key(k)
self.assertEqual(cursor.search(), 0)
diff --git a/test/suite/test_txn06.py b/test/suite/test_txn06.py
index e4636e40e2e..2bff97f6aac 100644
--- a/test/suite/test_txn06.py
+++ b/test/suite/test_txn06.py
@@ -30,7 +30,7 @@
# Transactions: test long-running snapshots
from suite_subprocess import suite_subprocess
-from helper import simple_populate
+from wtdataset import SimpleDataSet
import wiredtiger, wttest
class test_txn06(wttest.WiredTigerTestCase, suite_subprocess):
@@ -47,7 +47,7 @@ class test_txn06(wttest.WiredTigerTestCase, suite_subprocess):
def test_long_running(self):
# Populate a table
- simple_populate(self, self.source_uri, 'key_format=S', self.nrows)
+ SimpleDataSet(self, self.source_uri, self.nrows).populate()
# Now scan the table and copy the rows into a new table
c_src = self.session.create(self.uri, "key_format=S")
diff --git a/test/suite/test_txn11.py b/test/suite/test_txn11.py
index dfb02799434..147bf3a76c0 100644
--- a/test/suite/test_txn11.py
+++ b/test/suite/test_txn11.py
@@ -31,7 +31,7 @@
import fnmatch, os, time
from suite_subprocess import suite_subprocess
-from helper import simple_populate
+from wtdataset import SimpleDataSet
import wttest
class test_txn11(wttest.WiredTigerTestCase, suite_subprocess):
@@ -64,7 +64,7 @@ class test_txn11(wttest.WiredTigerTestCase, suite_subprocess):
def test_ops(self):
# Populate a table
- simple_populate(self, self.source_uri, 'key_format=S', self.nrows)
+ SimpleDataSet(self, self.source_uri, self.nrows).populate()
# Run forced checkpoints
self.run_checkpoints()
diff --git a/test/suite/test_upgrade.py b/test/suite/test_upgrade.py
index 1c2e3f6cda8..4eb6a9e6817 100644
--- a/test/suite/test_upgrade.py
+++ b/test/suite/test_upgrade.py
@@ -28,7 +28,7 @@
import os, time
import wiredtiger, wttest
-from helper import complex_populate, simple_populate
+from wtdataset import SimpleDataSet, ComplexDataSet
from wtscenario import make_scenarios
# test_upgrade.py
@@ -42,9 +42,9 @@ class test_upgrade(wttest.WiredTigerTestCase):
])
# Populate an object, then upgrade it.
- def upgrade(self, populate, with_cursor):
+ def upgrade(self, dataset, with_cursor):
uri = self.uri + self.name
- populate(self, uri, 'key_format=S', 10)
+ dataset(self, uri, 10).populate()
# Open cursors should cause failure.
if with_cursor:
@@ -59,13 +59,13 @@ class test_upgrade(wttest.WiredTigerTestCase):
# Test upgrade of an object.
def test_upgrade(self):
# Simple file or table object.
- self.upgrade(simple_populate, False)
- self.upgrade(simple_populate, True)
+ self.upgrade(SimpleDataSet, False)
+ self.upgrade(SimpleDataSet, True)
# A complex, multi-file table object.
if self.uri == "table:":
- self.upgrade(complex_populate, False)
- self.upgrade(complex_populate, True)
+ self.upgrade(ComplexDataSet, False)
+ self.upgrade(ComplexDataSet, True)
if __name__ == '__main__':
wttest.run()
diff --git a/test/suite/test_util02.py b/test/suite/test_util02.py
index 13f52d6aeea..7aa24605ed1 100644
--- a/test/suite/test_util02.py
+++ b/test/suite/test_util02.py
@@ -29,8 +29,8 @@
import string, os
import wiredtiger, wttest
from suite_subprocess import suite_subprocess
+from wtdataset import ComplexDataSet
from wtscenario import make_scenarios
-from helper import complex_populate
# test_util02.py
# Utilities: wt load
@@ -169,7 +169,7 @@ class test_load_commandline(wttest.WiredTigerTestCase, suite_subprocess):
def load_commandline(self, args, fail):
errfile= "errfile"
- complex_populate(self, self.uri, "key_format=S,value_format=S", 20)
+ ComplexDataSet(self, self.uri, 20).populate()
self.runWt(["dump", self.uri], outfilename="dump.out")
loadargs = ["load", "-f", "dump.out"] + args
self.runWt(loadargs, errfilename=errfile, failure=fail)
diff --git a/test/suite/test_util13.py b/test/suite/test_util13.py
index 9804dc700ba..7890d4fdb1b 100644
--- a/test/suite/test_util13.py
+++ b/test/suite/test_util13.py
@@ -28,11 +28,9 @@
import os, re, string
from suite_subprocess import suite_subprocess
-import itertools, wiredtiger, wttest
+from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet
+import wiredtiger, wttest
-from helper import complex_populate_cgconfig, complex_populate_cgconfig_lsm
-from helper import simple_populate
-from helper import complex_populate_check, simple_populate_check
from wtscenario import make_scenarios
# test_util13.py
@@ -52,25 +50,23 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess):
# Select table configuration settings that are not the default.
#
types = [
- ('file-simple', dict(uri='file:' + pfx, pop=simple_populate,
- populate_check=simple_populate_check,
- table_config='prefix_compression_min=3', cfg='')),
- ('lsm-simple', dict(uri='lsm:' + pfx, pop=simple_populate,
- populate_check=simple_populate_check,
- table_config='lsm=(bloom_bit_count=29)',
- cfg='bloom_bit_count=29')),
- ('table-simple', dict(uri='table:' + pfx, pop=simple_populate,
- populate_check=simple_populate_check,
- table_config='split_pct=50', cfg='')),
+ ('file-simple', dict(uri='file:' + pfx, dataset=SimpleDataSet,
+ table_config='prefix_compression_min=3', cfg='',
+ cg_config='')),
+ ('lsm-simple', dict(uri='lsm:' + pfx, dataset=SimpleDataSet,
+ table_config='lsm=(bloom_bit_count=29)', cfg='bloom_bit_count=29',
+ cg_config='')),
+ ('table-simple', dict(uri='table:' + pfx, dataset=SimpleDataSet,
+ table_config='split_pct=50', cfg='',
+ cg_config='')),
('table-complex',
- dict(uri='table:' + pfx, pop=complex_populate_cgconfig,
- populate_check=complex_populate_check,
- table_config='allocation_size=512B', cfg='')),
+ dict(uri='table:' + pfx, dataset=ComplexDataSet,
+ table_config='allocation_size=512B', cfg='',
+ cg_config='allocation_size=512B')),
('table-complex-lsm',
- dict(uri='table:' + pfx, pop=complex_populate_cgconfig_lsm,
- populate_check=complex_populate_check,
- table_config='lsm=(merge_max=5)',
- cfg='merge_max=5')),
+ dict(uri='table:' + pfx, dataset=ComplexLSMDataSet,
+ table_config='lsm=(merge_max=5)', cfg='merge_max=5',
+ cg_config='lsm=(merge_max=5)'))
]
scenarios = make_scenarios(types)
@@ -83,7 +79,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess):
#print "compare_config Expected config "
#print expected_cfg
cfg_orig = actual_cfg
- if self.pop != simple_populate:
+ if self.dataset != SimpleDataSet:
#
# If we have a complex config, strip out the colgroups and
# columns from the config. Doing so allows us to keep the
@@ -127,7 +123,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess):
break
return True
- def load_recheck(self, expect_subset, dump_out):
+ def load_recheck(self, ds, expect_subset, dump_out):
newdump = "newdump.out"
os.mkdir(self.dir)
self.runWt(['-h', self.dir, 'load', '-f', dump_out])
@@ -135,7 +131,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess):
conn = self.wiredtiger_open(self.dir)
session = conn.open_session()
cursor = session.open_cursor(self.uri, None, None)
- self.populate_check
+ ds.check()
conn.close()
dumpargs = ["-h"]
dumpargs.append(self.dir)
@@ -150,8 +146,9 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess):
# The number of btree_entries reported is influenced by the
# number of column groups and indices. Each insert will have
# a multiplied effect.
- self.pop(self, self.uri,
- 'key_format=S,value_format=S,' + self.table_config, self.nentries)
+ ds = self.dataset(self, self.uri, self.nentries,
+ config=self.table_config, cgconfig=self.cg_config)
+ ds.populate()
ver = wiredtiger.wiredtiger_version()
verstring = str(ver[1]) + '.' + str(ver[2]) + '.' + str(ver[3])
@@ -166,7 +163,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess):
expectout.write('Header\n')
expectout.write(self.uri + '\n')
# Check the config on the colgroup itself for complex tables.
- if self.pop != simple_populate:
+ if self.dataset != SimpleDataSet:
expectout.write('key_format=S\n')
expectout.write('colgroup:' + self.pfx + ':cgroup1\n')
if self.cfg == '':
@@ -182,7 +179,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess):
self.runWt(dumpargs, outfilename=outfile)
self.assertTrue(self.compare_files(expectfile, outfile))
- self.assertTrue(self.load_recheck(expectfile, outfile))
+ self.assertTrue(self.load_recheck(ds, expectfile, outfile))
if __name__ == '__main__':
wttest.run()
diff --git a/test/suite/wtdataset.py b/test/suite/wtdataset.py
new file mode 100644
index 00000000000..cb77735510c
--- /dev/null
+++ b/test/suite/wtdataset.py
@@ -0,0 +1,302 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-2016 MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+
+class BaseDataSet(object):
+ """
+ BaseDataSet is an abstract base class for other *DataSet classes.
+ An object of this type should not be created directly. These classes
+ represent test data sets that can be used to populate tables and
+ to check the contents of existing tables.
+ """
+ def __init__(self, testcase, uri, rows, **kwargs):
+ self.testcase = testcase
+ self.uri = uri
+ self.rows = rows
+ self.key_format = kwargs.get('key_format', 'S')
+ self.value_format = kwargs.get('value_format', 'S')
+ self.config = kwargs.get('config', '')
+
+ def create(self):
+ self.testcase.session.create(self.uri, 'key_format=' + self.key_format
+ + ',value_format=' + self.value_format
+ + ',' + self.config)
+
+ def fill(self):
+ c = self.testcase.session.open_cursor(self.uri, None)
+ for i in range(1, self.rows + 1):
+ c[self.key(i)] = self.value(i)
+ c.close()
+
+ def postfill(self):
+ pass
+
+ @classmethod
+ def is_lsm(cls):
+ return False
+
+ def populate(self):
+ self.testcase.pr('populate: ' + self.uri + ' with '
+ + str(self.rows) + ' rows')
+ self.create()
+ self.fill()
+ self.postfill()
+
+ # Create a key for a Simple or Complex data set.
+ @staticmethod
+ def key_by_format(i, key_format):
+ if key_format == 'i' or key_format == 'r' or key_format == 'u':
+ return i
+ elif key_format == 'S':
+ return str('%015d' % i)
+ else:
+ raise AssertionError(
+ 'key: object has unexpected format: ' + key_format)
+
+ # Create a value for a Simple data set.
+ @staticmethod
+ def value_by_format(i, value_format):
+ if value_format == 'i' or value_format == 'r' or value_format == 'u':
+ return i
+ elif value_format == 'S':
+ return str(i) + ': abcdefghijklmnopqrstuvwxyz'
+ elif value_format == '8t':
+ value = (
+ 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xaa, 0xab,
+ 0xac, 0xad, 0xae, 0xaf, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6,
+ 0xb7, 0xb8, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf)
+ return value[i % len(value)]
+ else:
+ raise AssertionError(
+ 'value: object has unexpected format: '
+ + value_format)
+
+ # Create a key for this data set. Simple and Complex data sets have
+ # the same key space.
+ def key(self, i):
+ return BaseDataSet.key_by_format(i, self.key_format)
+
+ def check(self):
+ self.testcase.pr('check: ' + self.uri)
+ cursor = self.testcase.session.open_cursor(self.uri, None)
+ self.check_cursor(cursor)
+ cursor.close()
+
+class SimpleDataSet(BaseDataSet):
+ """
+ SimpleDataSet creates a table with a single key and value that is
+ populated with predefined data, up to the requested number of rows.
+ key_format and value_format may be set in the constructor to
+ override the simple string defaults.
+ """
+ def __init__(self, testcase, uri, rows, **kwargs):
+ super(SimpleDataSet, self).__init__(testcase, uri, rows, **kwargs)
+
+ def value(self, i):
+ return BaseDataSet.value_by_format(i, self.value_format)
+
+ def check_cursor(self, cursor):
+ i = 0
+ for key, val in cursor:
+ i += 1
+ self.testcase.assertEqual(key, self.key(i))
+ if cursor.value_format == '8t' and val == 0: # deleted
+ continue
+ self.testcase.assertEqual(val, self.value(i))
+ self.testcase.assertEqual(i, self.rows)
+
+class SimpleLSMDataSet(SimpleDataSet):
+ """
+ SimpleLSMDataSet is identical to SimpleDataSet, but using LSM files
+ via the type=lsm configuration.
+ """
+ def __init__(self, testcase, uri, rows, **kwargs):
+ kwargs['config'] = kwargs.get('config', '') + ',type=lsm'
+ super(SimpleLSMDataSet, self).__init__(
+ testcase, uri, rows, **kwargs)
+
+ @classmethod
+ def is_lsm(cls):
+ return True
+
+class SimpleIndexDataSet(SimpleDataSet):
+ """
+ SimpleIndexDataSet is identical to SimpleDataSet, adding one index
+ that maps the value to the key.
+ """
+ def __init__(self, testcase, uri, rows, **kwargs):
+ self.indexname = 'index:' + uri.split(":")[1] + ':index1'
+ self.origconfig = kwargs.get('config', '')
+ kwargs['config'] = self.origconfig + ',columns=(key0,value0)'
+ super(SimpleIndexDataSet, self).__init__(
+ testcase, uri, rows, **kwargs)
+
+ def create(self):
+ super(SimpleIndexDataSet, self).create()
+ self.testcase.session.create(self.indexname, 'columns=(value0,key0),' +
+ self.origconfig)
+
+ def check(self):
+ BaseDataSet.check(self)
+
+ # Check values in the index.
+ idxcursor = self.testcase.session.open_cursor(self.indexname)
+ for i in range(1, self.rows + 1):
+ k = self.key(i)
+ v = self.value(i)
+ ik = (v, k) # The index key is columns=(v,k).
+ self.testcase.assertEqual(v, idxcursor[ik])
+ idxcursor.close()
+
+class SimpleIndexLSMDataSet(SimpleIndexDataSet):
+ """
+ SimpleIndexLSMDataSet is identical to SimpleIndexDataSet, but
+ using LSM files.
+ """
+ def __init__(self, testcase, uri, rows, **kwargs):
+ kwargs['config'] = kwargs.get('config', '') + ',type=lsm'
+ super(SimpleIndexLSMDataSet, self).__init__(
+ testcase, uri, rows, **kwargs)
+
+ @classmethod
+ def is_lsm(cls):
+ return True
+
+class ComplexDataSet(BaseDataSet):
+ """
+ ComplexDataSet populates a table with a mixed set of indices
+ and column groups. Some indices are created before the
+ table is populated, some after.
+ """
+ def __init__(self, testcase, uri, rows, **kwargs):
+ self.indexlist = [
+ ['indx1', 'column2'],
+ ['indx2', 'column3'],
+ ['indx3', 'column4'],
+ ['indx4', 'column2,column4'],
+ ['indx5', 'column3,column5'],
+ ['indx6', 'column3,column5,column4']]
+ self.cglist = [
+ ['cgroup1', 'column2'],
+ ['cgroup2', 'column3'],
+ ['cgroup3', 'column4'],
+ ['cgroup4', 'column2,column3'],
+ ['cgroup5', 'column3,column4'],
+ ['cgroup6', 'column2,column4,column5']]
+ self.cgconfig = kwargs.pop('cgconfig', '')
+ config = kwargs.get('config', '')
+ config += ',columns=(record,column2,column3,column4,column5),' + \
+ 'colgroups=(cgroup1,cgroup2,cgroup3,cgroup4,cgroup5,cgroup6)'
+ kwargs['config'] = config
+ kwargs['value_format'] = 'SiSS'
+ super(ComplexDataSet, self).__init__(testcase, uri, rows, **kwargs)
+
+ def create(self):
+ config = 'key_format=' + self.key_format + \
+ ',value_format=' + self.value_format + ',' + self.config
+ session = self.testcase.session
+ ##self.testcase.tty('URI=' + self.uri + 'CONFIG=' + config)
+ session.create(self.uri, config)
+ tablepart = self.uri.split(":")[1] + ':'
+ for cg in self.cglist:
+ session.create('colgroup:' + tablepart + cg[0],
+ ',columns=(' + cg[1] + '),' + self.cgconfig)
+ for index in self.indexlist[0:4]:
+ session.create('index:' + tablepart + index[0],
+ ',columns=(' + index[1] + '),' + self.config)
+
+ def postfill(self):
+ # add some indices after filling the table
+ tablepart = self.uri.split(":")[1] + ':'
+ session = self.testcase.session
+ for index in self.indexlist[4:]:
+ session.create('index:' + tablepart + index[0],
+ ',columns=(' + index[1] + ')')
+
+ def colgroup_count(self):
+ return len(self.cglist)
+
+ def colgroup_name(self, i):
+ return 'colgroup:' + self.uri.split(":")[1] + ':' + self.cglist[i][0]
+
+ def index_count(self):
+ return len(self.indexlist)
+
+ def index_name(self, i):
+ return 'index:' + self.uri.split(":")[1] + ':' + self.indexlist[i][0]
+
+ # A value suitable for checking the value returned by a cursor, as
+ # cursor.get_value() returns a list.
+ def comparable_value(self, i):
+ return [str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%26],
+ i,
+ str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%23],
+ str(i) + ': abcdefghijklmnopqrstuvwxyz'[0:i%18]]
+
+ # A value suitable for assigning to a cursor, as
+ # cursor.set_value() expects a tuple when there it is used with
+ # a single argument and the value is composite.
+ def value(self, i):
+ return tuple(self.comparable_value(i))
+
+ def check_cursor(self, cursor):
+ i = 0
+ for key, s1, i2, s3, s4 in cursor:
+ i += 1
+ self.testcase.assertEqual(key, self.key(i))
+ v = self.value(i)
+ self.testcase.assertEqual(s1, v[0])
+ self.testcase.assertEqual(i2, v[1])
+ self.testcase.assertEqual(s3, v[2])
+ self.testcase.assertEqual(s4, v[3])
+ self.testcase.assertEqual(i, self.rows)
+
+class ComplexLSMDataSet(ComplexDataSet):
+ """
+ ComplexLSMDataSet is identical to ComplexDataSet, but using LSM files.
+ """
+ def __init__(self, testcase, uri, rows, **kwargs):
+ kwargs['cgconfig'] = kwargs.get('cgconfig', '') + ',type=lsm'
+ super(ComplexLSMDataSet, self).__init__(
+ testcase, uri, rows, **kwargs)
+
+ @classmethod
+ def is_lsm(cls):
+ return True
+
+# create a key based on a cursor as a shortcut to creating a SimpleDataSet
+def simple_key(cursor, i):
+ return BaseDataSet.key_by_format(i, cursor.key_format)
+
+# create a value based on a cursor as a shortcut to creating a SimpleDataSet
+def simple_value(cursor, i):
+ return BaseDataSet.value_by_format(i, cursor.value_format)
+
+# create a key based on a cursor as a shortcut to creating a ComplexDataSet
+def complex_key(cursor, i):
+ return BaseDataSet.key_by_format(i, cursor.key_format)