Merge branch 'develop' into overflow-relax

6 files changed, 404 insertions, 7 deletions

diff --git a/test/suite/test_compress01.py b/test/suite/test_compress01.py
index 5f7e4e5680d..43dfca0515c 100644
--- a/test/suite/test_compress01.py
+++ b/test/suite/test_compress01.py
@@ -26,7 +26,7 @@
 # OTHER DEALINGS IN THE SOFTWARE.
 #
 # test_compress01.py
-#   Basic compression operations
+#   Basic block compression operations
 #
 
 import os, run
diff --git a/test/suite/test_drop.py b/test/suite/test_drop.py
index 11d69b159c7..dcda3998436 100644
--- a/test/suite/test_drop.py
+++ b/test/suite/test_drop.py
@@ -71,10 +71,22 @@ class test_drop(wttest.WiredTigerTestCase):
     # Test drop of a non-existent object: force succeeds, without force fails.
     def test_drop_dne(self):
         uri = self.uri + self.name
+        cguri = 'colgroup:' + self.name
+        idxuri = 'index:' + self.name + ':indexname'
+        lsmuri = 'lsm:' + self.name
         confirm_does_not_exist(self, uri)
         self.session.drop(uri, 'force')
         self.assertRaises(
             wiredtiger.WiredTigerError, lambda: self.session.drop(uri, None))
+        self.session.drop(cguri, 'force')
+        self.assertRaises(
+            wiredtiger.WiredTigerError, lambda: self.session.drop(cguri, None))
+        self.session.drop(idxuri, 'force')
+        self.assertRaises(
+            wiredtiger.WiredTigerError, lambda: self.session.drop(idxuri, None))
+        self.session.drop(lsmuri, 'force')
+        self.assertRaises(
+            wiredtiger.WiredTigerError, lambda: self.session.drop(lsmuri, None))
 
 
 if __name__ == '__main__':
diff --git a/test/suite/test_txn02.py b/test/suite/test_txn02.py
index 3eb35de8691..8e6da405860 100644
--- a/test/suite/test_txn02.py
+++ b/test/suite/test_txn02.py
@@ -29,7 +29,7 @@
 #   Transactions: commits and rollbacks
 #
 
-import fnmatch, os, shutil
+import fnmatch, os, shutil, time
 from suite_subprocess import suite_subprocess
 from wiredtiger import wiredtiger_open
 from wtscenario import multiply_scenarios, number_scenarios
@@ -162,8 +162,9 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess):
                 self.check(backup_conn.open_session(), None, committed)
             finally:
                 # Yield so that the archive thread gets a chance to run
-                # before we close the connection.
-                yield
+                # before we close the connection.  time.sleep(0) is not
+                # guaranteed to force a context switch.  So use a small time.
+                time.sleep(0.01)
                 backup_conn.close()
             count += 1
         #
diff --git a/test/suite/test_txn05.py b/test/suite/test_txn05.py
index 83218dba27b..738e3eb9e7a 100644
--- a/test/suite/test_txn05.py
+++ b/test/suite/test_txn05.py
@@ -29,7 +29,7 @@
 # Transactions: commits and rollbacks
 #
 
-import fnmatch, os, shutil
+import fnmatch, os, shutil, time
 from suite_subprocess import suite_subprocess
 from wiredtiger import wiredtiger_open
 from wtscenario import multiply_scenarios, number_scenarios
@@ -142,8 +142,10 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess):
                  self.check(backup_conn.open_session(), None, committed)
             finally:
                  # Let other threads like archive run before closing.
-                 yield
-                 backup_conn.close()
+                 # time.sleep(0) is not guaranteed to force a context switch.
+                 # Use a small timeout.
+                time.sleep(0.01)
+                backup_conn.close()
             count += 1
         #
         # Check logs after repeated openings. The first log should
diff --git a/test/suite/test_txn07.py b/test/suite/test_txn07.py
new file mode 100644
index 00000000000..08dd331a086
--- /dev/null
+++ b/test/suite/test_txn07.py
@@ -0,0 +1,294 @@
+#!/usr/bin/env python
+#
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# test_txn07.py
+# Transactions: commits and rollbacks
+#
+
+import fnmatch, os, shutil, run, time
+from suite_subprocess import suite_subprocess
+from wiredtiger import wiredtiger_open, stat
+from wtscenario import multiply_scenarios, number_scenarios
+import wttest
+
+class test_txn07(wttest.WiredTigerTestCase, suite_subprocess):
+    logmax = "100K"
+    tablename = 'test_txn07'
+    uri = 'table:' + tablename
+    archive_list = ['true', 'false']
+    sync_list = [
+        '(method=dsync,enabled)',
+        '(method=fsync,enabled)',
+        '(method=none,enabled)',
+        '(enabled=false)'
+    ]
+
+    types = [
+        ('row', dict(tabletype='row',
+                    create_params = 'key_format=i,value_format=S')),
+        ('var', dict(tabletype='var',
+                    create_params = 'key_format=r,value_format=S')),
+        ('fix', dict(tabletype='fix',
+                    create_params = 'key_format=r,value_format=8t')),
+    ]
+    op1s = [
+        ('trunc-all', dict(op1=('all', 0))),
+        ('trunc-both', dict(op1=('both', 2))),
+        ('trunc-start', dict(op1=('start', 2))),
+        ('trunc-stop', dict(op1=('stop', 2))),
+    ]
+    txn1s = [('t1c', dict(txn1='commit')), ('t1r', dict(txn1='rollback'))]
+    compress = [
+        ('bzip2', dict(compress='bzip2')),
+        ('nop', dict(compress='nop')),
+        ('snappy', dict(compress='snappy')),
+        ('zlib', dict(compress='zlib')),
+        ('none', dict(compress='')),
+    ]
+
+    scenarios = number_scenarios(multiply_scenarios('.', types, op1s, txn1s,
+                                                    compress))
+    # Overrides WiredTigerTestCase
+    def setUpConnectionOpen(self, dir):
+        self.home = dir
+        # Cycle through the different transaction_sync values in a
+        # deterministic manner.
+        self.txn_sync = self.sync_list[
+            self.scenario_number % len(self.sync_list)]
+        self.backup_dir = os.path.join(self.home, "WT_BACKUP")
+        conn_params = \
+                'log=(archive=false,enabled,file_max=%s,' % self.logmax + \
+                'compressor=%s)' % self.compress + \
+                self.extensionArg(self.compress) + \
+                ',create,error_prefix="%s: ",' % self.shortid() + \
+                "statistics=(fast)," + \
+                'transaction_sync="%s",' % self.txn_sync
+        # print "Creating conn at '%s' with config '%s'" % (dir, conn_params)
+        try:
+            conn = wiredtiger_open(dir, conn_params)
+        except wiredtiger.WiredTigerError as e:
+            print "Failed conn at '%s' with config '%s'" % (dir, conn_params)
+        self.pr(`conn`)
+        self.session2 = conn.open_session()
+        return conn
+
+    # Return the wiredtiger_open extension argument for a shared library.
+    def extensionArg(self, name):
+        if name == None or name == '':
+            return ''
+
+        testdir = os.path.dirname(__file__)
+        extdir = os.path.join(run.wt_builddir, 'ext/compressors')
+        extfile = os.path.join(
+            extdir, name, '.libs', 'libwiredtiger_' + name + '.so')
+        if not os.path.exists(extfile):
+            self.skipTest('compression extension "' + extfile + '" not built')
+        return ',extensions=["' + extfile + '"]'
+
+    # Check that a cursor (optionally started in a new transaction), sees the
+    # expected values.
+    def check(self, session, txn_config, expected):
+        if txn_config:
+            session.begin_transaction(txn_config)
+        c = session.open_cursor(self.uri, None)
+        actual = dict((k, v) for k, v in c if v != 0)
+        # Search for the expected items as well as iterating
+        for k, v in expected.iteritems():
+            self.assertEqual(c[k], v)
+        c.close()
+        if txn_config:
+            session.commit_transaction()
+        self.assertEqual(actual, expected)
+
+    # Check the state of the system with respect to the current cursor and
+    # different isolation levels.
+    def check_all(self, current, committed):
+        # Transactions see their own changes.
+        # Read-uncommitted transactions see all changes.
+        # Snapshot and read-committed transactions should not see changes.
+        self.check(self.session, None, current)
+        self.check(self.session2, "isolation=snapshot", committed)
+        self.check(self.session2, "isolation=read-committed", committed)
+        self.check(self.session2, "isolation=read-uncommitted", current)
+
+        # Opening a clone of the database home directory should run
+        # recovery and see the committed results.
+        self.backup(self.backup_dir)
+        backup_conn_params = 'log=(enabled,file_max=%s,' % self.logmax + \
+                'compressor=%s)' % self.compress + \
+                self.extensionArg(self.compress)
+        backup_conn = wiredtiger_open(self.backup_dir, backup_conn_params)
+        try:
+            self.check(backup_conn.open_session(), None, committed)
+        finally:
+            backup_conn.close()
+
+    def check_log(self, committed):
+        self.backup(self.backup_dir)
+        #
+        # Open and close the backup connection a few times to force
+        # repeated recovery and log archiving even if later recoveries
+        # are essentially no-ops. Confirm that the backup contains
+        # the committed operations after recovery.
+        #
+        # Cycle through the different archive values in a
+        # deterministic manner.
+        self.archive = self.archive_list[
+            self.scenario_number % len(self.archive_list)]
+        backup_conn_params = \
+            'log=(enabled,file_max=%s,archive=%s)' % (self.logmax, self.archive)
+        orig_logs = fnmatch.filter(os.listdir(self.backup_dir), "*Log*")
+        endcount = 2
+        count = 0
+        while count < endcount:
+            backup_conn = wiredtiger_open(self.backup_dir, backup_conn_params)
+            try:
+                 self.check(backup_conn.open_session(), None, committed)
+            finally:
+                 # Let other threads like archive run before closing.
+                 # time.sleep(0) is not guaranteed to force a context switch.
+                 # Use a small timeout.
+                 time.sleep(0.01)
+                 backup_conn.close()
+            count += 1
+        #
+        # Check logs after repeated openings. The first log should
+        # have been archived if configured. Subsequent openings would not
+        # archive because no checkpoint is written due to no modifications.
+        #
+        cur_logs = fnmatch.filter(os.listdir(self.backup_dir), "*Log*")
+        for o in orig_logs:
+            if self.archive == 'true':
+                self.assertEqual(False, o in cur_logs)
+            else:
+                self.assertEqual(True, o in cur_logs)
+        #
+        # Run printlog and make sure it exits with zero status.
+        #
+        self.runWt(['-h', self.backup_dir, 'printlog'], outfilename='printlog.out')
+
+    def test_ops(self):
+        # print "Creating %s with config '%s'" % (self.uri, self.create_params)
+        self.session.create(self.uri, self.create_params)
+        # Set up the table with entries for 1-5.
+        # We then truncate starting or ending in various places.
+        c = self.session.open_cursor(self.uri, None)
+        if self.tabletype == 'fix':
+            value = 1
+        else:
+            # Choose large compressible values for the string cases.
+            value = 'abc' * 1000000
+        current = {1:value, 2:value, 3:value, 4:value, 5:value}
+        c.set_value(value)
+        for k in current:
+            c.set_key(k)
+            c.insert()
+        committed = current.copy()
+
+        ops = (self.op1, )
+        txns = (self.txn1, )
+        for i, ot in enumerate(zip(ops, txns)):
+            self.session.begin_transaction()
+            ok, txn = ot
+            # print '%d: %s(%d)[%s]' % (i, ok[0], ok[1], txn)
+            op, k = ok
+            
+            # print '%d: %s(%d)[%s]' % (i, ok[0], ok[1], txn)
+            if op == 'stop':
+                c.set_key(k)
+                self.session.truncate(None, None, c, None)
+                kstart = 1
+                kstop = k
+            elif op == 'start':
+                c.set_key(k)
+                self.session.truncate(None, c, None, None)
+                kstart = k
+                kstop = len(current)
+            elif op == 'both':
+                c2 = self.session.open_cursor(self.uri, None)
+                # For both, the key given is the start key.  Add 2
+                # for the stop key.
+                kstart = k
+                kstop = k + 2
+                c.set_key(kstart)
+                c2.set_key(kstop)
+                self.session.truncate(None, c, c2, None)
+                c2.close()
+            elif op == 'all':
+                c2 = self.session.open_cursor(self.uri, None)
+                kstart = 1
+                kstop = len(current)
+                c.set_key(kstart)
+                c2.set_key(kstop)
+                self.session.truncate(None, c, c2, None)
+                c2.close()
+
+            while (kstart <= kstop):
+                del current[kstart]
+                kstart += 1
+
+            # print current
+            # Check the state after each operation.
+            self.check_all(current, committed)
+
+            if txn == 'commit':
+                committed = current.copy()
+                self.session.commit_transaction()
+            elif txn == 'rollback':
+                current = committed.copy()
+                self.session.rollback_transaction()
+
+            # Check the state after each commit/rollback.
+            self.check_all(current, committed)
+
+        stat_cursor = self.session.open_cursor('statistics:', None, None)
+        clen = stat_cursor[stat.conn.log_compress_len][2]
+        cmem = stat_cursor[stat.conn.log_compress_mem][2]
+        cwrites = stat_cursor[stat.conn.log_compress_writes][2]
+        cfails = stat_cursor[stat.conn.log_compress_write_fails][2]
+        csmall = stat_cursor[stat.conn.log_compress_small][2]
+        stat_cursor.close()
+
+        # Check the log state after the entire op completes
+        # and run recovery.
+        self.check_log(committed)
+
+        if self.compress == '':
+            self.assertEqual(clen, cmem)
+            self.assertEqual(cwrites, 0)
+            self.assertEqual(cfails, 0)
+        elif self.compress == 'nop':
+            self.assertEqual(clen, cmem)
+            self.assertEqual(cwrites, 0)
+            self.assertEqual((cfails > 0 or csmall > 0), True)
+        else:
+            self.assertEqual(clen < cmem, True)
+            self.assertEqual(cwrites > 0, True)
+            self.assertEqual((cfails > 0 or csmall > 0), True)
+
+if __name__ == '__main__':
+    wttest.run()
diff --git a/test/suite/test_txn08.py b/test/suite/test_txn08.py
new file mode 100644
index 00000000000..a2f08a2686b
--- /dev/null
+++ b/test/suite/test_txn08.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+#
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# test_txn08.py
+# Printlog: test Unicode output
+#
+
+import fnmatch, os, shutil, run, time
+from suite_subprocess import suite_subprocess
+from wiredtiger import wiredtiger_open, stat
+from wtscenario import multiply_scenarios, number_scenarios
+import wttest
+
+class test_txn08(wttest.WiredTigerTestCase, suite_subprocess):
+    logmax = "100K"
+    tablename = 'test_txn08'
+    uri = 'table:' + tablename
+
+    # Overrides WiredTigerTestCase
+    def setUpConnectionOpen(self, dir):
+        self.home = dir
+        # Cycle through the different transaction_sync values in a
+        # deterministic manner.
+        self.txn_sync = '(method=dsync,enabled)'
+        conn_params = \
+                'log=(archive=false,enabled,file_max=%s)' % self.logmax + \
+                ',create,error_prefix="%s: ",' % self.shortid() + \
+                'transaction_sync="%s",' % self.txn_sync
+        # print "Creating conn at '%s' with config '%s'" % (dir, conn_params)
+        try:
+            conn = wiredtiger_open(dir, conn_params)
+        except wiredtiger.WiredTigerError as e:
+            print "Failed conn at '%s' with config '%s'" % (dir, conn_params)
+        self.pr(`conn`)
+        self.session2 = conn.open_session()
+        return conn
+
+    def test_printlog_unicode(self):
+        # print "Creating %s with config '%s'" % (self.uri, self.create_params)
+        create_params = 'key_format=i,value_format=S'
+        self.session.create(self.uri, create_params)
+        c = self.session.open_cursor(self.uri, None)
+
+        # We want to test some chars that produce Unicode encoding
+        # for printlog output.
+        value = u'\u0001\u0002abcd\u0003\u0004'
+
+        self.session.begin_transaction()
+        for k in range(5):
+            c.set_key(k)
+            c.set_value(value)
+            c.insert()
+
+        self.session.commit_transaction()
+
+        #
+        # Run printlog and make sure it exits with zero status.
+        #
+        self.runWt(['printlog'], outfilename='printlog.out')
+        self.check_file_contains('printlog.out',
+            '\\u0001\\u0002abcd\\u0003\\u0004')
+
+if __name__ == '__main__':
+    wttest.run()