Import wiredtiger-wiredtiger-2.7.0-505-g7fea169.tar.gz from wiredtiger branch mongodb-3.4

ref: 44463c5..7fea169

WT-2355       Fix minor scratch buffer usage in logging.
WT-2348       xargs -P isn't portable
WT-2347       Java: schema format edge cases
WT-2344       OS X compiler warning
WT-2342       Enhance wtperf to support background create and drop operations
WT-2340       Add logging guarantee assertions, whitespace
WT-2339       format post-rebalance verify failure (stress run #11586)
WT-2338       Disable using pre-allocated log files when backup cursor is open
WT-2335       NULL pointer crash in config_check_search with invalid configuration string
WT-2333       Add a flag so drop doesn't block
WT-2332       Bug in logging write-no-sync mode
WT-2331       Checking of search() result for reference cursors before join()
WT-2328       schema drop does direct unlink, it should use a block manager interface.
WT-2326       Change WTPERF to use new memory allocation functions instead of the standard
WT-2321       WT-2321: race between eviction and worker threads on the eviction queue
WT-2320       Only check copyright when cutting releases
WT-2316       stress test failure: WT_CURSOR.prev out-of-order returns
WT-2314       page-swap error handling is inconsistent
WT-2313       sweep-server: conn_dhandle.c, 610: dhandle != conn->cache->evict_file_next
WT-2312       re-creating a deleted column-store page can corrupt the in-memory tree
WT-2308       custom extractor for ref_cursors in join cursor
WT-2305       Fix coverity scan issues on 23/12/2015
WT-2296       New log algorithm needs improving for sync/flush settings
WT-2295       WT_SESSION.create does a full-scan of the main table
WT-2287       WT_SESSION.rebalance
WT-2275       broken DB after application crash
WT-2267       Improve wtperf throttling implementation to provide steady load
WT-2247       variable-length column-store in-memory page splits
WT-2242       WiredTiger treats dead trees the same as other trees in eviction
WT-2142       Connection cleanup in Python tests
WT-2073       metadata cleanups
WT-1801       Add a directory sync after rollback of a WT_SESSION::rename operation
WT-1517       schema format edge cases
SERVER-22064  Coverity analysis defect 77699: Unchecked return value
SERVER-21619  sys-perf: WT crash during core_workloads_WT execution

1 files changed, 192 insertions, 0 deletions

diff --git a/src/third_party/wiredtiger/test/suite/test_sweep01.py b/src/third_party/wiredtiger/test/suite/test_sweep01.py
new file mode 100644
index 00000000000..f996dbfa06d
--- /dev/null
+++ b/src/third_party/wiredtiger/test/suite/test_sweep01.py
@@ -0,0 +1,192 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-2016 MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# test_sweep01.py
+# Test lots of tables, number of open files and sweeping.  Run both
+# with and without checkpoints.
+#
+
+import fnmatch, os, shutil, run, time
+from suite_subprocess import suite_subprocess
+from wiredtiger import stat
+from wtscenario import multiply_scenarios, number_scenarios, prune_scenarios
+import wttest
+
+class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess):
+    tablebase = 'test_sweep01'
+    uri = 'table:' + tablebase
+    numfiles = 50
+    numkv = 1000
+    conn_config = 'file_manager=(close_handle_minimum=0,' + \
+                  'close_idle_time=6,close_scan_interval=2),' + \
+                  'statistics=(fast),'
+
+    types = [
+        ('row', dict(tabletype='row',
+                    create_params = 'key_format=i,value_format=i')),
+        ('var', dict(tabletype='var',
+                    create_params = 'key_format=r,value_format=i')),
+        ('fix', dict(tabletype='fix',
+                    create_params = 'key_format=r,value_format=8t')),
+    ]
+
+    scenarios = types
+
+    def test_ops(self):
+        #
+        # Set up numfiles with numkv entries.  We just want some data in there
+        # we don't care what it is.
+        #
+        for f in range(self.numfiles):
+            uri = '%s.%d' % (self.uri, f)
+            # print "Creating %s with config '%s'" % (uri, self.create_params)
+            self.session.create(uri, self.create_params)
+            c = self.session.open_cursor(uri, None)
+            for k in range(self.numkv):
+                c[k+1] = 1
+            c.close()
+            if f % 20 == 0:
+                time.sleep(1)
+
+        stat_cursor = self.session.open_cursor('statistics:', None, None)
+        close1 = stat_cursor[stat.conn.dh_sweep_close][2]
+        remove1 = stat_cursor[stat.conn.dh_sweep_remove][2]
+        sweep1 = stat_cursor[stat.conn.dh_sweeps][2]
+        sclose1 = stat_cursor[stat.conn.dh_session_handles][2]
+        ssweep1 = stat_cursor[stat.conn.dh_session_sweeps][2]
+        tod1 = stat_cursor[stat.conn.dh_sweep_tod][2]
+        ref1 = stat_cursor[stat.conn.dh_sweep_ref][2]
+        nfile1 = stat_cursor[stat.conn.file_open][2]
+        stat_cursor.close()
+
+        #
+        # We've configured checkpoints to run every 5 seconds, sweep server to
+        # run every 2 seconds and idle time to be 6 seconds. It should take
+        # about 8 seconds for a handle to be closed. Sleep for 12 seconds to be
+        # safe.
+        #
+        uri = '%s.test' % self.uri
+        self.session.create(uri, self.create_params)
+
+        #
+        # Keep inserting data to keep at least one handle active and give
+        # checkpoint something to do.  Make sure checkpoint doesn't adjust
+        # the time of death for inactive handles.
+        #
+        # Note that we do checkpoints inline because that has the side effect
+        # of sweeping the session cache, which will allow handles to be
+        # removed.
+        #
+        c = self.session.open_cursor(uri, None)
+        k = 0
+        sleep = 0
+        while sleep < 12:
+            self.session.checkpoint()
+            k = k+1
+            c[k] = 1
+            sleep += 2
+            time.sleep(2)
+        c.close()
+
+        stat_cursor = self.session.open_cursor('statistics:', None, None)
+        close2 = stat_cursor[stat.conn.dh_sweep_close][2]
+        remove2 = stat_cursor[stat.conn.dh_sweep_remove][2]
+        sweep2 = stat_cursor[stat.conn.dh_sweeps][2]
+        sclose2 = stat_cursor[stat.conn.dh_session_handles][2]
+        ssweep2 = stat_cursor[stat.conn.dh_session_sweeps][2]
+        nfile2 = stat_cursor[stat.conn.file_open][2]
+        tod2 = stat_cursor[stat.conn.dh_sweep_tod][2]
+        ref2 = stat_cursor[stat.conn.dh_sweep_ref][2]
+        stat_cursor.close()
+        # print "checkpoint: " + str(self.ckpt)
+        # print "nfile1: " + str(nfile1) + " nfile2: " + str(nfile2)
+        # print "close1: " + str(close1) + " close2: " + str(close2)
+        # print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)
+        # print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)
+        # print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)
+        # print "tod1: " + str(tod1) + " tod2: " + str(tod2)
+        # print "ref1: " + str(ref1) + " ref2: " + str(ref2)
+
+        #
+        # The files are all closed.  Check that sweep did its work even
+        # in the presence of recent checkpoints.
+        #
+        if (close1 >= close2):
+            print "XX: close1: " + str(close1) + " close2: " + str(close2)
+            print "remove1: " + str(remove1) + " remove2: " + str(remove2)
+            print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)
+            print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)
+            print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)
+            print "tod1: " + str(tod1) + " tod2: " + str(tod2)
+            print "ref1: " + str(ref1) + " ref2: " + str(ref2)
+            print "nfile1: " + str(nfile1) + " nfile2: " + str(nfile2)
+        self.assertEqual(close1 < close2, True)
+        if (remove1 >= remove2):
+            print "close1: " + str(close1) + " close2: " + str(close2)
+            print "XX: remove1: " + str(remove1) + " remove2: " + str(remove2)
+            print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)
+            print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)
+            print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)
+            print "tod1: " + str(tod1) + " tod2: " + str(tod2)
+            print "ref1: " + str(ref1) + " ref2: " + str(ref2)
+            print "nfile1: " + str(nfile1) + " nfile2: " + str(nfile2)
+        self.assertEqual(remove1 < remove2, True)
+        if (sweep1 >= sweep2):
+            print "close1: " + str(close1) + " close2: " + str(close2)
+            print "remove1: " + str(remove1) + " remove2: " + str(remove2)
+            print "XX: sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)
+            print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)
+            print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)
+            print "tod1: " + str(tod1) + " tod2: " + str(tod2)
+            print "ref1: " + str(ref1) + " ref2: " + str(ref2)
+        self.assertEqual(sweep1 < sweep2, True)
+        if (nfile2 >= nfile1):
+            print "close1: " + str(close1) + " close2: " + str(close2)
+            print "remove1: " + str(remove1) + " remove2: " + str(remove2)
+            print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)
+            print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)
+            print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)
+            print "tod1: " + str(tod1) + " tod2: " + str(tod2)
+            print "ref1: " + str(ref1) + " ref2: " + str(ref2)
+            print "XX: nfile1: " + str(nfile1) + " nfile2: " + str(nfile2)
+        self.assertEqual(nfile2 < nfile1, True)
+        # The only files that should be left are the metadata, the lookaside
+        # file, the lock file, and the active file.
+        if (nfile2 != 4):
+            print "close1: " + str(close1) + " close2: " + str(close2)
+            print "remove1: " + str(remove1) + " remove2: " + str(remove2)
+            print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)
+            print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)
+            print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)
+            print "tod1: " + str(tod1) + " tod2: " + str(tod2)
+            print "ref1: " + str(ref1) + " ref2: " + str(ref2)
+            print "XX2: nfile1: " + str(nfile1) + " nfile2: " + str(nfile2)
+        self.assertEqual(nfile2 == 4, True)
+
+if __name__ == '__main__':
+    wttest.run()