WT-2737 Scrub dirty pages rather than evicting them (#2889)

* Link/include test/testutils with wtperf so we can use the test utility macros and functions in wtperf.

* Minor changes to wtperf variables, sort some options, make some things boolean.

* Merge the stub memory allocation functions used by the test code and wtperf.

* Remove the WT_EVICT_CLEAN flag, it's never used.

* __wt_page_can_evict() doesn't need to set WT_EVICTING, the only place
that cares is __evict_review(), and it already sets that flag.

* Add a new reconciliation flag WT_EVICT_SCRUB; it causes reconciliation
to save disk images it creates. Change the eviction of multi-block pages to
optionally re-instantiate pages in memory instead of evicting them.

* When instantiating pages in-memory, set the read-generation to the same
value as the read-generation of the original page (unless it's set to
WT_READGEN_OLDEST, in which case leave the page's read-generation unset.

* Simplify the in-memory tests some.

* Turn on fixed-length store tests, they're no longer slow (no clue what changed).

* Sue suggested removing Helium support from wtperf.

* Replace dcalloc() calls before snprintf() calls with dmalloc.

* Cosmetic cleanup: set the boundary structure's entries when initializing at the start of __rec_split_write(), don't set it at some random spot in the code.

* After writing a block during checkpoint, we potentially re-use that block during eviction, and during eviction we'll want a disk image for potential re-instantiation in memory. If re-using a block but the boundary structure doesn't already have a disk image, create one.

  I don't think the boundary structure will ever already have a disk image in the current code, but future versions of reconciliation might, and the test is the same as the one we have to do for raw compression, which has already created the disk image.

* If we're closing, don't instantiate any disk images, additionally, free any disk images we don't use. Both changes should fix the same problem where we have a disk image when we're truly discarding the page during a close, one should prevent us from ever having a disk image at that point, the other should prevent use from using any disk image we have.

* Fixes for wtperf directory create changes: ensure the directory exists before the configuration file is dumped and create the right directories in multi-database mode.

3 files changed, 47 insertions, 53 deletions

diff --git a/test/suite/test_inmem01.py b/test/suite/test_inmem01.py
index 875ebb2bfa7..fd2b04bf7e3 100644
--- a/test/suite/test_inmem01.py
+++ b/test/suite/test_inmem01.py
@@ -35,90 +35,68 @@ from wtscenario import check_scenarios
 # test_inmem01.py
 #    Test in-memory configuration.
 class test_inmem01(wttest.WiredTigerTestCase):
-    name = 'inmem01'
-    """
-    In memory configuration still creates files on disk, but has limits
-    in terms of how much data can be written.
-    Test various scenarios including:
-     - Add a small amount of data, ensure it is present.
-     - Add more data than would fit into the configured cache.
-     - Fill the cache with data, remove some data, ensure more data can be
-       inserted (after a reasonable amount of time for space to be reclaimed)
-     - Run queries after adding, removing and re-inserting data.
-     - Try out keeping a cursor open while adding new data.
-    """
+    uri = 'table:inmem01'
+    conn_config = \
+        'cache_size=5MB,file_manager=(close_idle_time=0),in_memory=true'
+    table_config = ',memory_page_max=32k,leaf_page_max=4k'
+
     scenarios = check_scenarios([
-        ('col', dict(tablekind='col')),
-        # Fixed length is very slow, disable it for now
-        #('fix', dict(tablekind='fix')),
-        ('row', dict(tablekind='row'))
+        ('col', dict(fmt='key_format=r,value_format=S')),
+        ('fix', dict(fmt='key_format=r,value_format=8t')),
+        ('row', dict(fmt='key_format=S,value_format=S'))
     ])
 
-    # create an in-memory database
-    conn_config = 'cache_size=5MB,' + \
-                  'file_manager=(close_idle_time=0),in_memory=true'
-
-    def get_table_config(self):
-        kf = 'key_format='
-        vf = 'value_format='
-        if self.tablekind == 'row':
-            kf = kf + 'S'
-        else:
-            kf = kf + 'r'  # record format
-        if self.tablekind == 'fix':
-            vf = vf + '8t'
-        else:
-            vf = vf + 'S'
-        return 'memory_page_max=32k,leaf_page_max=4k,' + kf + ',' + vf
-
+    # Smoke-test in-memory configurations, add a small amount of data and
+    # ensure it's visible.
     def test_insert(self):
-        table_config = self.get_table_config()
-        simple_populate(self,
-            "table:" + self.name, table_config, 1000)
-        # Ensure the data is visible.
-        simple_populate_check(self, 'table:' + self.name, 1000)
+        config = self.fmt + self.table_config
+        simple_populate(self, self.uri, config, 1000)
+        simple_populate_check(self, self.uri, 1000)
 
+    # Add more data than fits into the configured cache and verify it fails.
     def test_insert_over_capacity(self):
-        table_config = self.get_table_config()
+        config = self.fmt + self.table_config
         msg = '/WT_CACHE_FULL.*/'
         self.assertRaisesHavingMessage(wiredtiger.WiredTigerError,
-            lambda:simple_populate(self,
-                "table:" + self.name, table_config, 10000000), msg)
+            lambda:simple_populate(self, self.uri, config, 10000000), msg)
 
-        # Figure out the last key we inserted.
-        cursor = self.session.open_cursor('table:' + self.name, None)
+        # Figure out the last key we successfully inserted, and check all
+        # previous inserts are still there.
+        cursor = self.session.open_cursor(self.uri, None)
         cursor.prev()
         last_key = int(cursor.get_key())
-        simple_populate_check(self, 'table:' + self.name, last_key)
+        simple_populate_check(self, self.uri, last_key)
 
+    # Fill the cache with data, remove some data, ensure more data can be
+    # inserted (after a reasonable amount of time for space to be reclaimed).
     def test_insert_over_delete(self):
-        table_config = self.get_table_config()
+        config = self.fmt + self.table_config
         msg = '/WT_CACHE_FULL.*/'
         self.assertRaisesHavingMessage(wiredtiger.WiredTigerError,
-            lambda:simple_populate(self,
-                "table:" + self.name, table_config, 10000000), msg)
+            lambda:simple_populate(self, self.uri, config, 10000000), msg)
 
         # Now that the database contains as much data as will fit into
         # the configured cache, verify removes succeed.
-        cursor = self.session.open_cursor('table:' + self.name, None)
+        cursor = self.session.open_cursor(self.uri, None)
         for i in range(1, 100):
             cursor.set_key(key_populate(cursor, i))
             cursor.remove()
 
+    # Run queries after adding, removing and re-inserting data.
+    # Try out keeping a cursor open while adding new data.
     def test_insert_over_delete_replace(self):
-        table_config = self.get_table_config()
+        config = self.fmt + self.table_config
         msg = '/WT_CACHE_FULL.*/'
         self.assertRaisesHavingMessage(wiredtiger.WiredTigerError,
-            lambda:simple_populate(self,
-                "table:" + self.name, table_config, 10000000), msg)
+            lambda:simple_populate(self, self.uri, config, 10000000), msg)
 
-        cursor = self.session.open_cursor('table:' + self.name, None)
+        cursor = self.session.open_cursor(self.uri, None)
         cursor.prev()
         last_key = int(cursor.get_key())
 
         # Now that the database contains as much data as will fit into
         # the configured cache, verify removes succeed.
-        cursor = self.session.open_cursor('table:' + self.name, None)
+        cursor = self.session.open_cursor(self.uri, None)
         for i in range(1, last_key / 4, 1):
             cursor.set_key(key_populate(cursor, i))
             cursor.remove()
diff --git a/test/utility/misc.c b/test/utility/misc.c
index dfc655dec1a..dffd29a5b6a 100644
--- a/test/utility/misc.c
+++ b/test/utility/misc.c
@@ -192,3 +192,18 @@ dstrdup(const void *str)
 		return (p);
 	testutil_die(errno, "strdup");
 }
+
+/*
+ * dstrndup --
+ *      Call emulating strndup, dying on failure. Don't use actual strndup here
+ *	as it is not supported within MSVC.
+ */
+void *
+dstrndup(const char *str, size_t len)
+{
+	char *p;
+
+	p = dcalloc(len + 1, sizeof(char));
+	memcpy(p, str, len);
+	return (p);
+}
diff --git a/test/utility/test_util.h b/test/utility/test_util.h
index 66ff8de2d19..821e06084d2 100644
--- a/test/utility/test_util.h
+++ b/test/utility/test_util.h
@@ -115,6 +115,7 @@ void *dcalloc(size_t, size_t);
 void *dmalloc(size_t);
 void *drealloc(void *, size_t);
 void *dstrdup(const void *);
+void *dstrndup(const char *, size_t);
 void  testutil_clean_work_dir(char *);
 void  testutil_cleanup(TEST_OPTS *);
 void  testutil_make_work_dir(char *);