WT-2737 Scrub dirty pages rather than evicting them (#2889)

* Link/include test/testutils with wtperf so we can use the test utility macros and functions in wtperf.

* Minor changes to wtperf variables, sort some options, make some things boolean.

* Merge the stub memory allocation functions used by the test code and wtperf.

* Remove the WT_EVICT_CLEAN flag, it's never used.

* __wt_page_can_evict() doesn't need to set WT_EVICTING, the only place
that cares is __evict_review(), and it already sets that flag.

* Add a new reconciliation flag WT_EVICT_SCRUB; it causes reconciliation
to save disk images it creates. Change the eviction of multi-block pages to
optionally re-instantiate pages in memory instead of evicting them.

* When instantiating pages in-memory, set the read-generation to the same
value as the read-generation of the original page (unless it's set to
WT_READGEN_OLDEST, in which case leave the page's read-generation unset.

* Simplify the in-memory tests some.

* Turn on fixed-length store tests, they're no longer slow (no clue what changed).

* Sue suggested removing Helium support from wtperf.

* Replace dcalloc() calls before snprintf() calls with dmalloc.

* Cosmetic cleanup: set the boundary structure's entries when initializing at the start of __rec_split_write(), don't set it at some random spot in the code.

* After writing a block during checkpoint, we potentially re-use that block during eviction, and during eviction we'll want a disk image for potential re-instantiation in memory. If re-using a block but the boundary structure doesn't already have a disk image, create one.

  I don't think the boundary structure will ever already have a disk image in the current code, but future versions of reconciliation might, and the test is the same as the one we have to do for raw compression, which has already created the disk image.

* If we're closing, don't instantiate any disk images, additionally, free any disk images we don't use. Both changes should fix the same problem where we have a disk image when we're truly discarding the page during a close, one should prevent us from ever having a disk image at that point, the other should prevent use from using any disk image we have.

* Fixes for wtperf directory create changes: ensure the directory exists before the configuration file is dumped and create the right directories in multi-database mode.

3 files changed, 66 insertions, 145 deletions

diff --git a/bench/wtperf/Makefile.am b/bench/wtperf/Makefile.am
index cc1f84b5406..57792e3887f 100644
--- a/bench/wtperf/Makefile.am
+++ b/bench/wtperf/Makefile.am
@@ -1,13 +1,17 @@
-AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include
-
-LDADD = $(top_builddir)/libwiredtiger.la -lm
+AM_CPPFLAGS = -I$(top_builddir)
+AM_CPPFLAGS +=-I$(top_srcdir)/src/include
+AM_CPPFLAGS +=-I$(top_srcdir)/test/utility
 
 noinst_PROGRAMS = wtperf
-wtperf_LDFLAGS = -static
 wtperf_SOURCES =\
 	config.c idle_table_cycle.c misc.c track.c wtperf.c \
 	wtperf.h wtperf_opt.i wtperf_throttle.c wtperf_truncate.c
 
+wtperf_LDADD = $(top_builddir)/test/utility/libtest_util.la
+wtperf_LDADD +=$(top_builddir)/libwiredtiger.la
+wtperf_LDADD +=-lm
+wtperf_LDFLAGS = -static
+
 TESTS = smoke.sh
 AM_TESTS_ENVIRONMENT = rm -rf WT_TEST ; mkdir WT_TEST ;
 # automake 1.11 compatibility
diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c
index a2718bc4290..2dbf14e9f1c 100644
--- a/bench/wtperf/wtperf.c
+++ b/bench/wtperf/wtperf.c
@@ -36,7 +36,6 @@ static const CONFIG default_cfg = {
 	NULL,				/* reopen config */
 	NULL,				/* base_uri */
 	NULL,				/* uris */
-	NULL,				/* helium_mount */
 	NULL,				/* conn */
 	NULL,				/* logf */
 	NULL,				/* async */
@@ -73,14 +72,14 @@ static const char * const debug_cconfig = "";
 static const char * const debug_tconfig = "";
 
 static void	*checkpoint_worker(void *);
-static int	 create_tables(CONFIG *);
-static int	drop_all_tables(CONFIG *);
+static int	 drop_all_tables(CONFIG *);
 static int	 execute_populate(CONFIG *);
 static int	 execute_workload(CONFIG *);
 static int	 find_table_count(CONFIG *);
 static void	*monitor(void *);
 static void	*populate_thread(void *);
 static void	 randomize_value(CONFIG_THREAD *, char *);
+static int	 recreate_dir(const char *);
 static int	 start_all_runs(CONFIG *);
 static int	 start_run(CONFIG *);
 static int	 start_threads(CONFIG *,
@@ -93,10 +92,6 @@ static void	*worker(void *);
 static uint64_t	 wtperf_rand(CONFIG_THREAD *);
 static uint64_t	 wtperf_value_range(CONFIG *);
 
-#define	HELIUM_NAME	"dev1"
-#define	HELIUM_PATH							\
-	"../../ext/test/helium/.libs/libwiredtiger_helium.so"
-#define	HELIUM_CONFIG	",type=helium"
 #define	INDEX_COL_NAMES	",columns=(key,val)"
 
 /* Retrieve an ID for the next insert operation. */
@@ -1894,9 +1889,6 @@ create_tables(CONFIG *cfg)
 	int ret;
 	char buf[512];
 
-	if (cfg->create == 0)
-		return (0);
-
 	if ((ret = cfg->conn->open_session(
 	    cfg->conn, NULL, cfg->sess_config, &session)) != 0) {
 		lprintf(cfg, ret, 0,
@@ -1988,13 +1980,10 @@ start_all_runs(CONFIG *cfg)
 		if (strcmp(cfg->monitor_dir, cfg->home) == 0)
 			next_cfg->monitor_dir = new_home;
 
-		/* Create clean home directories. */
-		snprintf(cmd_buf, cmd_len, "rm -rf %s && mkdir %s",
-		    next_cfg->home, next_cfg->home);
-		if ((ret = system(cmd_buf)) != 0) {
-			fprintf(stderr, "%s: failed\n", cmd_buf);
-			goto err;
-		}
+		/* If creating the sub-database, recreate it's home */
+		if (cfg->create != 0)
+			recreate_dir(next_cfg->home);
+
 		if ((ret = pthread_create(
 		    &threads[i], NULL, thread_run_wtperf, next_cfg)) != 0) {
 			lprintf(cfg, ret, 0, "Error creating thread");
@@ -2043,7 +2032,6 @@ start_run(CONFIG *cfg)
 	uint64_t total_ops;
 	uint32_t run_time;
 	int monitor_created, ret, t_ret;
-	char helium_buf[256];
 
 	monitor_created = ret = 0;
 					/* [-Wconditional-uninitialized] */
@@ -2058,21 +2046,10 @@ start_run(CONFIG *cfg)
 		goto err;
 	}
 
-	/* Configure optional Helium volume. */
-	if (cfg->helium_mount != NULL) {
-		snprintf(helium_buf, sizeof(helium_buf),
-		    "entry=wiredtiger_extension_init,config=["
-		    "%s=[helium_devices=\"he://./%s\","
-		    "helium_o_volume_truncate=1]]",
-		    HELIUM_NAME, cfg->helium_mount);
-		if ((ret = cfg->conn->load_extension(
-		    cfg->conn, HELIUM_PATH, helium_buf)) != 0)
-			lprintf(cfg,
-			    ret, 0, "Error loading Helium: %s", helium_buf);
-	}
-
 	create_uris(cfg);
-	if ((ret = create_tables(cfg)) != 0)
+
+	/* If creating, create the tables. */
+	if (cfg->create != 0 && (ret = create_tables(cfg)) != 0)
 		goto err;
 
 	/* Start the monitor thread. */
@@ -2202,18 +2179,21 @@ err:		if (ret == 0)
 
 extern int __wt_optind, __wt_optreset;
 extern char *__wt_optarg;
+void (*custom_die)(void) = NULL;
 
 int
 main(int argc, char *argv[])
 {
 	CONFIG *cfg, _cfg;
 	size_t req_len, sreq_len;
-	int ch, monitor_set, ret;
-	const char *opts = "C:H:h:m:O:o:T:";
+	bool monitor_set;
+	int ch, ret;
+	const char *opts = "C:h:m:O:o:T:";
 	const char *config_opts;
 	char *cc_buf, *sess_cfg, *tc_buf, *user_cconfig, *user_tconfig;
 
-	monitor_set = ret = 0;
+	monitor_set = false;
+	ret = 0;
 	config_opts = NULL;
 	cc_buf = sess_cfg = tc_buf = user_cconfig = user_tconfig = NULL;
 
@@ -2239,8 +2219,12 @@ main(int argc, char *argv[])
 				strcat(user_cconfig, __wt_optarg);
 			}
 			break;
-		case 'H':
-			cfg->helium_mount = __wt_optarg;
+		case 'h':
+			cfg->home = __wt_optarg;
+			break;
+		case 'm':
+			cfg->monitor_dir = __wt_optarg;
+			monitor_set = true;
 			break;
 		case 'O':
 			config_opts = __wt_optarg;
@@ -2256,15 +2240,7 @@ main(int argc, char *argv[])
 				strcat(user_tconfig, __wt_optarg);
 			}
 			break;
-		case 'h':
-			cfg->home = __wt_optarg;
-			break;
-		case 'm':
-			cfg->monitor_dir = __wt_optarg;
-			monitor_set = 1;
-			break;
 		case '?':
-			fprintf(stderr, "Invalid option\n");
 			usage();
 			goto einval;
 		}
@@ -2320,7 +2296,7 @@ main(int argc, char *argv[])
 		 * to 4096 if needed.
 		 */
 		req_len = strlen(",async=(enabled=true,threads=)") + 4;
-		cfg->async_config = dcalloc(req_len, 1);
+		cfg->async_config = dmalloc(req_len);
 		snprintf(cfg->async_config, req_len,
 		    ",async=(enabled=true,threads=%" PRIu32 ")",
 		    cfg->async_threads);
@@ -2341,13 +2317,9 @@ main(int argc, char *argv[])
 	}
 
 	/* Build the URI from the table name. */
-	req_len = strlen("table:") +
-	    strlen(HELIUM_NAME) + strlen(cfg->table_name) + 2;
-	cfg->base_uri = dcalloc(req_len, 1);
-	snprintf(cfg->base_uri, req_len, "table:%s%s%s",
-	    cfg->helium_mount == NULL ? "" : HELIUM_NAME,
-	    cfg->helium_mount == NULL ? "" : "/",
-	    cfg->table_name);
+	req_len = strlen("table:") + strlen(cfg->table_name) + 2;
+	cfg->base_uri = dmalloc(req_len);
+	snprintf(cfg->base_uri, req_len, "table:%s", cfg->table_name);
 
 	/* Make stdout line buffered, so verbose output appears quickly. */
 	__wt_stream_set_line_buffer(stdout);
@@ -2366,13 +2338,13 @@ main(int argc, char *argv[])
 		if (cfg->session_count_idle > 0) {
 			sreq_len = strlen(",session_max=") + 6;
 			req_len += sreq_len;
-			sess_cfg = dcalloc(sreq_len, 1);
+			sess_cfg = dmalloc(sreq_len);
 			snprintf(sess_cfg, sreq_len,
 			    ",session_max=%" PRIu32,
 			    cfg->session_count_idle + cfg->workers_cnt +
 			    cfg->populate_threads + 10);
 		}
-		cc_buf = dcalloc(req_len, 1);
+		cc_buf = dmalloc(req_len);
 		/*
 		 * This is getting hard to parse.
 		 */
@@ -2388,36 +2360,34 @@ main(int argc, char *argv[])
 		if ((ret = config_opt_str(cfg, "conn_config", cc_buf)) != 0)
 			goto err;
 	}
-	if (cfg->verbose > 1 || cfg->index || cfg->helium_mount != NULL ||
+	if (cfg->verbose > 1 || cfg->index ||
 	    user_tconfig != NULL || cfg->compress_table != NULL) {
-		req_len = strlen(cfg->table_config) + strlen(HELIUM_CONFIG) +
-		    strlen(debug_tconfig) + 3;
+		req_len = strlen(cfg->table_config) + strlen(debug_tconfig) + 3;
 		if (user_tconfig != NULL)
 			req_len += strlen(user_tconfig);
 		if (cfg->compress_table != NULL)
 			req_len += strlen(cfg->compress_table);
 		if (cfg->index)
 			req_len += strlen(INDEX_COL_NAMES);
-		tc_buf = dcalloc(req_len, 1);
+		tc_buf = dmalloc(req_len);
 		/*
 		 * This is getting hard to parse.
 		 */
-		snprintf(tc_buf, req_len, "%s%s%s%s%s%s%s%s",
+		snprintf(tc_buf, req_len, "%s%s%s%s%s%s%s",
 		    cfg->table_config,
 		    cfg->index ? INDEX_COL_NAMES : "",
 		    cfg->compress_table ? cfg->compress_table : "",
 		    cfg->verbose > 1 ? ",": "",
 		    cfg->verbose > 1 ? debug_tconfig : "",
 		    user_tconfig ? ",": "",
-		    user_tconfig ? user_tconfig : "",
-		    cfg->helium_mount == NULL ? "" : HELIUM_CONFIG);
+		    user_tconfig ? user_tconfig : "");
 		if ((ret = config_opt_str(cfg, "table_config", tc_buf)) != 0)
 			goto err;
 	}
 	if (cfg->log_partial && cfg->table_count > 1) {
 		req_len = strlen(cfg->table_config) +
 		    strlen(LOG_PARTIAL_CONFIG) + 1;
-		cfg->partial_config = dcalloc(req_len, 1);
+		cfg->partial_config = dmalloc(req_len);
 		snprintf(cfg->partial_config, req_len, "%s%s",
 		    cfg->table_config, LOG_PARTIAL_CONFIG);
 	}
@@ -2430,7 +2400,7 @@ main(int argc, char *argv[])
 		    strlen(READONLY_CONFIG) + 1;
 	else
 		req_len = strlen(cfg->conn_config) + 1;
-	cfg->reopen_config = dcalloc(req_len, 1);
+	cfg->reopen_config = dmalloc(req_len);
 	if (cfg->readonly)
 		snprintf(cfg->reopen_config, req_len, "%s%s",
 		    cfg->conn_config, READONLY_CONFIG);
@@ -2442,6 +2412,12 @@ main(int argc, char *argv[])
 	if ((ret = config_sanity(cfg)) != 0)
 		goto err;
 
+	/* If creating, remove and re-create the home directory. */
+	if (cfg->create != 0 && (ret = recreate_dir(cfg->home)) != 0) {
+		lprintf(cfg, ret, 0, "Error re-creating home directory");
+		goto err;
+	}
+
 	/* Write a copy of the config. */
 	config_to_file(cfg);
 
@@ -2557,6 +2533,22 @@ stop_threads(CONFIG *cfg, u_int num, CONFIG_THREAD *threads)
 }
 
 static int
+recreate_dir(const char *name)
+{
+	char *buf;
+	size_t len;
+
+	len = strlen(name) * 2 + 100;
+	buf = dmalloc(len);
+	(void)snprintf(
+	    buf, len, "rm -rf %s && mkdir %s", name, name);
+	testutil_checkfmt(system(buf), "system: %s", buf);
+	free(buf);
+
+	return (0);
+}
+
+static int
 drop_all_tables(CONFIG *cfg)
 {
 	struct timespec start, stop;
diff --git a/bench/wtperf/wtperf.h b/bench/wtperf/wtperf.h
index a316b40d7fa..9f37d361755 100644
--- a/bench/wtperf/wtperf.h
+++ b/bench/wtperf/wtperf.h
@@ -29,14 +29,11 @@
 #ifndef	HAVE_WTPERF_H
 #define	HAVE_WTPERF_H
 
-#include <wt_internal.h>
+#include "test_util.h"
+
 #include <assert.h>
 #include <math.h>
 
-#ifdef _WIN32
-#include "windows_shim.h"
-#endif
-
 #include "config_opt.h"
 
 typedef struct __config CONFIG;
@@ -125,7 +122,6 @@ struct __config {			/* Configuration structure */
 	char *reopen_config;		/* Config string for conn reopen */
 	char *base_uri;			/* Object URI */
 	char **uris;			/* URIs if multiple tables */
-	const char *helium_mount;	/* Optional Helium mount point */
 
 	WT_CONNECTION *conn;		/* Database connection */
 
@@ -327,75 +323,4 @@ die(int e, const char *str)
 	fprintf(stderr, "Call to %s failed: %s", str, wiredtiger_strerror(e));
 	exit(EXIT_FAILURE);
 }
-
-/*
- * dmalloc --
- *      Call malloc, dying on failure.
- */
-static inline void *
-dmalloc(size_t len)
-{
-	void *p;
-
-	if ((p = malloc(len)) == NULL)
-		die(errno, "malloc");
-	return (p);
-}
-
-/*
- * dcalloc --
- *      Call calloc, dying on failure.
- */
-static inline void *
-dcalloc(size_t num, size_t size)
-{
-	void *p;
-
-	if ((p = calloc(num, size)) == NULL)
-		die(errno, "calloc");
-	return (p);
-}
-
-/*
- * drealloc --
- *      Call realloc, dying on failure.
- */
-static inline void *
-drealloc(void *p, size_t len)
-{
-	void *repl;
-
-	if ((repl = realloc(p, len)) == NULL)
-		die(errno, "realloc");
-	return (repl);
-}
-
-/*
- * dstrdup --
- *      Call strdup, dying on failure.
- */
-static inline char *
-dstrdup(const char *str)
-{
-	char *p;
-
-	if ((p = strdup(str)) == NULL)
-		die(errno, "strdup");
-	return (p);
-}
-
-/*
- * dstrndup --
- *      Call emulating strndup, dying on failure. Don't use actual strndup here
- *	as it is not supported within MSVC.
- */
-static inline char *
-dstrndup(const char *str, const size_t len)
-{
-	char *p;
-
-	p = dcalloc(len + 1, sizeof(char));
-	memcpy(p, str, len);
-	return (p);
-}
 #endif