WT-2535: Add an explicit test for lost updates

Switch to manipulating WT_ITEMs instead of individual buffers, it makes
some things in the ops code easier.
Print key-counts with u64's, not u32's.

6 files changed, 92 insertions, 101 deletions

diff --git a/test/format/bdb.c b/test/format/bdb.c
index 823fc8ff888..154a6fa3c1f 100644
--- a/test/format/bdb.c
+++ b/test/format/bdb.c
@@ -30,7 +30,7 @@
 #include "format.h"
 
 static DBT key, value;
-static uint8_t *keybuf;
+static WT_ITEM keyitem;
 
 static int
 bdb_compare_reverse(DB *dbp, const DBT *k1, const DBT *k2
@@ -78,7 +78,7 @@ bdb_open(void)
 	assert(db->cursor(db, NULL, &dbc, 0) == 0);
 	g.dbc = dbc;
 
-	key_gen_setup(&keybuf);
+	key_gen_setup(&keyitem);
 }
 
 void
@@ -95,8 +95,7 @@ bdb_close(void)
 	assert(db->close(db, 0) == 0);
 	assert(dbenv->close(dbenv, 0) == 0);
 
-	free(keybuf);
-	keybuf = NULL;
+	free(keyitem.mem);
 }
 
 void
@@ -144,12 +143,11 @@ void
 bdb_read(uint64_t keyno, void *valuep, size_t *valuesizep, int *notfoundp)
 {
 	DBC *dbc = g.dbc;
-	size_t size;
 	int ret;
 
-	key_gen(keybuf, &size, keyno);
-	key.data = keybuf;
-	key.size = (uint32_t)size;
+	key_gen(&keyitem, keyno);
+	key.data = (void *)keyitem.data;
+	key.size = keyitem.size;
 
 	*notfoundp = 0;
 	if ((ret = dbc->get(dbc, &key, &value, DB_SET)) != 0) {
@@ -193,9 +191,9 @@ bdb_remove(uint64_t keyno, int *notfoundp)
 	size_t size;
 	int ret;
 
-	key_gen(keybuf, &size, keyno);
-	key.data = keybuf;
-	key.size = (uint32_t)size;
+	key_gen(&keyitem, keyno);
+	key.data = (void *)keyitem.data;
+	key.size = keyitem.size;
 
 	bdb_read(keyno, &value.data, &size, notfoundp);
 	value.size = (uint32_t)size;
diff --git a/test/format/bulk.c b/test/format/bulk.c
index 64b005d294f..4266108dba3 100644
--- a/test/format/bulk.c
+++ b/test/format/bulk.c
@@ -35,11 +35,9 @@ wts_load(void)
 	WT_CURSOR *cursor;
 	WT_ITEM key, value;
 	WT_SESSION *session;
-	uint8_t *keybuf, *valbuf;
 	bool is_bulk;
 
 	conn = g.wts_conn;
-	keybuf = valbuf = NULL;
 
 	testutil_check(conn->open_session(conn, NULL, NULL, &session));
 
@@ -63,8 +61,8 @@ wts_load(void)
 	    is_bulk ? "bulk,append" : NULL, &cursor));
 
 	/* Set up the key/value buffers. */
-	key_gen_setup(&keybuf);
-	val_gen_setup(NULL, &valbuf);
+	key_gen_setup(&key);
+	val_gen_setup(NULL, &value);
 
 	for (;;) {
 		if (++g.key_cnt > g.c_rows) {
@@ -76,10 +74,8 @@ wts_load(void)
 		if (g.key_cnt % 100 == 0)
 			track("bulk load", g.key_cnt, NULL);
 
-		key_gen(keybuf, &key.size, (uint64_t)g.key_cnt);
-		key.data = keybuf;
-		val_gen(NULL, valbuf, &value.size, (uint64_t)g.key_cnt);
-		value.data = valbuf;
+		key_gen(&key, g.key_cnt);
+		val_gen(NULL, &value, g.key_cnt);
 
 		switch (g.type) {
 		case FIX:
@@ -88,7 +84,7 @@ wts_load(void)
 			cursor->set_value(cursor, *(uint8_t *)value.data);
 			if (g.logging == LOG_OPS)
 				(void)g.wt_api->msg_printf(g.wt_api, session,
-				    "%-10s %" PRIu32 " {0x%02" PRIx8 "}",
+				    "%-10s %" PRIu64 " {0x%02" PRIx8 "}",
 				    "bulk V",
 				    g.key_cnt, ((uint8_t *)value.data)[0]);
 			break;
@@ -98,7 +94,7 @@ wts_load(void)
 			cursor->set_value(cursor, &value);
 			if (g.logging == LOG_OPS)
 				(void)g.wt_api->msg_printf(g.wt_api, session,
-				    "%-10s %" PRIu32 " {%.*s}", "bulk V",
+				    "%-10s %" PRIu64 " {%.*s}", "bulk V",
 				    g.key_cnt,
 				    (int)value.size, (char *)value.data);
 			break;
@@ -106,12 +102,12 @@ wts_load(void)
 			cursor->set_key(cursor, &key);
 			if (g.logging == LOG_OPS)
 				(void)g.wt_api->msg_printf(g.wt_api, session,
-				    "%-10s %" PRIu32 " {%.*s}", "bulk K",
+				    "%-10s %" PRIu64 " {%.*s}", "bulk K",
 				    g.key_cnt, (int)key.size, (char *)key.data);
 			cursor->set_value(cursor, &value);
 			if (g.logging == LOG_OPS)
 				(void)g.wt_api->msg_printf(g.wt_api, session,
-				    "%-10s %" PRIu32 " {%.*s}", "bulk V",
+				    "%-10s %" PRIu64 " {%.*s}", "bulk V",
 				    g.key_cnt,
 				    (int)value.size, (char *)value.data);
 			break;
@@ -133,6 +129,6 @@ wts_load(void)
 
 	testutil_check(session->close(session, NULL));
 
-	free(keybuf);
-	free(valbuf);
+	free(key.mem);
+	free(value.mem);
 }
diff --git a/test/format/format.h b/test/format/format.h
index cce170da51b..5a421dd05fb 100644
--- a/test/format/format.h
+++ b/test/format/format.h
@@ -312,17 +312,17 @@ void	 config_single(const char *, int);
 void	*dmalloc(size_t);
 char	*dstrdup(const char *);
 void	 fclose_and_clear(FILE **);
-void	 key_gen(void *, size_t *, uint64_t);
-void	 key_gen_insert(WT_RAND_STATE *, void *, size_t *, uint64_t);
-void	 key_gen_setup(uint8_t **);
+void	 key_gen(WT_ITEM *, uint64_t);
+void	 key_gen_insert(WT_RAND_STATE *, WT_ITEM *, uint64_t);
+void	 key_gen_setup(WT_ITEM *);
 void	 key_len_setup(void);
 void	*lrt(void *);
 void	 path_setup(const char *);
 int	 read_row(WT_CURSOR *, WT_ITEM *, uint64_t, int);
 uint32_t rng(WT_RAND_STATE *);
 void	 track(const char *, uint64_t, TINFO *);
-void	 val_gen(WT_RAND_STATE *, void *, size_t *, uint64_t);
-void	 val_gen_setup(WT_RAND_STATE *, uint8_t **);
+void	 val_gen(WT_RAND_STATE *, WT_ITEM *, uint64_t);
+void	 val_gen_setup(WT_RAND_STATE *, WT_ITEM *);
 void	 wts_close(void);
 void	 wts_create(void);
 void	 wts_dump(const char *, int);
diff --git a/test/format/lrt.c b/test/format/lrt.c
index 451d2f4fa3c..d6cb5fa215a 100644
--- a/test/format/lrt.c
+++ b/test/format/lrt.c
@@ -43,16 +43,14 @@ lrt(void *arg)
 	uint64_t keyno, saved_keyno;
 	u_int period;
 	int pinned, ret;
-	uint8_t bitfield, *keybuf;
+	uint8_t bitfield;
 	void *buf;
 
 	(void)(arg);			/* Unused parameter */
 
 	saved_keyno = 0;		/* [-Werror=maybe-uninitialized] */
 
-	key_gen_setup(&keybuf);
-	memset(&key, 0, sizeof(key));
-	key.data = keybuf;
+	key_gen_setup(&key);
 	memset(&value, 0, sizeof(value));
 
 	buf = NULL;
@@ -165,7 +163,7 @@ lrt(void *arg)
 
 	testutil_check(session->close(session, NULL));
 
-	free(keybuf);
+	free(key.mem);
 	free(buf);
 
 	return (NULL);
diff --git a/test/format/ops.c b/test/format/ops.c
index 63c3aff8703..a62589bcde2 100644
--- a/test/format/ops.c
+++ b/test/format/ops.c
@@ -228,7 +228,6 @@ ops(void *arg)
 	WT_SESSION *session;
 	uint64_t keyno, ckpt_op, reset_op, session_op;
 	uint32_t op;
-	uint8_t *keybuf, *valbuf;
 	u_int np;
 	int dir, notfound;
 	char *ckpt_config, ckpt_name[64];
@@ -237,15 +236,14 @@ ops(void *arg)
 	tinfo = arg;
 
 	conn = g.wts_conn;
-	keybuf = valbuf = NULL;
 	readonly = false;		/* -Wconditional-uninitialized */
 
 	/* Initialize the per-thread random number generator. */
 	__wt_random_init(&tinfo->rnd);
 
 	/* Set up the default key and value buffers. */
-	key_gen_setup(&keybuf);
-	val_gen_setup(&tinfo->rnd, &valbuf);
+	key_gen_setup(&key);
+	val_gen_setup(&tinfo->rnd, &value);
 
 	/* Set the first operation where we'll create sessions and cursors. */
 	session_op = 0;
@@ -406,8 +404,6 @@ ops(void *arg)
 		notfound = 0;
 
 		keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows);
-		key.data = keybuf;
-		value.data = valbuf;
 
 		/*
 		 * Perform some number of operations: the percentage of deletes,
@@ -541,8 +537,8 @@ deadlock:			++tinfo->deadlock;
 	if (session != NULL)
 		testutil_check(session->close(session, NULL));
 
-	free(keybuf);
-	free(valbuf);
+	free(key.mem);
+	free(value.mem);
 
 	tinfo->state = TINFO_COMPLETE;
 	return (NULL);
@@ -560,12 +556,11 @@ wts_read_scan(void)
 	WT_ITEM key;
 	WT_SESSION *session;
 	uint64_t cnt, last_cnt;
-	uint8_t *keybuf;
 
 	conn = g.wts_conn;
 
 	/* Set up the default key buffer. */
-	key_gen_setup(&keybuf);
+	key_gen_setup(&key);
 
 	/* Open a session and cursor pair. */
 	testutil_check(conn->open_session(
@@ -583,14 +578,13 @@ wts_read_scan(void)
 			last_cnt = cnt;
 		}
 
-		key.data = keybuf;
 		testutil_checkfmt(
 		    read_row(cursor, &key, cnt, 0), "%s", "read_scan");
 	}
 
 	testutil_check(session->close(session, NULL));
 
-	free(keybuf);
+	free(key.mem);
 }
 
 /*
@@ -620,7 +614,7 @@ read_row(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int notfound_err)
 		cursor->set_key(cursor, keyno);
 		break;
 	case ROW:
-		key_gen(key->data, &key->size, keyno);
+		key_gen(key, keyno);
 		cursor->set_key(cursor, key);
 		break;
 	}
@@ -819,8 +813,8 @@ row_update(TINFO *tinfo,
 
 	session = cursor->session;
 
-	key_gen(key->data, &key->size, keyno);
-	val_gen(&tinfo->rnd, value->data, &value->size, keyno);
+	key_gen(key, keyno);
+	val_gen(&tinfo->rnd, value, keyno);
 
 	/* Log the operation */
 	if (g.logging == LOG_OPS)
@@ -865,7 +859,7 @@ col_update(TINFO *tinfo,
 
 	session = cursor->session;
 
-	val_gen(&tinfo->rnd, value->data, &value->size, keyno);
+	val_gen(&tinfo->rnd, value, keyno);
 
 	/* Log the operation */
 	if (g.logging == LOG_OPS) {
@@ -899,7 +893,7 @@ col_update(TINFO *tinfo,
 	{
 	int notfound;
 
-	key_gen(key->data, &key->size, keyno);
+	key_gen(key, keyno);
 	bdb_update(key->data, key->size, value->data, value->size, &notfound);
 	(void)notfound_chk("col_update", ret, notfound, keyno);
 	}
@@ -1022,8 +1016,8 @@ row_insert(TINFO *tinfo,
 
 	session = cursor->session;
 
-	key_gen_insert(&tinfo->rnd, key->data, &key->size, keyno);
-	val_gen(&tinfo->rnd, value->data, &value->size, keyno);
+	key_gen_insert(&tinfo->rnd, key, keyno);
+	val_gen(&tinfo->rnd, value, keyno);
 
 	/* Log the operation */
 	if (g.logging == LOG_OPS)
@@ -1069,7 +1063,7 @@ col_insert(TINFO *tinfo,
 
 	session = cursor->session;
 
-	val_gen(&tinfo->rnd, value->data, &value->size, g.rows + 1);
+	val_gen(&tinfo->rnd, value, g.rows + 1);
 
 	if (g.type == FIX)
 		cursor->set_value(cursor, *(uint8_t *)value->data);
@@ -1105,7 +1099,7 @@ col_insert(TINFO *tinfo,
 	{
 	int notfound;
 
-	key_gen(key->data, &key->size, keyno);
+	key_gen(key, keyno);
 	bdb_update(key->data, key->size, value->data, value->size, &notfound);
 	}
 #else
@@ -1126,7 +1120,7 @@ row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp)
 
 	session = cursor->session;
 
-	key_gen(key->data, &key->size, keyno);
+	key_gen(key, keyno);
 
 	/* Log the operation */
 	if (g.logging == LOG_OPS)
@@ -1200,7 +1194,7 @@ col_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp)
 	 * do the same thing for the BDB store.
 	 */
 	if (g.type == FIX) {
-		key_gen(key->data, &key->size, keyno);
+		key_gen(key, keyno);
 		bdb_update(key->data, key->size, "\0", 1, &notfound);
 	} else
 		bdb_remove(keyno, &notfound);
diff --git a/test/format/util.c b/test/format/util.c
index 31771de026f..dc5cb007e6c 100644
--- a/test/format/util.c
+++ b/test/format/util.c
@@ -98,68 +98,70 @@ key_len_setup(void)
 }
 
 void
-key_gen_setup(uint8_t **keyp)
+key_gen_setup(WT_ITEM *key)
 {
-	uint8_t *key;
 	size_t i, len;
-
-	*keyp = NULL;
+	char *p;
 
 	len = MAX(KILOBYTE(100), g.c_key_max);
-	key = dmalloc(len);
+	p = dmalloc(len);
 	for (i = 0; i < len; ++i)
-		key[i] = (uint8_t)("abcdefghijklmnopqrstuvwxyz"[i % 26]);
-	*keyp = key;
+		p[i] = "abcdefghijklmnopqrstuvwxyz"[i % 26];
+
+	key->mem = p;
+	key->memsize = len;
+	key->data = key->mem;
+	key->size = 0;
 }
 
 static void
-key_gen_common(void *keyarg, size_t *sizep, uint64_t keyno, int suffix)
+key_gen_common(WT_ITEM *key, uint64_t keyno, int suffix)
 {
 	int len;
-	uint8_t *key;
+	char *p;
 
-	key = keyarg;
+	p = key->mem;
 
 	/*
 	 * The key always starts with a 10-digit string (the specified cnt)
 	 * followed by two digits, a random number between 1 and 15 if it's
 	 * an insert, otherwise 00.
 	 */
-	len = sprintf((char *)key, "%010" PRIu64 ".%02d", keyno, suffix);
+	len = sprintf(p, "%010" PRIu64 ".%02d", keyno, suffix);
 
 	/*
 	 * In a column-store, the key is only used for BDB, and so it doesn't
 	 * need a random length.
 	 */
 	if (g.type == ROW) {
-		key[len] = '/';
+		p[len] = '/';
 		len = (int)g.key_rand_len[keyno %
 		    (sizeof(g.key_rand_len) / sizeof(g.key_rand_len[0]))];
 	}
-	*sizep = (size_t)len;
+	key->size = (size_t)len;
 }
 
 void
-key_gen(void *key, size_t *sizep, uint64_t keyno)
+key_gen(WT_ITEM *key, uint64_t keyno)
 {
-	key_gen_common(key, sizep, keyno, 0);
+	key_gen_common(key, keyno, 0);
 }
 
 void
-key_gen_insert(WT_RAND_STATE *rnd, void *key, size_t *sizep, uint64_t keyno)
+key_gen_insert(WT_RAND_STATE *rnd, WT_ITEM *key, uint64_t keyno)
 {
-	key_gen_common(key, sizep, keyno, (int)mmrand(rnd, 1, 15));
+	key_gen_common(key, keyno, (int)mmrand(rnd, 1, 15));
 }
 
 static uint32_t val_dup_data_len;	/* Length of duplicate data items */
 
 void
-val_gen_setup(WT_RAND_STATE *rnd, uint8_t **valp)
+val_gen_setup(WT_RAND_STATE *rnd, WT_ITEM *value)
 {
-	uint8_t *val;
 	size_t i, len;
+	char *p;
 
-	*valp = NULL;
+	memset(value, 0, sizeof(WT_ITEM));
 
 	/*
 	 * Set initial buffer contents to recognizable text.
@@ -169,22 +171,25 @@ val_gen_setup(WT_RAND_STATE *rnd, uint8_t **valp)
 	 * data for column-store run-length encoded files.
 	 */
 	len = MAX(KILOBYTE(100), g.c_value_max) + 20;
-	val = dmalloc(len);
+	p = dmalloc(len);
 	for (i = 0; i < len; ++i)
-		val[i] = (uint8_t)("ABCDEFGHIJKLMNOPQRSTUVWXYZ"[i % 26]);
+		p[i] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"[i % 26];
 
-	*valp = val;
+	value->mem = p;
+	value->memsize = len;
+	value->data = value->mem;
+	value->size = 0;
 
 	val_dup_data_len = kv_len(rnd,
 	    (uint64_t)mmrand(rnd, 1, 20), g.c_value_min, g.c_value_max);
 }
 
 void
-val_gen(WT_RAND_STATE *rnd, void *valarg, size_t *sizep, uint64_t keyno)
+val_gen(WT_RAND_STATE *rnd, WT_ITEM *value, uint64_t keyno)
 {
-	uint8_t *val;
+	char *p;
 
-	val = valarg;
+	p = value->mem;
 
 	/*
 	 * Fixed-length records: take the low N bits from the last digit of
@@ -192,16 +197,16 @@ val_gen(WT_RAND_STATE *rnd, void *valarg, size_t *sizep, uint64_t keyno)
 	 */
 	if (g.type == FIX) {
 		switch (g.c_bitcnt) {
-		case 8: val[0] = (uint8_t)mmrand(rnd, 1, 0xff); break;
-		case 7: val[0] = (uint8_t)mmrand(rnd, 1, 0x7f); break;
-		case 6: val[0] = (uint8_t)mmrand(rnd, 1, 0x3f); break;
-		case 5: val[0] = (uint8_t)mmrand(rnd, 1, 0x1f); break;
-		case 4: val[0] = (uint8_t)mmrand(rnd, 1, 0x0f); break;
-		case 3: val[0] = (uint8_t)mmrand(rnd, 1, 0x07); break;
-		case 2: val[0] = (uint8_t)mmrand(rnd, 1, 0x03); break;
-		case 1: val[0] = 1; break;
+		case 8: p[0] = (char)mmrand(rnd, 1, 0xff); break;
+		case 7: p[0] = (char)mmrand(rnd, 1, 0x7f); break;
+		case 6: p[0] = (char)mmrand(rnd, 1, 0x3f); break;
+		case 5: p[0] = (char)mmrand(rnd, 1, 0x1f); break;
+		case 4: p[0] = (char)mmrand(rnd, 1, 0x0f); break;
+		case 3: p[0] = (char)mmrand(rnd, 1, 0x07); break;
+		case 2: p[0] = (char)mmrand(rnd, 1, 0x03); break;
+		case 1: p[0] = 1; break;
 		}
-		*sizep = 1;
+		value->size = 1;
 		return;
 	}
 
@@ -210,8 +215,8 @@ val_gen(WT_RAND_STATE *rnd, void *valarg, size_t *sizep, uint64_t keyno)
 	 * test that by inserting a zero-length data item every so often.
 	 */
 	if (keyno % 63 == 0) {
-		val[0] = '\0';
-		*sizep = 0;
+		p[0] = '\0';
+		value->size = 0;
 		return;
 	}
 
@@ -226,13 +231,13 @@ val_gen(WT_RAND_STATE *rnd, void *valarg, size_t *sizep, uint64_t keyno)
 	if ((g.type == ROW || g.type == VAR) &&
 	    g.c_repeat_data_pct != 0 &&
 	    mmrand(rnd, 1, 100) < g.c_repeat_data_pct) {
-		(void)strcpy((char *)val, "DUPLICATEV");
-		val[10] = '/';
-		*sizep = val_dup_data_len;
+		(void)strcpy(p, "DUPLICATEV");
+		p[10] = '/';
+		value->size = val_dup_data_len;
 	} else {
-		(void)sprintf((char *)val, "%010" PRIu64, keyno);
-		val[10] = '/';
-		*sizep = kv_len(rnd, keyno, g.c_value_min, g.c_value_max);
+		(void)sprintf(p, "%010" PRIu64, keyno);
+		p[10] = '/';
+		value->size = kv_len(rnd, keyno, g.c_value_min, g.c_value_max);
 	}
 }