WT-2833 Add projections to wt dump utility (#3192)

When dumping with a projection, the dumped metadata associated with the
table is modified so that the list of columns matches the list from the
projection, and the value format corresponds to projected format.  We use an
open cursor using the projection to obtain the value format to show, that
requires a slight reordering of when cursors are opened in the dump utility.

Also fix a problem in the JSON dump cursor which did not handle projections.

Added tests for dump with projections, including tables and indices using
the normal dump formats, and tables dumped using JSON dump format. Tables
are also dumped with projections and then loaded into the modified table
format to be checked.

8 files changed, 348 insertions, 40 deletions

diff --git a/src/cursor/cur_index.c b/src/cursor/cur_index.c
index 0ab992bc88c..4786b0524bc 100644
--- a/src/cursor/cur_index.c
+++ b/src/cursor/cur_index.c
@@ -520,8 +520,8 @@ __wt_curindex_open(WT_SESSION_IMPL *session,
 	WT_ERR(__curindex_open_colgroups(session, cindex, cfg));
 
 	if (F_ISSET(cursor, WT_CURSTD_DUMP_JSON))
-		__wt_json_column_init(
-		    cursor, table->key_format, &idx->colconf, &table->colconf);
+		__wt_json_column_init(cursor, uri, table->key_format,
+		    &idx->colconf, &table->colconf);
 
 	if (0) {
 err:		WT_TRET(__curindex_close(cursor));
diff --git a/src/cursor/cur_json.c b/src/cursor/cur_json.c
index a0a3ffdd974..5870d14273e 100644
--- a/src/cursor/cur_json.c
+++ b/src/cursor/cur_json.c
@@ -369,11 +369,11 @@ __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode)
  *	of column names.
  */
 void
-__wt_json_column_init(WT_CURSOR *cursor, const char *keyformat,
+__wt_json_column_init(WT_CURSOR *cursor, const char *uri, const char *keyformat,
     const WT_CONFIG_ITEM *idxconf, const WT_CONFIG_ITEM *colconf)
 {
 	WT_CURSOR_JSON *json;
-	const char *p, *end, *beginkey;
+	const char *beginkey, *end, *lparen, *p;
 	uint32_t keycnt, nkeys;
 
 	json = (WT_CURSOR_JSON *)cursor->json_private;
@@ -400,8 +400,16 @@ __wt_json_column_init(WT_CURSOR *cursor, const char *keyformat,
 			keycnt++;
 		p++;
 	}
-	json->value_names.str = p;
-	json->value_names.len = WT_PTRDIFF(end, p);
+	if ((lparen = strchr(uri, '(')) != NULL) {
+		/* This cursor is a projection. */
+		json->value_names.str = lparen;
+		json->value_names.len = strlen(lparen) - 1;
+		WT_ASSERT((WT_SESSION_IMPL *)cursor->session,
+		    json->value_names.str[json->value_names.len] == ')');
+	} else {
+		json->value_names.str = p;
+		json->value_names.len = WT_PTRDIFF(end, p);
+	}
 	if (idxconf == NULL) {
 		if (p > beginkey)
 			p--;
diff --git a/src/cursor/cur_table.c b/src/cursor/cur_table.c
index fae7667e44f..76f7fc5865f 100644
--- a/src/cursor/cur_table.c
+++ b/src/cursor/cur_table.c
@@ -951,7 +951,7 @@ __wt_curtable_open(WT_SESSION_IMPL *session,
 
 	if (F_ISSET(cursor, WT_CURSTD_DUMP_JSON))
 		__wt_json_column_init(
-		    cursor, table->key_format, NULL, &table->colconf);
+		    cursor, uri, table->key_format, NULL, &table->colconf);
 
 	/*
 	 * Open the colgroup cursors immediately: we're going to need them for
diff --git a/src/include/extern.h b/src/include/extern.h
index be042bcd6cb..4824dc93d96 100644
--- a/src/include/extern.h
+++ b/src/include/extern.h
@@ -294,7 +294,7 @@ extern int __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT
 extern int __wt_json_alloc_unpack(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, WT_CURSOR_JSON *json, bool iskey, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern void __wt_json_close(WT_SESSION_IMPL *session, WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern size_t __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
-extern void __wt_json_column_init(WT_CURSOR *cursor, const char *keyformat, const WT_CONFIG_ITEM *idxconf, const WT_CONFIG_ITEM *colconf) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
+extern void __wt_json_column_init(WT_CURSOR *cursor, const char *uri, const char *keyformat, const WT_CONFIG_ITEM *idxconf, const WT_CONFIG_ITEM *colconf) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern int __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, const char **tokstart, size_t *toklen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern const char *__wt_json_tokname(int toktype) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
 extern int __wt_json_to_item(WT_SESSION_IMPL *session, const char *jstr, const char *format, WT_CURSOR_JSON *json, bool iskey, WT_ITEM *item) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
diff --git a/src/utilities/util_dump.c b/src/utilities/util_dump.c
index 7dde13ee837..651cc7acf9c 100644
--- a/src/utilities/util_dump.c
+++ b/src/utilities/util_dump.c
@@ -6,10 +6,14 @@
  * See the file LICENSE for redistribution information.
  */
 
+#include <assert.h>
 #include "util.h"
 #include "util_dump.h"
 
-static int dump_config(WT_SESSION *, const char *, bool, bool);
+#define	STRING_MATCH_CONFIG(s, item)					\
+	(strncmp(s, (item).str, (item).len) == 0 && (s)[(item).len] == '\0')
+
+static int dump_config(WT_SESSION *, const char *, WT_CURSOR *, bool, bool);
 static int dump_json_begin(WT_SESSION *);
 static int dump_json_end(WT_SESSION *);
 static int dump_json_separator(WT_SESSION *);
@@ -17,7 +21,8 @@ static int dump_json_table_end(WT_SESSION *);
 static int dump_prefix(WT_SESSION *, bool, bool);
 static int dump_record(WT_CURSOR *, bool, bool);
 static int dump_suffix(WT_SESSION *, bool);
-static int dump_table_config(WT_SESSION *, WT_CURSOR *, const char *, bool);
+static int dump_table_config(
+    WT_SESSION *, WT_CURSOR *, WT_CURSOR *, const char *, bool);
 static int dump_table_parts_config(
     WT_SESSION *, WT_CURSOR *, const char *, const char *, bool);
 static int dup_json_string(const char *, char **);
@@ -32,10 +37,11 @@ util_dump(WT_SESSION *session, int argc, char *argv[])
 	size_t len;
 	int ch, i;
 	bool hex, json, reverse;
-	char *checkpoint, *config, *name;
+	char *checkpoint, *config, *name, *p, *simplename;
 
 	hex = json = reverse = false;
-	checkpoint = config = name = NULL;
+	checkpoint = config = name = simplename = NULL;
+	cursor = NULL;
 	while ((ch = __wt_getopt(progname, argc, argv, "c:f:jrx")) != EOF)
 		switch (ch) {
 		case 'c':
@@ -84,14 +90,12 @@ util_dump(WT_SESSION *session, int argc, char *argv[])
 			if ((ret = dump_json_separator(session)) != 0)
 				goto err;
 		free(name);
-		name = NULL;
+		free(simplename);
+		name = simplename = NULL;
 
 		if ((name = util_name(session, argv[i], "table")) == NULL)
 			goto err;
 
-		if (dump_config(session, name, hex, json) != 0)
-			goto err;
-
 		len =
 		    checkpoint == NULL ? 0 : strlen("checkpoint=") +
 		    strlen(checkpoint) + 1;
@@ -115,10 +119,26 @@ util_dump(WT_SESSION *session, int argc, char *argv[])
 			goto err;
 		}
 
+		if ((simplename = strdup(name)) == NULL) {
+			ret = util_err(session, errno, NULL);
+			goto err;
+		}
+		if ((p = strchr(simplename, '(')) != NULL)
+			*p = '\0';
+		if (dump_config(session, simplename, cursor, hex, json) != 0)
+			goto err;
+
 		if ((ret = dump_record(cursor, reverse, json)) != 0)
 			goto err;
 		if (json && (ret = dump_json_table_end(session)) != 0)
 			goto err;
+
+		ret = cursor->close(cursor);
+		cursor = NULL;
+		if (ret != 0) {
+			ret = util_err(session, ret, NULL);
+			goto err;
+		}
 	}
 	if (json && ((ret = dump_json_end(session)) != 0))
 		goto err;
@@ -129,7 +149,11 @@ err:		ret = 1;
 
 	free(config);
 	free(name);
-
+	free(simplename);
+	if (cursor != NULL && (ret = cursor->close(cursor)) != 0) {
+		(void)util_err(session, ret, NULL);
+		ret = 1;
+	}
 	return (ret);
 }
 
@@ -138,15 +162,16 @@ err:		ret = 1;
  *	Dump the config for the uri.
  */
 static int
-dump_config(WT_SESSION *session, const char *uri, bool hex, bool json)
+dump_config(WT_SESSION *session, const char *uri, WT_CURSOR *cursor, bool hex,
+    bool json)
 {
-	WT_CURSOR *cursor;
+	WT_CURSOR *mcursor;
 	WT_DECL_RET;
 	int tret;
 
 	/* Open a metadata cursor. */
 	if ((ret = session->open_cursor(
-	    session, "metadata:create", NULL, NULL, &cursor)) != 0) {
+	    session, "metadata:create", NULL, NULL, &mcursor)) != 0) {
 		fprintf(stderr, "%s: %s: session.open_cursor: %s\n", progname,
 		    "metadata:create", session->strerror(session, ret));
 		return (1);
@@ -156,10 +181,11 @@ dump_config(WT_SESSION *session, const char *uri, bool hex, bool json)
 	 * want to output a header if the user entered the wrong name. This is
 	 * where we find out a table doesn't exist, use a simple error message.
 	 */
-	cursor->set_key(cursor, uri);
-	if ((ret = cursor->search(cursor)) == 0) {
+	mcursor->set_key(mcursor, uri);
+	if ((ret = mcursor->search(mcursor)) == 0) {
 		if ((!json && dump_prefix(session, hex, json) != 0) ||
-		    dump_table_config(session, cursor, uri, json) != 0 ||
+		    dump_table_config(session, mcursor, cursor,
+		    uri, json) != 0 ||
 		    dump_suffix(session, json) != 0)
 			ret = 1;
 	} else if (ret == WT_NOTFOUND)
@@ -167,8 +193,8 @@ dump_config(WT_SESSION *session, const char *uri, bool hex, bool json)
 	else
 		ret = util_err(session, ret, "%s", uri);
 
-	if ((tret = cursor->close(cursor)) != 0) {
-		tret = util_cerr(cursor, "close", tret);
+	if ((tret = mcursor->close(mcursor)) != 0) {
+		tret = util_cerr(mcursor, "close", tret);
 		if (ret == 0)
 			ret = tret;
 	}
@@ -225,16 +251,125 @@ dump_json_table_end(WT_SESSION *session)
 }
 
 /*
+ * dump_add_config
+ *	Add a formatted config string to an output buffer.
+ */
+static int
+dump_add_config(WT_SESSION *session, char **bufp, size_t *leftp,
+    const char *fmt, ...)
+	WT_GCC_FUNC_ATTRIBUTE((format (printf, 4, 5)))
+{
+	int n;
+	va_list ap;
+
+	va_start(ap, fmt);
+	n = vsnprintf(*bufp, *leftp, fmt, ap);
+	va_end(ap);
+	if (n < 0)
+		return (util_err(session, EINVAL, NULL));
+	*bufp += n;
+	*leftp -= n;
+	return (0);
+}
+
+/*
+ * dump_projection --
+ *	Create a new config containing projection information.
+ */
+static int
+dump_projection(WT_SESSION *session, const char *config, WT_CURSOR *cursor,
+    char **newconfigp)
+{
+	WT_DECL_RET;
+	WT_CONFIG_ITEM key, value;
+	WT_CONFIG_PARSER *parser;
+	WT_EXTENSION_API *wt_api;
+	size_t len, vallen;
+	int nkeys;
+	char *newconfig;
+	const char *keyformat, *p;
+
+	len = strlen(config) + strlen(cursor->value_format) +
+	    strlen(cursor->uri) + 20;
+	if ((newconfig = malloc(len)) == NULL)
+		return util_err(session, errno, NULL);
+	*newconfigp = newconfig;
+	wt_api = session->connection->get_extension_api(session->connection);
+	if ((ret = wt_api->config_parser_open(wt_api, session, config,
+	    strlen(config), &parser)) != 0)
+		return (util_err(
+		    session, ret, "WT_EXTENSION_API.config_parser_open"));
+	keyformat = cursor->key_format;
+	for (nkeys = 0; *keyformat; keyformat++)
+		if (!__wt_isdigit((u_char)*keyformat))
+			nkeys++;
+
+	/*
+	 * Copy the configuration, replacing some fields to match the
+	 * projection.
+	 */
+	while ((ret = parser->next(parser, &key, &value)) == 0) {
+		WT_RET(dump_add_config(session, &newconfig, &len,
+		    "%.*s=", (int)key.len, key.str));
+		if (STRING_MATCH_CONFIG("value_format", key))
+			WT_RET(dump_add_config(session, &newconfig, &len,
+			    "%s", cursor->value_format));
+		else if (STRING_MATCH_CONFIG("columns", key)) {
+			/* copy names of keys */
+			p = value.str;
+			vallen = value.len;
+			while (vallen > 0) {
+				if ((*p == ',' || *p == ')') && --nkeys == 0)
+					break;
+				p++;
+				vallen--;
+			}
+			WT_RET(dump_add_config(session, &newconfig, &len,
+			    "%.*s", (int)(p - value.str), value.str));
+
+			/* copy names of projected values */
+			p = strchr(cursor->uri, '(');
+			assert(p != NULL);
+			assert(p[strlen(p) - 1] == ')');
+			p++;
+			if (*p != ')')
+				WT_RET(dump_add_config(session, &newconfig,
+				    &len, "%s", ","));
+			WT_RET(dump_add_config(session, &newconfig, &len,
+			    "%.*s),", (int)(strlen(p) - 1), p));
+		} else if (value.type == WT_CONFIG_ITEM_STRING &&
+		    value.len != 0)
+			WT_RET(dump_add_config(session, &newconfig, &len,
+			    "\"%.*s\",", (int)value.len, value.str));
+		else
+			WT_RET(dump_add_config(session, &newconfig, &len,
+			    "%.*s,", (int)value.len, value.str));
+	}
+	if (ret != WT_NOTFOUND)
+		return (util_err(session, ret, "WT_CONFIG_PARSER.next"));
+
+	assert(len > 0);
+	if ((ret = parser->close(parser)) != 0)
+		return (util_err(
+		    session, ret, "WT_CONFIG_PARSER.close"));
+
+	return (0);
+}
+
+/*
  * dump_table_config --
  *	Dump the config for a table.
  */
 static int
 dump_table_config(
-    WT_SESSION *session, WT_CURSOR *cursor, const char *uri, bool json)
+    WT_SESSION *session, WT_CURSOR *mcursor, WT_CURSOR *cursor,
+    const char *uri, bool json)
 {
 	WT_DECL_RET;
+	char *proj_config;
 	const char *name, *v;
 
+	proj_config = NULL;
 	/* Get the table name. */
 	if ((name = strchr(uri, ':')) == NULL) {
 		fprintf(stderr, "%s: %s: corrupted uri\n", progname, uri);
@@ -246,20 +381,25 @@ dump_table_config(
 	 * Dump out the config information: first, dump the uri entry itself,
 	 * it overrides all subsequent configurations.
 	 */
-	cursor->set_key(cursor, uri);
-	if ((ret = cursor->search(cursor)) != 0)
-		return (util_cerr(cursor, "search", ret));
-	if ((ret = cursor->get_value(cursor, &v)) != 0)
-		return (util_cerr(cursor, "get_value", ret));
-
-	WT_RET(print_config(session, uri, v, json, true));
+	mcursor->set_key(mcursor, uri);
+	if ((ret = mcursor->search(mcursor)) != 0)
+		return (util_cerr(mcursor, "search", ret));
+	if ((ret = mcursor->get_value(mcursor, &v)) != 0)
+		return (util_cerr(mcursor, "get_value", ret));
+
+	if (strchr(cursor->uri, '(') != NULL) {
+		WT_ERR(dump_projection(session, v, cursor, &proj_config));
+		v = proj_config;
+	}
+	WT_ERR(print_config(session, uri, v, json, true));
 
-	WT_RET(dump_table_parts_config(
-	    session, cursor, name, "colgroup:", json));
-	WT_RET(dump_table_parts_config(
-	    session, cursor, name, "index:", json));
+	WT_ERR(dump_table_parts_config(
+	    session, mcursor, name, "colgroup:", json));
+	WT_ERR(dump_table_parts_config(
+	    session, mcursor, name, "index:", json));
 
-	return (0);
+err:	free(proj_config);
+	return (ret);
 }
 
 /*
diff --git a/test/suite/test_dump.py b/test/suite/test_dump.py
index f6a83c32489..3127c7aef00 100644
--- a/test/suite/test_dump.py
+++ b/test/suite/test_dump.py
@@ -32,7 +32,7 @@ import wiredtiger, wttest
 from suite_subprocess import suite_subprocess
 from wtscenario import make_scenarios
 from wtdataset import SimpleDataSet, SimpleIndexDataSet, SimpleLSMDataSet, \
-    ComplexDataSet, ComplexLSMDataSet
+    ComplexDataSet, ComplexLSMDataSet, ProjectionDataSet, ProjectionIndexDataSet
 
 # test_dump.py
 #    Utilities: wt dump
@@ -62,6 +62,10 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess):
         ('table-simple-lsm', dict(uri='table:', dataset=SimpleLSMDataSet)),
         ('table-complex', dict(uri='table:', dataset=ComplexDataSet)),
         ('table-complex-lsm', dict(uri='table:', dataset=ComplexLSMDataSet)),
+        ('table-simple-proj', dict(uri='table:',
+            dataset=ProjectionDataSet, projection=True)),
+        ('table-index-proj', dict(uri='table:',
+            dataset=ProjectionIndexDataSet, projection=True)),
     ]
     scenarios = make_scenarios(types, keyfmt, dumpfmt)
 
@@ -158,5 +162,53 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess):
         pop = self.dataset(self, uri2, self.nentries, key_format=self.keyfmt)
         pop.check()
 
+# test_dump_projection
+#    Utilities: wt dump
+# Test the dump utility with projections
+class test_dump_projection(wttest.WiredTigerTestCase, suite_subprocess):
+    dir = 'dump.dir'            # Backup directory name
+
+    name = 'test_dump'
+    nentries = 2500
+    uri = 'table:'
+
+    # Dump, re-load and do a content comparison.
+    def test_dump(self):
+
+        # Create the object.
+        uri = self.uri + self.name
+        pop = ProjectionDataSet(self, uri, self.nentries, key_format='S')
+        pop.populate()
+
+        # Check some cases with invalid projections.
+        self.runWt(['dump', '-x', uri + '('], \
+            outfilename='bad1.out', errfilename='err1.out', failure=True)
+        self.check_non_empty_file('err1.out')
+        self.runWt(['dump', '-x', uri + '(xx)'], \
+            outfilename='bad2.out', errfilename='err2.out', failure=True)
+        self.check_non_empty_file('err2.out')
+        self.runWt(['dump', '-x', uri + pop.projection[:-1]], \
+            outfilename='bad3.out', errfilename='err3.out', failure=True)
+        self.check_non_empty_file('err3.out')
+
+        # Dump the object with a valid projection.
+        self.runWt(['dump', '-x', uri + pop.projection], outfilename='dump.out')
+
+        # Re-load the object in a new home.
+        os.mkdir(self.dir)
+        self.runWt(['-h', self.dir, 'load', '-f', 'dump.out'])
+
+        # Check the database contents.
+        self.runWt(['list'], outfilename='list.out')
+        self.runWt(['-h', self.dir, 'list'], outfilename='list.out.new')
+        s1 = set(open('list.out').read().split())
+        s2 = set(open('list.out.new').read().split())
+        self.assertEqual(not s1.symmetric_difference(s2), True)
+
+        # Check the object's contents.
+        self.reopen_conn(self.dir)
+        pop_reload = ProjectionDataSet(self, uri, self.nentries, key_format='S')
+        pop_reload.check()
+
 if __name__ == '__main__':
     wttest.run()
diff --git a/test/suite/test_jsondump02.py b/test/suite/test_jsondump02.py
index 8482851fb94..60863c4aa97 100644
--- a/test/suite/test_jsondump02.py
+++ b/test/suite/test_jsondump02.py
@@ -234,6 +234,24 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess):
                 ('"ikey" : 4,\n"Skey" : "key4"',
                  '"S1" : "val16",\n"i2" : 16,\n"S3" : "val64",\n"i4" : 64'))
         self.check_json(self.table_uri4, table4_json)
+        # This projection has 3 value fields reversed with a key at the end.
+        table4_json_projection = (
+                ('"ikey" : 1,\n"Skey" : "key1"',
+                 '"i4" : 1,\n"S3" : "val1",\n"i2" : 1,\n"ikey" : 1'),
+                ('"ikey" : 2,\n"Skey" : "key2"',
+                 '"i4" : 8,\n"S3" : "val8",\n"i2" : 4,\n"ikey" : 2'),
+                ('"ikey" : 3,\n"Skey" : "key3"',
+                 '"i4" : 27,\n"S3" : "val27",\n"i2" : 9,\n"ikey" : 3'),
+                ('"ikey" : 4,\n"Skey" : "key4"',
+                 '"i4" : 64,\n"S3" : "val64",\n"i2" : 16,\n"ikey" : 4'))
+        # bad projection URI
+        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
+                 lambda: self.check_json(self.table_uri4 + '(i4,S3,i2,ikey',
+                        table4_json_projection),
+            '/Unbalanced brackets/')
+        # This projection should work.
+        self.check_json(self.table_uri4 + '(i4,S3,i2,ikey)',
+                        table4_json_projection)
         # The dump config currently is not supported for the index type.
         self.check_json(uri4index1, (
                 ('"Skey" : "key1"',
diff --git a/test/suite/wtdataset.py b/test/suite/wtdataset.py
index 74e07e24e93..946b97d995f 100644
--- a/test/suite/wtdataset.py
+++ b/test/suite/wtdataset.py
@@ -41,6 +41,7 @@ class BaseDataSet(object):
         self.key_format = kwargs.get('key_format', 'S')
         self.value_format = kwargs.get('value_format', 'S')
         self.config = kwargs.get('config', '')
+        self.projection = kwargs.get('projection', '')
 
     def create(self):
         self.testcase.session.create(self.uri, 'key_format=' + self.key_format
@@ -103,7 +104,8 @@ class BaseDataSet(object):
 
     def check(self):
         self.testcase.pr('check: ' + self.uri)
-        cursor = self.testcase.session.open_cursor(self.uri, None)
+        cursor = self.testcase.session.open_cursor(
+            self.uri + self.projection, None, None)
         self.check_cursor(cursor)
         cursor.close()
 
@@ -289,6 +291,94 @@ class ComplexLSMDataSet(ComplexDataSet):
     def is_lsm(cls):
         return True
 
+class ProjectionDataSet(SimpleDataSet):
+    """
+    ProjectionDataSet creates a table with predefined data identical to
+    SimpleDataSet (single key and value), but when checking it, uses
+    a cursor with a projection.
+    """
+    def __init__(self, testcase, uri, rows, **kwargs):
+        kwargs['config'] = kwargs.get('config', '') + ',columns=(k,v0)'
+        kwargs['projection'] = '(v0,v0,v0)'
+        super(ProjectionDataSet, self).__init__(testcase, uri, rows, **kwargs)
+
+    # A value suitable for checking the value returned by a cursor.
+    def comparable_value(self, i):
+        v0 = self.value(i)
+        return [v0, v0, v0]
+
+    def check_cursor(self, cursor):
+        i = 0
+        for key, got0, got1, got2 in cursor:
+            i += 1
+            self.testcase.assertEqual(key, self.key(i))
+            if cursor.value_format == '8t' and got0 == 0:    # deleted
+                continue
+            self.testcase.assertEqual([got0, got1, got2],
+                self.comparable_value(i))
+        self.testcase.assertEqual(i, self.rows)
+
+class ProjectionIndexDataSet(BaseDataSet):
+    """
+    ProjectionIndexDataSet creates a table with three values and
+    an index.  Checks are made against a projection of the main table
+    and a projection of the index.
+    """
+    def __init__(self, testcase, uri, rows, **kwargs):
+        self.origconfig = kwargs.get('config', '')
+        self.indexname = 'index:' + uri.split(":")[1] +  ':index0'
+        kwargs['config'] = self.origconfig + ',columns=(k,v0,v1,v2)'
+        kwargs['value_format'] = kwargs.get('value_format', 'SiS')
+        kwargs['projection'] = '(v1,v2,v0)'
+        super(ProjectionIndexDataSet, self).__init__(
+            testcase, uri, rows, **kwargs)
+
+    def value(self, i):
+        return ('v0:' + str(i), i*i, 'v2:' + str(i))
+
+    # Suitable for checking the value returned by a cursor using a projection.
+    def comparable_value(self, i):
+        return [i*i, 'v2:' + str(i), 'v0:' + str(i)]
+
+    def create(self):
+        super(ProjectionIndexDataSet, self).create()
+        self.testcase.session.create(self.indexname, 'columns=(v2,v1),' +
+            self.origconfig)
+
+    def check_cursor(self, cursor):
+        i = 0
+        for key, got0, got1, got2 in cursor:
+            i += 1
+            self.testcase.assertEqual(key, self.key(i))
+            if cursor.value_format == '8t' and got0 == 0:    # deleted
+                continue
+            self.testcase.assertEqual([got0, got1, got2],
+                self.comparable_value(i))
+        self.testcase.assertEqual(i, self.rows)
+
+    def check_index_cursor(self, cursor):
+        for i in xrange(1, self.rows + 1):
+            k = self.key(i)
+            v = self.value(i)
+            ik = (v[2], v[1])  # The index key is (v2,v2)
+            expect = [v[1],k,v[2],v[0]]
+            self.testcase.assertEqual(expect, cursor[ik])
+
+    def check(self):
+        BaseDataSet.check(self)
+
+        # Check values in the index.
+        idxcursor = self.testcase.session.open_cursor(
+            self.indexname + '(v1,k,v2,v0)')
+        self.check_index_cursor(idxcursor)
+        idxcursor.close()
+
+    def index_count(self):
+        return 1
+
+    def index_name(self, i):
+        return self.indexname
+
 # create a key based on a cursor as a shortcut to creating a SimpleDataSet
 def simple_key(cursor, i):
     return BaseDataSet.key_by_format(i, cursor.key_format)