Support logging the statistics of a type of object based on a partial

URI ("table:" will log statistics for all tables, "table:mytable" will
continue to log statistics for "mytable").

Includes a minor change: we now only log statistics for existing
handles, as opposed to actually opening the data sources if they aren't
yet open (the implementation changed to walk the list of open handles,
and that falls out of the implementation).

6 files changed, 117 insertions, 63 deletions

diff --git a/dist/api_data.py b/dist/api_data.py
index 163e2629c93..38ad59ab282 100644
--- a/dist/api_data.py
+++ b/dist/api_data.py
@@ -585,9 +585,12 @@ methods = {
 		the database home'''),
 		Config('sources', '', r'''
 		if non-empty, include statistics for the list of data source
-		URIs.  No statistics that require traversing a tree are
-		reported, as if the \c statistics_fast configuration string
-		were set''',
+		URIs, if they are open at the time of the statistics logging.
+		The list may include URIs matching a single data source
+		("table:mytable"), or a URI matching all data sources of a
+		particular type ("table:").  No statistics that require the 
+		traversal of a tree are reported, as if the \c statistics_fast
+		configuration string were set''',
 		type='list'),
 		Config('timestamp', '"%b %d %H:%M:%S"', r'''
 		a timestamp prepended to each log record, may contain strftime
diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c
index 540fc973c04..5ff99f3c1ef 100644
--- a/examples/c/ex_all.c
+++ b/examples/c/ex_all.c
@@ -1113,12 +1113,20 @@ main(void)
 	if (ret == 0)
 		(void)conn->close(conn, NULL);
 
-	/*! [Statistics logging with objects] */
+	/*! [Statistics logging with a table] */
 	ret = wiredtiger_open(home, NULL,
 	    "create,"
 	    "statistics_log=(sources=(\"table:table1\",\"table:table2\"))",
 	    &conn);
-	/*! [Statistics logging with objects] */
+	/*! [Statistics logging with a table] */
+	if (ret == 0)
+		(void)conn->close(conn, NULL);
+
+	/*! [Statistics logging with all tables] */
+	ret = wiredtiger_open(home, NULL,
+	    "create,statistics_log=(sources=(\"table:\"))",
+	    &conn);
+	/*! [Statistics logging with all tables] */
 	if (ret == 0)
 		(void)conn->close(conn, NULL);
 
diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c
index 0660ab2435c..8b87ecdb90b 100644
--- a/src/conn/conn_stat.c
+++ b/src/conn/conn_stat.c
@@ -29,7 +29,7 @@ __wt_conn_stat_init(WT_SESSION_IMPL *session, uint32_t flags)
 }
 
 /*
- * __wt_statlog_config --
+ * __statlog__config --
  *	Parse and setup the statistics server options.
  */
 static int
@@ -66,31 +66,12 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, int *runp)
 		;
 	WT_RET_NOTFOUND_OK(ret);
 	if (cnt != 0) {
-		WT_RET(
-		    __wt_calloc_def(session, cnt * 2 + 1, &conn->stat_sources));
+		WT_RET(__wt_calloc_def(session, cnt + 1, &conn->stat_sources));
 		WT_RET(__wt_config_subinit(session, &objectconf, &cval));
 		for (cnt = 0;
-		    (ret = __wt_config_next(&objectconf, &k, &v)) == 0;) {
-			/*
-			 * We close and re-open each statistics cursor each time
-			 * we dump statistics (the object may or may not exist
-			 * underneath at any point, and I don't want this code
-			 * to break if/when the lifetime of an underlying object
-			 * changes).  Create pairs of strings: the first is the
-			 * object uri, written into the output, the second is
-			 * the enhanced uri used to open the statistics cursor.
-			 */
+		    (ret = __wt_config_next(&objectconf, &k, &v)) == 0; ++cnt)
 			WT_RET(__wt_strndup(session,
 			    k.str, k.len, &conn->stat_sources[cnt]));
-			++cnt;
-
-			WT_RET(__wt_calloc_def(session,
-			    strlen("statistics:") + k.len + 1,
-			    &conn->stat_sources[cnt]));
-			strcpy(conn->stat_sources[cnt], "statistics:");
-			strncat(conn->stat_sources[cnt], k.str, k.len);
-			++cnt;
-		}
 		WT_RET_NOTFOUND_OK(ret);
 	}
 
@@ -99,28 +80,48 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, int *runp)
 
 	WT_RET(__wt_config_gets(
 	    session, cfg, "statistics_log.timestamp", &cval));
-	WT_RET(__wt_strndup(session, cval.str, cval.len, &conn->stat_stamp));
+	WT_RET(__wt_strndup(session, cval.str, cval.len, &conn->stat_format));
 
 	return (0);
 }
 
 /*
- * __stat_server_dump --
- *	Dump a single set of statistics.
+ * __statlog_dump --
+ *	Dump out the connection statistics.
  */
 static int
-__stat_server_dump(WT_SESSION_IMPL *session,
-    const char *name, const char *cursor_uri, const char *stamp, FILE *fp)
+__statlog_dump(WT_SESSION_IMPL *session, const char *name, int conn_stats)
 {
+	WT_CONNECTION_IMPL *conn;
 	WT_CURSOR *cursor;
+	WT_DECL_ITEM(tmp);
 	WT_DECL_RET;
 	WT_SESSION *wt_session;
 	uint64_t value;
-	const char *config, *desc, *pdesc;
+	const char *config, *desc, *pdesc, *uri;
+
+	conn = S2C(session);
 
+	/* Build the statistics cursor URI. */
+	if (conn_stats)
+		uri = "statistics:";
+	else {
+		WT_RET(__wt_scr_alloc(
+		    session, strlen("statistics:") + strlen(name) + 5, &tmp));
+		(void)strcpy(tmp->mem, "statistics:");
+		(void)strcat(tmp->mem, name);
+		uri = tmp->mem;
+	}
+
+	/*
+	 * Open the statistics cursor; immediately free any temporary buffer,
+	 * it makes error handling easier.
+	 */
 	wt_session = (WT_SESSION *)session;
 	config = S2C(session)->stat_clear ?
 	    "statistics_clear,statistics_fast" : "statistics_fast";
+	ret = wt_session->open_cursor(wt_session, uri, NULL, config, &cursor);
+	__wt_scr_free(&tmp);
 
 	/*
 	 * If we don't find an underlying object, silently ignore it, the object
@@ -128,17 +129,15 @@ __stat_server_dump(WT_SESSION_IMPL *session,
 	 * of WT_NOTFOUND for missing files, check both, as well as for EBUSY if
 	 * the handle is exclusively locked at the moment.
 	 */
-	ret = wt_session->open_cursor(
-	    wt_session, cursor_uri, NULL, config, &cursor);
 	if (ret == EBUSY || ret == ENOENT || ret == WT_NOTFOUND)
 		return (0);
 	WT_RET(ret);
 
 	while ((ret = cursor->next(cursor)) == 0 &&
 	    (ret = cursor->get_value(cursor, &desc, &pdesc, &value)) == 0)
-		WT_ERR_TEST((fprintf(fp,
+		WT_ERR_TEST((fprintf(conn->stat_fp,
 		    "%s %" PRIu64 " %s %s\n",
-		    stamp, value, name, desc) < 0), __wt_errno());
+		    conn->stat_stamp, value, name, desc) < 0), __wt_errno());
 	WT_ERR_NOTFOUND_OK(ret);
 
 err:	WT_TRET(cursor->close(cursor));
@@ -147,20 +146,40 @@ err:	WT_TRET(cursor->close(cursor));
 }
 
 /*
- * __stat_server --
+ * __statlog_apply --
+ *	Review a single open handle and dump statistics on demand.
+ */
+static int
+__statlog_apply(WT_SESSION_IMPL *session, const char *cfg[])
+{
+	WT_BTREE *btree;
+	char **p;
+
+	WT_UNUSED(cfg);
+
+	btree = session->btree;
+
+	/* Check for a match on the set of sources. */
+	for (p = S2C(session)->stat_sources; *p != NULL; ++p)
+		if (WT_PREFIX_MATCH(btree->name, *p))
+			return (__statlog_dump(session, btree->name, 0));
+	return (0);
+}
+
+/*
+ * __statlog_server --
  *	The statistics server thread.
  */
 static void *
-__stat_server(void *arg)
+__statlog_server(void *arg)
 {
 	struct timespec ts;
 	struct tm *tm, _tm;
 	FILE *fp;
 	WT_CONNECTION_IMPL *conn;
-	WT_ITEM path, tmp;
 	WT_DECL_RET;
+	WT_ITEM path, tmp;
 	WT_SESSION_IMPL *session;
-	char **p;
 
 	session = arg;
 	conn = S2C(session);
@@ -222,19 +241,25 @@ __stat_server(void *arg)
 		}
 
 		/* Create the entry prefix for this time of day. */
-		if (strftime(tmp.mem, tmp.memsize, conn->stat_stamp, tm) == 0)
+		if (strftime(tmp.mem, tmp.memsize, conn->stat_format, tm) == 0)
 			WT_ERR_MSG(
 			    session, ENOMEM, "strftime timestamp conversion");
 
+		/* Reference temporary values from the connection structure. */
+		conn->stat_fp = fp;
+		conn->stat_stamp = tmp.mem;
+
 		/* Dump the connection statistics. */
-		WT_ERR(__stat_server_dump(
-		    session, conn->home, "statistics:", tmp.mem, fp));
+		WT_ERR(__statlog_dump(session, conn->home, 1));
 
-		/* Dump the object list statistics. */
-		if ((p = conn->stat_sources) != NULL)
-			for (; *p != NULL; p += 2)
-				WT_ERR(__stat_server_dump(
-				    session, p[0], p[1], tmp.mem, fp));
+		/*
+		 * Lock the schema and walk the list of open handles, dumping
+		 * any that match the list of object sources.
+		 */
+		if (conn->stat_sources != NULL)
+			WT_WITH_SCHEMA_LOCK(session,
+			    ret = __wt_conn_btree_apply(
+			    session, __statlog_apply, NULL));
 
 		/* Flush. */
 		WT_ERR(fflush(fp) == 0 ? 0 : __wt_errno());
@@ -255,7 +280,7 @@ err:		__wt_err(session, ret, "statistics log server error");
 }
 
 /*
- * __wt_statlog_create -
+ * __wt_statlog_create --
  *	Start the statistics server thread.
  */
 int
@@ -291,14 +316,14 @@ __wt_statlog_create(WT_CONNECTION_IMPL *conn, const char *cfg[])
 	 * to figure out the scheduling.
 	 */
 	WT_RET(__wt_thread_create(
-	    session, &conn->stat_tid, __stat_server, conn->stat_session));
+	    session, &conn->stat_tid, __statlog_server, conn->stat_session));
 	conn->stat_tid_set = 1;
 
 	return (0);
 }
 
 /*
- * __wt_statlog_destroy -
+ * __wt_statlog_destroy --
  *	Destroy the statistics server thread.
  */
 int
@@ -325,7 +350,7 @@ __wt_statlog_destroy(WT_CONNECTION_IMPL *conn)
 		__wt_free(session, conn->stat_sources);
 	}
 	__wt_free(session, conn->stat_path);
-	__wt_free(session, conn->stat_stamp);
+	__wt_free(session, conn->stat_format);
 
 	/* Close the server thread's session, free its hazard array. */
 	if (conn->stat_session != NULL) {
diff --git a/src/docs/statistics.dox b/src/docs/statistics.dox
index 0258ae2d59d..3b2141bf8df 100644
--- a/src/docs/statistics.dox
+++ b/src/docs/statistics.dox
@@ -27,11 +27,18 @@ The timestamp format may be changed with the \c statistics_log.timestamp
 configuration string.  The \c timestamp value may contain ISO C90 standard
 strftime conversion specifications.
 
-By default, only the system's connection statistics are logged, but
-statistics may be optionally reported for underlying objects by adding
-a list of URIs to the \c statistics_log configuration string:
+By default, only the database statistics are logged.
 
-@snippet ex_all.c Statistics logging with objects
+Statistics for specific underlying data sources may be included by adding
+a list of data source URIs to the \c statistics_log configuration string:
+
+@snippet ex_all.c Statistics logging with a table
+
+Statistics for all underlying data sources of a particular type may be
+included by adding a partial data source URI to the \c statistics_log
+configuration string:
+
+@snippet ex_all.c Statistics logging with all tables
 
 When database statistics are logged, the database home will be the first
 space-separated entry for each record in the log file.  For example:
@@ -45,8 +52,9 @@ Mar 08 11:38:23 1 /database/home total heap memory re-allocations
 Mar 08 11:38:23 472 /database/home total read I/Os
 @endcode
 
-When object statistics are logged, the object URI will be the first
-space-separated entry for each record in the log file.  For example:
+When data source statistics are logged, the data source's URI will be
+the first space-separated entry for each record in the log file.  For
+example:
 
 @code
 Mar 20 10:42:36 21 table:mytable compressed pages written
@@ -56,6 +64,11 @@ Mar 20 10:42:36 586 table:mytable cursor insert calls
 Mar 20 10:42:36 0 table:mytable bulk-loaded cursor-insert calls
 @endcode
 
+No statistics are logged for any data source for which a handle is not
+currently open in the database, nor will any statistics requiring the
+traversal of a tree (as if the \c statistics_fast configuration string
+were set).
+
 The location of the log files may be changed with the \c statistics_log.path
 configuration string.  The \c path value value may contain ISO C90 standard
 strftime conversion specifications.  WiredTiger will not create non-existent
diff --git a/src/include/connection.h b/src/include/connection.h
index 67b87a1d606..271ba5f2cf7 100644
--- a/src/include/connection.h
+++ b/src/include/connection.h
@@ -147,10 +147,13 @@ struct __wt_connection_impl {
 	pthread_t	 stat_tid;	/* Statistics log thread */
 	int		 stat_tid_set;	/* Statistics log thread set */
 	WT_CONDVAR	*stat_cond;	/* Statistics log wait mutex */
+
 	int		 stat_clear;	/* Statistics log clear */
+	const char	*stat_format;	/* Statistics log timestamp format */
+	FILE		*stat_fp;	/* Statistics log file handle */
 	const char	*stat_path;	/* Statistics log path format */
 	char	       **stat_sources;	/* Statistics log list of objects */
-	const char	*stat_stamp;	/* Statistics log timestamp format */
+	const char	*stat_stamp;	/* Statistics log entry timestamp */
 	long		 stat_usecs;	/* Statistics log period */
 
 	WT_FH	   *log_fh;		/* Logging file handle */
diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in
index d61871af60a..77132e77e92 100644
--- a/src/include/wiredtiger.in
+++ b/src/include/wiredtiger.in
@@ -1310,9 +1310,11 @@ struct __wt_connection {
  * specifications.  If the value is not an absolute path name\, the file is
  * created relative to the database home.,a string; default \c
  * "WiredTigerStat.%H".}@config{&nbsp;&nbsp;&nbsp;&nbsp;sources, if non-empty\,
- * include statistics for the list of data source URIs.  No statistics that
- * require traversing a tree are reported\, as if the \c statistics_fast
- * configuration string were set.,a list of strings; default
+ * include statistics for the list of data source URIs.  The list may include
+ * URIs matching a single data source ("table:mytable")\, or a URI matching all
+ * data sources of a particular type ("table:").  No statistics that require the
+ * traversal of a tree are reported\, as if the \c statistics_fast configuration
+ * string were set.,a list of strings; default
  * empty.}@config{&nbsp;&nbsp;&nbsp;&nbsp;timestamp, a timestamp prepended to
  * each log record\, may contain strftime conversion specifications.,a string;
  * default \c "%b %d %H:%M:%S".}@config{&nbsp;&nbsp;&nbsp;&nbsp;wait, seconds to