Merge pull request #492 from wiredtiger/statlog-source-types

Support logging the statistics of a type of object based on a partial URI.

8 files changed, 121 insertions, 66 deletions

diff --git a/README b/README
index e215d642153..e733cafa064 100644
--- a/README
+++ b/README
@@ -1,8 +1,8 @@
-WiredTiger 1.5.1: (March 25, 2013)
+WiredTiger 1.5.1: (March 26, 2013)
 
 This is version 1.5.1 of WiredTiger.
 
-WiredTiger documentation can be found at:
+WiredTiger release packages and documentation can be found at:
 
 	http://source.wiredtiger.com/
 
diff --git a/dist/api_data.py b/dist/api_data.py
index 163e2629c93..38ad59ab282 100644
--- a/dist/api_data.py
+++ b/dist/api_data.py
@@ -585,9 +585,12 @@ methods = {
 		the database home'''),
 		Config('sources', '', r'''
 		if non-empty, include statistics for the list of data source
-		URIs.  No statistics that require traversing a tree are
-		reported, as if the \c statistics_fast configuration string
-		were set''',
+		URIs, if they are open at the time of the statistics logging.
+		The list may include URIs matching a single data source
+		("table:mytable"), or a URI matching all data sources of a
+		particular type ("table:").  No statistics that require the 
+		traversal of a tree are reported, as if the \c statistics_fast
+		configuration string were set''',
 		type='list'),
 		Config('timestamp', '"%b %d %H:%M:%S"', r'''
 		a timestamp prepended to each log record, may contain strftime
diff --git a/dist/s_readme b/dist/s_readme
index 521dc28d68a..65476d2ba73 100644
--- a/dist/s_readme
+++ b/dist/s_readme
@@ -18,7 +18,7 @@ $WIREDTIGER_VERSION_STRING
 
 This is version $WIREDTIGER_VERSION_MAJOR.$WIREDTIGER_VERSION_MINOR.$WIREDTIGER_VERSION_PATCH of WiredTiger.
 
-WiredTiger documentation can be found at:
+WiredTiger release packages and documentation can be found at:
 
 	http://source.wiredtiger.com/
 
diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c
index 540fc973c04..5ff99f3c1ef 100644
--- a/examples/c/ex_all.c
+++ b/examples/c/ex_all.c
@@ -1113,12 +1113,20 @@ main(void)
 	if (ret == 0)
 		(void)conn->close(conn, NULL);
 
-	/*! [Statistics logging with objects] */
+	/*! [Statistics logging with a table] */
 	ret = wiredtiger_open(home, NULL,
 	    "create,"
 	    "statistics_log=(sources=(\"table:table1\",\"table:table2\"))",
 	    &conn);
-	/*! [Statistics logging with objects] */
+	/*! [Statistics logging with a table] */
+	if (ret == 0)
+		(void)conn->close(conn, NULL);
+
+	/*! [Statistics logging with all tables] */
+	ret = wiredtiger_open(home, NULL,
+	    "create,statistics_log=(sources=(\"table:\"))",
+	    &conn);
+	/*! [Statistics logging with all tables] */
 	if (ret == 0)
 		(void)conn->close(conn, NULL);
 
diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c
index 0660ab2435c..8b87ecdb90b 100644
--- a/src/conn/conn_stat.c
+++ b/src/conn/conn_stat.c
@@ -29,7 +29,7 @@ __wt_conn_stat_init(WT_SESSION_IMPL *session, uint32_t flags)
 }
 
 /*
- * __wt_statlog_config --
+ * __statlog__config --
  *	Parse and setup the statistics server options.
  */
 static int
@@ -66,31 +66,12 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, int *runp)
 		;
 	WT_RET_NOTFOUND_OK(ret);
 	if (cnt != 0) {
-		WT_RET(
-		    __wt_calloc_def(session, cnt * 2 + 1, &conn->stat_sources));
+		WT_RET(__wt_calloc_def(session, cnt + 1, &conn->stat_sources));
 		WT_RET(__wt_config_subinit(session, &objectconf, &cval));
 		for (cnt = 0;
-		    (ret = __wt_config_next(&objectconf, &k, &v)) == 0;) {
-			/*
-			 * We close and re-open each statistics cursor each time
-			 * we dump statistics (the object may or may not exist
-			 * underneath at any point, and I don't want this code
-			 * to break if/when the lifetime of an underlying object
-			 * changes).  Create pairs of strings: the first is the
-			 * object uri, written into the output, the second is
-			 * the enhanced uri used to open the statistics cursor.
-			 */
+		    (ret = __wt_config_next(&objectconf, &k, &v)) == 0; ++cnt)
 			WT_RET(__wt_strndup(session,
 			    k.str, k.len, &conn->stat_sources[cnt]));
-			++cnt;
-
-			WT_RET(__wt_calloc_def(session,
-			    strlen("statistics:") + k.len + 1,
-			    &conn->stat_sources[cnt]));
-			strcpy(conn->stat_sources[cnt], "statistics:");
-			strncat(conn->stat_sources[cnt], k.str, k.len);
-			++cnt;
-		}
 		WT_RET_NOTFOUND_OK(ret);
 	}
 
@@ -99,28 +80,48 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, int *runp)
 
 	WT_RET(__wt_config_gets(
 	    session, cfg, "statistics_log.timestamp", &cval));
-	WT_RET(__wt_strndup(session, cval.str, cval.len, &conn->stat_stamp));
+	WT_RET(__wt_strndup(session, cval.str, cval.len, &conn->stat_format));
 
 	return (0);
 }
 
 /*
- * __stat_server_dump --
- *	Dump a single set of statistics.
+ * __statlog_dump --
+ *	Dump out the connection statistics.
  */
 static int
-__stat_server_dump(WT_SESSION_IMPL *session,
-    const char *name, const char *cursor_uri, const char *stamp, FILE *fp)
+__statlog_dump(WT_SESSION_IMPL *session, const char *name, int conn_stats)
 {
+	WT_CONNECTION_IMPL *conn;
 	WT_CURSOR *cursor;
+	WT_DECL_ITEM(tmp);
 	WT_DECL_RET;
 	WT_SESSION *wt_session;
 	uint64_t value;
-	const char *config, *desc, *pdesc;
+	const char *config, *desc, *pdesc, *uri;
+
+	conn = S2C(session);
 
+	/* Build the statistics cursor URI. */
+	if (conn_stats)
+		uri = "statistics:";
+	else {
+		WT_RET(__wt_scr_alloc(
+		    session, strlen("statistics:") + strlen(name) + 5, &tmp));
+		(void)strcpy(tmp->mem, "statistics:");
+		(void)strcat(tmp->mem, name);
+		uri = tmp->mem;
+	}
+
+	/*
+	 * Open the statistics cursor; immediately free any temporary buffer,
+	 * it makes error handling easier.
+	 */
 	wt_session = (WT_SESSION *)session;
 	config = S2C(session)->stat_clear ?
 	    "statistics_clear,statistics_fast" : "statistics_fast";
+	ret = wt_session->open_cursor(wt_session, uri, NULL, config, &cursor);
+	__wt_scr_free(&tmp);
 
 	/*
 	 * If we don't find an underlying object, silently ignore it, the object
@@ -128,17 +129,15 @@ __stat_server_dump(WT_SESSION_IMPL *session,
 	 * of WT_NOTFOUND for missing files, check both, as well as for EBUSY if
 	 * the handle is exclusively locked at the moment.
 	 */
-	ret = wt_session->open_cursor(
-	    wt_session, cursor_uri, NULL, config, &cursor);
 	if (ret == EBUSY || ret == ENOENT || ret == WT_NOTFOUND)
 		return (0);
 	WT_RET(ret);
 
 	while ((ret = cursor->next(cursor)) == 0 &&
 	    (ret = cursor->get_value(cursor, &desc, &pdesc, &value)) == 0)
-		WT_ERR_TEST((fprintf(fp,
+		WT_ERR_TEST((fprintf(conn->stat_fp,
 		    "%s %" PRIu64 " %s %s\n",
-		    stamp, value, name, desc) < 0), __wt_errno());
+		    conn->stat_stamp, value, name, desc) < 0), __wt_errno());
 	WT_ERR_NOTFOUND_OK(ret);
 
 err:	WT_TRET(cursor->close(cursor));
@@ -147,20 +146,40 @@ err:	WT_TRET(cursor->close(cursor));
 }
 
 /*
- * __stat_server --
+ * __statlog_apply --
+ *	Review a single open handle and dump statistics on demand.
+ */
+static int
+__statlog_apply(WT_SESSION_IMPL *session, const char *cfg[])
+{
+	WT_BTREE *btree;
+	char **p;
+
+	WT_UNUSED(cfg);
+
+	btree = session->btree;
+
+	/* Check for a match on the set of sources. */
+	for (p = S2C(session)->stat_sources; *p != NULL; ++p)
+		if (WT_PREFIX_MATCH(btree->name, *p))
+			return (__statlog_dump(session, btree->name, 0));
+	return (0);
+}
+
+/*
+ * __statlog_server --
  *	The statistics server thread.
  */
 static void *
-__stat_server(void *arg)
+__statlog_server(void *arg)
 {
 	struct timespec ts;
 	struct tm *tm, _tm;
 	FILE *fp;
 	WT_CONNECTION_IMPL *conn;
-	WT_ITEM path, tmp;
 	WT_DECL_RET;
+	WT_ITEM path, tmp;
 	WT_SESSION_IMPL *session;
-	char **p;
 
 	session = arg;
 	conn = S2C(session);
@@ -222,19 +241,25 @@ __stat_server(void *arg)
 		}
 
 		/* Create the entry prefix for this time of day. */
-		if (strftime(tmp.mem, tmp.memsize, conn->stat_stamp, tm) == 0)
+		if (strftime(tmp.mem, tmp.memsize, conn->stat_format, tm) == 0)
 			WT_ERR_MSG(
 			    session, ENOMEM, "strftime timestamp conversion");
 
+		/* Reference temporary values from the connection structure. */
+		conn->stat_fp = fp;
+		conn->stat_stamp = tmp.mem;
+
 		/* Dump the connection statistics. */
-		WT_ERR(__stat_server_dump(
-		    session, conn->home, "statistics:", tmp.mem, fp));
+		WT_ERR(__statlog_dump(session, conn->home, 1));
 
-		/* Dump the object list statistics. */
-		if ((p = conn->stat_sources) != NULL)
-			for (; *p != NULL; p += 2)
-				WT_ERR(__stat_server_dump(
-				    session, p[0], p[1], tmp.mem, fp));
+		/*
+		 * Lock the schema and walk the list of open handles, dumping
+		 * any that match the list of object sources.
+		 */
+		if (conn->stat_sources != NULL)
+			WT_WITH_SCHEMA_LOCK(session,
+			    ret = __wt_conn_btree_apply(
+			    session, __statlog_apply, NULL));
 
 		/* Flush. */
 		WT_ERR(fflush(fp) == 0 ? 0 : __wt_errno());
@@ -255,7 +280,7 @@ err:		__wt_err(session, ret, "statistics log server error");
 }
 
 /*
- * __wt_statlog_create -
+ * __wt_statlog_create --
  *	Start the statistics server thread.
  */
 int
@@ -291,14 +316,14 @@ __wt_statlog_create(WT_CONNECTION_IMPL *conn, const char *cfg[])
 	 * to figure out the scheduling.
 	 */
 	WT_RET(__wt_thread_create(
-	    session, &conn->stat_tid, __stat_server, conn->stat_session));
+	    session, &conn->stat_tid, __statlog_server, conn->stat_session));
 	conn->stat_tid_set = 1;
 
 	return (0);
 }
 
 /*
- * __wt_statlog_destroy -
+ * __wt_statlog_destroy --
  *	Destroy the statistics server thread.
  */
 int
@@ -325,7 +350,7 @@ __wt_statlog_destroy(WT_CONNECTION_IMPL *conn)
 		__wt_free(session, conn->stat_sources);
 	}
 	__wt_free(session, conn->stat_path);
-	__wt_free(session, conn->stat_stamp);
+	__wt_free(session, conn->stat_format);
 
 	/* Close the server thread's session, free its hazard array. */
 	if (conn->stat_session != NULL) {
diff --git a/src/docs/statistics.dox b/src/docs/statistics.dox
index 0258ae2d59d..3b2141bf8df 100644
--- a/src/docs/statistics.dox
+++ b/src/docs/statistics.dox
@@ -27,11 +27,18 @@ The timestamp format may be changed with the \c statistics_log.timestamp
 configuration string.  The \c timestamp value may contain ISO C90 standard
 strftime conversion specifications.
 
-By default, only the system's connection statistics are logged, but
-statistics may be optionally reported for underlying objects by adding
-a list of URIs to the \c statistics_log configuration string:
+By default, only the database statistics are logged.
 
-@snippet ex_all.c Statistics logging with objects
+Statistics for specific underlying data sources may be included by adding
+a list of data source URIs to the \c statistics_log configuration string:
+
+@snippet ex_all.c Statistics logging with a table
+
+Statistics for all underlying data sources of a particular type may be
+included by adding a partial data source URI to the \c statistics_log
+configuration string:
+
+@snippet ex_all.c Statistics logging with all tables
 
 When database statistics are logged, the database home will be the first
 space-separated entry for each record in the log file.  For example:
@@ -45,8 +52,9 @@ Mar 08 11:38:23 1 /database/home total heap memory re-allocations
 Mar 08 11:38:23 472 /database/home total read I/Os
 @endcode
 
-When object statistics are logged, the object URI will be the first
-space-separated entry for each record in the log file.  For example:
+When data source statistics are logged, the data source's URI will be
+the first space-separated entry for each record in the log file.  For
+example:
 
 @code
 Mar 20 10:42:36 21 table:mytable compressed pages written
@@ -56,6 +64,11 @@ Mar 20 10:42:36 586 table:mytable cursor insert calls
 Mar 20 10:42:36 0 table:mytable bulk-loaded cursor-insert calls
 @endcode
 
+No statistics are logged for any data source for which a handle is not
+currently open in the database, nor will any statistics requiring the
+traversal of a tree (as if the \c statistics_fast configuration string
+were set).
+
 The location of the log files may be changed with the \c statistics_log.path
 configuration string.  The \c path value value may contain ISO C90 standard
 strftime conversion specifications.  WiredTiger will not create non-existent
diff --git a/src/include/connection.h b/src/include/connection.h
index 67b87a1d606..271ba5f2cf7 100644
--- a/src/include/connection.h
+++ b/src/include/connection.h
@@ -147,10 +147,13 @@ struct __wt_connection_impl {
 	pthread_t	 stat_tid;	/* Statistics log thread */
 	int		 stat_tid_set;	/* Statistics log thread set */
 	WT_CONDVAR	*stat_cond;	/* Statistics log wait mutex */
+
 	int		 stat_clear;	/* Statistics log clear */
+	const char	*stat_format;	/* Statistics log timestamp format */
+	FILE		*stat_fp;	/* Statistics log file handle */
 	const char	*stat_path;	/* Statistics log path format */
 	char	       **stat_sources;	/* Statistics log list of objects */
-	const char	*stat_stamp;	/* Statistics log timestamp format */
+	const char	*stat_stamp;	/* Statistics log entry timestamp */
 	long		 stat_usecs;	/* Statistics log period */
 
 	WT_FH	   *log_fh;		/* Logging file handle */
diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in
index d61871af60a..234aa9c67c8 100644
--- a/src/include/wiredtiger.in
+++ b/src/include/wiredtiger.in
@@ -1310,9 +1310,12 @@ struct __wt_connection {
  * specifications.  If the value is not an absolute path name\, the file is
  * created relative to the database home.,a string; default \c
  * "WiredTigerStat.%H".}@config{&nbsp;&nbsp;&nbsp;&nbsp;sources, if non-empty\,
- * include statistics for the list of data source URIs.  No statistics that
- * require traversing a tree are reported\, as if the \c statistics_fast
- * configuration string were set.,a list of strings; default
+ * include statistics for the list of data source URIs\, if they are open at the
+ * time of the statistics logging. The list may include URIs matching a single
+ * data source ("table:mytable")\, or a URI matching all data sources of a
+ * particular type ("table:").  No statistics that require the  traversal of a
+ * tree are reported\, as if the \c statistics_fast configuration string were
+ * set.,a list of strings; default
  * empty.}@config{&nbsp;&nbsp;&nbsp;&nbsp;timestamp, a timestamp prepended to
  * each log record\, may contain strftime conversion specifications.,a string;
  * default \c "%b %d %H:%M:%S".}@config{&nbsp;&nbsp;&nbsp;&nbsp;wait, seconds to