Merge branch 'develop' into merge-opt

8 files changed, 171 insertions, 58 deletions

diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c
index 3e64e63107a..7b8eeb05600 100644
--- a/src/conn/conn_api.c
+++ b/src/conn/conn_api.c
@@ -991,7 +991,7 @@ err:	if (cbuf != NULL)
 	__wt_buf_free(session, &exconfig);
 
 	if (ret != 0 && conn != NULL)
-		WT_TRET(__wt_connection_destroy(conn));
+		WT_TRET(__wt_connection_close(conn));
 
 	/* Let the server threads proceed. */
 	if (ret == 0)
diff --git a/src/conn/conn_cache_pool.c b/src/conn/conn_cache_pool.c
index d570c6b3593..02f54e8bad0 100644
--- a/src/conn/conn_cache_pool.c
+++ b/src/conn/conn_cache_pool.c
@@ -254,7 +254,15 @@ __wt_conn_cache_pool_destroy(WT_CONNECTION_IMPL *conn)
 		cp->currently_used -= conn->cache_size;
 	}
 
-	WT_ASSERT(session, cp->refs > 0);
+	/*
+	 * If there are no references, we are cleaning up after a failed
+	 * wiredtiger_open, there is nothing further to do.
+	 */
+	if (cp->refs < 1) {
+		__wt_spin_unlock(session, &cp->cache_pool_lock);
+		return (0);
+	}
+
 	if (--cp->refs == 0) {
 		WT_ASSERT(session, TAILQ_EMPTY(&cp->cache_pool_qh));
 		F_CLR(cp, WT_CACHE_POOL_RUN);
diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c
index ef4c3b5f441..a24c3055422 100644
--- a/src/conn/conn_stat.c
+++ b/src/conn/conn_stat.c
@@ -58,7 +58,7 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, int *runp)
 	conn->stat_clear = cval.val != 0;
 
 	WT_RET(__wt_config_gets(session, cfg, "statistics_log.path", &cval));
-	WT_RET(__wt_strndup(session, cval.str, cval.len, &conn->stat_path));
+	WT_RET(__wt_nfilename(session, cval.str, cval.len, &conn->stat_path));
 
 	WT_RET(__wt_config_gets(
 	    session, cfg, "statistics_log.timestamp", &cval));
@@ -84,7 +84,7 @@ __stat_server(void *arg)
 	WT_SESSION *wt_session;
 	WT_SESSION_IMPL *session;
 	uint64_t value;
-	const char *config, *desc, *pdesc, *p;
+	const char *config, *desc, *pdesc;
 
 	session = arg;
 	conn = S2C(session);
@@ -96,16 +96,6 @@ __stat_server(void *arg)
 	fp = NULL;
 
 	/*
-	 * If the logging file name beings with an absolute path, use it as is,
-	 * otherwise build a version relative to the database home directory.
-	 */
-	if (!__wt_absolute_path(conn->stat_path)) {
-		WT_ERR(__wt_filename(session, conn->stat_path, &p));
-		__wt_free(session, conn->stat_path);
-		conn->stat_path = p;
-	}
-
-	/*
 	 * We need a temporary place to build a path and an entry prefix.
 	 * The length of the path plus 128 should be more than enough.
 	 *
diff --git a/src/docs/spell.ok b/src/docs/spell.ok
index 814b38d7c1f..3aff5b4dedc 100644
--- a/src/docs/spell.ok
+++ b/src/docs/spell.ok
@@ -41,6 +41,7 @@ WiredTigerCheckpoint
 aR
 ack'ed
 alloc
+allocator
 allocsize
 ao
 api
@@ -117,6 +118,7 @@ env
 eof
 erlang
 errno
+fadvise
 failchk
 fd's
 fieldname
@@ -225,6 +227,7 @@ objectsin
 ol
 oltp
 oob
+os
 ovfl
 pcoll
 pdf
diff --git a/src/docs/tuning.dox b/src/docs/tuning.dox
index 7d4b61466b0..ebfbd4d57a1 100644
--- a/src/docs/tuning.dox
+++ b/src/docs/tuning.dox
@@ -1,6 +1,8 @@
 /*! @page tuning Performance Tuning
 
-@section tuning_cache_size Cache size
+@section tuning_cache WiredTiger's cache
+
+@subsection tuning_cache_size Cache size
 
 The cache size for the database is configurable by setting the \c
 cache_size configuration string when calling the ::wiredtiger_open
@@ -13,18 +15,7 @@ An example of setting a cache size to 500MB:
 
 @snippet ex_config.c configure cache size
 
-@section tuning_memory_allocation Memory allocation
-
-The performance of heavily-threaded WiredTiger applications can be
-dominated by memory allocation because the WiredTiger engine has to free
-and re-allocate memory as part of many queries.  Replacing the system's
-malloc implementation with one that has better threaded performance (for
-example, Google's
-<a href="http://goog-perftools.sourceforge.net/doc/tcmalloc.html">tcmalloc</a>,
-or <a href="http://www.canonware.com/jemalloc">jemalloc</a>),
-can dramatically improve throughput.
-
-@section tuning_read_only_objects Read-only objects
+@subsection tuning_read_only_objects Read-only objects
 
 Cursors opened on checkpoints (either named, or using the special "last
 checkpoint" name "WiredTigerCheckpoint") are read-only objects.  Unless
@@ -45,7 +36,7 @@ string "checkpoint" with the name "WiredTigerCheckpoint" to the
 WT_SESSION::open_cursor method:
 @snippet ex_all.c open the default checkpoint
 
-@section tuning_cache_resident Cache resident objects
+@subsection tuning_cache_resident Cache resident objects
 
 Cache resident objects (objects never considered for the purposes of
 cache eviction), can be configured with the WT_SESSION::create
@@ -63,6 +54,36 @@ An example of configuring a cache-resident object:
 
 @snippet ex_all.c Create a cache-resident object
 
+@section tuning_memory_allocator Memory allocator
+
+The performance of heavily-threaded WiredTiger applications can be
+dominated by memory allocation because the WiredTiger engine has to free
+and re-allocate memory as part of many queries.  Replacing the system's
+malloc implementation with one that has better threaded performance (for
+example, Google's
+<a href="http://goog-perftools.sourceforge.net/doc/tcmalloc.html">tcmalloc</a>,
+or <a href="http://www.canonware.com/jemalloc">jemalloc</a>),
+can dramatically improve throughput.
+
+@section tuning_cursor_persistence Cursor persistence
+
+Opening a new cursor is a relatively expensive operation in WiredTiger
+(especially in table objects and Log-Structured Merge Trees (LSM) trees,
+where a logical cursor may require multiple, underlying object cursors),
+and caching cursors can improve performance.  On the other hand, cursors
+hold positions in objects, and therefore long-lived cursor positions can
+decrease performance.  The best combination is to cache cursors, but use
+the WT_CURSOR::reset method to discard the cursor's position in the
+object when the position is no longer needed.
+
+Additionally, cursors are automatically reset whenever a transaction
+boundary is crossed; when a transaction is started with the
+WT_SESSION::begin_transaction or ended with either
+WT_SESSION::commit_transaction or WT_SESSION::rollback_transaction, all
+open cursors are automatically reset, there is no need to call the
+WT_CURSOR::reset method explicitly, and the cursor can be immediately
+reused.
+
 @section tuning_page_size  Page and overflow sizes
 
 There are four page and item size configuration values: \c internal_page_max,
@@ -140,6 +161,82 @@ An example of configuring page sizes:
 
 @snippet ex_file.c file create
 
+@section tuning_system_buffer_cache System buffer cache
+
+@subsection tuning_system_buffer_cache_direct_io Direct I/O
+
+WiredTiger optionally supports direct I/O.  Configuring direct I/O may
+be useful for applications wanting to:
+- minimize the operating system cache effects of I/O to and from
+WiredTiger's buffer cache,
+- avoid double-buffering of blocks in WiredTiger's cache and the
+operating system buffer cache, and
+- avoid stalling underlying solid-state drives by writing a large number
+of dirty blocks.
+
+Direct I/O is configured using the "direct_io" configuration string to
+the ::wiredtiger_open function.  An example of configuring direct I/O
+for WiredTiger's data files:
+
+@snippet ex_all.c Configure direct_io for data files
+
+Direct I/O implies a writing thread waits for the write to complete
+(which is a slower operation than writing into the system buffer cache),
+and configuring direct I/O is likely to decrease overall application
+performance.
+
+Direct I/O is based on the non-standard \c O_DIRECT flag to the POSIX
+1003.1 open system call and may not available on all platforms.
+
+@subsection tuning_system_buffer_cache_os_cache_dirty_max os_cache_dirty_max
+
+As well as direct I/O, WiredTiger supports two additional configuration
+options related to the system buffer cache:
+
+The first is \c os_cache_dirty_max, the maximum dirty bytes an object
+is allowed to have in the system buffer cache.  Once this many bytes
+from an object are written into the system buffer cache, WiredTiger will
+attempt to schedule writes for all of the dirty blocks the object has
+in the system buffer cache.  This configuration option allows
+applications to flush dirty blocks from the object, avoiding stalling
+any underlying drives when the object is subsequently flushed to disk
+as part of a durability operation.
+
+An example of configuring \c os_cache_dirty_max:
+
+@snippet ex_all.c os_cache_dirty_max configuration
+
+The \c os_cache_dirty_max configuration may not be used in combination
+with direct I/O.
+
+The \c os_cache_dirty_max configuration is based on the non-standard
+Linux \c sync_file_range system call and may not available on all
+platforms.
+
+@subsection tuning_system_buffer_cache_os_cache_max os_cache_max
+
+The second configuration option related to the system buffer cache is
+\c os_cache_max, the maximum bytes an object is allowed to have in the
+system buffer cache.  Once this many bytes from an object are either
+read into or written from the system buffer cache, WiredTiger will
+attempt to evict all of the object's blocks from the buffer cache.  This
+configuration option allows applications to evict blocks from the system
+buffer cache to limit double-buffering and system buffer cache overhead.
+
+An example of configuring \c os_cache_max:
+
+@snippet ex_all.c os_cache_max configuration
+
+The \c os_cache_max configuration may not be used in combination with
+direct I/O.
+
+The \c os_cache_max configuration is based on the POSIX 1003.1 standard
+\c posix_fadvise system call and may not available on all platforms.
+
+Configuring direct I/O, \c os_cache_dirty_max or \c os_cache_max all
+have the side effect of turning off memory-mapping of objects in
+WiredTiger.
+
 @section tuning_checksums Checksums
 
 WiredTiger checksums file reads and writes, by default.  In read-only
@@ -158,39 +255,31 @@ blocks which are not compressed:
 
 @snippet ex_all.c Configure checksums to uncompressed
 
-@section tuning_direct_io Direct I/O
-
-WiredTiger optionally supports direct I/O, based on the non-standard \c
-O_DIRECT flag to the POSIX 1003.1 open system call.   Configuring direct
-I/O may be useful for applications wanting to minimize the operating
-system cache effects of I/O to and from WiredTiger's buffer cache.
-
-Direct I/O is configured using the "direct_io" configuration string to
-the ::wiredtiger_open function.  An example of configuring direct I/O
-for WiredTiger's data files:
-
-@snippet ex_all.c Configure direct_io for data files
-
 @section tuning_compression Compression
 
-WiredTiger configures key prefix compression for row-store objects, and
-column-store compression for both row-store and column-store objects,
-by default.
-These forms of compression minimize in-memory and on-disk space, but at
-some CPU cost when rows are read and written.   Turning these forms of
-compression off may increase application throughput.
+WiredTiger configures key prefix compression for row-store objects by
+default.  Additional forms of compression for both row- and column-store
+objects, including dictionary and block compression, and Huffman
+encoding, are optional.  Compression minimizes in-memory and on-disk
+resource requirements and decreases the amount of I/O, at some CPU cost
+when rows are read and written.
+
+Configuring compression on or off may change application throughput.
+For example, in applications using solid-state drives (where I/O is less
+expensive), turning off compression may increase application performance
+by reducing CPU costs; in applications where I/O costs are more
+expensive, turning on compression may increase application performance
+by reducing the overall number of I/O operations.
 
 For example, turning off row-store key prefix compression:
 
 @snippet ex_all.c Configure key prefix compression off
 
-For example, turning off row-store or column-store dictionary compression:
+For example, turning on row-store or column-store dictionary compression:
 
-@snippet ex_all.c Configure dictionary compression off
+@snippet ex_all.c Configure dictionary compression on
 
-WiredTiger does not configure Huffman encoding or block compression by
-default, but these forms of compression can also impact overall
-throughput.  See @ref file_formats_compression for more information.
+See @ref file_formats_compression for more information.
 
 @section tuning_statistics Performance monitoring with statistics
 
diff --git a/src/include/extern.h b/src/include/extern.h
index 6da82c23fa4..d555bf4867e 100644
--- a/src/include/extern.h
+++ b/src/include/extern.h
@@ -1140,6 +1140,10 @@ extern int __wt_absolute_path(const char *path);
 extern int __wt_filename(WT_SESSION_IMPL *session,
     const char *name,
     const char **path);
+extern int __wt_nfilename(WT_SESSION_IMPL *session,
+    const char *name,
+    size_t namelen,
+    const char **path);
 extern int __wt_library_init(void);
 extern int __wt_breakpoint(void);
 extern void __wt_attach(WT_SESSION_IMPL *session);
diff --git a/src/support/filename.c b/src/support/filename.c
index 57f3de97d65..6b3d7ff17d2 100644
--- a/src/support/filename.c
+++ b/src/support/filename.c
@@ -19,11 +19,25 @@ __wt_absolute_path(const char *path)
 
 /*
  * __wt_filename --
- *	Build a filename in a scratch buffer.
+ *	Build a file name in a scratch buffer, automatically calculate the
+ *	length of the file name.
  */
 int
 __wt_filename(WT_SESSION_IMPL *session, const char *name, const char **path)
 {
+	return __wt_nfilename(session, name, strlen(name), path);
+}
+
+/*
+ * __wt_nfilename --
+ *	Build a file name in a scratch buffer. If the name is already an
+ *	absolute path duplicate it, otherwise generate a path relative to the
+ *	connection home directory.
+ */
+int
+__wt_nfilename(WT_SESSION_IMPL *session,
+    const char *name, size_t namelen, const char **path)
+{
 	WT_CONNECTION_IMPL *conn;
 	size_t len;
 	char *buf;
@@ -31,10 +45,14 @@ __wt_filename(WT_SESSION_IMPL *session, const char *name, const char **path)
 	conn = S2C(session);
 	*path = NULL;
 
-	len = strlen(conn->home) + 1 + strlen(name) + 1;
-	WT_RET(__wt_calloc(session, 1, len, &buf));
-	snprintf(buf, len, "%s/%s", conn->home, name);
+	if (__wt_absolute_path(name))
+		WT_RET(__wt_strndup(session, name, namelen, path));
+	else {
+		len = strlen(conn->home) + 1 + namelen + 1;
+		WT_RET(__wt_calloc(session, 1, len, &buf));
+		snprintf(buf, len, "%s/%.*s", conn->home, (int)namelen, name);
+		*path = buf;
+	}
 
-	*path = buf;
 	return (0);
 }
diff --git a/src/txn/txn.c b/src/txn/txn.c
index 2b0bef4f44f..f3bf4831d46 100644
--- a/src/txn/txn.c
+++ b/src/txn/txn.c
@@ -461,5 +461,6 @@ __wt_txn_global_destroy(WT_CONNECTION_IMPL *conn)
 	session = conn->default_session;
 	txn_global = &conn->txn_global;
 
-	__wt_free(session, txn_global->states);
+	if (txn_global != NULL)
+		__wt_free(session, txn_global->states);
 }