Merge branch 'develop' into lsm-ckpt-experiments

Conflicts:
	src/bloom/bloom.c
	src/config/config_def.c
	src/lsm/lsm_cursor.c
	src/lsm/lsm_merge.c
	src/lsm/lsm_tree.c
	src/lsm/lsm_worker.c

5 files changed, 148 insertions, 69 deletions

diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c
index 3946fd93483..92dc7382de5 100644
--- a/src/conn/conn_api.c
+++ b/src/conn/conn_api.c
@@ -10,26 +10,32 @@
 static int __conn_verbose_config(WT_SESSION_IMPL *, const char *[]);
 
 /*
- * api_err_printf --
- *	Extension API call to print to the error stream.
+ * __conn_get_extension_api --
+ *	WT_CONNECTION.get_extension_api method.
  */
-static int
-__api_err_printf(WT_SESSION *wt_session, const char *fmt, ...)
+static WT_EXTENSION_API *
+__conn_get_extension_api(WT_CONNECTION *wt_conn)
 {
-	WT_DECL_RET;
-	va_list ap;
+	WT_CONNECTION_IMPL *conn;
 
-	va_start(ap, fmt);
-	ret = __wt_verrx((WT_SESSION_IMPL *)wt_session, fmt, ap);
-	va_end(ap);
-	return (ret);
-}
+	conn = (WT_CONNECTION_IMPL *)wt_conn;
 
-static WT_EXTENSION_API __api = {
-	__api_err_printf,
-	__wt_scr_alloc_ext,
-	__wt_scr_free_ext
-};
+	conn->extension_api.conn = wt_conn;
+	conn->extension_api.err_printf = __wt_ext_err_printf;
+	conn->extension_api.msg_printf = __wt_ext_msg_printf;
+	conn->extension_api.strerror = wiredtiger_strerror;
+	conn->extension_api.scr_alloc = __wt_ext_scr_alloc;
+	conn->extension_api.scr_free = __wt_ext_scr_free;
+	conn->extension_api.config_get = __wt_ext_config_get;
+	conn->extension_api.config_scan_begin = __wt_ext_config_scan_begin;
+	conn->extension_api.config_scan_end = __wt_ext_config_scan_end;
+	conn->extension_api.config_scan_next = __wt_ext_config_scan_next;
+	conn->extension_api.struct_pack = __wt_ext_struct_pack;
+	conn->extension_api.struct_size = __wt_ext_struct_size;
+	conn->extension_api.struct_unpack = __wt_ext_struct_unpack;
+
+	return (&conn->extension_api);
+}
 
 /*
  * __conn_load_extension --
@@ -44,17 +50,17 @@ __conn_load_extension(
 	WT_DECL_RET;
 	WT_DLH *dlh;
 	WT_SESSION_IMPL *session;
-	int (*entry)(WT_SESSION *, WT_EXTENSION_API *, const char *);
-	const char *entry_name;
+	int (*load)(WT_CONNECTION *, WT_CONFIG_ARG *);
+	const char *init_name, *terminate_name;
 
 	dlh = NULL;
+	init_name = terminate_name = NULL;
 
 	conn = (WT_CONNECTION_IMPL *)wt_conn;
 	CONNECTION_API_CALL(conn, session, load_extension, config, cfg);
 
-	entry_name = NULL;
 	WT_ERR(__wt_config_gets(session, cfg, "entry", &cval));
-	WT_ERR(__wt_strndup(session, cval.str, cval.len, &entry_name));
+	WT_ERR(__wt_strndup(session, cval.str, cval.len, &init_name));
 
 	/*
 	 * This assumes the underlying shared libraries are reference counted,
@@ -62,23 +68,28 @@ __conn_load_extension(
 	 * count, and closing it simply decrements the ref count, and the last
 	 * close discards the reference entirely -- in other words, we do not
 	 * check to see if we've already opened this shared library.
+	 *
+	 * Fill in the extension structure and call the load function.
 	 */
 	WT_ERR(__wt_dlopen(session, path, &dlh));
-	WT_ERR(__wt_dlsym(session, dlh, entry_name, &entry));
+	WT_ERR(__wt_dlsym(session, dlh, init_name, 1, &load));
+	WT_ERR(load(wt_conn, (WT_CONFIG_ARG *)cfg));
 
-	/* Call the entry function. */
-	WT_ERR(entry(&session->iface, &__api, config));
+	/* Remember the unload function for when we close. */
+	WT_ERR(__wt_config_gets(session, cfg, "terminate", &cval));
+	WT_ERR(__wt_strndup(session, cval.str, cval.len, &terminate_name));
+	WT_ERR(__wt_dlsym(session, dlh, terminate_name, 0, &dlh->terminate));
 
 	/* Link onto the environment's list of open libraries. */
 	__wt_spin_lock(session, &conn->api_lock);
 	TAILQ_INSERT_TAIL(&conn->dlhqh, dlh, q);
 	__wt_spin_unlock(session, &conn->api_lock);
+	dlh = NULL;
 
-	if (0) {
-err:		if (dlh != NULL)
-			WT_TRET(__wt_dlclose(session, dlh));
-	}
-	__wt_free(session, entry_name);
+err:	if (dlh != NULL)
+		WT_TRET(__wt_dlclose(session, dlh));
+	__wt_free(session, init_name);
+	__wt_free(session, terminate_name);
 
 	API_END_NOTFOUND_MAP(session, ret);
 }
@@ -96,6 +107,8 @@ __conn_add_collator(WT_CONNECTION *wt_conn,
 	WT_NAMED_COLLATOR *ncoll;
 	WT_SESSION_IMPL *session;
 
+	ncoll = NULL;
+
 	conn = (WT_CONNECTION_IMPL *)wt_conn;
 	CONNECTION_API_CALL(conn, session, add_collator, config, cfg);
 	WT_UNUSED(cfg);
@@ -106,9 +119,13 @@ __conn_add_collator(WT_CONNECTION *wt_conn,
 
 	__wt_spin_lock(session, &conn->api_lock);
 	TAILQ_INSERT_TAIL(&conn->collqh, ncoll, q);
-	__wt_spin_unlock(session, &conn->api_lock);
 	ncoll = NULL;
-err:	__wt_free(session, ncoll);
+	__wt_spin_unlock(session, &conn->api_lock);
+
+err:	if (ncoll != NULL) {
+		__wt_free(session, ncoll->name);
+		__wt_free(session, ncoll);
+	}
 
 	API_END_NOTFOUND_MAP(session, ret);
 }
@@ -148,6 +165,7 @@ __conn_add_compressor(WT_CONNECTION *wt_conn,
 
 	WT_UNUSED(name);
 	WT_UNUSED(compressor);
+	ncomp = NULL;
 
 	conn = (WT_CONNECTION_IMPL *)wt_conn;
 	CONNECTION_API_CALL(conn, session, add_compressor, config, cfg);
@@ -159,9 +177,13 @@ __conn_add_compressor(WT_CONNECTION *wt_conn,
 
 	__wt_spin_lock(session, &conn->api_lock);
 	TAILQ_INSERT_TAIL(&conn->compqh, ncomp, q);
-	__wt_spin_unlock(session, &conn->api_lock);
 	ncomp = NULL;
-err:	__wt_free(session, ncomp);
+	__wt_spin_unlock(session, &conn->api_lock);
+
+err:	if (ncomp != NULL) {
+		__wt_free(session, ncomp->name);
+		__wt_free(session, ncomp);
+	}
 
 	API_END_NOTFOUND_MAP(session, ret);
 }
@@ -212,11 +234,11 @@ __conn_add_data_source(WT_CONNECTION *wt_conn,
 	/* Link onto the environment's list of data sources. */
 	__wt_spin_lock(session, &conn->api_lock);
 	TAILQ_INSERT_TAIL(&conn->dsrcqh, ndsrc, q);
+	ndsrc = NULL;
 	__wt_spin_unlock(session, &conn->api_lock);
 
-	if (0) {
-err:		if (ndsrc != NULL)
-			__wt_free(session, ndsrc->prefix);
+err:	if (ndsrc != NULL) {
+		__wt_free(session, ndsrc->prefix);
 		__wt_free(session, ndsrc);
 	}
 
@@ -265,6 +287,10 @@ __conn_add_extractor(WT_CONNECTION *wt_conn,
 err:	API_END_NOTFOUND_MAP(session, ret);
 }
 
+/*
+ * __conn_get_home --
+ *	WT_CONNECTION.get_home method.
+ */
 static const char *
 __conn_get_home(WT_CONNECTION *wt_conn)
 {
@@ -272,6 +298,26 @@ __conn_get_home(WT_CONNECTION *wt_conn)
 }
 
 /*
+ * __conn_configure_method --
+ *	WT_CONNECTION.configure_method method.
+ */
+static int
+__conn_configure_method(WT_CONNECTION *wt_conn, const char *method,
+    const char *uri, const char *config, const char *type, const char *check)
+{
+	WT_CONNECTION_IMPL *conn;
+	WT_DECL_RET;
+	WT_SESSION_IMPL *session;
+
+	conn = (WT_CONNECTION_IMPL *)wt_conn;
+	CONNECTION_API_CALL_NOCONF(conn, session, configure_method);
+
+	ret = __wt_configure_method(session, method, uri, config, type, check);
+
+err:	API_END_NOTFOUND_MAP(session, ret);
+}
+
+/*
  * __conn_is_new --
  *	WT_CONNECTION->is_new method.
  */
@@ -330,7 +376,7 @@ __conn_close(WT_CONNECTION *wt_conn, const char *config)
 	WT_TRET(__wt_statlog_destroy(conn));
 
 	/* Clean up open LSM handles. */
-	WT_ERR(__wt_lsm_cleanup(&conn->iface));
+	WT_ERR(__wt_lsm_tree_close_all(session));
 
 	/* Close open data handles. */
 	WT_TRET(__wt_conn_dhandle_discard(conn));
@@ -554,8 +600,8 @@ __conn_config_file(WT_SESSION_IMPL *session, const char **cfg, WT_ITEM **cbufp)
 #endif
 
 	/* Check the configuration string. */
-	WT_ERR(__wt_config_check(
-	    session, __wt_confchk_wiredtiger_open, cbuf->data, 0));
+	WT_ERR(__wt_config_check(session,
+	    WT_CONFIG_REF(session, wiredtiger_open), cbuf->data, 0));
 
 	/*
 	 * The configuration file falls between the default configuration and
@@ -605,8 +651,8 @@ __conn_config_env(WT_SESSION_IMPL *session, const char **cfg)
 		    "lacks privileges to use that environment variable");
 
 	/* Check the configuration string. */
-	WT_RET(__wt_config_check(
-	    session, __wt_confchk_wiredtiger_open, env_config, 0));
+	WT_RET(__wt_config_check(session,
+	    WT_CONFIG_REF(session, wiredtiger_open), env_config, 0));
 
 	/*
 	 * The environment setting comes second-to-last: it overrides the
@@ -751,7 +797,7 @@ __conn_verbose_config(WT_SESSION_IMPL *session, const char *cfg[])
 	WT_CONFIG_ITEM cval, sval;
 	WT_CONNECTION_IMPL *conn;
 	WT_DECL_RET;
-	static struct {
+	static const struct {
 		const char *name;
 		uint32_t flag;
 	} *ft, verbtypes[] = {
@@ -797,19 +843,21 @@ int
 wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler,
     const char *config, WT_CONNECTION **wt_connp)
 {
-	static WT_CONNECTION stdc = {
+	static const WT_CONNECTION stdc = {
 		__conn_close,
 		__conn_reconfigure,
 		__conn_get_home,
+		__conn_configure_method,
 		__conn_is_new,
 		__conn_open_session,
 		__conn_load_extension,
 		__conn_add_data_source,
 		__conn_add_collator,
 		__conn_add_compressor,
-		__conn_add_extractor
+		__conn_add_extractor,
+		__conn_get_extension_api
 	};
-	static struct {
+	static const struct {
 		const char *name;
 		uint32_t flag;
 	} *ft, directio_types[] = {
@@ -823,8 +871,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler,
 	WT_DECL_RET;
 	WT_ITEM *cbuf, expath, exconfig;
 	WT_SESSION_IMPL *session;
-	const char *cfg[] =
-	    { __wt_confdfl_wiredtiger_open, config, NULL, NULL, NULL };
+	const char *cfg[5];
 	int exist;
 
 	*wt_connp = NULL;
@@ -855,9 +902,13 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler,
 	/* Remaining basic initialization of the connection structure. */
 	WT_ERR(__wt_connection_init(conn));
 
-	/* Check the configuration strings. */
-	WT_ERR(__wt_config_check(
-	    session, __wt_confchk_wiredtiger_open, config, 0));
+	/* Check/set the configuration strings. */
+	WT_ERR(__wt_config_check(session,
+	    WT_CONFIG_REF(session, wiredtiger_open), config, 0));
+	cfg[0] = WT_CONFIG_BASE(session, wiredtiger_open);
+	cfg[1] = config;
+	/* Leave space for optional additional configuration. */
+	cfg[2] = cfg[3] = cfg[4] = NULL;
 
 	/* Get the database home. */
 	WT_ERR(__conn_home(session, home, cfg));
@@ -972,12 +1023,6 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler,
 	/* If there's a hot-backup file, load it. */
 	WT_ERR(__wt_metadata_load_backup(session));
 
-	/*
-	 * XXX LSM initialization.
-	 * This is structured so that it could be moved to an extension.
-	 */
-	WT_ERR(__wt_lsm_init(&conn->iface, NULL));
-
 	STATIC_ASSERT(offsetof(WT_CONNECTION_IMPL, iface) == 0);
 	*wt_connp = &conn->iface;
 
diff --git a/src/conn/conn_cache_pool.c b/src/conn/conn_cache_pool.c
index 405be9c6d66..fc22e437b64 100644
--- a/src/conn/conn_cache_pool.c
+++ b/src/conn/conn_cache_pool.c
@@ -19,7 +19,7 @@
 /* Balancing passes after a reduction before a connection is a candidate. */
 #define	WT_CACHE_POOL_REDUCE_SKIPS	5
 
-static int __cache_pool_adjust(uint64_t, int);
+static int __cache_pool_adjust(uint64_t, uint64_t);
 static int __cache_pool_assess(uint64_t *);
 static int  __cache_pool_balance(void);
 
@@ -328,7 +328,7 @@ __cache_pool_balance(void)
 	WT_CACHE_POOL *cp;
 	WT_CONNECTION_IMPL *entry;
 	WT_DECL_RET;
-	uint64_t highest;
+	uint64_t bump_threshold, highest, last_used;
 
 	cp = __wt_process.cache_pool;
 	highest = 0;
@@ -349,9 +349,18 @@ __cache_pool_balance(void)
 		    "Failed to create session for cache pool");
 
 	WT_ERR(__cache_pool_assess(&highest));
-	WT_ERR(__cache_pool_adjust(highest, 0));
-	while (cp->currently_used > cp->size)
-		WT_ERR(__cache_pool_adjust(highest, 1));
+	last_used = cp->currently_used;
+	bump_threshold = WT_CACHE_POOL_BUMP_THRESHOLD;
+	/*
+	 * Actively attempt to:
+	 * - Reduce the amount allocated, if we are over the budget
+	 * - Increase the amount used if there is capacity and any pressure.
+	 */
+	do {
+		WT_ERR(__cache_pool_adjust(highest, --bump_threshold));
+	} while (cp->currently_used > cp->size ||
+	    (cp->currently_used == last_used && bump_threshold > 0 &&
+	    cp->currently_used < cp->size));
 
 err:	__wt_spin_unlock(NULL, &cp->cache_pool_lock);
 	return (ret);
@@ -404,29 +413,40 @@ __cache_pool_assess(uint64_t *phighest)
  * allocated more than their reserved size.
  */
 static int
-__cache_pool_adjust(uint64_t highest, int force)
+__cache_pool_adjust(uint64_t highest, uint64_t bump_threshold)
 {
 	WT_CACHE_POOL *cp;
 	WT_CACHE *cache;
 	WT_CONNECTION_IMPL *entry;
 	WT_SESSION_IMPL *session;
 	uint64_t adjusted, reserved, read_pressure;
-	int grew;
+	int force, grew;
 
 	cp = __wt_process.cache_pool;
 	session = cp->session;
 	adjusted = reserved = read_pressure = 0;
 	grew = 0;
+	force = (cp->currently_used > cp->size);
+	if (WT_VERBOSE_ISSET(session, shared_cache)) {
+		WT_VERBOSE_RET(session, shared_cache,
+		    "Cache pool distribution: ");
+		WT_VERBOSE_RET(session, shared_cache,
+		    "\t" "cache_size, read_pressure, skips: ");
+	}
 
 	TAILQ_FOREACH(entry, &cp->cache_pool_qh, cpq) {
 		cache = entry->cache;
 		reserved = cache->cp_reserved;
 		adjusted = 0;
+
+		read_pressure = cache->cp_current_evict / highest;
+		WT_VERBOSE_RET(session, shared_cache,
+		    "\t%"PRIu64", %"PRIu64", %d",
+		    entry->cache_size, read_pressure, cache->cp_skip_count);
+
 		/* Allow to stabilize after changes. */
 		if (cache->cp_skip_count > 0 && --cache->cp_skip_count > 0)
 			continue;
-
-		read_pressure = cache->cp_current_evict / highest;
 		/*
 		 * TODO: Use __wt_cache_bytes_inuse instead of eviction_target
 		 * which doesn't do the right thing at the moment.
@@ -451,7 +471,7 @@ __cache_pool_adjust(uint64_t highest, int force)
 		     cache->bytes_inmem >=
 		     (entry->cache_size * cache->eviction_target) / 100 &&
 		     cp->currently_used < cp->size &&
-		     read_pressure > WT_CACHE_POOL_BUMP_THRESHOLD) {
+		     read_pressure > bump_threshold) {
 			grew = 1;
 			adjusted = WT_MIN(cp->chunk,
 			    cp->size - cp->currently_used);
@@ -468,8 +488,8 @@ __cache_pool_adjust(uint64_t highest, int force)
 				cp->currently_used -= adjusted;
 			}
 			WT_VERBOSE_RET(session, shared_cache,
-			    "Allocated %" PRId64 " to %s",
-			    adjusted, entry->home);
+			    "Allocated %s%" PRId64 " to %s",
+			    grew ? "" : "-", adjusted, entry->home);
 			/*
 			 * TODO: Add a loop waiting for connection to give up
 			 * cache.
diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c
index 8b48b6cfe5c..fc4d0f2898f 100644
--- a/src/conn/conn_dhandle.c
+++ b/src/conn/conn_dhandle.c
@@ -257,7 +257,8 @@ __conn_btree_config_set(WT_SESSION_IMPL *session)
 	 * it, after the copy, we don't want to free it.
 	 */
 	WT_ERR(__wt_calloc_def(session, 3, &dhandle->cfg));
-	WT_ERR(__wt_strdup(session, __wt_confdfl_file_meta, &dhandle->cfg[0]));
+	WT_ERR(__wt_strdup(
+	    session, WT_CONFIG_BASE(session, file_meta), &dhandle->cfg[0]));
 	dhandle->cfg[1] = metaconf;
 	metaconf = NULL;
 	return (0);
diff --git a/src/conn/conn_handle.c b/src/conn/conn_handle.c
index fb59812d75d..b7ea447a419 100644
--- a/src/conn/conn_handle.c
+++ b/src/conn/conn_handle.c
@@ -27,6 +27,9 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn)
 
 	TAILQ_INIT(&conn->lsmqh);		/* WT_LSM_TREE list */
 
+	/* Configuration. */
+	WT_RET(__wt_conn_config_init(session));
+
 	/* Statistics. */
 	__wt_stat_init_connection_stats(&conn->stats);
 
@@ -81,6 +84,11 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn)
 	TAILQ_REMOVE(&__wt_process.connqh, conn, q);
 	__wt_spin_unlock(session, &__wt_process.spinlock);
 
+	/* Configuration */
+	__wt_conn_config_discard(session);		/* configuration */
+
+	__wt_conn_foc_discard(session);			/* free-on-close */
+
 	__wt_spin_destroy(session, &conn->api_lock);
 	__wt_spin_destroy(session, &conn->fh_lock);
 	__wt_spin_destroy(session, &conn->metadata_lock);
diff --git a/src/conn/conn_open.c b/src/conn/conn_open.c
index 2bafe877767..f810c147057 100644
--- a/src/conn/conn_open.c
+++ b/src/conn/conn_open.c
@@ -72,11 +72,13 @@ err:	WT_TRET(__wt_connection_close(conn));
 int
 __wt_connection_close(WT_CONNECTION_IMPL *conn)
 {
+	WT_CONNECTION *wt_conn;
 	WT_SESSION_IMPL *session;
 	WT_DECL_RET;
 	WT_DLH *dlh;
 	WT_FH *fh;
 
+	wt_conn = (WT_CONNECTION *)conn;
 	session = conn->default_session;
 
 	/*
@@ -110,9 +112,12 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn)
 	/* Discard transaction state. */
 	__wt_txn_global_destroy(conn);
 
-	/* Close extensions. */
+	/* Close extensions, first calling any unload entry point. */
 	while ((dlh = TAILQ_FIRST(&conn->dlhqh)) != NULL) {
 		TAILQ_REMOVE(&conn->dlhqh, dlh, q);
+
+		if (dlh->terminate != NULL)
+			WT_TRET(dlh->terminate(wt_conn));
 		WT_TRET(__wt_dlclose(session, dlh));
 	}