Merge pull request #411 from wiredtiger/panic

Add a __wt_panic function that shuts down all of the WiredTiger APIs.

17 files changed, 96 insertions, 51 deletions

diff --git a/dist/api_data.py b/dist/api_data.py
index 634edf4ca30..996206337ad 100644
--- a/dist/api_data.py
+++ b/dist/api_data.py
@@ -25,9 +25,9 @@ errors = [
 		record to return.  This includes search and other
 		operations where no record matched the cursor's search
 		key such as WT_CURSOR::update or WT_CURSOR::remove.'''),
-	Error('WT_PANIC', 'the application must exit the database', '''
+	Error('WT_PANIC', 'WiredTiger library panic', '''
 		This error indicates an underlying problem that requires the
-		application exit the database and restart.'''),
+		application exit and restart.'''),
 	Error('WT_RESTART', 'restart the operation (internal)', undoc=True),
 ]
 
@@ -572,6 +572,7 @@ flags = {
 	'conn' : [
 		'CONN_CACHE_POOL',
 		'CONN_LSM_MERGE',
+		'CONN_PANIC',
 		'CONN_SERVER_RUN',
 		'CONN_SYNC',
 		'CONN_TRANSACTIONAL',
diff --git a/src/api/api_strerror.c b/src/api/api_strerror.c
index fa09398626e..150621e2333 100644
--- a/src/api/api_strerror.c
+++ b/src/api/api_strerror.c
@@ -25,7 +25,7 @@ wiredtiger_strerror(int error)
 	case WT_NOTFOUND:
 		return ("WT_NOTFOUND: cursor item not found");
 	case WT_PANIC:
-		return ("WT_PANIC: the application must exit the database");
+		return ("WT_PANIC: WiredTiger library panic");
 	case WT_RESTART:
 		return ("WT_RESTART: restart the operation (internal)");
 	default:
diff --git a/src/btree/bt_evict.c b/src/btree/bt_evict.c
index b9769b32397..0df38d79d57 100644
--- a/src/btree/bt_evict.c
+++ b/src/btree/bt_evict.c
@@ -227,8 +227,10 @@ __wt_cache_evict_server(void *arg)
 			    __wt_cache_pages_inuse(cache),
 			    __wt_cache_bytes_inuse(cache));
 		}
-	} else
+	} else {
 err:		__wt_err(session, ret, "eviction server error");
+		(void)__wt_panic(session);
+	}
 
 	__wt_free(session, cache->evict);
 
diff --git a/src/cursor/cur_backup.c b/src/cursor/cur_backup.c
index 6fdaf072c5a..4bb3065c643 100644
--- a/src/cursor/cur_backup.c
+++ b/src/cursor/cur_backup.c
@@ -59,6 +59,7 @@ static int
 __curbackup_reset(WT_CURSOR *cursor)
 {
 	WT_CURSOR_BACKUP *cb;
+	WT_DECL_RET;
 	WT_SESSION_IMPL *session;
 
 	cb = (WT_CURSOR_BACKUP *)cursor;
@@ -67,8 +68,8 @@ __curbackup_reset(WT_CURSOR *cursor)
 	cb->next = 0;
 	F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET);
 
-	API_END(session);
-	return (0);
+err:	API_END(session);
+	return (ret);
 }
 
 /*
@@ -100,7 +101,7 @@ __curbackup_close(WT_CURSOR *cursor)
 	    tret = __backup_stop(session));		/* Stop the backup. */
 	WT_TRET(tret);
 
-	API_END(session);
+err:	API_END(session);
 	return (ret);
 }
 
diff --git a/src/cursor/cur_bulk.c b/src/cursor/cur_bulk.c
index e1e05d0a5ca..ac9dc9a7b51 100644
--- a/src/cursor/cur_bulk.c
+++ b/src/cursor/cur_bulk.c
@@ -61,7 +61,7 @@ __curbulk_close(WT_CURSOR *cursor)
 	cursor->uri = NULL;
 	WT_TRET(__wt_cursor_close(cursor));
 
-	API_END(session);
+err:	API_END(session);
 	return (ret);
 }
 
diff --git a/src/cursor/cur_dump.c b/src/cursor/cur_dump.c
index ba5d5bb57d7..52265c2a5a9 100644
--- a/src/cursor/cur_dump.c
+++ b/src/cursor/cur_dump.c
@@ -278,7 +278,7 @@ __curdump_close(WT_CURSOR *cursor)
 	cursor->uri = NULL;
 	WT_TRET(__wt_cursor_close(cursor));
 
-	API_END(session);
+err:	API_END(session);
 	return (ret);
 }
 
diff --git a/src/cursor/cur_file.c b/src/cursor/cur_file.c
index 2aeaa1c8592..0e311438e2e 100644
--- a/src/cursor/cur_file.c
+++ b/src/cursor/cur_file.c
@@ -54,7 +54,7 @@ __curfile_next(WT_CURSOR *cursor)
 	CURSOR_API_CALL(cursor, session, next, cbt->btree);
 	ret = __wt_btcur_next((WT_CURSOR_BTREE *)cursor, 0);
 
-	API_END(session);
+err:	API_END(session);
 	return (ret);
 }
 
@@ -74,7 +74,7 @@ __curfile_next_random(WT_CURSOR *cursor)
 	CURSOR_API_CALL(cursor, session, next, cbt->btree);
 	ret = __wt_btcur_next_random(cbt);
 
-	API_END(session);
+err:	API_END(session);
 	return (ret);
 }
 
@@ -93,7 +93,7 @@ __curfile_prev(WT_CURSOR *cursor)
 	CURSOR_API_CALL(cursor, session, prev, cbt->btree);
 	ret = __wt_btcur_prev((WT_CURSOR_BTREE *)cursor, 0);
 
-	API_END(session);
+err:	API_END(session);
 	return (ret);
 }
 
@@ -112,7 +112,7 @@ __curfile_reset(WT_CURSOR *cursor)
 	CURSOR_API_CALL(cursor, session, reset, cbt->btree);
 	ret = __wt_btcur_reset(cbt);
 
-	API_END(session);
+err:	API_END(session);
 	return (ret);
 }
 
@@ -267,7 +267,7 @@ __curfile_close(WT_CURSOR *cursor)
 	cursor->uri = NULL;
 	WT_TRET(__wt_cursor_close(cursor));
 
-	API_END(session);
+err:	API_END(session);
 	return (ret);
 }
 
diff --git a/src/cursor/cur_index.c b/src/cursor/cur_index.c
index 5e9ef2d968f..c440861fa67 100644
--- a/src/cursor/cur_index.c
+++ b/src/cursor/cur_index.c
@@ -54,9 +54,13 @@ __curindex_set_value(WT_CURSOR *cursor, ...)
 	WT_SESSION_IMPL *session;
 
 	CURSOR_API_CALL(cursor, session, set_value, NULL);
-	WT_UNUSED(ret);
 	cursor->saved_err = ENOTSUP;
 	F_CLR(cursor, WT_CURSTD_VALUE_SET);
+
+	if (0) {
+err:		cursor->saved_err = ret;
+		F_CLR(cursor, WT_CURSTD_VALUE_SET);
+	}
 	API_END(session);
 }
 
@@ -129,7 +133,7 @@ __curindex_next(WT_CURSOR *cursor)
 	else
 		F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET);
 
-	API_END(session);
+err:	API_END(session);
 	return (ret);
 }
 
@@ -151,7 +155,7 @@ __curindex_prev(WT_CURSOR *cursor)
 	else
 		F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET);
 
-	API_END(session);
+err:	API_END(session);
 	return (ret);
 }
 
@@ -180,7 +184,7 @@ __curindex_reset(WT_CURSOR *cursor)
 		WT_TRET((*cp)->reset(*cp));
 	}
 
-	API_END(session);
+err:	API_END(session);
 	return (ret);
 }
 
@@ -253,7 +257,7 @@ __curindex_search_near(WT_CURSOR *cursor, int *exact)
 	else
 		F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET);
 
-	API_END(session);
+err:	API_END(session);
 	return (ret);
 }
 
@@ -298,7 +302,7 @@ __curindex_close(WT_CURSOR *cursor)
 	cursor->uri = NULL;
 	WT_TRET(__wt_cursor_close(cursor));
 
-	API_END(session);
+err:	API_END(session);
 	return (ret);
 }
 
diff --git a/src/cursor/cur_stat.c b/src/cursor/cur_stat.c
index abff1a6de13..783f7abd6c7 100644
--- a/src/cursor/cur_stat.c
+++ b/src/cursor/cur_stat.c
@@ -44,8 +44,8 @@ __curstat_get_key(WT_CURSOR *cursor, ...)
 	va_list ap;
 
 	cst = (WT_CURSOR_STAT *)cursor;
-	CURSOR_API_CALL(cursor, session, get_key, cst->btree);
 	va_start(ap, cursor);
+	CURSOR_API_CALL(cursor, session, get_key, cst->btree);
 
 	WT_CURSOR_NEEDKEY(cursor);
 
@@ -82,8 +82,8 @@ __curstat_get_value(WT_CURSOR *cursor, ...)
 	size_t size;
 
 	cst = (WT_CURSOR_STAT *)cursor;
-	CURSOR_API_CALL(cursor, session, get_value, cst->btree);
 	va_start(ap, cursor);
+	CURSOR_API_CALL(cursor, session, get_value, cst->btree);
 
 	WT_CURSOR_NEEDVALUE(cursor);
 
@@ -139,7 +139,7 @@ __curstat_set_key(WT_CURSOR *cursor, ...)
 	else
 		F_CLR(cursor, WT_CURSTD_KEY_SET);
 
-	API_END(session);
+err:	API_END(session);
 }
 
 /*
@@ -226,6 +226,7 @@ static int
 __curstat_reset(WT_CURSOR *cursor)
 {
 	WT_CURSOR_STAT *cst;
+	WT_DECL_RET;
 	WT_SESSION_IMPL *session;
 
 	cst = (WT_CURSOR_STAT *)cursor;
@@ -234,8 +235,8 @@ __curstat_reset(WT_CURSOR *cursor)
 	cst->notpositioned = 1;
 	F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET);
 
-	API_END(session);
-	return (0);
+err:	API_END(session);
+	return (ret);
 }
 
 /*
@@ -293,7 +294,7 @@ __curstat_close(WT_CURSOR *cursor)
 	__wt_free(session, cst->stats);
 	WT_TRET(__wt_cursor_close(cursor));
 
-	API_END(session);
+err:	API_END(session);
 	return (ret);
 }
 
diff --git a/src/cursor/cur_std.c b/src/cursor/cur_std.c
index a77b2c76159..3a724524ddb 100644
--- a/src/cursor/cur_std.c
+++ b/src/cursor/cur_std.c
@@ -288,9 +288,9 @@ __wt_cursor_set_value(WT_CURSOR *cursor, ...)
 	size_t sz;
 	va_list ap;
 
+	va_start(ap, cursor);
 	CURSOR_API_CALL(cursor, session, set_value, NULL);
 
-	va_start(ap, cursor);
 	fmt = F_ISSET(cursor, WT_CURSOR_RAW_OK) ? "u" : cursor->value_format;
 
 	/* Fast path some common cases: single strings, byte arrays and bits. */
@@ -363,7 +363,7 @@ __wt_cursor_close(WT_CURSOR *cursor)
 	__wt_free(session, cursor->uri);
 	__wt_free(session, cursor);
 
-	API_END(session);
+err:	API_END(session);
 	return (ret);
 }
 
diff --git a/src/cursor/cur_table.c b/src/cursor/cur_table.c
index 0c5f2586bc2..25c9b45a8b2 100644
--- a/src/cursor/cur_table.c
+++ b/src/cursor/cur_table.c
@@ -94,6 +94,7 @@ __wt_curtable_get_value(WT_CURSOR *cursor, ...)
 		ret = __wt_schema_project_out(session,
 		    ctable->cg_cursors, ctable->plan, ap);
 	va_end(ap);
+
 err:	API_END(session);
 	return (ret);
 }
@@ -170,7 +171,7 @@ __wt_curtable_set_value(WT_CURSOR *cursor, ...)
 			F_CLR(*cp, WT_CURSTD_VALUE_SET);
 		}
 
-	API_END(session);
+err:	API_END(session);
 }
 
 /*
@@ -216,7 +217,8 @@ __curtable_next(WT_CURSOR *cursor)
 	ctable = (WT_CURSOR_TABLE *)cursor;
 	CURSOR_API_CALL(cursor, session, next, NULL);
 	APPLY_CG(ctable, next);
-	API_END(session);
+
+err:	API_END(session);
 	return (ret);
 }
 
@@ -269,7 +271,8 @@ __curtable_prev(WT_CURSOR *cursor)
 	ctable = (WT_CURSOR_TABLE *)cursor;
 	CURSOR_API_CALL(cursor, session, prev, NULL);
 	APPLY_CG(ctable, prev);
-	API_END(session);
+
+err:	API_END(session);
 	return (ret);
 }
 
@@ -287,7 +290,8 @@ __curtable_reset(WT_CURSOR *cursor)
 	ctable = (WT_CURSOR_TABLE *)cursor;
 	CURSOR_API_CALL(cursor, session, reset, NULL);
 	APPLY_CG(ctable, reset);
-	API_END(session);
+
+err:	API_END(session);
 	return (ret);
 }
 
@@ -305,7 +309,8 @@ __curtable_search(WT_CURSOR *cursor)
 	ctable = (WT_CURSOR_TABLE *)cursor;
 	CURSOR_API_CALL(cursor, session, search, NULL);
 	APPLY_CG(ctable, search);
-	API_END(session);
+
+err:	API_END(session);
 	return (ret);
 }
 
@@ -621,7 +626,7 @@ __curtable_close(WT_CURSOR *cursor)
 	cursor->uri = NULL;
 	WT_TRET(__wt_cursor_close(cursor));
 
-	API_END(session);
+err:	API_END(session);
 	return (ret);
 }
 
diff --git a/src/docs/error.dox b/src/docs/error.dox
index db1013b4c8b..4fa48a7bf86 100644
--- a/src/docs/error.dox
+++ b/src/docs/error.dox
@@ -27,7 +27,7 @@ This error is returned when an error is not covered by a specific error return.
 This error indicates a cursor operation did not find a record to return. This includes search and other operations where no record matched the cursor's search key such as WT_CURSOR::update or WT_CURSOR::remove.
 
 @par <code>WT_PANIC</code>
-This error indicates an underlying problem that requires the application exit the database and restart.
+This error indicates an underlying problem that requires the application exit and restart.
 
 @if IGNORE_BUILT_BY_API_ERR_END
 @endif
diff --git a/src/include/api.h b/src/include/api.h
index 0c5b02e27b7..918e0a32d29 100644
--- a/src/include/api.h
+++ b/src/include/api.h
@@ -283,11 +283,13 @@ struct __wt_connection_impl {
 	(s)->name = #h "." #n;
 
 #define	API_CALL_NOCONF(s, h, n, cur, bt) do {				\
-	API_SESSION_INIT(s, h, n, cur, bt);
+	API_SESSION_INIT(s, h, n, cur, bt);				\
+	WT_ERR(F_ISSET(S2C(s), WT_CONN_PANIC) ? __wt_panic(s) : 0)
 
 #define	API_CALL(s, h, n, cur, bt, cfg, cfgvar) do {			\
 	const char *cfgvar[] = API_CONF_DEFAULTS(h, n, cfg);		\
 	API_SESSION_INIT(s, h, n, cur, bt);				\
+	WT_ERR(F_ISSET(S2C(s), WT_CONN_PANIC) ? __wt_panic(s) : 0);	\
 	WT_ERR(((cfg) != NULL) ?					\
 	    __wt_config_check((s), __wt_confchk_##h##_##n, (cfg), 0) : 0)
 
@@ -386,8 +388,9 @@ extern WT_PROCESS __wt_process;
  * API flags section: BEGIN
  */
 #define	WT_CACHE_POOL_RUN				0x00000001
-#define	WT_CONN_CACHE_POOL				0x00000010
-#define	WT_CONN_LSM_MERGE				0x00000008
+#define	WT_CONN_CACHE_POOL				0x00000020
+#define	WT_CONN_LSM_MERGE				0x00000010
+#define	WT_CONN_PANIC					0x00000008
 #define	WT_CONN_SERVER_RUN				0x00000004
 #define	WT_CONN_SYNC					0x00000002
 #define	WT_CONN_TRANSACTIONAL				0x00000001
diff --git a/src/include/extern.h b/src/include/extern.h
index 6bd9a5f4626..fe62d0ae763 100644
--- a/src/include/extern.h
+++ b/src/include/extern.h
@@ -1165,6 +1165,7 @@ extern void __wt_assert(WT_SESSION_IMPL *session,
     ...) WT_GCC_ATTRIBUTE((format (printf,
     5,
     6)));
+extern int __wt_panic(WT_SESSION_IMPL *session);
 extern int __wt_illegal_value(WT_SESSION_IMPL *session, const char *name);
 extern int __wt_bad_object_type(WT_SESSION_IMPL *session, const char *uri);
 extern int __wt_filename(WT_SESSION_IMPL *session,
diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in
index 3068f61b363..40b6e7ee00b 100644
--- a/src/include/wiredtiger.in
+++ b/src/include/wiredtiger.in
@@ -1450,9 +1450,9 @@ const char *wiredtiger_version(int *majorp, int *minorp, int *patchp);
  * cursor's search key such as WT_CURSOR::update or WT_CURSOR::remove.
  */
 #define	WT_NOTFOUND	-31803
-/*! The application must exit the database.
+/*! WiredTiger library panic.
  * This error indicates an underlying problem that requires the application exit
- * the database and restart.
+ * and restart.
  */
 #define	WT_PANIC	-31804
 /*! @cond internal */
diff --git a/src/lsm/lsm_cursor.c b/src/lsm/lsm_cursor.c
index fc63671b069..8702dab0e22 100644
--- a/src/lsm/lsm_cursor.c
+++ b/src/lsm/lsm_cursor.c
@@ -516,7 +516,8 @@ __clsm_reset(WT_CURSOR *cursor)
 	}
 	F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET);
 	F_CLR(clsm, WT_CLSM_ITERATE_NEXT | WT_CLSM_ITERATE_PREV);
-	API_END(session);
+
+err:	API_END(session);
 	return (ret);
 }
 
@@ -945,7 +946,7 @@ __clsm_close(WT_CURSOR *cursor)
 		__wt_lsm_tree_release(session, clsm->lsm_tree);
 	WT_TRET(__wt_cursor_close(cursor));
 
-	API_END(session);
+err:	API_END(session);
 	return (ret);
 }
 
diff --git a/src/support/err.c b/src/support/err.c
index c332d18478e..768a01c35e1 100644
--- a/src/support/err.c
+++ b/src/support/err.c
@@ -348,25 +348,51 @@ __wt_assert(WT_SESSION_IMPL *session,
 	va_end(ap);
 
 #ifdef HAVE_DIAGNOSTIC
-	__wt_abort(session);
+	__wt_abort(session);			/* Drop core if testing. */
 	/* NOTREACHED */
 #endif
 }
 
 /*
+ * __wt_panic --
+ *	A standard error message when we panic.
+ */
+int
+__wt_panic(WT_SESSION_IMPL *session)
+{
+	F_SET(S2C(session), WT_CONN_PANIC);
+	__wt_errx(session, "%s",
+	    "the WiredTiger library cannot continue; the process must exit "
+	    "and restart");
+
+#ifdef HAVE_DIAGNOSTIC
+	__wt_abort(session);			/* Drop core if testing. */
+	/* NOTREACHED */
+#endif
+	/*
+	 * Chaos reigns within.
+	 * Reflect, repent, and reboot.
+	 * Order shall return.
+	 */
+	return (WT_PANIC);
+}
+
+/*
  * __wt_illegal_value --
- *	Print a standard error message when we detect an illegal value.
+ *	A standard error message when we detect an illegal value.
  */
 int
 __wt_illegal_value(WT_SESSION_IMPL *session, const char *name)
 {
-	WT_RET_MSG(session, WT_ERROR,
-	    "%s%s"
-	    "encountered an illegal file format or internal value; restart "
-	    "the system and verify the underlying files, if corruption is "
-	    "detected use the WT_SESSION salvage method or the wt utility's "
-	    "salvage command to repair the file",
-	    name == NULL ? "" : name, name == NULL ? "" : " ");
+	__wt_errx(session, "%s%s%s",
+	    name == NULL ? "" : name, name == NULL ? "" : ": ",
+	    "encountered an illegal file format or internal value");
+
+#ifdef HAVE_DIAGNOSTIC
+	__wt_abort(session);			/* Drop core if testing. */
+	/* NOTREACHED */
+#endif
+	return (__wt_panic(session));
 }
 
 /*