Issue #310 needs to be able to call a serialization function from inside

reconciliation, which won't work at the moment, because the thread calling
sync sets session.wq_sleeping, the eviction server uses that session
to call reconciliation, reconciliation calls a serialization function,
that clears session.wq_sleeping, and the sync thread never wakes up.
    We don't need any of the workQ stuff any more: remove all of the workQ
data structures from WT_SESSION_IMPL.  Remove the serialization functions,
instead, wrap the function call itself with calls to the serialized
function spinlock.  The serialization functions had some side effects:
it optionally woke the eviction server and then slept on the eviction
server's completion of the task: move that code into the btree sync code,
that's the only place that used it.  Also, the serialization functions
had support for optionally dirtying the page and tree after updating a
page: create a new fuction that does that and call it explicitly from
the serialization functions that need it.
    I'm going to push this separately from issue #310, it's a clean up we
might as well have regardless of where that issue goes.

6 files changed, 84 insertions, 145 deletions

diff --git a/src/include/api.h b/src/include/api.h
index 11ba9656cc7..ebc658fa327 100644
--- a/src/include/api.h
+++ b/src/include/api.h
@@ -99,10 +99,6 @@ struct __wt_session_impl {
 		int line;
 	} *scratch_track;
 #endif
-					/* Serialized operation state */
-	void	*wq_args;		/* Operation arguments */
-	int	wq_sleeping;		/* Thread is blocked */
-	int	wq_ret;			/* Return value */
 
 	WT_TXN_ISOLATION isolation;
 	WT_TXN	txn;			/* Transaction state */
@@ -118,6 +114,7 @@ struct __wt_session_impl {
 #define	WT_SYNC_DISCARD		2	/* Sync the file, discard pages */
 #define	WT_SYNC_DISCARD_NOWRITE	3	/* Discard the file */
 	int syncop;			/* File operation */
+	int syncop_ret;			/* Return value */
 
 	uint32_t id;			/* Offset in conn->session_array */
 
diff --git a/src/include/btree.i b/src/include/btree.i
index e5f01bcb2e6..7335d29a9bf 100644
--- a/src/include/btree.i
+++ b/src/include/btree.i
@@ -155,6 +155,26 @@ __wt_page_modify_set(WT_PAGE *page)
 }
 
 /*
+ * __wt_page_and_tree_modify_set --
+ *	Mark both the page and tree dirty.
+ */
+static inline void
+__wt_page_and_tree_modify_set(WT_SESSION_IMPL *session, WT_PAGE *page)
+{
+	WT_BTREE *btree;
+
+	btree = session->btree;
+
+	/*
+	 * A memory barrier is required for setting the tree's modified value,
+	 * we depend on the barrier called in setting the page's modified value.
+	 */
+	btree->modified = 1;
+
+	__wt_page_modify_set(page);
+}
+
+/*
  * __wt_page_is_modified --
  *	Return if the page is dirty.
  */
diff --git a/src/include/extern.h b/src/include/extern.h
index 25c1b20041c..86f16de8c32 100644
--- a/src/include/extern.h
+++ b/src/include/extern.h
@@ -291,7 +291,7 @@ extern void __wt_page_out(WT_SESSION_IMPL *session,
     uint32_t flags);
 extern void __wt_evict_list_clr_page(WT_SESSION_IMPL *session, WT_PAGE *page);
 extern void __wt_evict_server_wake(WT_SESSION_IMPL *session);
-extern void __wt_sync_file_serial_func(WT_SESSION_IMPL *session);
+extern int __wt_sync_file_serial_func(WT_SESSION_IMPL *session, void *args);
 extern void __wt_evict_page_request(WT_SESSION_IMPL *session, WT_PAGE *page);
 extern void *__wt_cache_evict_server(void *arg);
 extern int __wt_evict_lru_page(WT_SESSION_IMPL *session, int is_app);
@@ -366,7 +366,7 @@ extern int __wt_tree_walk(WT_SESSION_IMPL *session,
 extern int __wt_col_modify(WT_SESSION_IMPL *session,
     WT_CURSOR_BTREE *cbt,
     int op);
-extern void __wt_col_append_serial_func(WT_SESSION_IMPL *session);
+extern int __wt_col_append_serial_func(WT_SESSION_IMPL *session, void *args);
 extern void __wt_col_leaf_obsolete(WT_SESSION_IMPL *session, WT_PAGE *page);
 extern int __wt_col_search(WT_SESSION_IMPL *session,
     WT_CURSOR_BTREE *cbt,
@@ -428,7 +428,7 @@ extern int __wt_row_ikey_alloc(WT_SESSION_IMPL *session,
     const void *key,
     uint32_t size,
     void *ikeyp);
-extern void __wt_row_key_serial_func(WT_SESSION_IMPL *session);
+extern int __wt_row_key_serial_func(WT_SESSION_IMPL *session, void *args);
 extern int __wt_row_modify(WT_SESSION_IMPL *session,
     WT_CURSOR_BTREE *cbt,
     int is_remove);
@@ -437,7 +437,7 @@ extern int __wt_row_insert_alloc(WT_SESSION_IMPL *session,
     u_int skipdepth,
     WT_INSERT **insp,
     size_t *ins_sizep);
-extern void __wt_insert_serial_func(WT_SESSION_IMPL *session);
+extern int __wt_insert_serial_func(WT_SESSION_IMPL *session, void *args);
 extern int __wt_update_check(WT_SESSION_IMPL *session,
     WT_PAGE *page,
     WT_UPDATE *next);
@@ -449,7 +449,7 @@ extern void __wt_update_obsolete(WT_SESSION_IMPL *session,
     WT_PAGE *page,
     WT_UPDATE *upd);
 extern void __wt_row_leaf_obsolete(WT_SESSION_IMPL *session, WT_PAGE *page);
-extern void __wt_update_serial_func(WT_SESSION_IMPL *session);
+extern int __wt_update_serial_func(WT_SESSION_IMPL *session, void *args);
 extern int __wt_search_insert(WT_SESSION_IMPL *session,
     WT_CURSOR_BTREE *cbt,
     WT_INSERT_HEAD *inshead,
diff --git a/src/include/serial.i b/src/include/serial.i
deleted file mode 100644
index 0d04e1f716b..00000000000
--- a/src/include/serial.i
+++ /dev/null
@@ -1,98 +0,0 @@
-/*-
- * Copyright (c) 2008-2012 WiredTiger, Inc.
- *	All rights reserved.
- *
- * See the file LICENSE for redistribution information.
- */
-
-/*
- * Serialization: serialization support allows scheduling operations requiring
- * serialized access to a piece of memory, normally by a different thread of
- * control.  This includes updating and evicting pages from trees.
- *
- * __wt_session_serialize_func --
- *	Schedule a serialization request, and block or spin until it completes.
- */
-static inline int
-__wt_session_serialize_func(WT_SESSION_IMPL *session,
-    wq_state_t op, void (*func)(WT_SESSION_IMPL *), void *args)
-{
-	WT_CONNECTION_IMPL *conn;
-
-	conn = S2C(session);
-
-	/*
-	 * Threads serializing access to data using a function:
-	 *	call the function while holding a spinlock
-	 *	update the session sleeping state, and
-	 *	if necessary, block until an async action completes.
-	 */
-	session->wq_args = args;
-	session->wq_sleeping = (op == WT_SERIAL_EVICT);
-
-	/* Functions are serialized by holding a spinlock. */
-	__wt_spin_lock(session, &conn->serial_lock);
-
-	func(session);
-
-	__wt_spin_unlock(session, &conn->serial_lock);
-
-	switch (op) {
-	case WT_SERIAL_EVICT:
-		__wt_evict_server_wake(session);
-		break;
-	default:
-		break;
-	}
-
-	/*
-	 * If we are waiting on a server thread, block on the session condition
-	 * variable: when the operation is complete, this will be notified and
-	 * we can continue.
-	 */
-	if (session->wq_sleeping)
-		__wt_cond_wait(session, session->cond);
-	return (session->wq_ret);
-}
-
-/*
- * __wt_session_serialize_wrapup --
- *	Server function cleanup.
- */
-static inline void
-__wt_session_serialize_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page, int ret)
-{
-	WT_BTREE *btree;
-
-	btree = session->btree;
-
-	/*
-	 * If passed a page and the return value is OK, we modified the tree
-	 * and the page.
-	 */
-	if (page != NULL && ret == 0) {
-		/*
-		 * A memory barrier is required for setting the tree's modified
-		 * value, we depend on the barrier called in setting the page's
-		 * modified value.
-		 */
-		btree->modified = 1;
-
-		__wt_page_modify_set(page);
-	}
-
-	/*
-	 * Publish: there must be a barrier to ensure the return value is set
-	 * before the calling thread can see its results, and the page's new
-	 * write generation makes it to memory.  The latter isn't a correctness
-	 * issue, the write generation just needs to be updated so that readers
-	 * get credit for reading the right version of the page, otherwise,
-	 * they will have to retry their update for reading an old version of
-	 * the page.
-	 */
-	WT_PUBLISH(session->wq_ret, ret);
-
-	/* If the calling thread is sleeping, wake it up. */
-	if (session->wq_sleeping)
-		__wt_cond_signal(session, session->cond);
-}
diff --git a/src/include/serial_funcs.i b/src/include/serial_funcs.i
index 96f6d613e45..39c1ece4061 100644
--- a/src/include/serial_funcs.i
+++ b/src/include/serial_funcs.i
@@ -65,8 +65,9 @@ __wt_col_append_serial(
 
 	args->skipdepth = skipdepth;
 
-	ret = __wt_session_serialize_func(session,
-	    WT_SERIAL_FUNC, __wt_col_append_serial_func, args);
+	__wt_spin_lock(session, &S2C(session)->serial_lock);
+	ret = __wt_col_append_serial_func(session, args);
+	__wt_spin_unlock(session, &S2C(session)->serial_lock);
 
 	if (!args->new_inslist_taken)
 		__wt_free(session, args->new_inslist);
@@ -79,13 +80,13 @@ __wt_col_append_serial(
 
 static inline void
 __wt_col_append_unpack(
-	WT_SESSION_IMPL *session, WT_PAGE **pagep, uint32_t *write_genp,
+	void *untyped_args, WT_PAGE **pagep, uint32_t *write_genp,
 	WT_INSERT_HEAD ***insheadpp, WT_INSERT ****ins_stackp, WT_INSERT_HEAD
 	***new_inslistp, WT_INSERT_HEAD **new_insheadp, WT_INSERT **new_insp,
 	u_int *skipdepthp)
 {
 	__wt_col_append_args *args =
-	    (__wt_col_append_args *)session->wq_args;
+	    (__wt_col_append_args *)untyped_args;
 
 	*pagep = args->page;
 	*write_genp = args->write_gen;
@@ -98,10 +99,12 @@ __wt_col_append_unpack(
 }
 
 static inline void
-__wt_col_append_new_inslist_taken(WT_SESSION_IMPL *session, WT_PAGE *page)
+__wt_col_append_new_inslist_taken(
+
+	    WT_SESSION_IMPL *session, void *untyped_args, WT_PAGE *page)
 {
 	__wt_col_append_args *args =
-	    (__wt_col_append_args *)session->wq_args;
+	    (__wt_col_append_args *)untyped_args;
 
 	args->new_inslist_taken = 1;
 
@@ -110,10 +113,12 @@ __wt_col_append_new_inslist_taken(WT_SESSION_IMPL *session, WT_PAGE *page)
 }
 
 static inline void
-__wt_col_append_new_inshead_taken(WT_SESSION_IMPL *session, WT_PAGE *page)
+__wt_col_append_new_inshead_taken(
+
+	    WT_SESSION_IMPL *session, void *untyped_args, WT_PAGE *page)
 {
 	__wt_col_append_args *args =
-	    (__wt_col_append_args *)session->wq_args;
+	    (__wt_col_append_args *)untyped_args;
 
 	args->new_inshead_taken = 1;
 
@@ -122,10 +127,12 @@ __wt_col_append_new_inshead_taken(WT_SESSION_IMPL *session, WT_PAGE *page)
 }
 
 static inline void
-__wt_col_append_new_ins_taken(WT_SESSION_IMPL *session, WT_PAGE *page)
+__wt_col_append_new_ins_taken(
+
+	    WT_SESSION_IMPL *session, void *untyped_args, WT_PAGE *page)
 {
 	__wt_col_append_args *args =
-	    (__wt_col_append_args *)session->wq_args;
+	    (__wt_col_append_args *)untyped_args;
 
 	args->new_ins_taken = 1;
 
@@ -198,8 +205,9 @@ __wt_insert_serial(
 
 	args->skipdepth = skipdepth;
 
-	ret = __wt_session_serialize_func(session,
-	    WT_SERIAL_FUNC, __wt_insert_serial_func, args);
+	__wt_spin_lock(session, &S2C(session)->serial_lock);
+	ret = __wt_insert_serial_func(session, args);
+	__wt_spin_unlock(session, &S2C(session)->serial_lock);
 
 	if (!args->new_inslist_taken)
 		__wt_free(session, args->new_inslist);
@@ -212,13 +220,13 @@ __wt_insert_serial(
 
 static inline void
 __wt_insert_unpack(
-	WT_SESSION_IMPL *session, WT_PAGE **pagep, uint32_t *write_genp,
+	void *untyped_args, WT_PAGE **pagep, uint32_t *write_genp,
 	WT_INSERT_HEAD ***insheadp, WT_INSERT ****ins_stackp, WT_INSERT_HEAD
 	***new_inslistp, WT_INSERT_HEAD **new_insheadp, WT_INSERT **new_insp,
 	u_int *skipdepthp)
 {
 	__wt_insert_args *args =
-	    (__wt_insert_args *)session->wq_args;
+	    (__wt_insert_args *)untyped_args;
 
 	*pagep = args->page;
 	*write_genp = args->write_gen;
@@ -231,10 +239,12 @@ __wt_insert_unpack(
 }
 
 static inline void
-__wt_insert_new_inslist_taken(WT_SESSION_IMPL *session, WT_PAGE *page)
+__wt_insert_new_inslist_taken(
+
+	    WT_SESSION_IMPL *session, void *untyped_args, WT_PAGE *page)
 {
 	__wt_insert_args *args =
-	    (__wt_insert_args *)session->wq_args;
+	    (__wt_insert_args *)untyped_args;
 
 	args->new_inslist_taken = 1;
 
@@ -243,10 +253,12 @@ __wt_insert_new_inslist_taken(WT_SESSION_IMPL *session, WT_PAGE *page)
 }
 
 static inline void
-__wt_insert_new_inshead_taken(WT_SESSION_IMPL *session, WT_PAGE *page)
+__wt_insert_new_inshead_taken(
+
+	    WT_SESSION_IMPL *session, void *untyped_args, WT_PAGE *page)
 {
 	__wt_insert_args *args =
-	    (__wt_insert_args *)session->wq_args;
+	    (__wt_insert_args *)untyped_args;
 
 	args->new_inshead_taken = 1;
 
@@ -255,10 +267,12 @@ __wt_insert_new_inshead_taken(WT_SESSION_IMPL *session, WT_PAGE *page)
 }
 
 static inline void
-__wt_insert_new_ins_taken(WT_SESSION_IMPL *session, WT_PAGE *page)
+__wt_insert_new_ins_taken(
+
+	    WT_SESSION_IMPL *session, void *untyped_args, WT_PAGE *page)
 {
 	__wt_insert_args *args =
-	    (__wt_insert_args *)session->wq_args;
+	    (__wt_insert_args *)untyped_args;
 
 	args->new_ins_taken = 1;
 
@@ -286,19 +300,20 @@ __wt_row_key_serial(
 
 	args->ikey = ikey;
 
-	ret = __wt_session_serialize_func(session,
-	    WT_SERIAL_FUNC, __wt_row_key_serial_func, args);
+	__wt_spin_lock(session, &S2C(session)->serial_lock);
+	ret = __wt_row_key_serial_func(session, args);
+	__wt_spin_unlock(session, &S2C(session)->serial_lock);
 
 	return (ret);
 }
 
 static inline void
 __wt_row_key_unpack(
-	WT_SESSION_IMPL *session, WT_PAGE **pagep, WT_ROW **row_argp, WT_IKEY
+	void *untyped_args, WT_PAGE **pagep, WT_ROW **row_argp, WT_IKEY
 	**ikeyp)
 {
 	__wt_row_key_args *args =
-	    (__wt_row_key_args *)session->wq_args;
+	    (__wt_row_key_args *)untyped_args;
 
 	*pagep = args->page;
 	*row_argp = args->row_arg;
@@ -318,18 +333,19 @@ __wt_sync_file_serial(
 
 	args->syncop = syncop;
 
-	ret = __wt_session_serialize_func(session,
-	    WT_SERIAL_EVICT, __wt_sync_file_serial_func, args);
+	__wt_spin_lock(session, &S2C(session)->serial_lock);
+	ret = __wt_sync_file_serial_func(session, args);
+	__wt_spin_unlock(session, &S2C(session)->serial_lock);
 
 	return (ret);
 }
 
 static inline void
 __wt_sync_file_unpack(
-	WT_SESSION_IMPL *session, int *syncopp)
+	void *untyped_args, int *syncopp)
 {
 	__wt_sync_file_args *args =
-	    (__wt_sync_file_args *)session->wq_args;
+	    (__wt_sync_file_args *)untyped_args;
 
 	*syncopp = args->syncop;
 }
@@ -379,8 +395,9 @@ __wt_update_serial(
 	}
 	args->upd_taken = 0;
 
-	ret = __wt_session_serialize_func(session,
-	    WT_SERIAL_FUNC, __wt_update_serial_func, args);
+	__wt_spin_lock(session, &S2C(session)->serial_lock);
+	ret = __wt_update_serial_func(session, args);
+	__wt_spin_unlock(session, &S2C(session)->serial_lock);
 
 	if (!args->new_upd_taken)
 		__wt_free(session, args->new_upd);
@@ -391,11 +408,11 @@ __wt_update_serial(
 
 static inline void
 __wt_update_unpack(
-	WT_SESSION_IMPL *session, WT_PAGE **pagep, uint32_t *write_genp,
-	WT_UPDATE ***srch_updp, WT_UPDATE ***new_updp, WT_UPDATE **updp)
+	void *untyped_args, WT_PAGE **pagep, uint32_t *write_genp, WT_UPDATE
+	***srch_updp, WT_UPDATE ***new_updp, WT_UPDATE **updp)
 {
 	__wt_update_args *args =
-	    (__wt_update_args *)session->wq_args;
+	    (__wt_update_args *)untyped_args;
 
 	*pagep = args->page;
 	*write_genp = args->write_gen;
@@ -405,10 +422,12 @@ __wt_update_unpack(
 }
 
 static inline void
-__wt_update_new_upd_taken(WT_SESSION_IMPL *session, WT_PAGE *page)
+__wt_update_new_upd_taken(
+
+	    WT_SESSION_IMPL *session, void *untyped_args, WT_PAGE *page)
 {
 	__wt_update_args *args =
-	    (__wt_update_args *)session->wq_args;
+	    (__wt_update_args *)untyped_args;
 
 	args->new_upd_taken = 1;
 
@@ -417,10 +436,12 @@ __wt_update_new_upd_taken(WT_SESSION_IMPL *session, WT_PAGE *page)
 }
 
 static inline void
-__wt_update_upd_taken(WT_SESSION_IMPL *session, WT_PAGE *page)
+__wt_update_upd_taken(
+
+	    WT_SESSION_IMPL *session, void *untyped_args, WT_PAGE *page)
 {
 	__wt_update_args *args =
-	    (__wt_update_args *)session->wq_args;
+	    (__wt_update_args *)untyped_args;
 
 	args->upd_taken = 1;
 
diff --git a/src/include/wt_internal.h b/src/include/wt_internal.h
index b09873e68d4..890ba369c7c 100644
--- a/src/include/wt_internal.h
+++ b/src/include/wt_internal.h
@@ -227,7 +227,6 @@ struct __wt_update;
 #include "log.i"
 #include "mutex.i"
 #include "packing.i"
-#include "serial.i"
 #include "serial_funcs.i"
 
 #if defined(__cplusplus)