Split part of the Task struct into a separate struct InCall

The idea is that this leaves Tasks and OSThread in one-to-one
correspondence.  The part of a Task that represents a call into
Haskell from C is split into a separate struct InCall, pointed to by
the Task and the TSO bound to it.  A given OSThread/Task thus always
uses the same mutex and condition variable, rather than getting a new
one for each callback.  Conceptually it is simpler, although there are
more types and indirections in a few places now.

This improves callback performance by removing some of the locks that
we had to take when making in-calls.  Now we also keep the current Task
in a thread-local variable if supported by the OS and gcc (currently
only Linux).

1 files changed, 68 insertions, 65 deletions

diff --git a/rts/Task.h b/rts/Task.h
index 9b5f0253cd..c2b58f2c45 100644
--- a/rts/Task.h
+++ b/rts/Task.h
@@ -17,24 +17,20 @@ BEGIN_RTS_PRIVATE
    Definition of a Task
    --------------------
  
-   A task is an OSThread that runs Haskell code.  Every OSThread
-   created by the RTS for the purposes of running Haskell code is a
-   Task, and OS threads that enter the Haskell RTS for the purposes of
-   making a call-in are also Tasks.
+   A task is an OSThread that runs Haskell code.  Every OSThread that
+   runs inside the RTS, whether as a worker created by the RTS or via
+   an in-call from C to Haskell, has an associated Task.  The first
+   time an OS thread calls into Haskell it is allocated a Task, which
+   remains until the RTS is shut down.
+
+   There is a one-to-one relationship between OSThreads and Tasks.
+   The Task for an OSThread is kept in thread-local storage, and can
+   be retrieved at any time using myTask().
    
    In the THREADED_RTS build, multiple Tasks may all be running
    Haskell code simultaneously. A task relinquishes its Capability
    when it is asked to evaluate an external (C) call.
 
-   In general, there may be multiple Tasks associated with a given OS
-   thread.  A second Task is created when one Task makes a foreign
-   call from Haskell, and subsequently calls back in to Haskell,
-   creating a new bound thread.
-
-   A particular Task structure can belong to more than one OS thread
-   over its lifetime.  This is to avoid creating an unbounded number
-   of Task structures.  The stats just accumulate.
-
    Ownership of Task
    -----------------
 
@@ -59,8 +55,8 @@ BEGIN_RTS_PRIVATE
          (1) a bound Task, the TSO will be on a queue somewhere
 	 (2) a worker task, on the spare_workers queue of task->cap.
 
-     (b) making a foreign call.  The Task will be on the
-         suspended_ccalling_tasks list.
+     (b) making a foreign call.  The InCall will be on the
+         suspended_ccalls list.
 
    We re-establish ownership in each case by respectively
 
@@ -73,9 +69,46 @@ BEGIN_RTS_PRIVATE
           ownership of the Task and a Capability.
 */
 
+// The InCall structure represents either a single in-call from C to
+// Haskell, or a worker thread.
+typedef struct InCall_ {
+    StgTSO *   tso;             // the bound TSO (or NULL for a worker)
+
+    StgTSO *   suspended_tso;   // the TSO is stashed here when we
+				// make a foreign call (NULL otherwise);
+
+    Capability *suspended_cap;  // The capability that the
+                                // suspended_tso is on, because
+                                // we can't read this from the TSO
+                                // without owning a Capability in the
+                                // first place.
+
+    struct Task_ *task;
+
+    // When a Haskell thread makes a foreign call that re-enters
+    // Haskell, we end up with another Task associated with the
+    // current thread.  We have to remember the whole stack of InCalls
+    // associated with the current Task so that we can correctly
+    // save & restore the InCall on entry to and exit from Haskell.
+    struct InCall_ *prev_stack;
+
+    // Links InCalls onto suspended_ccalls, spare_incalls
+    struct InCall_ *prev;
+    struct InCall_ *next;
+} InCall;
+
 typedef struct Task_ {
 #if defined(THREADED_RTS)
     OSThreadId id;		// The OS Thread ID of this task
+
+    Condition cond;             // used for sleeping & waking up this task
+    Mutex lock;			// lock for the condition variable
+
+    // this flag tells the task whether it should wait on task->cond
+    // or just continue immediately.  It's a workaround for the fact
+    // that signalling a condition variable doesn't do anything if the
+    // thread is already running, but we want it to be sticky.
+    rtsBool wakeup;
 #endif
 
     // This points to the Capability that the Task "belongs" to.  If
@@ -92,26 +125,18 @@ typedef struct Task_ {
     // must be held when modifying task->cap.
     struct Capability_ *cap;
 
+    // The current top-of-stack InCall
+    struct InCall_ *incall;
+
+    nat n_spare_incalls;
+    struct InCall_ *spare_incalls;
+
+    rtsBool    worker;          // == rtsTrue if this is a worker Task
     rtsBool    stopped;         // this task has stopped or exited Haskell
-    StgTSO *   suspended_tso;   // the TSO is stashed here when we
-				// make a foreign call (NULL otherwise);
 
-    // The following 3 fields are used by bound threads:
-    StgTSO *   tso;             // the bound TSO (or NULL)
     SchedulerStatus  stat;      // return status
     StgClosure **    ret;       // return value
 
-#if defined(THREADED_RTS)
-    Condition cond;             // used for sleeping & waking up this task
-    Mutex lock;			// lock for the condition variable
-
-    // this flag tells the task whether it should wait on task->cond
-    // or just continue immediately.  It's a workaround for the fact
-    // that signalling a condition variable doesn't do anything if the
-    // thread is already running, but we want it to be sticky.
-    rtsBool wakeup;
-#endif
-
     // Stats that we collect about this task
     // ToDo: we probably want to put this in a separate TaskStats
     // structure, so we can share it between multiple Tasks.  We don't
@@ -125,29 +150,19 @@ typedef struct Task_ {
     Ticks       gc_time;
     Ticks       gc_etime;
 
-    // Links tasks onto various lists. (ToDo: do we need double
-    // linking now?)
-    struct Task_ *prev;
+    // Links tasks on the returning_tasks queue of a Capability, and
+    // on spare_workers.
     struct Task_ *next;
 
-    // Links tasks on the returning_tasks queue of a Capability.
-    struct Task_ *return_link;
-
     // Links tasks on the all_tasks list
     struct Task_ *all_link;
 
-    // When a Haskell thread makes a foreign call that re-enters
-    // Haskell, we end up with another Task associated with the
-    // current thread.  We have to remember the whole stack of Tasks
-    // associated with the current thread so that we can correctly
-    // save & restore the thread-local current task pointer.
-    struct Task_ *prev_stack;
 } Task;
 
 INLINE_HEADER rtsBool
 isBoundTask (Task *task) 
 {
-    return (task->tso != NULL);
+    return (task->incall->tso != NULL);
 }
 
 
@@ -171,11 +186,6 @@ Task *newBoundTask (void);
 //
 void boundTaskExiting (Task *task);
 
-// This must be called when a new Task is associated with the current
-// thread.  It sets up the thread-local current task pointer so that
-// myTask() can work.
-INLINE_HEADER void taskEnter (Task *task);
-
 // Notify the task manager that a task has stopped.  This is used
 // mainly for stats-gathering purposes.
 // Requires: sched_mutex.
@@ -194,7 +204,7 @@ void taskTimeStamp (Task *task);
 // Put the task back on the free list, mark it stopped.  Used by
 // forkProcess().
 //
-void discardTask (Task *task);
+void discardTasksExcept (Task *keep);
 
 // Get the Task associated with the current OS thread (or NULL if none).
 //
@@ -207,8 +217,7 @@ INLINE_HEADER Task *myTask (void);
 // will become the running_task for that Capability.
 // Requires: sched_mutex.
 //
-void startWorkerTask  (struct Capability_ *cap, 
-		       void OSThreadProcAttr (*taskStart)(Task *task));
+void startWorkerTask (Capability *cap);
 
 #endif /* THREADED_RTS */
 
@@ -218,7 +227,13 @@ void startWorkerTask  (struct Capability_ *cap,
 // A thread-local-storage key that we can use to get access to the
 // current thread's Task structure.
 #if defined(THREADED_RTS)
+#if defined(linux_HOST_OS) && \
+    (defined(i386_HOST_ARCH) || defined(x86_64_HOST_ARCH))
+#define MYTASK_USE_TLV
+extern __thread Task *my_task;
+#else
 extern ThreadLocalKey currentTaskKey;
+#endif
 #else
 extern Task *my_task;
 #endif
@@ -232,7 +247,7 @@ extern Task *my_task;
 INLINE_HEADER Task *
 myTask (void)
 {
-#if defined(THREADED_RTS)
+#if defined(THREADED_RTS) && !defined(MYTASK_USE_TLV)
     return getThreadLocalVar(&currentTaskKey);
 #else
     return my_task;
@@ -242,25 +257,13 @@ myTask (void)
 INLINE_HEADER void
 setMyTask (Task *task)
 {
-#if defined(THREADED_RTS)
+#if defined(THREADED_RTS) && !defined(MYTASK_USE_TLV)
     setThreadLocalVar(&currentTaskKey,task);
 #else
     my_task = task;
 #endif
 }
 
-// This must be called when a new Task is associated with the current
-// thread.  It sets up the thread-local current task pointer so that
-// myTask() can work.
-INLINE_HEADER void
-taskEnter (Task *task)
-{
-    // save the current value, just in case this Task has been created
-    // as a result of re-entering the RTS (defaults to NULL):
-    task->prev_stack = myTask();
-    setMyTask(task);
-}
-
 END_RTS_PRIVATE
 
 #endif /* TASK_H */