Expose all EventLog events as DTrace probes

- Defines a DTrace provider, called 'HaskellEvent', that provides a probe
  for every event of the eventlog framework.
- In contrast to the original eventlog, the DTrace probes are available in
  all flavours of the runtime system (DTrace probes have virtually no
  overhead if not enabled); when -DTRACING is defined both the regular
  event log as well as DTrace probes can be used.
- Currently, Mac OS X only.  User-space DTrace probes are implemented
  differently on Mac OS X than in the original DTrace implementation.
  Nevertheless, it shouldn't be too hard to enable these probes on other
  platforms, too.
- Documentation is at http://hackage.haskell.org/trac/ghc/wiki/DTrace

13 files changed, 381 insertions, 29 deletions

diff --git a/rts/Capability.c b/rts/Capability.c
index f4fdd70ed9..bd781e9cd7 100644
--- a/rts/Capability.c
+++ b/rts/Capability.c
@@ -98,7 +98,7 @@ findSpark (Capability *cap)
           cap->sparks_converted++;
 
           // Post event for running a spark from capability's own pool.
-          traceSchedEvent(cap, EVENT_RUN_SPARK, cap->r.rCurrentTSO, 0);
+          traceEventRunSpark(cap, cap->r.rCurrentTSO);
 
           return spark;
       }
@@ -132,8 +132,7 @@ findSpark (Capability *cap)
           if (spark != NULL) {
               cap->sparks_converted++;
 
-              traceSchedEvent(cap, EVENT_STEAL_SPARK, 
-                              cap->r.rCurrentTSO, robbed->no);
+              traceEventStealSpark(cap, cap->r.rCurrentTSO, robbed->no);
               
               return spark;
           }
@@ -579,9 +578,9 @@ yieldCapability (Capability** pCap, Task *task)
     Capability *cap = *pCap;
 
     if (waiting_for_gc == PENDING_GC_PAR) {
-        traceSchedEvent(cap, EVENT_GC_START, 0, 0);
+        traceEventGcStart(cap);
         gcWorkerThread(cap);
-        traceSchedEvent(cap, EVENT_GC_END, 0, 0);
+        traceEventGcEnd(cap);
         return;
     }
 
@@ -788,7 +787,7 @@ shutdownCapability (Capability *cap, Task *task, rtsBool safe)
             continue;
         }
             
-        traceSchedEvent(cap, EVENT_SHUTDOWN, 0, 0);
+        traceEventShutdown(cap);
 	RELEASE_LOCK(&cap->lock);
 	break;
     }
diff --git a/rts/PrimOps.cmm b/rts/PrimOps.cmm
index b4dfb6ddc2..377418af41 100644
--- a/rts/PrimOps.cmm
+++ b/rts/PrimOps.cmm
@@ -1903,8 +1903,23 @@ stg_traceEventzh
 {
    W_ msg;
    msg = R1;
+
 #if defined(TRACING) || defined(DEBUG)
+
    foreign "C" traceUserMsg(MyCapability() "ptr", msg "ptr") [];
+
+#elif defined(DTRACE)
+
+   W_ enabled;
+
+   // We should go through the macro HASKELLEVENT_USER_MSG_ENABLED from
+   // RtsProbes.h, but that header file includes unistd.h, which doesn't
+   // work in Cmm
+   (enabled) = foreign "C" __dtrace_isenabled$HaskellEvent$user__msg$v1() [];
+   if (enabled != 0) {
+     foreign "C" dtraceUserMsgWrapper(MyCapability() "ptr", msg "ptr") [];
+   }
+
 #endif
    jump %ENTRY_CODE(Sp(0));
 }
diff --git a/rts/RtsProbes.d b/rts/RtsProbes.d
new file mode 100644
index 0000000000..87a34c8dca
--- /dev/null
+++ b/rts/RtsProbes.d
@@ -0,0 +1,62 @@
+/* -----------------------------------------------------------------------------
+ *
+ * (c) The GHC Team, 2009
+ *
+ * User-space dtrace probes for the runtime system.
+ *
+ * ---------------------------------------------------------------------------*/
+
+#include "HsFFI.h"
+#include "rts/EventLogFormat.h"
+
+
+// -----------------------------------------------------------------------------
+// Payload datatypes for Haskell events
+// -----------------------------------------------------------------------------
+
+// We effectively have:
+//
+// typedef uint16_t EventTypeNum;
+// typedef uint64_t EventTimestamp;   // in nanoseconds
+// typedef uint32_t EventThreadID;
+// typedef uint16_t EventCapNo;
+// typedef uint16_t EventPayloadSize; // variable-size events
+// typedef uint16_t EventThreadStatus;
+
+
+// -----------------------------------------------------------------------------
+// The HaskellEvent provider captures everything from eventlog for use with
+// dtrace
+// -----------------------------------------------------------------------------
+
+// These probes correspond to the events defined in EventLogFormat.h
+//
+provider HaskellEvent {
+
+  // scheduler events
+  probe create__thread (EventCapNo, EventThreadID);
+  probe run__thread (EventCapNo, EventThreadID);
+  probe stop__thread (EventCapNo, EventThreadID, EventThreadStatus);
+  probe thread__runnable (EventCapNo, EventThreadID);
+  probe migrate__thread (EventCapNo, EventThreadID, EventCapNo);
+  probe run__spark (EventCapNo, EventThreadID);
+  probe steal__spark (EventCapNo, EventThreadID, EventCapNo);
+  probe shutdown (EventCapNo);
+  probe thread_wakeup (EventCapNo, EventThreadID, EventCapNo);
+  probe gc__start (EventCapNo);
+  probe gc__end (EventCapNo);
+  probe request__seq__gc (EventCapNo);
+  probe request__par__gc (EventCapNo);
+  probe create__spark__thread (EventCapNo, EventThreadID);
+
+  // other events
+//This one doesn't seem to be used at all at the moment:
+//  probe log__msg (char *);
+  probe startup (EventCapNo);
+  // we don't need EVENT_BLOCK_MARKER with dtrace
+  probe user__msg (EventCapNo, char *);
+  probe gc__idle (EventCapNo);
+  probe gc__work (EventCapNo);
+  probe gc__done (EventCapNo);
+
+};
diff --git a/rts/RtsStartup.c b/rts/RtsStartup.c
index afa38aaf2c..b85b15305f 100644
--- a/rts/RtsStartup.c
+++ b/rts/RtsStartup.c
@@ -149,6 +149,9 @@ hs_init(int *argc, char **argv[])
 #ifdef TRACING
     initTracing();
 #endif
+    /* Dtrace events are always enabled
+     */
+    dtraceEventStartup();
 
     /* initialise scheduler data structures (needs to be done before
      * initStorage()).
diff --git a/rts/Schedule.c b/rts/Schedule.c
index 3f07d3cb11..cfdb392168 100644
--- a/rts/Schedule.c
+++ b/rts/Schedule.c
@@ -468,7 +468,7 @@ run_thread:
     }
 #endif
 
-    traceSchedEvent(cap, EVENT_RUN_THREAD, t, 0);
+    traceEventRunThread(cap, t);
 
     switch (prev_what_next) {
 	
@@ -518,7 +518,7 @@ run_thread:
     t->saved_winerror = GetLastError();
 #endif
 
-    traceSchedEvent (cap, EVENT_STOP_THREAD, t, ret);
+    traceEventStopThread(cap, t, ret);
 
 #if defined(THREADED_RTS)
     // If ret is ThreadBlocked, and this Task is bound to the TSO that
@@ -778,7 +778,7 @@ schedulePushWork(Capability *cap USED_IF_THREADS,
 		    debugTrace(DEBUG_sched, "pushing thread %lu to capability %d", (unsigned long)t->id, free_caps[i]->no);
 		    appendToRunQueue(free_caps[i],t);
 
-                    traceSchedEvent (cap, EVENT_MIGRATE_THREAD, t, free_caps[i]->no);
+            traceEventMigrateThread (cap, t, free_caps[i]->no);
 
 		    if (t->bound) { t->bound->cap = free_caps[i]; }
 		    t->cap = free_caps[i];
@@ -802,7 +802,7 @@ schedulePushWork(Capability *cap USED_IF_THREADS,
 		    if (spark != NULL) {
 			debugTrace(DEBUG_sched, "pushing spark %p to capability %d", spark, free_caps[i]->no);
 
-      traceSchedEvent(free_caps[i], EVENT_STEAL_SPARK, t, cap->no);
+            traceEventStealSpark(free_caps[i], t, cap->no);
 
 			newSpark(&(free_caps[i]->r), spark);
 		    }
@@ -1418,11 +1418,11 @@ scheduleDoGC (Capability *cap, Task *task USED_IF_THREADS, rtsBool force_major)
     
     if (gc_type == PENDING_GC_SEQ)
     {
-        traceSchedEvent(cap, EVENT_REQUEST_SEQ_GC, 0, 0);
+        traceEventRequestSeqGc(cap);
     }
     else
     {
-        traceSchedEvent(cap, EVENT_REQUEST_PAR_GC, 0, 0);
+        traceEventRequestParGc(cap);
         debugTrace(DEBUG_sched, "ready_to_gc, grabbing GC threads");
     }
 
@@ -1478,8 +1478,8 @@ delete_threads_and_gc:
     
     heap_census = scheduleNeedHeapProfile(rtsTrue);
 
+    traceEventGcStart(cap);
 #if defined(THREADED_RTS)
-    traceSchedEvent(cap, EVENT_GC_START, 0, 0);
     // reset waiting_for_gc *before* GC, so that when the GC threads
     // emerge they don't immediately re-enter the GC.
     waiting_for_gc = 0;
@@ -1487,7 +1487,7 @@ delete_threads_and_gc:
 #else
     GarbageCollect(force_major || heap_census, 0, cap);
 #endif
-    traceSchedEvent(cap, EVENT_GC_END, 0, 0);
+    traceEventGcEnd(cap);
 
     if (recent_activity == ACTIVITY_INACTIVE && force_major)
     {
@@ -1806,7 +1806,7 @@ suspendThread (StgRegTable *reg)
   task = cap->running_task;
   tso = cap->r.rCurrentTSO;
 
-  traceSchedEvent(cap, EVENT_STOP_THREAD, tso, THREAD_SUSPENDED_FOREIGN_CALL);
+  traceEventStopThread(cap, tso, THREAD_SUSPENDED_FOREIGN_CALL);
 
   // XXX this might not be necessary --SDM
   tso->what_next = ThreadRunGHC;
@@ -1869,7 +1869,7 @@ resumeThread (void *task_)
     task->suspended_tso = NULL;
     tso->_link = END_TSO_QUEUE; // no write barrier reqd
 
-    traceSchedEvent(cap, EVENT_RUN_THREAD, tso, tso->what_next);
+    traceEventRunThread(cap, tso);
     
     if (tso->why_blocked == BlockedOnCCall) {
         // avoid locking the TSO if we don't have to
@@ -1925,7 +1925,7 @@ scheduleThreadOn(Capability *cap, StgWord cpu USED_IF_THREADS, StgTSO *tso)
     if (cpu == cap->no) {
 	appendToRunQueue(cap,tso);
     } else {
-        traceSchedEvent (cap, EVENT_MIGRATE_THREAD, tso, capabilities[cpu].no);
+        traceEventMigrateThread (cap, tso, capabilities[cpu].no);
 	wakeupThreadOnCapability(cap, &capabilities[cpu], tso);
     }
 #else
diff --git a/rts/Schedule.h b/rts/Schedule.h
index 5f669b3d83..6751144be8 100644
--- a/rts/Schedule.h
+++ b/rts/Schedule.h
@@ -139,7 +139,7 @@ appendToRunQueue (Capability *cap, StgTSO *tso)
 	setTSOLink(cap, cap->run_queue_tl, tso);
     }
     cap->run_queue_tl = tso;
-    traceSchedEvent (cap, EVENT_THREAD_RUNNABLE, tso, 0);
+    traceEventThreadRunnable (cap, tso);
 }
 
 /* Push a thread on the beginning of the run queue.
diff --git a/rts/Sparks.c b/rts/Sparks.c
index a17b78ce57..e5e6d7e955 100644
--- a/rts/Sparks.c
+++ b/rts/Sparks.c
@@ -47,7 +47,7 @@ createSparkThread (Capability *cap)
     tso = createIOThread (cap, RtsFlags.GcFlags.initialStkSize, 
                           &base_GHCziConc_runSparks_closure);
 
-    traceSchedEvent(cap, EVENT_CREATE_SPARK_THREAD, 0, tso->id);
+    traceEventCreateSparkThread(cap, tso->id);
 
     appendToRunQueue(cap,tso);
 }
diff --git a/rts/Threads.c b/rts/Threads.c
index 799cf9062a..8eaa951a2d 100644
--- a/rts/Threads.c
+++ b/rts/Threads.c
@@ -107,7 +107,7 @@ createThread(Capability *cap, nat size)
     RELEASE_LOCK(&sched_mutex);
     
     // ToDo: report the stack size in the event?
-    traceSchedEvent (cap, EVENT_CREATE_THREAD, tso, tso->stack_size);
+    traceEventCreateThread(cap, tso);
 
     return tso;
 }
@@ -254,7 +254,7 @@ unblockOne_ (Capability *cap, StgTSO *tso,
   cap->context_switch = 1;
 #endif
 
-  traceSchedEvent (cap, EVENT_THREAD_WAKEUP, tso, tso->cap->no);
+  traceEventThreadWakeup (cap, tso, tso->cap->no);
 
   return next;
 }
diff --git a/rts/Trace.c b/rts/Trace.c
index 7cfb78cc9a..a1da9911a7 100644
--- a/rts/Trace.c
+++ b/rts/Trace.c
@@ -9,10 +9,11 @@
 // external headers
 #include "Rts.h"
 
-#ifdef TRACING
-
 // internal headers
 #include "Trace.h"
+
+#ifdef TRACING
+
 #include "GetTime.h"
 #include "Stats.h"
 #include "eventlog/EventLog.h"
@@ -310,6 +311,7 @@ void traceUserMsg(Capability *cap, char *msg)
             postUserMsg(cap, msg);
         }
     }
+    dtraceUserMsg(cap->no, msg);
 }
 
 void traceThreadStatus_ (StgTSO *tso USED_IF_DEBUG)
@@ -345,3 +347,15 @@ void traceEnd (void)
 #endif /* DEBUG */
 
 #endif /* TRACING */
+
+// If DTRACE is enabled, but neither DEBUG nor TRACING, we need a C land
+// wrapper for the user-msg probe (as we can't expand that in PrimOps.cmm)
+//
+#if !defined(DEBUG) && !defined(TRACING) && defined(DTRACE)
+
+void dtraceUserMsgWrapper(Capability *cap, char *msg)
+{
+    dtraceUserMsg(cap->no, msg);
+}
+
+#endif /* !defined(DEBUG) && !defined(TRACING) && defined(DTRACE) */
diff --git a/rts/Trace.h b/rts/Trace.h
index 21828d261f..f8b6ad497d 100644
--- a/rts/Trace.h
+++ b/rts/Trace.h
@@ -2,7 +2,7 @@
  *
  * (c) The GHC Team, 2008-2009
  *
- * Support for fast binary event logging.
+ * Support for fast binary event logging and user-space dtrace probes.
  *
  * ---------------------------------------------------------------------------*/
 
@@ -12,6 +12,10 @@
 #include "rts/EventLogFormat.h"
 #include "Capability.h"
 
+#if defined(DTRACE)
+#include "RtsProbes.h"
+#endif /* defined(DTRACE) */
+
 BEGIN_RTS_PRIVATE
 
 // -----------------------------------------------------------------------------
@@ -152,6 +156,237 @@ void traceThreadStatus_ (StgTSO *tso);
 
 #endif /* TRACING */
 
+// If DTRACE is enabled, but neither DEBUG nor TRACING, we need a C land
+// wrapper for the user-msg probe (as we can't expand that in PrimOps.cmm)
+//
+#if !defined(DEBUG) && !defined(TRACING) && defined(DTRACE)
+
+void dtraceUserMsgWrapper(Capability *cap, char *msg);
+
+#endif /* !defined(DEBUG) && !defined(TRACING) && defined(DTRACE) */
+
+// -----------------------------------------------------------------------------
+// Aliases for static dtrace probes if dtrace is available
+// -----------------------------------------------------------------------------
+
+#if defined(DTRACE)
+
+#define dtraceCreateThread(cap, tid)                    \
+    HASKELLEVENT_CREATE_THREAD(cap, tid)
+#define dtraceRunThread(cap, tid)                       \
+    HASKELLEVENT_RUN_THREAD(cap, tid)
+#define dtraceStopThread(cap, tid, status)              \
+    HASKELLEVENT_STOP_THREAD(cap, tid, status)
+#define dtraceThreadRunnable(cap, tid)                  \
+    HASKELLEVENT_THREAD_RUNNABLE(cap, tid)
+#define dtraceMigrateThread(cap, tid, new_cap)          \
+    HASKELLEVENT_MIGRATE_THREAD(cap, tid, new_cap)
+#define dtraceRunSpark(cap, tid)                        \
+    HASKELLEVENT_RUN_SPARK(cap, tid)
+#define dtraceStealSpark(cap, tid, victim_cap)          \
+    HASKELLEVENT_STEAL_SPARK(cap, tid, victim_cap)
+#define dtraceShutdown(cap)                             \
+    HASKELLEVENT_SHUTDOWN(cap)
+#define dtraceThreadWakeup(cap, tid, other_cap)         \
+    HASKELLEVENT_THREAD_WAKEUP(cap, tid, other_cap)
+#define dtraceGcStart(cap)                              \
+    HASKELLEVENT_GC_START(cap)
+#define dtraceGcEnd(cap)                                \
+    HASKELLEVENT_GC_END(cap)
+#define dtraceRequestSeqGc(cap)                         \
+    HASKELLEVENT_REQUEST_SEQ_GC(cap)
+#define dtraceRequestParGc(cap)                         \
+    HASKELLEVENT_REQUEST_PAR_GC(cap)
+#define dtraceCreateSparkThread(cap, spark_tid)         \
+    HASKELLEVENT_CREATE_SPARK_THREAD(cap, spark_tid)
+#define dtraceStartup(num_caps)                         \
+    HASKELLEVENT_STARTUP(num_caps)
+#define dtraceUserMsg(cap, msg)                         \
+    HASKELLEVENT_USER_MSG(cap, msg)
+#define dtraceGcIdle(cap)                               \
+    HASKELLEVENT_GC_IDLE(cap)
+#define dtraceGcWork(cap)                               \
+    HASKELLEVENT_GC_WORK(cap)
+#define dtraceGcDone(cap)                               \
+    HASKELLEVENT_GC_DONE(cap)
+
+#else /* !defined(DTRACE) */
+
+#define dtraceCreateThread(cap, tid)                    /* nothing */
+#define dtraceRunThread(cap, tid)                       /* nothing */
+#define dtraceStopThread(cap, tid, status)              /* nothing */
+#define dtraceThreadRunnable(cap, tid)                  /* nothing */
+#define dtraceMigrateThread(cap, tid, new_cap)          /* nothing */
+#define dtraceRunSpark(cap, tid)                        /* nothing */
+#define dtraceStealSpark(cap, tid, victim_cap)          /* nothing */
+#define dtraceShutdown(cap)                             /* nothing */
+#define dtraceThreadWakeup(cap, tid, other_cap)         /* nothing */
+#define dtraceGcStart(cap)                              /* nothing */
+#define dtraceGcEnd(cap)                                /* nothing */
+#define dtraceRequestSeqGc(cap)                         /* nothing */
+#define dtraceRequestParGc(cap)                         /* nothing */
+#define dtraceCreateSparkThread(cap, spark_tid)         /* nothing */
+#define dtraceStartup(num_caps)                         /* nothing */
+#define dtraceUserMsg(cap, msg)                         /* nothing */
+#define dtraceGcIdle(cap)                               /* nothing */
+#define dtraceGcWork(cap)                               /* nothing */
+#define dtraceGcDone(cap)                               /* nothing */
+
+#endif
+
+// -----------------------------------------------------------------------------
+// Trace probes dispatching to various tracing frameworks
+//
+// In order to avoid accumulating multiple calls to tracing calls at trace
+// points, we define inline probe functions that contain the various
+// invocations.
+//
+// Dtrace - dtrace probes are unconditionally added as probe activation is
+//   handled by the dtrace component of the kernel, and inactive probes are
+//   very cheap — usually, one no-op.  Consequently, dtrace can be used with
+//   all flavours of the RTS.  In addition, we still support logging events to
+//   a file, even in the presence of dtrace.  This is, eg, useful when tracing
+//   on a server, but browsing trace information with ThreadScope on a local
+//   client.
+// 
+// -----------------------------------------------------------------------------
+
+INLINE_HEADER void traceEventCreateThread(Capability *cap STG_UNUSED, 
+                                          StgTSO     *tso STG_UNUSED)
+{
+    traceSchedEvent(cap, EVENT_CREATE_THREAD, tso, tso->stack_size);
+    dtraceCreateThread((EventCapNo)cap->no, (EventThreadID)tso->id);
+}
+
+INLINE_HEADER void traceEventRunThread(Capability *cap STG_UNUSED, 
+                                       StgTSO     *tso STG_UNUSED)
+{
+    traceSchedEvent(cap, EVENT_RUN_THREAD, tso, tso->what_next);
+    dtraceRunThread((EventCapNo)cap->no, (EventThreadID)tso->id);
+}
+
+INLINE_HEADER void traceEventStopThread(Capability          *cap    STG_UNUSED, 
+                                        StgTSO              *tso    STG_UNUSED, 
+                                        StgThreadReturnCode  status STG_UNUSED)
+{
+    traceSchedEvent(cap, EVENT_STOP_THREAD, tso, status);
+    dtraceStopThread((EventCapNo)cap->no, (EventThreadID)tso->id,
+                     (EventThreadStatus)status);
+}
+
+// needs to be EXTERN_INLINE as it is used in another EXTERN_INLINE function
+EXTERN_INLINE void traceEventThreadRunnable(Capability *cap STG_UNUSED, 
+                                            StgTSO     *tso STG_UNUSED);
+
+EXTERN_INLINE void traceEventThreadRunnable(Capability *cap STG_UNUSED, 
+                                            StgTSO     *tso STG_UNUSED)
+{
+    traceSchedEvent(cap, EVENT_THREAD_RUNNABLE, tso, 0);
+    dtraceThreadRunnable((EventCapNo)cap->no, (EventThreadID)tso->id);
+}
+
+INLINE_HEADER void traceEventMigrateThread(Capability *cap     STG_UNUSED, 
+                                           StgTSO     *tso     STG_UNUSED,
+                                           nat         new_cap STG_UNUSED)
+{
+    traceSchedEvent(cap, EVENT_MIGRATE_THREAD, tso, new_cap);
+    dtraceMigrateThread((EventCapNo)cap->no, (EventThreadID)tso->id,
+                        (EventCapNo)new_cap);
+}
+
+INLINE_HEADER void traceEventRunSpark(Capability *cap STG_UNUSED, 
+                                      StgTSO     *tso STG_UNUSED)
+{
+    traceSchedEvent(cap, EVENT_RUN_SPARK, tso, 0);
+    dtraceRunSpark((EventCapNo)cap->no, (EventThreadID)tso->id);
+}
+
+INLINE_HEADER void traceEventStealSpark(Capability *cap        STG_UNUSED, 
+                                        StgTSO     *tso        STG_UNUSED,
+                                        nat         victim_cap STG_UNUSED)
+{
+    traceSchedEvent(cap, EVENT_STEAL_SPARK, tso, victim_cap);
+    dtraceStealSpark((EventCapNo)cap->no, (EventThreadID)tso->id,
+                     (EventCapNo)victim_cap);
+}
+
+INLINE_HEADER void traceEventShutdown(Capability *cap STG_UNUSED)
+{
+    traceSchedEvent(cap, EVENT_SHUTDOWN, 0, 0);
+    dtraceShutdown((EventCapNo)cap->no);
+}
+
+INLINE_HEADER void traceEventThreadWakeup(Capability *cap       STG_UNUSED, 
+                                          StgTSO     *tso       STG_UNUSED,
+                                          nat         other_cap STG_UNUSED)
+{
+    traceSchedEvent(cap, EVENT_THREAD_WAKEUP, tso, other_cap);
+    dtraceThreadWakeup((EventCapNo)cap->no, (EventThreadID)tso->id,
+                       (EventCapNo)other_cap);
+}
+
+INLINE_HEADER void traceEventGcStart(Capability *cap STG_UNUSED)
+{
+    traceSchedEvent(cap, EVENT_GC_START, 0, 0);
+    dtraceGcStart((EventCapNo)cap->no);
+}
+
+INLINE_HEADER void traceEventGcEnd(Capability *cap STG_UNUSED)
+{
+    traceSchedEvent(cap, EVENT_GC_END, 0, 0);
+    dtraceGcEnd((EventCapNo)cap->no);
+}
+
+INLINE_HEADER void traceEventRequestSeqGc(Capability *cap STG_UNUSED)
+{
+    traceSchedEvent(cap, EVENT_REQUEST_SEQ_GC, 0, 0);
+    dtraceRequestSeqGc((EventCapNo)cap->no);
+}
+
+INLINE_HEADER void traceEventRequestParGc(Capability *cap STG_UNUSED)
+{
+    traceSchedEvent(cap, EVENT_REQUEST_PAR_GC, 0, 0);
+    dtraceRequestParGc((EventCapNo)cap->no);
+}
+
+INLINE_HEADER void traceEventCreateSparkThread(Capability  *cap      STG_UNUSED, 
+                                               StgThreadID spark_tid STG_UNUSED)
+{
+    traceSchedEvent(cap, EVENT_CREATE_SPARK_THREAD, 0, spark_tid);
+    dtraceCreateSparkThread((EventCapNo)cap->no, (EventThreadID)spark_tid);
+}
+
+// This applies only to dtrace as EVENT_STARTUP in the logging framework is
+// handled specially in 'EventLog.c'.
+//
+INLINE_HEADER void dtraceEventStartup(void)
+{
+#ifdef THREADED_RTS
+    // XXX n_capabilities hasn't been initislised yet
+    dtraceStartup(RtsFlags.ParFlags.nNodes);
+#else
+    dtraceStartup(1);
+#endif
+}
+
+INLINE_HEADER void traceEventGcIdle(Capability *cap STG_UNUSED)
+{
+    traceEvent(cap, EVENT_GC_IDLE);
+    dtraceGcIdle((EventCapNo)cap->no);
+}
+
+INLINE_HEADER void traceEventGcWork(Capability *cap STG_UNUSED)
+{
+    traceEvent(cap, EVENT_GC_WORK);
+    dtraceGcWork((EventCapNo)cap->no);
+}
+
+INLINE_HEADER void traceEventGcDone(Capability *cap STG_UNUSED)
+{
+    traceEvent(cap, EVENT_GC_DONE);
+    dtraceGcDone((EventCapNo)cap->no);
+}
+
 END_RTS_PRIVATE
 
 #endif /* TRACE_H */
diff --git a/rts/eventlog/EventLog.c b/rts/eventlog/EventLog.c
index a48c92e457..ede1615139 100644
--- a/rts/eventlog/EventLog.c
+++ b/rts/eventlog/EventLog.c
@@ -13,7 +13,6 @@
 
 #include "Trace.h"
 #include "Capability.h"
-#include "Trace.h"
 #include "RtsUtils.h"
 #include "Stats.h"
 #include "EventLog.h"
diff --git a/rts/ghc.mk b/rts/ghc.mk
index c3d7eb6e6b..bffdaaedf1 100644
--- a/rts/ghc.mk
+++ b/rts/ghc.mk
@@ -70,6 +70,11 @@ rts/dist/build/sm/Scav_thr.c : rts/sm/Scav.c | $$(dir $$@)/.
 
 rts_H_FILES = $(wildcard includes/*.h) $(wildcard rts/*.h)
 
+ifeq "$(HaveDtrace)" "YES"
+DTRACEPROBES_H = rts/dist/build/RtsProbes.h
+rts_H_FILES += $(DTRACEPROBES_H)
+endif
+
 # collect the -l flags that we need to link the rts dyn lib.
 rts/libs.depend : $(GHC_PKG_INPLACE)
 	"$(GHC_PKG_INPLACE)" field rts extra-libraries \
@@ -362,9 +367,15 @@ rts_dist_C_SRCS  = $(rts_C_SRCS) $(rts_thr_EXTRA_C_SRCS)
 rts_dist_S_SRCS =  $(rts_S_SRCS)
 rts_dist_C_FILES = $(rts_C_SRCS) $(rts_thr_EXTRA_C_SRCS) $(rts_S_SRCS)
 
+ifeq "$(HaveDtrace)" "YES"
+
+rts_dist_MKDEPENDC_OPTS += -Irts/dist/build
+
+endif
+
 $(eval $(call build-dependencies,rts,dist))
 
-$(rts_dist_depfile_c_asm) : libffi/dist-install/build/ffi.h
+$(rts_dist_depfile_c_asm) : libffi/dist-install/build/ffi.h $(DTRACEPROBES_H)
 
 #-----------------------------------------------------------------------------
 # libffi stuff
@@ -383,6 +394,20 @@ $(DYNWRAPPER_PROG): $(DYNWRAPPER_SRC)
 	"$(HC)" -cpp -optc-include -optcdyn-wrapper-patchable-behaviour.h $(INPLACE_EXTRA_FLAGS) $< -o $@
 
 # -----------------------------------------------------------------------------
+# compile dtrace probes if dtrace is supported
+
+ifeq "$(HaveDtrace)" "YES"
+
+rts_CC_OPTS		+= -DDTRACE
+rts_HC_OPTS		+= -DDTRACE
+
+DTRACEPROBES_SRC = rts/RtsProbes.d
+$(DTRACEPROBES_H): $(DTRACEPROBES_SRC) | $(dir $@)/.
+	"$(DTRACE)" $(filter -I%,$(rts_CC_OPTS)) -C -h -o $@ -s $<
+
+endif
+
+# -----------------------------------------------------------------------------
 # build the static lib containing the C main symbol
 
 rts/dist/build/libHSrtsmain.a : rts/dist/build/Main.o
diff --git a/rts/sm/GC.c b/rts/sm/GC.c
index 63023b6d93..3bd50171c3 100644
--- a/rts/sm/GC.c
+++ b/rts/sm/GC.c
@@ -1008,7 +1008,7 @@ scavenge_until_all_done (void)
 	
 
 loop:
-    traceEvent(&capabilities[gct->thread_index], EVENT_GC_WORK);
+    traceEventGcWork(&capabilities[gct->thread_index]);
 
 #if defined(THREADED_RTS)
     if (n_gc_threads > 1) {
@@ -1023,7 +1023,7 @@ loop:
     // scavenge_loop() only exits when there's no work to do
     r = dec_running();
     
-    traceEvent(&capabilities[gct->thread_index], EVENT_GC_IDLE);
+    traceEventGcIdle(&capabilities[gct->thread_index]);
 
     debugTrace(DEBUG_gc, "%d GC threads still running", r);
     
@@ -1039,7 +1039,7 @@ loop:
         // scavenge_loop() to perform any pending work.
     }
     
-    traceEvent(&capabilities[gct->thread_index], EVENT_GC_DONE);
+    traceEventGcDone(&capabilities[gct->thread_index]);
 }
 
 #if defined(THREADED_RTS)