Add capability sets to the tracing/events system

We trace the creation and shutdown of capabilities. All the capabilities
in the process are assigned to one capabilitiy set of OS-process type.

This is a second version of the patch. Includes work by Spencer Janssen.

6 files changed, 183 insertions, 2 deletions

diff --git a/rts/Capability.c b/rts/Capability.c
index 3e1dd972c7..fe5dbdca40 100644
--- a/rts/Capability.c
+++ b/rts/Capability.c
@@ -253,6 +253,8 @@ initCapability( Capability *cap, nat i )
     cap->transaction_tokens = 0;
     cap->context_switch = 0;
     cap->pinned_object_block = NULL;
+
+    traceCapsetAssignCap(CAPSET_OSPROCESS_DEFAULT, i);
 }
 
 /* ---------------------------------------------------------------------------
@@ -266,6 +268,10 @@ initCapability( Capability *cap, nat i )
 void
 initCapabilities( void )
 {
+    /* Declare a single capability set representing the process. 
+       Each capability will get added to this capset. */ 
+    traceCapsetCreate(CAPSET_OSPROCESS_DEFAULT, CapsetTypeOsProcess);
+
 #if defined(THREADED_RTS)
     nat i;
 
@@ -717,7 +723,9 @@ tryGrabCapability (Capability *cap, Task *task)
  * ------------------------------------------------------------------------- */
 
 void
-shutdownCapability (Capability *cap USED_IF_THREADS, Task *task USED_IF_THREADS, rtsBool safe USED_IF_THREADS)
+shutdownCapability (Capability *cap,
+                    Task *task USED_IF_THREADS,
+                    rtsBool safe USED_IF_THREADS)
 {
 #if defined(THREADED_RTS)
     nat i;
@@ -813,6 +821,8 @@ shutdownCapability (Capability *cap USED_IF_THREADS, Task *task USED_IF_THREADS,
     // closeMutex(&cap->lock);
     
 #endif /* THREADED_RTS */
+
+    traceCapsetRemoveCap(CAPSET_OSPROCESS_DEFAULT, cap->no);
 }
 
 void
@@ -823,6 +833,7 @@ shutdownCapabilities(Task *task, rtsBool safe)
         ASSERT(task->incall->tso == NULL);
         shutdownCapability(&capabilities[i], task, safe);
     }
+    traceCapsetDelete(CAPSET_OSPROCESS_DEFAULT);
 }
 
 static void
diff --git a/rts/RtsProbes.d b/rts/RtsProbes.d
index dbc5111e46..bd32fca385 100644
--- a/rts/RtsProbes.d
+++ b/rts/RtsProbes.d
@@ -23,6 +23,8 @@
  * typedef uint16_t EventCapNo;
  * typedef uint16_t EventPayloadSize; // variable-size events
  * typedef uint16_t EventThreadStatus;
+ * typedef uint32_t EventCapsetID;
+ * typedef uint16_t EventCapsetType;  // types for EVENT_CAPSET_CREATE
  */
 
 /* -----------------------------------------------------------------------------
@@ -60,5 +62,9 @@ provider HaskellEvent {
   probe gc__idle (EventCapNo);
   probe gc__work (EventCapNo);
   probe gc__done (EventCapNo);
+  probe capset__create(EventCapsetID, EventCapsetType);
+  probe capset__delete(EventCapsetID);
+  probe capset__assign__cap(EventCapsetID, EventCapNo);
+  probe capset__remove__cap(EventCapsetID, EventCapNo);
 
 };
diff --git a/rts/Trace.c b/rts/Trace.c
index 6c69cbc6dc..faa54d7741 100644
--- a/rts/Trace.c
+++ b/rts/Trace.c
@@ -251,6 +251,39 @@ void traceSchedEvent_ (Capability *cap, EventTypeNum tag,
     }
 }
 
+void traceCapsetModify_ (EventTypeNum tag,
+                         CapsetID capset,
+                         StgWord32 other)
+{
+#ifdef DEBUG
+    if (RtsFlags.TraceFlags.tracing == TRACE_STDERR) {
+        ACQUIRE_LOCK(&trace_utx);
+
+        tracePreface();
+        switch (tag) {
+        case EVENT_CAPSET_CREATE:   // (capset, capset_type)
+            debugBelch("created capset %d of type %d\n", capset, other);
+            break;
+        case EVENT_CAPSET_DELETE:   // (capset)
+            debugBelch("deleted capset %d\n", capset);
+            break;
+        case EVENT_CAPSET_ASSIGN_CAP:  // (capset, capno)
+            debugBelch("assigned cap %d to capset %d\n", other, capset);
+            break;
+        case EVENT_CAPSET_REMOVE_CAP:  // (capset, capno)
+            debugBelch("removed cap %d from capset %d\n", other, capset);
+            break;
+        }
+        RELEASE_LOCK(&trace_utx);
+    } else
+#endif
+    {
+        if (eventlog_enabled) {
+            postCapsetModifyEvent(tag, capset, other);
+        }
+    }
+}
+
 void traceEvent_ (Capability *cap, EventTypeNum tag)
 {
 #ifdef DEBUG
diff --git a/rts/Trace.h b/rts/Trace.h
index b264394399..f253280b26 100644
--- a/rts/Trace.h
+++ b/rts/Trace.h
@@ -31,6 +31,13 @@ void resetTracing (void);
 
 #endif /* TRACING */
 
+typedef StgWord32 CapsetID;
+typedef StgWord16 CapsetType;
+enum CapsetType { CapsetTypeCustom = CAPSET_TYPE_CUSTOM,
+                  CapsetTypeOsProcess = CAPSET_TYPE_OSPROCESS,
+                  CapsetTypeClockdomain = CAPSET_TYPE_CLOCKDOMAIN };
+#define CAPSET_OSPROCESS_DEFAULT 0
+
 // -----------------------------------------------------------------------------
 // Message classes
 // -----------------------------------------------------------------------------
@@ -162,6 +169,19 @@ void traceThreadStatus_ (StgTSO *tso);
 
 void traceEventStartup_ (int n_caps);
 
+/*
+ * Events for describing capability sets in the eventlog
+ *
+ * Note: unlike other events, these are not conditional on TRACE_sched or
+ * similar because they are not "real" events themselves but provide
+ * information and context for other "real" events. Other events depend on
+ * the capset info events so for simplicity, rather than working out if
+ * they're necessary we always emit them. They should be very low volume.
+ */
+void traceCapsetModify_ (EventTypeNum tag,
+                         CapsetID capset,
+                         StgWord32 other);
+
 #else /* !TRACING */
 
 #define traceSchedEvent(cap, tag, tso, other) /* nothing */
@@ -173,6 +193,7 @@ void traceEventStartup_ (int n_caps);
 #define debugTraceCap(class, cap, str, ...) /* nothing */
 #define traceThreadStatus(class, tso) /* nothing */
 #define traceEventStartup_(n_caps) /* nothing */
+#define traceCapsetModify_(tag, capset, other) /* nothing */
 
 #endif /* TRACING */
 
@@ -229,6 +250,14 @@ void dtraceUserMsgWrapper(Capability *cap, char *msg);
     HASKELLEVENT_GC_WORK(cap)
 #define dtraceGcDone(cap)                               \
     HASKELLEVENT_GC_DONE(cap)
+#define dtraceCapsetCreate(capset, capset_type)         \
+    HASKELLEVENT_CAPSET_CREATE(capset, capset_type)
+#define dtraceCapsetDelete(capset)                      \
+    HASKELLEVENT_CAPSET_DELETE(capset)
+#define dtraceCapsetAssignCap(capset, capno)            \
+    HASKELLEVENT_CAPSET_ASSIGN_CAP(capset, capno)
+#define dtraceCapsetRemoveCap(capset, capno)            \
+    HASKELLEVENT_CAPSET_REMOVE_CAP(capset, capno)
 
 #else /* !defined(DTRACE) */
 
@@ -251,6 +280,10 @@ void dtraceUserMsgWrapper(Capability *cap, char *msg);
 #define dtraceGcIdle(cap)                               /* nothing */
 #define dtraceGcWork(cap)                               /* nothing */
 #define dtraceGcDone(cap)                               /* nothing */
+#define dtraceCapsetCreate(capset, capset_type)         /* nothing */
+#define dtraceCapsetDelete(capset)                      /* nothing */
+#define dtraceCapsetAssignCap(capset, capno)            /* nothing */
+#define dtraceCapsetRemoveCap(capset, capno)            /* nothing */
 
 #endif
 
@@ -409,6 +442,33 @@ INLINE_HEADER void traceEventGcDone(Capability *cap STG_UNUSED)
     dtraceGcDone((EventCapNo)cap->no);
 }
 
+INLINE_HEADER void traceCapsetCreate(CapsetID   capset      STG_UNUSED,
+                                     CapsetType capset_type STG_UNUSED)
+{
+    traceCapsetModify_(EVENT_CAPSET_CREATE, capset, capset_type);
+    dtraceCapsetCreate(capset, capset_type);
+}
+
+INLINE_HEADER void traceCapsetDelete(CapsetID capset STG_UNUSED)
+{
+    traceCapsetModify_(EVENT_CAPSET_DELETE, capset, 0);
+    dtraceCapsetDelete(capset);
+}
+
+INLINE_HEADER void traceCapsetAssignCap(CapsetID capset STG_UNUSED,
+                                        nat      capno  STG_UNUSED)
+{
+    traceCapsetModify_(EVENT_CAPSET_ASSIGN_CAP, capset, capno);
+    dtraceCapsetAssignCap(capset, capno);
+}
+
+INLINE_HEADER void traceCapsetRemoveCap(CapsetID capset STG_UNUSED,
+                                        nat      capno  STG_UNUSED)
+{
+    traceCapsetModify_(EVENT_CAPSET_REMOVE_CAP, capset, capno);
+    dtraceCapsetRemoveCap(capset, capno);
+}
+
 #include "EndPrivate.h"
 
 #endif /* TRACE_H */
diff --git a/rts/eventlog/EventLog.c b/rts/eventlog/EventLog.c
index 2884ad9f10..b5c2ef63bb 100644
--- a/rts/eventlog/EventLog.c
+++ b/rts/eventlog/EventLog.c
@@ -75,7 +75,11 @@ char *EventDesc[] = {
   [EVENT_GC_IDLE]             = "GC idle",
   [EVENT_GC_WORK]             = "GC working",
   [EVENT_GC_DONE]             = "GC done",
-  [EVENT_BLOCK_MARKER]        = "Block marker"
+  [EVENT_BLOCK_MARKER]        = "Block marker",
+  [EVENT_CAPSET_CREATE]       = "Create capability set",
+  [EVENT_CAPSET_DELETE]       = "Delete capability set",
+  [EVENT_CAPSET_ASSIGN_CAP]   = "Add capability to capability set",
+  [EVENT_CAPSET_REMOVE_CAP]   = "Remove capability from capability set"
 };
 
 // Event type. 
@@ -146,6 +150,12 @@ static inline void postThreadID(EventsBuf *eb, EventThreadID id)
 static inline void postCapNo(EventsBuf *eb, EventCapNo no)
 { postWord16(eb,no); }
 
+static inline void postCapsetID(EventsBuf *eb, EventCapsetID id)
+{ postWord32(eb,id); }
+
+static inline void postCapsetType(EventsBuf *eb, EventCapsetType type)
+{ postWord16(eb,type); }
+
 static inline void postPayloadSize(EventsBuf *eb, EventPayloadSize size)
 { postWord16(eb,size); }
 
@@ -259,6 +269,21 @@ initEventLogging(void)
             eventTypes[t].size = sizeof(EventCapNo);
             break;
 
+        case EVENT_CAPSET_CREATE:   // (capset, capset_type)
+            eventTypes[t].size =
+                sizeof(EventCapsetID) + sizeof(EventCapsetType);
+            break;
+
+        case EVENT_CAPSET_DELETE:   // (capset)
+            eventTypes[t].size = sizeof(EventCapsetID);
+            break;
+
+        case EVENT_CAPSET_ASSIGN_CAP:  // (capset, cap)
+        case EVENT_CAPSET_REMOVE_CAP:
+            eventTypes[t].size =
+                sizeof(EventCapsetID) + sizeof(EventCapNo);
+            break;
+
         case EVENT_SHUTDOWN:        // (cap)
         case EVENT_REQUEST_SEQ_GC:  // (cap)
         case EVENT_REQUEST_PAR_GC:  // (cap)
@@ -439,6 +464,45 @@ postSchedEvent (Capability *cap,
     }
 }
 
+void postCapsetModifyEvent (EventTypeNum tag,
+                            EventCapsetID capset,
+                            StgWord32 other)
+{
+    ACQUIRE_LOCK(&eventBufMutex);
+
+    if (!hasRoomForEvent(&eventBuf, tag)) {
+        // Flush event buffer to make room for new event.
+        printAndClearEventBuf(&eventBuf);
+    }
+
+    postEventHeader(&eventBuf, tag);
+    postCapsetID(&eventBuf, capset);
+
+    switch (tag) {
+    case EVENT_CAPSET_CREATE:   // (capset, capset_type)
+    {
+        postCapsetType(&eventBuf, other /* capset_type */);
+        break;
+    }
+
+    case EVENT_CAPSET_DELETE:   // (capset)
+    {
+        break;
+    }
+
+    case EVENT_CAPSET_ASSIGN_CAP:  // (capset, capno)
+    case EVENT_CAPSET_REMOVE_CAP:  // (capset, capno)
+    {
+        postCapNo(&eventBuf, other /* capno */);
+        break;
+    }
+    default:
+        barf("postCapsetModifyEvent: unknown event tag %d", tag);
+    }
+
+    RELEASE_LOCK(&eventBufMutex);
+}
+
 void
 postEvent (Capability *cap, EventTypeNum tag)
 {
diff --git a/rts/eventlog/EventLog.h b/rts/eventlog/EventLog.h
index 22a034c2d4..116b532c1f 100644
--- a/rts/eventlog/EventLog.h
+++ b/rts/eventlog/EventLog.h
@@ -47,6 +47,13 @@ void postCapMsg(Capability *cap, char *msg, va_list ap);
 
 void postEventStartup(EventCapNo n_caps);
 
+/*
+ * Post a capability set modification event
+ */
+void postCapsetModifyEvent (EventTypeNum tag,
+                            EventCapsetID capset,
+                            StgWord32 other);
+
 #else /* !TRACING */
 
 INLINE_HEADER void postSchedEvent (Capability *cap  STG_UNUSED,