rts: TraverseHeap: Move "flip" bit into traverseState struct

6 files changed, 67 insertions, 57 deletions

diff --git a/rts/ProfHeap.c b/rts/ProfHeap.c
index 195a6b6af2..336013f255 100644
--- a/rts/ProfHeap.c
+++ b/rts/ProfHeap.c
@@ -215,7 +215,7 @@ closureIdentity( const StgClosure *p )
     case HEAP_BY_RETAINER:
         // AFAIK, the only closures in the heap which might not have a
         // valid retainer set are DEAD_WEAK closures.
-        if (isTravDataValid(p))
+        if (isRetainerSetValid(p))
             return retainerSetOf(p);
         else
             return NULL;
@@ -739,7 +739,7 @@ closureSatisfiesConstraints( const StgClosure* p )
        // reason it might not be valid is if this closure is a
        // a newly deceased weak pointer (i.e. a DEAD_WEAK), since
        // these aren't reached by the retainer profiler's traversal.
-       if (isTravDataValid((StgClosure *)p)) {
+       if (isRetainerSetValid((StgClosure *)p)) {
            rs = retainerSetOf((StgClosure *)p);
            if (rs != NULL) {
                for (i = 0; i < rs->num; i++) {
diff --git a/rts/RetainerProfile.c b/rts/RetainerProfile.c
index 7222bed0ad..81b41b8207 100644
--- a/rts/RetainerProfile.c
+++ b/rts/RetainerProfile.c
@@ -251,13 +251,18 @@ associate( StgClosure *c, RetainerSet *s )
 {
     // StgWord has the same size as pointers, so the following type
     // casting is okay.
-    setTravData(c, (StgWord)s);
+    setTravData(&g_retainerTraverseState, c, (StgWord)s);
+}
+
+bool isRetainerSetValid( const StgClosure *c )
+{
+    return isTravDataValid(&g_retainerTraverseState, c);
 }
 
 inline RetainerSet*
 retainerSetOf( const StgClosure *c )
 {
-    ASSERT(isTravDataValid(c));
+    ASSERT(isRetainerSetValid(c));
     return (RetainerSet*)getTravData(c);
 }
 
@@ -355,7 +360,7 @@ retainRoot(void *user, StgClosure **tl)
     // be a root.
 
     c = UNTAG_CLOSURE(*tl);
-    traverseMaybeInitClosureData(c);
+    traverseMaybeInitClosureData(&g_retainerTraverseState, c);
     if (c != &stg_END_TSO_QUEUE_closure && isRetainer(c)) {
         traversePushRoot(ts, c, c, (stackData)getRetainerFrom(c));
     } else {
diff --git a/rts/RetainerProfile.h b/rts/RetainerProfile.h
index 6a2dc5e846..f15f0aa716 100644
--- a/rts/RetainerProfile.h
+++ b/rts/RetainerProfile.h
@@ -20,10 +20,12 @@ void initRetainerProfiling ( void );
 void endRetainerProfiling  ( void );
 void retainerProfile       ( void );
 
+bool isRetainerSetValid( const StgClosure *c );
 RetainerSet* retainerSetOf( const StgClosure *c );
 
 // Used by GC.c
 W_ retainerStackBlocks(void);
+extern traverseState g_retainerTraverseState;
 
 #include "EndPrivate.h"
 
diff --git a/rts/TraverseHeap.c b/rts/TraverseHeap.c
index 110cae3b34..281bf4edc2 100644
--- a/rts/TraverseHeap.c
+++ b/rts/TraverseHeap.c
@@ -16,52 +16,20 @@
 
 #include "TraverseHeap.h"
 
-/** Note [Profiling heap traversal visited bit]
- *
- * If the RTS is compiled with profiling enabled StgProfHeader can be used by
- * profiling code to store per-heap object information. Specifically the
- * 'hp_hdr' field is used to store heap profiling information.
- *
- * The generic heap traversal code reserves the least significant bit of the
- * heap profiling word to decide whether we've already visited a given closure
- * in the current pass or not. The rest of the field is free to be used by the
- * calling profiler.
- *
- * By doing things this way we implicitly assume that the LSB is not used by the
- * user. This is true at least for the word aligned pointers which the retainer
- * profiler currently stores there and should be maintained by new users for
- * example by shifting the real data up by one bit.
- *
- * Since we don't want to have to scan the entire heap a second time just to
- * reset the per-object visitied bit before/after the real traversal we make the
- * interpretation of this bit dependent on the value of a global variable,
- * 'flip'.
- *
- * When the 'trav' bit is equal to the value of 'flip' the closure data is
- * valid otherwise not (see isTravDataValid). We then invert the value of 'flip'
- * on each heap traversal (see traverseWorkStack), in effect marking all
- * closure's data as invalid at once.
- *
- * There are some complications with this approach, namely: static objects and
- * mutable data. There we do just go over all existing objects to reset the bit
- * manually. See 'resetStaticObjectForProfiling' and 'resetMutableObjects'.
- */
-static StgWord flip = 0;
-
 StgWord getTravData(const StgClosure *c)
 {
     const StgWord hp_hdr = c->header.prof.hp.trav;
     return hp_hdr & (STG_WORD_MAX ^ 1);
 }
 
-void setTravData(StgClosure *c, StgWord w)
+void setTravData(const traverseState *ts, StgClosure *c, StgWord w)
 {
-    c->header.prof.hp.trav = w | flip;
+    c->header.prof.hp.trav = w | ts->flip;
 }
 
-bool isTravDataValid(const StgClosure *c)
+bool isTravDataValid(const traverseState *ts, const StgClosure *c)
 {
-    return ((c->header.prof.hp.trav & 1) ^ flip) == 0;
+    return (c->header.prof.hp.trav & 1) == ts->flip;
 }
 
 typedef enum {
@@ -959,10 +927,10 @@ out:
  * See Note [Profiling heap traversal visited bit].
  */
 bool
-traverseMaybeInitClosureData(StgClosure *c)
+traverseMaybeInitClosureData(const traverseState* ts, StgClosure *c)
 {
-    if (!isTravDataValid(c)) {
-        setTravData(c, 0);
+    if (!isTravDataValid(ts, c)) {
+        setTravData(ts, c, 0);
         return true;
     }
     return false;
@@ -1176,7 +1144,7 @@ traversePAP (traverseState *ts,
 }
 
 static void
-resetMutableObjects(void)
+resetMutableObjects(traverseState* ts)
 {
     uint32_t g, n;
     bdescr *bd;
@@ -1195,7 +1163,7 @@ resetMutableObjects(void)
           for (bd = capabilities[n]->mut_lists[g]; bd != NULL; bd = bd->link) {
             for (ml = bd->start; ml < bd->free; ml++) {
 
-                traverseMaybeInitClosureData((StgClosure *)*ml);
+                traverseMaybeInitClosureData(ts, (StgClosure *)*ml);
             }
           }
         }
@@ -1219,7 +1187,7 @@ traverseWorkStack(traverseState *ts, visitClosure_cb visit_cb)
     bool other_children;
 
     // Now we flip the flip bit.
-    flip = flip ^ 1;
+    ts->flip = ts->flip ^ 1;
 
     // c = Current closure                           (possibly tagged)
     // cp = Current closure's Parent                 (NOT tagged)
@@ -1231,7 +1199,7 @@ loop:
 
     if (c == NULL) {
         debug("maxStackSize= %d\n", ts->maxStackSize);
-        resetMutableObjects();
+        resetMutableObjects(ts);
         return;
     }
 
@@ -1309,7 +1277,7 @@ inner_loop:
     stackAccum accum = {};
 
     // If this is the first visit to c, initialize its data.
-    bool first_visit = traverseMaybeInitClosureData(c);
+    bool first_visit = traverseMaybeInitClosureData(ts, c);
     bool traverse_children
         = visit_cb(c, cp, data, first_visit, &accum, &child_data);
     if(!traverse_children)
@@ -1458,7 +1426,7 @@ inner_loop:
  * encountering.
  */
 void
-resetStaticObjectForProfiling( StgClosure *static_objects )
+resetStaticObjectForProfiling( const traverseState *ts, StgClosure *static_objects )
 {
     uint32_t count = 0;
     StgClosure *p;
@@ -1476,7 +1444,7 @@ resetStaticObjectForProfiling( StgClosure *static_objects )
             p = (StgClosure*)*IND_STATIC_LINK(p);
             break;
         case THUNK_STATIC:
-            traverseMaybeInitClosureData(p);
+            traverseMaybeInitClosureData(ts, p);
             p = (StgClosure*)*THUNK_STATIC_LINK(p);
             break;
         case FUN_STATIC:
@@ -1485,7 +1453,7 @@ resetStaticObjectForProfiling( StgClosure *static_objects )
         case CONSTR_2_0:
         case CONSTR_1_1:
         case CONSTR_NOCAF:
-            traverseMaybeInitClosureData(p);
+            traverseMaybeInitClosureData(ts, p);
             p = (StgClosure*)*STATIC_LINK(get_itbl(p), p);
             break;
         default:
diff --git a/rts/TraverseHeap.h b/rts/TraverseHeap.h
index 3855123323..b0ecbd2a02 100644
--- a/rts/TraverseHeap.h
+++ b/rts/TraverseHeap.h
@@ -16,7 +16,6 @@
 
 #include "BeginPrivate.h"
 
-void resetStaticObjectForProfiling(StgClosure *static_objects);
 
 typedef struct traverseState_ traverseState;
 
@@ -34,6 +33,40 @@ typedef union stackAccum_ {
 typedef struct stackElement_ stackElement;
 
 typedef struct traverseState_ {
+    /** Note [Profiling heap traversal visited bit]
+     * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+     *
+     * If the RTS is compiled with profiling enabled StgProfHeader can be used
+     * by profiling code to store per-heap object information. Specifically the
+     * 'hp_hdr' field is used to store heap profiling information.
+     *
+     * The generic heap traversal code reserves the least significant bit of the
+     * heap profiling word to decide whether we've already visited a given
+     * closure in the current pass or not. The rest of the field is free to be
+     * used by the calling profiler.
+     *
+     * By doing things this way we implicitly assume that the LSB is not used by
+     * the user. This is true at least for the word aligned pointers which the
+     * retainer profiler currently stores there and should be maintained by new
+     * users for example by shifting the real data up by one bit.
+     *
+     * Since we don't want to have to scan the entire heap a second time just to
+     * reset the per-object visitied bit before/after the real traversal we make
+     * the interpretation of this bit dependent on the value of a global
+     * variable, 'flip' and "flip" this variable when we want to invalidate all
+     * objects.
+     *
+     * When the visited bit is equal to the value of 'flip' the closure data
+     * is valid otherwise not (see isTravDataValid). We then invert the value of
+     * 'flip' after each each profiling run (see traverseWorkStack).
+     *
+     * There are some complications with this approach, namely: static objects
+     * and mutable data. There we do just go over all existing objects to reset
+     * the bit manually. See 'resetStaticObjectForProfiling' and
+     * 'resetMutableObjects'.
+     */
+    StgWord flip;
+
     /**
      * Invariants:
      *
@@ -123,18 +156,20 @@ typedef bool (*visitClosure_cb) (
     stackData *child_data);
 
 StgWord getTravData(const StgClosure *c);
-void setTravData(StgClosure *c, StgWord w);
-bool isTravDataValid(const StgClosure *c);
+void setTravData(const traverseState *ts, StgClosure *c, StgWord w);
+bool isTravDataValid(const traverseState *ts, const StgClosure *c);
 
 void traverseWorkStack(traverseState *ts, visitClosure_cb visit_cb);
 void traversePushRoot(traverseState *ts, StgClosure *c, StgClosure *cp, stackData data);
-bool traverseMaybeInitClosureData(StgClosure *c);
+bool traverseMaybeInitClosureData(const traverseState* ts, StgClosure *c);
 
 void initializeTraverseStack(traverseState *ts);
 void closeTraverseStack(traverseState *ts);
 int getTraverseStackMaxSize(traverseState *ts);
 
+// for GC.c
 W_ traverseWorkStackBlocks(traverseState *ts);
+void resetStaticObjectForProfiling(const traverseState *ts, StgClosure *static_objects);
 
 #include "EndPrivate.h"
 
diff --git a/rts/sm/GC.c b/rts/sm/GC.c
index 6d222da02a..55e57a58b2 100644
--- a/rts/sm/GC.c
+++ b/rts/sm/GC.c
@@ -916,7 +916,7 @@ GarbageCollect (uint32_t collect_gen,
   // zeroing below.
 
   // ToDo: fix the gct->scavenged_static_objects below
-  resetStaticObjectForProfiling(gct->scavenged_static_objects);
+  resetStaticObjectForProfiling(&g_retainerTraverseState, gct->scavenged_static_objects);
 #endif
 
   // Start any pending finalizers.  Must be after