Nonmoving: Ensure write barrier vanishes in non-threaded RTS

12 files changed, 65 insertions, 36 deletions

diff --git a/includes/Cmm.h b/includes/Cmm.h
index 42aa0bf550..546e81e8f6 100644
--- a/includes/Cmm.h
+++ b/includes/Cmm.h
@@ -950,19 +950,23 @@
     return (dst);
 
 
+//
+// Nonmoving write barrier helpers
+//
+// See Note [Update remembered set] in NonMovingMark.c.
+
 #if defined(THREADED_RTS)
-#define IF_WRITE_BARRIER_ENABLED                               \
+#define IF_NONMOVING_WRITE_BARRIER_ENABLED                     \
     if (W_[nonmoving_write_barrier_enabled] != 0) (likely: False)
 #else
 // A similar measure is also taken in rts/NonMoving.h, but that isn't visible from C--
-#define IF_WRITE_BARRIER_ENABLED                               \
+#define IF_NONMOVING_WRITE_BARRIER_ENABLED                     \
     if (0)
 #define nonmoving_write_barrier_enabled 0
 #endif
 
 // A useful helper for pushing a pointer to the update remembered set.
-// See Note [Update remembered set] in NonMovingMark.c.
 #define updateRemembSetPushPtr(p)                                    \
-    IF_WRITE_BARRIER_ENABLED {                                       \
+    IF_NONMOVING_WRITE_BARRIER_ENABLED {                             \
       ccall updateRemembSetPushClosure_(BaseReg "ptr", p "ptr");     \
     }
diff --git a/includes/Stg.h b/includes/Stg.h
index 73de97055f..8f0abdbf6d 100644
--- a/includes/Stg.h
+++ b/includes/Stg.h
@@ -597,3 +597,21 @@ typedef union {
   c;                                            \
 })
 #endif
+
+/* -----------------------------------------------------------------------------
+   Nonmoving GC write barrier
+   -------------------------------------------------------------------------- */
+
+// Note that RTS code should not condition on this directly by rather
+// use the IF_NONMOVING_WRITE_BARRIER_ENABLED macro to ensure that
+// the barrier is eliminated in the non-threaded RTS.
+extern StgWord DLL_IMPORT_DATA_VAR(nonmoving_write_barrier_enabled);
+
+// A similar macro is defined in includes/Cmm.h for C-- code.
+#if defined(THREADED_RTS)
+#define IF_NONMOVING_WRITE_BARRIER_ENABLED \
+    if (RTS_UNLIKELY(nonmoving_write_barrier_enabled))
+#else
+#define IF_NONMOVING_WRITE_BARRIER_ENABLED \
+    if (0)
+#endif
diff --git a/includes/rts/NonMoving.h b/includes/rts/NonMoving.h
index 6a6d96b2c8..f64769e8a9 100644
--- a/includes/rts/NonMoving.h
+++ b/includes/rts/NonMoving.h
@@ -20,5 +20,3 @@ void updateRemembSetPushClosure_(StgRegTable *reg, StgClosure *p);
 void updateRemembSetPushClosure(Capability *cap, StgClosure *p);
 
 void updateRemembSetPushThunk_(StgRegTable *reg, StgThunk *p);
-
-extern StgWord DLL_IMPORT_DATA_VAR(nonmoving_write_barrier_enabled);
diff --git a/rts/Messages.c b/rts/Messages.c
index b16ada53e1..374f3d673e 100644
--- a/rts/Messages.c
+++ b/rts/Messages.c
@@ -262,7 +262,7 @@ loop:
 
         // point to the BLOCKING_QUEUE from the BLACKHOLE
         write_barrier(); // make the BQ visible, see Note [Heap memory barriers].
-        if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
+        IF_NONMOVING_WRITE_BARRIER_ENABLED {
             updateRemembSetPushClosure(cap, (StgClosure*)p);
         }
         ((StgInd*)bh)->indirectee = (StgClosure *)bq;
@@ -293,7 +293,7 @@ loop:
         }
 #endif
 
-        if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
+        IF_NONMOVING_WRITE_BARRIER_ENABLED {
             // We are about to overwrite bq->queue; make sure its current value
             // makes it into the update remembered set
             updateRemembSetPushClosure(cap, (StgClosure*)bq->queue);
diff --git a/rts/PrimOps.cmm b/rts/PrimOps.cmm
index 6d3df0700c..c076641152 100644
--- a/rts/PrimOps.cmm
+++ b/rts/PrimOps.cmm
@@ -478,7 +478,7 @@ stg_copyArray_barrier ( W_ hdr_size, gcptr dst, W_ dst_off, W_ n)
     end = p + WDS(n);
 
 again:
-    IF_WRITE_BARRIER_ENABLED {
+    IF_NONMOVING_WRITE_BARRIER_ENABLED {
         ccall updateRemembSetPushClosure_(BaseReg "ptr", W_[p] "ptr");
     }
     p = p + WDS(1);
@@ -494,7 +494,7 @@ stg_copySmallArrayzh ( gcptr src, W_ src_off, gcptr dst, W_ dst_off, W_ n)
     W_ dst_p, src_p, bytes;
 
     if (n > 0) {
-        IF_WRITE_BARRIER_ENABLED {
+        IF_NONMOVING_WRITE_BARRIER_ENABLED {
             call stg_copyArray_barrier(SIZEOF_StgSmallMutArrPtrs,
                                       dst, dst_off, n);
         }
@@ -515,7 +515,7 @@ stg_copySmallMutableArrayzh ( gcptr src, W_ src_off, gcptr dst, W_ dst_off, W_ n
     W_ dst_p, src_p, bytes;
 
     if (n > 0) {
-        IF_WRITE_BARRIER_ENABLED {
+        IF_NONMOVING_WRITE_BARRIER_ENABLED {
             call stg_copyArray_barrier(SIZEOF_StgSmallMutArrPtrs,
                                       dst, dst_off, n);
         }
diff --git a/rts/STM.c b/rts/STM.c
index c17f33aaa0..1dde70b485 100644
--- a/rts/STM.c
+++ b/rts/STM.c
@@ -297,8 +297,10 @@ static StgClosure *lock_tvar(Capability *cap,
   } while (cas((void *)&(s -> current_value),
                (StgWord)result, (StgWord)trec) != (StgWord)result);
 
-  if (RTS_UNLIKELY(nonmoving_write_barrier_enabled && result)) {
-      updateRemembSetPushClosure(cap, result);
+
+  IF_NONMOVING_WRITE_BARRIER_ENABLED {
+      if (result)
+          updateRemembSetPushClosure(cap, result);
   }
   return result;
 }
@@ -323,8 +325,9 @@ static StgBool cond_lock_tvar(Capability *cap,
   TRACE("%p : cond_lock_tvar(%p, %p)", trec, s, expected);
   w = cas((void *)&(s -> current_value), (StgWord)expected, (StgWord)trec);
   result = (StgClosure *)w;
-  if (RTS_UNLIKELY(nonmoving_write_barrier_enabled && result)) {
-      updateRemembSetPushClosure(cap, expected);
+  IF_NONMOVING_WRITE_BARRIER_ENABLED {
+      if (result)
+          updateRemembSetPushClosure(cap, expected);
   }
   TRACE("%p : %s", trec, result ? "success" : "failure");
   return (result == expected);
diff --git a/rts/Schedule.c b/rts/Schedule.c
index f867d7f359..f66a276602 100644
--- a/rts/Schedule.c
+++ b/rts/Schedule.c
@@ -2500,7 +2500,7 @@ resumeThread (void *task_)
     incall->suspended_tso = NULL;
     incall->suspended_cap = NULL;
     // we will modify tso->_link
-    if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
+    IF_NONMOVING_WRITE_BARRIER_ENABLED {
         updateRemembSetPushClosure(cap, (StgClosure *)tso->_link);
     }
     tso->_link = END_TSO_QUEUE;
diff --git a/rts/ThreadPaused.c b/rts/ThreadPaused.c
index 5cbb3f5595..368f99a4f9 100644
--- a/rts/ThreadPaused.c
+++ b/rts/ThreadPaused.c
@@ -334,15 +334,16 @@ threadPaused(Capability *cap, StgTSO *tso)
             }
 #endif
 
-            if (RTS_UNLIKELY(nonmoving_write_barrier_enabled
-                             && ip_THUNK(INFO_PTR_TO_STRUCT(bh_info)))) {
-                // We are about to replace a thunk with a blackhole.
-                // Add the free variables of the closure we are about to
-                // overwrite to the update remembered set.
-                // N.B. We caught the WHITEHOLE case above.
-                updateRemembSetPushThunkEager(cap,
-                                             THUNK_INFO_PTR_TO_STRUCT(bh_info),
-                                             (StgThunk *) bh);
+            IF_NONMOVING_WRITE_BARRIER_ENABLED {
+                if (ip_THUNK(INFO_PTR_TO_STRUCT(bh_info))) {
+                    // We are about to replace a thunk with a blackhole.
+                    // Add the free variables of the closure we are about to
+                    // overwrite to the update remembered set.
+                    // N.B. We caught the WHITEHOLE case above.
+                    updateRemembSetPushThunkEager(cap,
+                                                 THUNK_INFO_PTR_TO_STRUCT(bh_info),
+                                                 (StgThunk *) bh);
+                }
             }
 
             // The payload of the BLACKHOLE points to the TSO
diff --git a/rts/Threads.c b/rts/Threads.c
index 2b11a1eb90..8334c5a5ac 100644
--- a/rts/Threads.c
+++ b/rts/Threads.c
@@ -723,7 +723,7 @@ threadStackUnderflow (Capability *cap, StgTSO *tso)
             barf("threadStackUnderflow: not enough space for return values");
         }
 
-        if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
+        IF_NONMOVING_WRITE_BARRIER_ENABLED {
             // ensure that values that we copy into the new stack are marked
             // for the nonmoving collector. Note that these values won't
             // necessarily form a full closure so we need to handle them
diff --git a/rts/Updates.h b/rts/Updates.h
index 84d9162868..91d1b0b1cb 100644
--- a/rts/Updates.h
+++ b/rts/Updates.h
@@ -50,7 +50,7 @@
                                                                 \
     prim_write_barrier;                                         \
     OVERWRITING_CLOSURE(p1);                                    \
-    IF_WRITE_BARRIER_ENABLED {                                  \
+    IF_NONMOVING_WRITE_BARRIER_ENABLED {                        \
       ccall updateRemembSetPushThunk_(BaseReg, p1 "ptr");       \
     }                                                           \
     StgInd_indirectee(p1) = p2;                                 \
@@ -81,7 +81,7 @@ INLINE_HEADER void updateWithIndirection (Capability *cap,
     /* See Note [Heap memory barriers] in SMP.h */
     write_barrier();
     OVERWRITING_CLOSURE(p1);
-    if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
+    IF_NONMOVING_WRITE_BARRIER_ENABLED {
         updateRemembSetPushThunk(cap, (StgThunk*)p1);
     }
     ((StgInd *)p1)->indirectee = p2;
diff --git a/rts/sm/NonMovingMark.h b/rts/sm/NonMovingMark.h
index d7066e56d6..54baa936db 100644
--- a/rts/sm/NonMovingMark.h
+++ b/rts/sm/NonMovingMark.h
@@ -115,6 +115,7 @@ extern StgIndStatic *debug_caf_list_snapshot;
 extern MarkQueue *current_mark_queue;
 extern bdescr *upd_rem_set_block_list;
 
+
 void nonmovingMarkInitUpdRemSet(void);
 
 void init_upd_rem_set(UpdRemSet *rset);
diff --git a/rts/sm/Storage.c b/rts/sm/Storage.c
index 7ed4896301..44163a353f 100644
--- a/rts/sm/Storage.c
+++ b/rts/sm/Storage.c
@@ -430,7 +430,7 @@ lockCAF (StgRegTable *reg, StgIndStatic *caf)
     // reference should be in SRTs
     ASSERT(orig_info_tbl->layout.payload.ptrs == 0);
     // Becuase the payload is empty we just push the SRT
-    if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
+    IF_NONMOVING_WRITE_BARRIER_ENABLED {
         StgThunkInfoTable *thunk_info = itbl_to_thunk_itbl(orig_info_tbl);
         if (thunk_info->i.srt) {
             updateRemembSetPushClosure(cap, GET_SRT(thunk_info));
@@ -1144,7 +1144,7 @@ dirty_MUT_VAR(StgRegTable *reg, StgMutVar *mvar, StgClosure *old)
     if (mvar->header.info == &stg_MUT_VAR_CLEAN_info) {
         mvar->header.info = &stg_MUT_VAR_DIRTY_info;
         recordClosureMutated(cap, (StgClosure *) mvar);
-        if (RTS_UNLIKELY(nonmoving_write_barrier_enabled != 0)) {
+        IF_NONMOVING_WRITE_BARRIER_ENABLED {
             updateRemembSetPushClosure_(reg, old);
         }
     }
@@ -1166,7 +1166,7 @@ dirty_TVAR(Capability *cap, StgTVar *p,
     if (p->header.info == &stg_TVAR_CLEAN_info) {
         p->header.info = &stg_TVAR_DIRTY_info;
         recordClosureMutated(cap,(StgClosure*)p);
-        if (RTS_UNLIKELY(nonmoving_write_barrier_enabled != 0)) {
+        IF_NONMOVING_WRITE_BARRIER_ENABLED {
             updateRemembSetPushClosure(cap, old);
         }
     }
@@ -1183,8 +1183,9 @@ setTSOLink (Capability *cap, StgTSO *tso, StgTSO *target)
     if (tso->dirty == 0) {
         tso->dirty = 1;
         recordClosureMutated(cap,(StgClosure*)tso);
-        if (RTS_UNLIKELY(nonmoving_write_barrier_enabled))
+        IF_NONMOVING_WRITE_BARRIER_ENABLED {
             updateRemembSetPushClosure(cap, (StgClosure *) tso->_link);
+        }
     }
     tso->_link = target;
 }
@@ -1195,8 +1196,9 @@ setTSOPrev (Capability *cap, StgTSO *tso, StgTSO *target)
     if (tso->dirty == 0) {
         tso->dirty = 1;
         recordClosureMutated(cap,(StgClosure*)tso);
-        if (RTS_UNLIKELY(nonmoving_write_barrier_enabled))
+        IF_NONMOVING_WRITE_BARRIER_ENABLED {
             updateRemembSetPushClosure(cap, (StgClosure *) tso->block_info.prev);
+        }
     }
     tso->block_info.prev = target;
 }
@@ -1209,8 +1211,9 @@ dirty_TSO (Capability *cap, StgTSO *tso)
         recordClosureMutated(cap,(StgClosure*)tso);
     }
 
-    if (RTS_UNLIKELY(nonmoving_write_barrier_enabled))
+    IF_NONMOVING_WRITE_BARRIER_ENABLED {
         updateRemembSetPushTSO(cap, tso);
+    }
 }
 
 void
@@ -1218,8 +1221,9 @@ dirty_STACK (Capability *cap, StgStack *stack)
 {
     // First push to upd_rem_set before we set stack->dirty since we
     // the nonmoving collector may already be marking the stack.
-    if (RTS_UNLIKELY(nonmoving_write_barrier_enabled))
+    IF_NONMOVING_WRITE_BARRIER_ENABLED {
         updateRemembSetPushStack(cap, stack);
+    }
 
     if (! (stack->dirty & STACK_DIRTY)) {
         stack->dirty = STACK_DIRTY;
@@ -1243,7 +1247,7 @@ void
 update_MVAR(StgRegTable *reg, StgClosure *p, StgClosure *old_val)
 {
     Capability *cap = regTableToCapability(reg);
-    if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
+    IF_NONMOVING_WRITE_BARRIER_ENABLED {
         StgMVar *mvar = (StgMVar *) p;
         updateRemembSetPushClosure(cap, old_val);
         updateRemembSetPushClosure(cap, (StgClosure *) mvar->head);