Rename updateRemembSetPushClosureRegs

12 files changed, 55 insertions, 31 deletions

diff --git a/compiler/GHC/StgToCmm/Utils.hs b/compiler/GHC/StgToCmm/Utils.hs
index 8cca28cc5a..5e9daa8c1d 100644
--- a/compiler/GHC/StgToCmm/Utils.hs
+++ b/compiler/GHC/StgToCmm/Utils.hs
@@ -617,7 +617,7 @@ emitUpdRemSetPush :: CmmExpr   -- ^ value of pointer which was overwritten
 emitUpdRemSetPush ptr =
     emitRtsCall
       rtsUnitId
-      (fsLit "updateRemembSetPushClosure_")
+      (fsLit "updateRemembSetPushClosureRegs")
       [(CmmReg (CmmGlobal BaseReg), AddrHint),
        (ptr, AddrHint)]
       False
@@ -627,7 +627,7 @@ emitUpdRemSetPushThunk :: CmmExpr -- ^ the thunk
 emitUpdRemSetPushThunk ptr =
     emitRtsCall
       rtsUnitId
-      (fsLit "updateRemembSetPushThunk_")
+      (fsLit "updateRemembSetPushThunkRegs")
       [(CmmReg (CmmGlobal BaseReg), AddrHint),
        (ptr, AddrHint)]
       False
diff --git a/includes/Cmm.h b/includes/Cmm.h
index 574c60a1b3..f6330c1f37 100644
--- a/includes/Cmm.h
+++ b/includes/Cmm.h
@@ -903,5 +903,5 @@
 // A useful helper for pushing a pointer to the update remembered set.
 #define updateRemembSetPushPtr(p)                                    \
     IF_NONMOVING_WRITE_BARRIER_ENABLED {                             \
-      ccall updateRemembSetPushClosure_(BaseReg "ptr", p "ptr");     \
+      ccall updateRemembSetPushClosureRegs(BaseReg "ptr", p "ptr");     \
     }
diff --git a/includes/rts/NonMoving.h b/includes/rts/NonMoving.h
index 314c582a1e..c4450f13f6 100644
--- a/includes/rts/NonMoving.h
+++ b/includes/rts/NonMoving.h
@@ -20,10 +20,10 @@ struct Capability_;
 
 /* This is called by the code generator */
 extern DLL_IMPORT_RTS
-void updateRemembSetPushClosure_(StgRegTable *reg, struct StgClosure_ *p);
+void updateRemembSetPushClosureRegs(StgRegTable *reg, struct StgClosure_ *p);
 
 extern DLL_IMPORT_RTS
-void updateRemembSetPushThunk_(StgRegTable *reg, struct StgThunk_ *p);
+void updateRemembSetPushThunkRegs(StgRegTable *reg, struct StgThunk_ *p);
 
 // Forward declaration for unregisterised backend.
 EF_(stg_copyArray_barrier);
diff --git a/rts/Messages.c b/rts/Messages.c
index 8cd0a5570f..4a4ff4adec 100644
--- a/rts/Messages.c
+++ b/rts/Messages.c
@@ -258,7 +258,7 @@ loop:
         // RELEASE to make the BQ visible, see Note [Heap memory barriers].
         RELEASE_STORE(&((StgInd*)bh)->indirectee, (StgClosure *)bq);
         IF_NONMOVING_WRITE_BARRIER_ENABLED {
-            updateRemembSetPushClosure(cap, (StgClosure*)p);
+            updateRemembSetPushClosure(&cap->upd_rem_set, (StgClosure*)p);
         }
         recordClosureMutated(cap,bh); // bh was mutated
 
@@ -290,7 +290,7 @@ loop:
         IF_NONMOVING_WRITE_BARRIER_ENABLED {
             // We are about to overwrite bq->queue; make sure its current value
             // makes it into the update remembered set
-            updateRemembSetPushClosure(cap, (StgClosure*)bq->queue);
+            updateRemembSetPushClosure(&cap->upd_rem_set, (StgClosure*)bq->queue);
         }
         RELAXED_STORE(&msg->link, bq->queue);
         bq->queue = msg;
diff --git a/rts/PrimOps.cmm b/rts/PrimOps.cmm
index 7ee77c7216..d9973ed89b 100644
--- a/rts/PrimOps.cmm
+++ b/rts/PrimOps.cmm
@@ -505,7 +505,7 @@ stg_copyArray_barrier ( W_ hdr_size, gcptr dst, W_ dst_off, W_ n)
 
 again:
     IF_NONMOVING_WRITE_BARRIER_ENABLED {
-        ccall updateRemembSetPushClosure_(BaseReg "ptr", W_[p] "ptr");
+        ccall updateRemembSetPushClosureRegs(BaseReg "ptr", W_[p] "ptr");
     }
     p = p + WDS(1);
     if (p < end) {
diff --git a/rts/RtsSymbols.c b/rts/RtsSymbols.c
index 7cd62faabe..00fb7c6d29 100644
--- a/rts/RtsSymbols.c
+++ b/rts/RtsSymbols.c
@@ -733,11 +733,11 @@
       SymI_HasProto(stg_isMutableByteArrayPinnedzh)                     \
       SymI_HasProto(stg_shrinkMutableByteArrayzh)                       \
       SymI_HasProto(stg_resizzeMutableByteArrayzh)                      \
-      SymI_HasProto(stg_shrinkSmallMutableArrayzh)                       \
+      SymI_HasProto(stg_shrinkSmallMutableArrayzh)                      \
       SymI_HasProto(newSpark)                                           \
-      SymI_HasProto(updateRemembSetPushThunk)                             \
-      SymI_HasProto(updateRemembSetPushThunk_)                            \
-      SymI_HasProto(updateRemembSetPushClosure_)                          \
+      SymI_HasProto(updateRemembSetPushThunk)                           \
+      SymI_HasProto(updateRemembSetPushThunkRegs)                       \
+      SymI_HasProto(updateRemembSetPushClosureRegs)                     \
       SymI_HasProto(performGC)                                          \
       SymI_HasProto(performMajorGC)                                     \
       SymI_HasProto(prog_argc)                                          \
diff --git a/rts/STM.c b/rts/STM.c
index 16dd029aea..9c012e1821 100644
--- a/rts/STM.c
+++ b/rts/STM.c
@@ -300,7 +300,7 @@ static StgClosure *lock_tvar(Capability *cap,
 
   IF_NONMOVING_WRITE_BARRIER_ENABLED {
       if (result)
-          updateRemembSetPushClosure(cap, result);
+          updateRemembSetPushClosure(cap->upd_rem_set, result);
   }
   return result;
 }
@@ -327,7 +327,7 @@ static StgBool cond_lock_tvar(Capability *cap,
   result = (StgClosure *)w;
   IF_NONMOVING_WRITE_BARRIER_ENABLED {
       if (result)
-          updateRemembSetPushClosure(cap, expected);
+          updateRemembSetPushClosure(cap->upd_rem_set, expected);
   }
   TRACE("%p : %s", trec, result ? "success" : "failure");
   return (result == expected);
diff --git a/rts/ThreadPaused.c b/rts/ThreadPaused.c
index 9f41fa1ab3..f75821314f 100644
--- a/rts/ThreadPaused.c
+++ b/rts/ThreadPaused.c
@@ -245,7 +245,7 @@ threadPaused(Capability *cap, StgTSO *tso)
             bh = ((StgUpdateFrame *)frame)->updatee;
             bh_info = ACQUIRE_LOAD(&bh->header.info);
             IF_NONMOVING_WRITE_BARRIER_ENABLED {
-                updateRemembSetPushClosure(cap, (StgClosure *) bh);
+                updateRemembSetPushClosure(&cap->upd_rem_set, (StgClosure *) bh);
             }
 
 #if defined(THREADED_RTS)
diff --git a/rts/Updates.h b/rts/Updates.h
index f2ca437dfc..ea7fbb26fb 100644
--- a/rts/Updates.h
+++ b/rts/Updates.h
@@ -52,7 +52,7 @@
     bd = Bdescr(p1);                                            \
     if (bdescr_gen_no(bd) != 0 :: bits16) {                     \
       IF_NONMOVING_WRITE_BARRIER_ENABLED {                      \
-        ccall updateRemembSetPushThunk_(BaseReg, p1 "ptr");     \
+        ccall updateRemembSetPushThunkRegs(BaseReg, p1 "ptr");  \
       }                                                         \
       recordMutableCap(p1, TO_W_(bdescr_gen_no(bd)));           \
       TICK_UPD_OLD_IND();                                       \
diff --git a/rts/sm/NonMovingMark.c b/rts/sm/NonMovingMark.c
index 8775156fa5..a5010d99e8 100644
--- a/rts/sm/NonMovingMark.c
+++ b/rts/sm/NonMovingMark.c
@@ -639,22 +639,44 @@ void updateRemembSetPushThunkEager(Capability *cap,
     }
 }
 
-void updateRemembSetPushThunk_(StgRegTable *reg, StgThunk *p)
+void updateRemembSetPushThunkRegs(StgRegTable *reg, StgThunk *p)
 {
     updateRemembSetPushThunk(regTableToCapability(reg), p);
 }
 
-inline void updateRemembSetPushClosure(Capability *cap, StgClosure *p)
+STATIC_INLINE void updateRemembSetPushClosure_(UpdRemSet *rs, StgClosure *p)
+{
+    if (markQueueBlockIsFull(rs->block)) {
+        // Yes, this block is full.
+        // Allocate a fresh block.
+        ACQUIRE_SM_LOCK;
+        nonmovingAddUpdRemSetBlocks(rs);
+        bdescr *bd = allocGroup(MARK_QUEUE_BLOCKS);
+        bd->link = markQueueBlockBdescr(rs->block);
+        rs->block = (MarkQueueBlock *) bd->start;
+        rs->block->head = 0;
+        RELEASE_SM_LOCK;
+    }
+
+    MarkQueueEnt ent = {
+        .mark_closure = {
+            .p = TAG_CLOSURE(MARK_CLOSURE, UNTAG_CLOSURE(p)),
+            .origin = NULL,
+        }
+    };
+    markQueueBlockPush(rs->block, &ent);
+}
+
+void updateRemembSetPushClosure(UpdRemSet *rs, StgClosure *p)
 {
     if (check_in_nonmoving_heap(p)) {
-        MarkQueue *queue = &cap->upd_rem_set.queue;
-        push_closure(queue, p, NULL);
+        updateRemembSetPushClosure_(rs, p);
     }
 }
 
-void updateRemembSetPushClosure_(StgRegTable *reg, struct StgClosure_ *p)
+void updateRemembSetPushClosureRegs(StgRegTable *reg, struct StgClosure_ *p)
 {
-    updateRemembSetPushClosure(regTableToCapability(reg), p);
+    updateRemembSetPushClosure(&regTableToCapability(reg)->upd_rem_set, p);
 }
 
 STATIC_INLINE bool needs_upd_rem_set_mark(StgClosure *p)
diff --git a/rts/sm/NonMovingMark.h b/rts/sm/NonMovingMark.h
index 37f58cd172..87399d541b 100644
--- a/rts/sm/NonMovingMark.h
+++ b/rts/sm/NonMovingMark.h
@@ -159,7 +159,7 @@ void nonmovingMarkInitUpdRemSet(void);
 
 void init_upd_rem_set(UpdRemSet *rset);
 void reset_upd_rem_set(UpdRemSet *rset);
-void updateRemembSetPushClosure(Capability *cap, StgClosure *p);
+void updateRemembSetPushClosure(UpdRemSet *rset, StgClosure *p);
 void updateRemembSetPushThunk(Capability *cap, StgThunk *p);
 void updateRemembSetPushTSO(Capability *cap, StgTSO *tso);
 void updateRemembSetPushStack(Capability *cap, StgStack *stack);
diff --git a/rts/sm/Storage.c b/rts/sm/Storage.c
index b9ae97af4c..86147f3e07 100644
--- a/rts/sm/Storage.c
+++ b/rts/sm/Storage.c
@@ -482,7 +482,7 @@ lockCAF (StgRegTable *reg, StgIndStatic *caf)
     IF_NONMOVING_WRITE_BARRIER_ENABLED {
         StgThunkInfoTable *thunk_info = itbl_to_thunk_itbl(orig_info_tbl);
         if (thunk_info->i.srt) {
-            updateRemembSetPushClosure(cap, GET_SRT(thunk_info));
+            updateRemembSetPushClosure(&cap->upd_rem_set, GET_SRT(thunk_info));
         }
     }
 
@@ -1311,7 +1311,7 @@ dirty_MUT_VAR(StgRegTable *reg, StgMutVar *mvar, StgClosure *old)
         recordClosureMutated(cap, (StgClosure *) mvar);
         IF_NONMOVING_WRITE_BARRIER_ENABLED {
             // See Note [Dirty flags in the non-moving collector] in NonMoving.c
-            updateRemembSetPushClosure_(reg, old);
+            updateRemembSetPushClosureRegs(reg, old);
         }
     }
 }
@@ -1334,7 +1334,7 @@ dirty_TVAR(Capability *cap, StgTVar *p,
         recordClosureMutated(cap,(StgClosure*)p);
         IF_NONMOVING_WRITE_BARRIER_ENABLED {
             // See Note [Dirty flags in the non-moving collector] in NonMoving.c
-            updateRemembSetPushClosure(cap, old);
+            updateRemembSetPushClosure(&cap->upd_rem_set, old);
         }
     }
 }
@@ -1351,7 +1351,8 @@ setTSOLink (Capability *cap, StgTSO *tso, StgTSO *target)
         RELAXED_STORE(&tso->dirty, 1);
         recordClosureMutated(cap,(StgClosure*)tso);
         IF_NONMOVING_WRITE_BARRIER_ENABLED {
-            updateRemembSetPushClosure(cap, (StgClosure *) tso->_link);
+            // See Note [Dirty flags in the non-moving collector] in NonMoving.c
+            updateRemembSetPushClosure(&cap->upd_rem_set, (StgClosure *) tso->_link);
         }
     }
     tso->_link = target;
@@ -1364,7 +1365,8 @@ setTSOPrev (Capability *cap, StgTSO *tso, StgTSO *target)
         RELAXED_STORE(&tso->dirty, 1);
         recordClosureMutated(cap,(StgClosure*)tso);
         IF_NONMOVING_WRITE_BARRIER_ENABLED {
-            updateRemembSetPushClosure(cap, (StgClosure *) tso->block_info.prev);
+            // See Note [Dirty flags in the non-moving collector] in NonMoving.c
+            updateRemembSetPushClosure(&cap->upd_rem_set, (StgClosure *) tso->block_info.prev);
         }
     }
     tso->block_info.prev = target;
@@ -1417,9 +1419,9 @@ update_MVAR(StgRegTable *reg, StgClosure *p, StgClosure *old_val)
     IF_NONMOVING_WRITE_BARRIER_ENABLED {
         // See Note [Dirty flags in the non-moving collector] in NonMoving.c
         StgMVar *mvar = (StgMVar *) p;
-        updateRemembSetPushClosure(cap, old_val);
-        updateRemembSetPushClosure(cap, (StgClosure *) mvar->head);
-        updateRemembSetPushClosure(cap, (StgClosure *) mvar->tail);
+        updateRemembSetPushClosure(&cap->upd_rem_set, old_val);
+        updateRemembSetPushClosure(&cap->upd_rem_set, (StgClosure *) mvar->head);
+        updateRemembSetPushClosure(&cap->upd_rem_set, (StgClosure *) mvar->tail);
     }
 }