summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--includes/Cmm.h12
-rw-r--r--includes/Stg.h18
-rw-r--r--includes/rts/NonMoving.h2
-rw-r--r--rts/Messages.c4
-rw-r--r--rts/PrimOps.cmm6
-rw-r--r--rts/STM.c11
-rw-r--r--rts/Schedule.c2
-rw-r--r--rts/ThreadPaused.c19
-rw-r--r--rts/Threads.c2
-rw-r--r--rts/Updates.h4
-rw-r--r--rts/sm/NonMovingMark.h1
-rw-r--r--rts/sm/Storage.c20
12 files changed, 65 insertions, 36 deletions
diff --git a/includes/Cmm.h b/includes/Cmm.h
index 42aa0bf550..546e81e8f6 100644
--- a/includes/Cmm.h
+++ b/includes/Cmm.h
@@ -950,19 +950,23 @@
return (dst);
+//
+// Nonmoving write barrier helpers
+//
+// See Note [Update remembered set] in NonMovingMark.c.
+
#if defined(THREADED_RTS)
-#define IF_WRITE_BARRIER_ENABLED \
+#define IF_NONMOVING_WRITE_BARRIER_ENABLED \
if (W_[nonmoving_write_barrier_enabled] != 0) (likely: False)
#else
// A similar measure is also taken in rts/NonMoving.h, but that isn't visible from C--
-#define IF_WRITE_BARRIER_ENABLED \
+#define IF_NONMOVING_WRITE_BARRIER_ENABLED \
if (0)
#define nonmoving_write_barrier_enabled 0
#endif
// A useful helper for pushing a pointer to the update remembered set.
-// See Note [Update remembered set] in NonMovingMark.c.
#define updateRemembSetPushPtr(p) \
- IF_WRITE_BARRIER_ENABLED { \
+ IF_NONMOVING_WRITE_BARRIER_ENABLED { \
ccall updateRemembSetPushClosure_(BaseReg "ptr", p "ptr"); \
}
diff --git a/includes/Stg.h b/includes/Stg.h
index 73de97055f..8f0abdbf6d 100644
--- a/includes/Stg.h
+++ b/includes/Stg.h
@@ -597,3 +597,21 @@ typedef union {
c; \
})
#endif
+
+/* -----------------------------------------------------------------------------
+ Nonmoving GC write barrier
+ -------------------------------------------------------------------------- */
+
+// Note that RTS code should not condition on this directly by rather
+// use the IF_NONMOVING_WRITE_BARRIER_ENABLED macro to ensure that
+// the barrier is eliminated in the non-threaded RTS.
+extern StgWord DLL_IMPORT_DATA_VAR(nonmoving_write_barrier_enabled);
+
+// A similar macro is defined in includes/Cmm.h for C-- code.
+#if defined(THREADED_RTS)
+#define IF_NONMOVING_WRITE_BARRIER_ENABLED \
+ if (RTS_UNLIKELY(nonmoving_write_barrier_enabled))
+#else
+#define IF_NONMOVING_WRITE_BARRIER_ENABLED \
+ if (0)
+#endif
diff --git a/includes/rts/NonMoving.h b/includes/rts/NonMoving.h
index 6a6d96b2c8..f64769e8a9 100644
--- a/includes/rts/NonMoving.h
+++ b/includes/rts/NonMoving.h
@@ -20,5 +20,3 @@ void updateRemembSetPushClosure_(StgRegTable *reg, StgClosure *p);
void updateRemembSetPushClosure(Capability *cap, StgClosure *p);
void updateRemembSetPushThunk_(StgRegTable *reg, StgThunk *p);
-
-extern StgWord DLL_IMPORT_DATA_VAR(nonmoving_write_barrier_enabled);
diff --git a/rts/Messages.c b/rts/Messages.c
index b16ada53e1..374f3d673e 100644
--- a/rts/Messages.c
+++ b/rts/Messages.c
@@ -262,7 +262,7 @@ loop:
// point to the BLOCKING_QUEUE from the BLACKHOLE
write_barrier(); // make the BQ visible, see Note [Heap memory barriers].
- if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
updateRemembSetPushClosure(cap, (StgClosure*)p);
}
((StgInd*)bh)->indirectee = (StgClosure *)bq;
@@ -293,7 +293,7 @@ loop:
}
#endif
- if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
// We are about to overwrite bq->queue; make sure its current value
// makes it into the update remembered set
updateRemembSetPushClosure(cap, (StgClosure*)bq->queue);
diff --git a/rts/PrimOps.cmm b/rts/PrimOps.cmm
index 6d3df0700c..c076641152 100644
--- a/rts/PrimOps.cmm
+++ b/rts/PrimOps.cmm
@@ -478,7 +478,7 @@ stg_copyArray_barrier ( W_ hdr_size, gcptr dst, W_ dst_off, W_ n)
end = p + WDS(n);
again:
- IF_WRITE_BARRIER_ENABLED {
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
ccall updateRemembSetPushClosure_(BaseReg "ptr", W_[p] "ptr");
}
p = p + WDS(1);
@@ -494,7 +494,7 @@ stg_copySmallArrayzh ( gcptr src, W_ src_off, gcptr dst, W_ dst_off, W_ n)
W_ dst_p, src_p, bytes;
if (n > 0) {
- IF_WRITE_BARRIER_ENABLED {
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
call stg_copyArray_barrier(SIZEOF_StgSmallMutArrPtrs,
dst, dst_off, n);
}
@@ -515,7 +515,7 @@ stg_copySmallMutableArrayzh ( gcptr src, W_ src_off, gcptr dst, W_ dst_off, W_ n
W_ dst_p, src_p, bytes;
if (n > 0) {
- IF_WRITE_BARRIER_ENABLED {
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
call stg_copyArray_barrier(SIZEOF_StgSmallMutArrPtrs,
dst, dst_off, n);
}
diff --git a/rts/STM.c b/rts/STM.c
index c17f33aaa0..1dde70b485 100644
--- a/rts/STM.c
+++ b/rts/STM.c
@@ -297,8 +297,10 @@ static StgClosure *lock_tvar(Capability *cap,
} while (cas((void *)&(s -> current_value),
(StgWord)result, (StgWord)trec) != (StgWord)result);
- if (RTS_UNLIKELY(nonmoving_write_barrier_enabled && result)) {
- updateRemembSetPushClosure(cap, result);
+
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
+ if (result)
+ updateRemembSetPushClosure(cap, result);
}
return result;
}
@@ -323,8 +325,9 @@ static StgBool cond_lock_tvar(Capability *cap,
TRACE("%p : cond_lock_tvar(%p, %p)", trec, s, expected);
w = cas((void *)&(s -> current_value), (StgWord)expected, (StgWord)trec);
result = (StgClosure *)w;
- if (RTS_UNLIKELY(nonmoving_write_barrier_enabled && result)) {
- updateRemembSetPushClosure(cap, expected);
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
+ if (result)
+ updateRemembSetPushClosure(cap, expected);
}
TRACE("%p : %s", trec, result ? "success" : "failure");
return (result == expected);
diff --git a/rts/Schedule.c b/rts/Schedule.c
index f867d7f359..f66a276602 100644
--- a/rts/Schedule.c
+++ b/rts/Schedule.c
@@ -2500,7 +2500,7 @@ resumeThread (void *task_)
incall->suspended_tso = NULL;
incall->suspended_cap = NULL;
// we will modify tso->_link
- if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
updateRemembSetPushClosure(cap, (StgClosure *)tso->_link);
}
tso->_link = END_TSO_QUEUE;
diff --git a/rts/ThreadPaused.c b/rts/ThreadPaused.c
index 5cbb3f5595..368f99a4f9 100644
--- a/rts/ThreadPaused.c
+++ b/rts/ThreadPaused.c
@@ -334,15 +334,16 @@ threadPaused(Capability *cap, StgTSO *tso)
}
#endif
- if (RTS_UNLIKELY(nonmoving_write_barrier_enabled
- && ip_THUNK(INFO_PTR_TO_STRUCT(bh_info)))) {
- // We are about to replace a thunk with a blackhole.
- // Add the free variables of the closure we are about to
- // overwrite to the update remembered set.
- // N.B. We caught the WHITEHOLE case above.
- updateRemembSetPushThunkEager(cap,
- THUNK_INFO_PTR_TO_STRUCT(bh_info),
- (StgThunk *) bh);
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
+ if (ip_THUNK(INFO_PTR_TO_STRUCT(bh_info))) {
+ // We are about to replace a thunk with a blackhole.
+ // Add the free variables of the closure we are about to
+ // overwrite to the update remembered set.
+ // N.B. We caught the WHITEHOLE case above.
+ updateRemembSetPushThunkEager(cap,
+ THUNK_INFO_PTR_TO_STRUCT(bh_info),
+ (StgThunk *) bh);
+ }
}
// The payload of the BLACKHOLE points to the TSO
diff --git a/rts/Threads.c b/rts/Threads.c
index 2b11a1eb90..8334c5a5ac 100644
--- a/rts/Threads.c
+++ b/rts/Threads.c
@@ -723,7 +723,7 @@ threadStackUnderflow (Capability *cap, StgTSO *tso)
barf("threadStackUnderflow: not enough space for return values");
}
- if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
// ensure that values that we copy into the new stack are marked
// for the nonmoving collector. Note that these values won't
// necessarily form a full closure so we need to handle them
diff --git a/rts/Updates.h b/rts/Updates.h
index 84d9162868..91d1b0b1cb 100644
--- a/rts/Updates.h
+++ b/rts/Updates.h
@@ -50,7 +50,7 @@
\
prim_write_barrier; \
OVERWRITING_CLOSURE(p1); \
- IF_WRITE_BARRIER_ENABLED { \
+ IF_NONMOVING_WRITE_BARRIER_ENABLED { \
ccall updateRemembSetPushThunk_(BaseReg, p1 "ptr"); \
} \
StgInd_indirectee(p1) = p2; \
@@ -81,7 +81,7 @@ INLINE_HEADER void updateWithIndirection (Capability *cap,
/* See Note [Heap memory barriers] in SMP.h */
write_barrier();
OVERWRITING_CLOSURE(p1);
- if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
updateRemembSetPushThunk(cap, (StgThunk*)p1);
}
((StgInd *)p1)->indirectee = p2;
diff --git a/rts/sm/NonMovingMark.h b/rts/sm/NonMovingMark.h
index d7066e56d6..54baa936db 100644
--- a/rts/sm/NonMovingMark.h
+++ b/rts/sm/NonMovingMark.h
@@ -115,6 +115,7 @@ extern StgIndStatic *debug_caf_list_snapshot;
extern MarkQueue *current_mark_queue;
extern bdescr *upd_rem_set_block_list;
+
void nonmovingMarkInitUpdRemSet(void);
void init_upd_rem_set(UpdRemSet *rset);
diff --git a/rts/sm/Storage.c b/rts/sm/Storage.c
index 7ed4896301..44163a353f 100644
--- a/rts/sm/Storage.c
+++ b/rts/sm/Storage.c
@@ -430,7 +430,7 @@ lockCAF (StgRegTable *reg, StgIndStatic *caf)
// reference should be in SRTs
ASSERT(orig_info_tbl->layout.payload.ptrs == 0);
// Becuase the payload is empty we just push the SRT
- if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
StgThunkInfoTable *thunk_info = itbl_to_thunk_itbl(orig_info_tbl);
if (thunk_info->i.srt) {
updateRemembSetPushClosure(cap, GET_SRT(thunk_info));
@@ -1144,7 +1144,7 @@ dirty_MUT_VAR(StgRegTable *reg, StgMutVar *mvar, StgClosure *old)
if (mvar->header.info == &stg_MUT_VAR_CLEAN_info) {
mvar->header.info = &stg_MUT_VAR_DIRTY_info;
recordClosureMutated(cap, (StgClosure *) mvar);
- if (RTS_UNLIKELY(nonmoving_write_barrier_enabled != 0)) {
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
updateRemembSetPushClosure_(reg, old);
}
}
@@ -1166,7 +1166,7 @@ dirty_TVAR(Capability *cap, StgTVar *p,
if (p->header.info == &stg_TVAR_CLEAN_info) {
p->header.info = &stg_TVAR_DIRTY_info;
recordClosureMutated(cap,(StgClosure*)p);
- if (RTS_UNLIKELY(nonmoving_write_barrier_enabled != 0)) {
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
updateRemembSetPushClosure(cap, old);
}
}
@@ -1183,8 +1183,9 @@ setTSOLink (Capability *cap, StgTSO *tso, StgTSO *target)
if (tso->dirty == 0) {
tso->dirty = 1;
recordClosureMutated(cap,(StgClosure*)tso);
- if (RTS_UNLIKELY(nonmoving_write_barrier_enabled))
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
updateRemembSetPushClosure(cap, (StgClosure *) tso->_link);
+ }
}
tso->_link = target;
}
@@ -1195,8 +1196,9 @@ setTSOPrev (Capability *cap, StgTSO *tso, StgTSO *target)
if (tso->dirty == 0) {
tso->dirty = 1;
recordClosureMutated(cap,(StgClosure*)tso);
- if (RTS_UNLIKELY(nonmoving_write_barrier_enabled))
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
updateRemembSetPushClosure(cap, (StgClosure *) tso->block_info.prev);
+ }
}
tso->block_info.prev = target;
}
@@ -1209,8 +1211,9 @@ dirty_TSO (Capability *cap, StgTSO *tso)
recordClosureMutated(cap,(StgClosure*)tso);
}
- if (RTS_UNLIKELY(nonmoving_write_barrier_enabled))
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
updateRemembSetPushTSO(cap, tso);
+ }
}
void
@@ -1218,8 +1221,9 @@ dirty_STACK (Capability *cap, StgStack *stack)
{
// First push to upd_rem_set before we set stack->dirty since we
// the nonmoving collector may already be marking the stack.
- if (RTS_UNLIKELY(nonmoving_write_barrier_enabled))
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
updateRemembSetPushStack(cap, stack);
+ }
if (! (stack->dirty & STACK_DIRTY)) {
stack->dirty = STACK_DIRTY;
@@ -1243,7 +1247,7 @@ void
update_MVAR(StgRegTable *reg, StgClosure *p, StgClosure *old_val)
{
Capability *cap = regTableToCapability(reg);
- if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
+ IF_NONMOVING_WRITE_BARRIER_ENABLED {
StgMVar *mvar = (StgMVar *) p;
updateRemembSetPushClosure(cap, old_val);
updateRemembSetPushClosure(cap, (StgClosure *) mvar->head);
View Lorry Controller Status