diff options
Diffstat (limited to 'rts')
-rw-r--r-- | rts/Messages.c | 4 | ||||
-rw-r--r-- | rts/PrimOps.cmm | 6 | ||||
-rw-r--r-- | rts/STM.c | 11 | ||||
-rw-r--r-- | rts/Schedule.c | 2 | ||||
-rw-r--r-- | rts/ThreadPaused.c | 19 | ||||
-rw-r--r-- | rts/Threads.c | 2 | ||||
-rw-r--r-- | rts/Updates.h | 4 | ||||
-rw-r--r-- | rts/sm/NonMovingMark.h | 1 | ||||
-rw-r--r-- | rts/sm/Storage.c | 20 |
9 files changed, 39 insertions, 30 deletions
diff --git a/rts/Messages.c b/rts/Messages.c index b16ada53e1..374f3d673e 100644 --- a/rts/Messages.c +++ b/rts/Messages.c @@ -262,7 +262,7 @@ loop: // point to the BLOCKING_QUEUE from the BLACKHOLE write_barrier(); // make the BQ visible, see Note [Heap memory barriers]. - if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) { + IF_NONMOVING_WRITE_BARRIER_ENABLED { updateRemembSetPushClosure(cap, (StgClosure*)p); } ((StgInd*)bh)->indirectee = (StgClosure *)bq; @@ -293,7 +293,7 @@ loop: } #endif - if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) { + IF_NONMOVING_WRITE_BARRIER_ENABLED { // We are about to overwrite bq->queue; make sure its current value // makes it into the update remembered set updateRemembSetPushClosure(cap, (StgClosure*)bq->queue); diff --git a/rts/PrimOps.cmm b/rts/PrimOps.cmm index 6d3df0700c..c076641152 100644 --- a/rts/PrimOps.cmm +++ b/rts/PrimOps.cmm @@ -478,7 +478,7 @@ stg_copyArray_barrier ( W_ hdr_size, gcptr dst, W_ dst_off, W_ n) end = p + WDS(n); again: - IF_WRITE_BARRIER_ENABLED { + IF_NONMOVING_WRITE_BARRIER_ENABLED { ccall updateRemembSetPushClosure_(BaseReg "ptr", W_[p] "ptr"); } p = p + WDS(1); @@ -494,7 +494,7 @@ stg_copySmallArrayzh ( gcptr src, W_ src_off, gcptr dst, W_ dst_off, W_ n) W_ dst_p, src_p, bytes; if (n > 0) { - IF_WRITE_BARRIER_ENABLED { + IF_NONMOVING_WRITE_BARRIER_ENABLED { call stg_copyArray_barrier(SIZEOF_StgSmallMutArrPtrs, dst, dst_off, n); } @@ -515,7 +515,7 @@ stg_copySmallMutableArrayzh ( gcptr src, W_ src_off, gcptr dst, W_ dst_off, W_ n W_ dst_p, src_p, bytes; if (n > 0) { - IF_WRITE_BARRIER_ENABLED { + IF_NONMOVING_WRITE_BARRIER_ENABLED { call stg_copyArray_barrier(SIZEOF_StgSmallMutArrPtrs, dst, dst_off, n); } @@ -297,8 +297,10 @@ static StgClosure *lock_tvar(Capability *cap, } while (cas((void *)&(s -> current_value), (StgWord)result, (StgWord)trec) != (StgWord)result); - if (RTS_UNLIKELY(nonmoving_write_barrier_enabled && result)) { - updateRemembSetPushClosure(cap, result); + + IF_NONMOVING_WRITE_BARRIER_ENABLED { + if (result) + updateRemembSetPushClosure(cap, result); } return result; } @@ -323,8 +325,9 @@ static StgBool cond_lock_tvar(Capability *cap, TRACE("%p : cond_lock_tvar(%p, %p)", trec, s, expected); w = cas((void *)&(s -> current_value), (StgWord)expected, (StgWord)trec); result = (StgClosure *)w; - if (RTS_UNLIKELY(nonmoving_write_barrier_enabled && result)) { - updateRemembSetPushClosure(cap, expected); + IF_NONMOVING_WRITE_BARRIER_ENABLED { + if (result) + updateRemembSetPushClosure(cap, expected); } TRACE("%p : %s", trec, result ? "success" : "failure"); return (result == expected); diff --git a/rts/Schedule.c b/rts/Schedule.c index f867d7f359..f66a276602 100644 --- a/rts/Schedule.c +++ b/rts/Schedule.c @@ -2500,7 +2500,7 @@ resumeThread (void *task_) incall->suspended_tso = NULL; incall->suspended_cap = NULL; // we will modify tso->_link - if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) { + IF_NONMOVING_WRITE_BARRIER_ENABLED { updateRemembSetPushClosure(cap, (StgClosure *)tso->_link); } tso->_link = END_TSO_QUEUE; diff --git a/rts/ThreadPaused.c b/rts/ThreadPaused.c index 5cbb3f5595..368f99a4f9 100644 --- a/rts/ThreadPaused.c +++ b/rts/ThreadPaused.c @@ -334,15 +334,16 @@ threadPaused(Capability *cap, StgTSO *tso) } #endif - if (RTS_UNLIKELY(nonmoving_write_barrier_enabled - && ip_THUNK(INFO_PTR_TO_STRUCT(bh_info)))) { - // We are about to replace a thunk with a blackhole. - // Add the free variables of the closure we are about to - // overwrite to the update remembered set. - // N.B. We caught the WHITEHOLE case above. - updateRemembSetPushThunkEager(cap, - THUNK_INFO_PTR_TO_STRUCT(bh_info), - (StgThunk *) bh); + IF_NONMOVING_WRITE_BARRIER_ENABLED { + if (ip_THUNK(INFO_PTR_TO_STRUCT(bh_info))) { + // We are about to replace a thunk with a blackhole. + // Add the free variables of the closure we are about to + // overwrite to the update remembered set. + // N.B. We caught the WHITEHOLE case above. + updateRemembSetPushThunkEager(cap, + THUNK_INFO_PTR_TO_STRUCT(bh_info), + (StgThunk *) bh); + } } // The payload of the BLACKHOLE points to the TSO diff --git a/rts/Threads.c b/rts/Threads.c index 2b11a1eb90..8334c5a5ac 100644 --- a/rts/Threads.c +++ b/rts/Threads.c @@ -723,7 +723,7 @@ threadStackUnderflow (Capability *cap, StgTSO *tso) barf("threadStackUnderflow: not enough space for return values"); } - if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) { + IF_NONMOVING_WRITE_BARRIER_ENABLED { // ensure that values that we copy into the new stack are marked // for the nonmoving collector. Note that these values won't // necessarily form a full closure so we need to handle them diff --git a/rts/Updates.h b/rts/Updates.h index 84d9162868..91d1b0b1cb 100644 --- a/rts/Updates.h +++ b/rts/Updates.h @@ -50,7 +50,7 @@ \ prim_write_barrier; \ OVERWRITING_CLOSURE(p1); \ - IF_WRITE_BARRIER_ENABLED { \ + IF_NONMOVING_WRITE_BARRIER_ENABLED { \ ccall updateRemembSetPushThunk_(BaseReg, p1 "ptr"); \ } \ StgInd_indirectee(p1) = p2; \ @@ -81,7 +81,7 @@ INLINE_HEADER void updateWithIndirection (Capability *cap, /* See Note [Heap memory barriers] in SMP.h */ write_barrier(); OVERWRITING_CLOSURE(p1); - if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) { + IF_NONMOVING_WRITE_BARRIER_ENABLED { updateRemembSetPushThunk(cap, (StgThunk*)p1); } ((StgInd *)p1)->indirectee = p2; diff --git a/rts/sm/NonMovingMark.h b/rts/sm/NonMovingMark.h index d7066e56d6..54baa936db 100644 --- a/rts/sm/NonMovingMark.h +++ b/rts/sm/NonMovingMark.h @@ -115,6 +115,7 @@ extern StgIndStatic *debug_caf_list_snapshot; extern MarkQueue *current_mark_queue; extern bdescr *upd_rem_set_block_list; + void nonmovingMarkInitUpdRemSet(void); void init_upd_rem_set(UpdRemSet *rset); diff --git a/rts/sm/Storage.c b/rts/sm/Storage.c index 7ed4896301..44163a353f 100644 --- a/rts/sm/Storage.c +++ b/rts/sm/Storage.c @@ -430,7 +430,7 @@ lockCAF (StgRegTable *reg, StgIndStatic *caf) // reference should be in SRTs ASSERT(orig_info_tbl->layout.payload.ptrs == 0); // Becuase the payload is empty we just push the SRT - if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) { + IF_NONMOVING_WRITE_BARRIER_ENABLED { StgThunkInfoTable *thunk_info = itbl_to_thunk_itbl(orig_info_tbl); if (thunk_info->i.srt) { updateRemembSetPushClosure(cap, GET_SRT(thunk_info)); @@ -1144,7 +1144,7 @@ dirty_MUT_VAR(StgRegTable *reg, StgMutVar *mvar, StgClosure *old) if (mvar->header.info == &stg_MUT_VAR_CLEAN_info) { mvar->header.info = &stg_MUT_VAR_DIRTY_info; recordClosureMutated(cap, (StgClosure *) mvar); - if (RTS_UNLIKELY(nonmoving_write_barrier_enabled != 0)) { + IF_NONMOVING_WRITE_BARRIER_ENABLED { updateRemembSetPushClosure_(reg, old); } } @@ -1166,7 +1166,7 @@ dirty_TVAR(Capability *cap, StgTVar *p, if (p->header.info == &stg_TVAR_CLEAN_info) { p->header.info = &stg_TVAR_DIRTY_info; recordClosureMutated(cap,(StgClosure*)p); - if (RTS_UNLIKELY(nonmoving_write_barrier_enabled != 0)) { + IF_NONMOVING_WRITE_BARRIER_ENABLED { updateRemembSetPushClosure(cap, old); } } @@ -1183,8 +1183,9 @@ setTSOLink (Capability *cap, StgTSO *tso, StgTSO *target) if (tso->dirty == 0) { tso->dirty = 1; recordClosureMutated(cap,(StgClosure*)tso); - if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) + IF_NONMOVING_WRITE_BARRIER_ENABLED { updateRemembSetPushClosure(cap, (StgClosure *) tso->_link); + } } tso->_link = target; } @@ -1195,8 +1196,9 @@ setTSOPrev (Capability *cap, StgTSO *tso, StgTSO *target) if (tso->dirty == 0) { tso->dirty = 1; recordClosureMutated(cap,(StgClosure*)tso); - if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) + IF_NONMOVING_WRITE_BARRIER_ENABLED { updateRemembSetPushClosure(cap, (StgClosure *) tso->block_info.prev); + } } tso->block_info.prev = target; } @@ -1209,8 +1211,9 @@ dirty_TSO (Capability *cap, StgTSO *tso) recordClosureMutated(cap,(StgClosure*)tso); } - if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) + IF_NONMOVING_WRITE_BARRIER_ENABLED { updateRemembSetPushTSO(cap, tso); + } } void @@ -1218,8 +1221,9 @@ dirty_STACK (Capability *cap, StgStack *stack) { // First push to upd_rem_set before we set stack->dirty since we // the nonmoving collector may already be marking the stack. - if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) + IF_NONMOVING_WRITE_BARRIER_ENABLED { updateRemembSetPushStack(cap, stack); + } if (! (stack->dirty & STACK_DIRTY)) { stack->dirty = STACK_DIRTY; @@ -1243,7 +1247,7 @@ void update_MVAR(StgRegTable *reg, StgClosure *p, StgClosure *old_val) { Capability *cap = regTableToCapability(reg); - if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) { + IF_NONMOVING_WRITE_BARRIER_ENABLED { StgMVar *mvar = (StgMVar *) p; updateRemembSetPushClosure(cap, old_val); updateRemembSetPushClosure(cap, (StgClosure *) mvar->head); |