rts: Address failures to inline

8 files changed, 98 insertions, 76 deletions

diff --git a/rts/ClosureSize.c b/rts/ClosureSize.c
new file mode 100644
index 0000000000..8fa99c6e6e
--- /dev/null
+++ b/rts/ClosureSize.c
@@ -0,0 +1,69 @@
+/* -----------------------------------------------------------------------------
+ *
+ * (c) The GHC Team 2021
+ *
+ * Implementation of closure_sizeW_
+ *
+ * ---------------------------------------------------------------------------*/
+
+#include "Rts.h"
+
+uint32_t
+closure_sizeW_ (const StgClosure *p, const StgInfoTable *info)
+{
+    switch (info->type) {
+    case THUNK_0_1:
+    case THUNK_1_0:
+        return sizeofW(StgThunk) + 1;
+    case FUN_0_1:
+    case CONSTR_0_1:
+    case FUN_1_0:
+    case CONSTR_1_0:
+        return sizeofW(StgHeader) + 1;
+    case THUNK_0_2:
+    case THUNK_1_1:
+    case THUNK_2_0:
+        return sizeofW(StgThunk) + 2;
+    case FUN_0_2:
+    case CONSTR_0_2:
+    case FUN_1_1:
+    case CONSTR_1_1:
+    case FUN_2_0:
+    case CONSTR_2_0:
+        return sizeofW(StgHeader) + 2;
+    case THUNK:
+        return thunk_sizeW_fromITBL(info);
+    case THUNK_SELECTOR:
+        return THUNK_SELECTOR_sizeW();
+    case AP_STACK:
+        return ap_stack_sizeW((StgAP_STACK *)p);
+    case AP:
+        return ap_sizeW((StgAP *)p);
+    case PAP:
+        return pap_sizeW((StgPAP *)p);
+    case IND:
+        return sizeofW(StgInd);
+    case ARR_WORDS:
+        return arr_words_sizeW((StgArrBytes *)p);
+    case MUT_ARR_PTRS_CLEAN:
+    case MUT_ARR_PTRS_DIRTY:
+    case MUT_ARR_PTRS_FROZEN_CLEAN:
+    case MUT_ARR_PTRS_FROZEN_DIRTY:
+        return mut_arr_ptrs_sizeW((StgMutArrPtrs*)p);
+    case SMALL_MUT_ARR_PTRS_CLEAN:
+    case SMALL_MUT_ARR_PTRS_DIRTY:
+    case SMALL_MUT_ARR_PTRS_FROZEN_CLEAN:
+    case SMALL_MUT_ARR_PTRS_FROZEN_DIRTY:
+        return small_mut_arr_ptrs_sizeW((StgSmallMutArrPtrs*)p);
+    case TSO:
+        return sizeofW(StgTSO);
+    case STACK:
+        return stack_sizeW((StgStack*)p);
+    case BCO:
+        return bco_sizeW((StgBCO *)p);
+    case TREC_CHUNK:
+        return sizeofW(StgTRecChunk);
+    default:
+        return sizeW_fromITBL(info);
+    }
+}
diff --git a/rts/Messages.h b/rts/Messages.h
index fef4510740..8cefcafd97 100644
--- a/rts/Messages.h
+++ b/rts/Messages.h
@@ -29,10 +29,7 @@ doneWithMsgThrowTo (Capability *cap, MessageThrowTo *m)
     // where we are a bit more lenient (#19075).
     ASSERT(n_capabilities == 1 || m->header.info == &stg_WHITEHOLE_info);
     IF_NONMOVING_WRITE_BARRIER_ENABLED {
-      updateRemembSetPushClosure(cap, (StgClosure *) m->link);
-      updateRemembSetPushClosure(cap, (StgClosure *) m->source);
-      updateRemembSetPushClosure(cap, (StgClosure *) m->target);
-      updateRemembSetPushClosure(cap, (StgClosure *) m->exception);
+      updateRemembSetPushMessageThrowTo(cap, m);
     }
     OVERWRITING_CLOSURE((StgClosure*)m);
     unlockClosure((StgClosure*)m, &stg_MSG_NULL_info);
diff --git a/rts/TraverseHeap.c b/rts/TraverseHeap.c
index 3b92bb9f98..43318541f8 100644
--- a/rts/TraverseHeap.c
+++ b/rts/TraverseHeap.c
@@ -1036,7 +1036,7 @@ traversePushStack(traverseState *ts, StgClosure *cp, stackElement *sep,
 /**
  * Call traversePushClosure for each of the children of a PAP/AP
  */
-STATIC_INLINE StgPtr
+static StgPtr
 traversePAP (traverseState *ts,
                     StgClosure *pap,    /* NOT tagged */
                     stackElement *sep,
diff --git a/rts/include/rts/storage/ClosureMacros.h b/rts/include/rts/storage/ClosureMacros.h
index 393bee3a9f..2ba84d1279 100644
--- a/rts/include/rts/storage/ClosureMacros.h
+++ b/rts/include/rts/storage/ClosureMacros.h
@@ -353,67 +353,8 @@ EXTERN_INLINE StgWord compact_nfdata_full_sizeW ( StgCompactNFData *str )
  *
  * (Also for 'closure_sizeW' below)
  */
-EXTERN_INLINE uint32_t
+uint32_t
 closure_sizeW_ (const StgClosure *p, const StgInfoTable *info);
-EXTERN_INLINE uint32_t
-closure_sizeW_ (const StgClosure *p, const StgInfoTable *info)
-{
-    switch (info->type) {
-    case THUNK_0_1:
-    case THUNK_1_0:
-        return sizeofW(StgThunk) + 1;
-    case FUN_0_1:
-    case CONSTR_0_1:
-    case FUN_1_0:
-    case CONSTR_1_0:
-        return sizeofW(StgHeader) + 1;
-    case THUNK_0_2:
-    case THUNK_1_1:
-    case THUNK_2_0:
-        return sizeofW(StgThunk) + 2;
-    case FUN_0_2:
-    case CONSTR_0_2:
-    case FUN_1_1:
-    case CONSTR_1_1:
-    case FUN_2_0:
-    case CONSTR_2_0:
-        return sizeofW(StgHeader) + 2;
-    case THUNK:
-        return thunk_sizeW_fromITBL(info);
-    case THUNK_SELECTOR:
-        return THUNK_SELECTOR_sizeW();
-    case AP_STACK:
-        return ap_stack_sizeW((StgAP_STACK *)p);
-    case AP:
-        return ap_sizeW((StgAP *)p);
-    case PAP:
-        return pap_sizeW((StgPAP *)p);
-    case IND:
-        return sizeofW(StgInd);
-    case ARR_WORDS:
-        return arr_words_sizeW((StgArrBytes *)p);
-    case MUT_ARR_PTRS_CLEAN:
-    case MUT_ARR_PTRS_DIRTY:
-    case MUT_ARR_PTRS_FROZEN_CLEAN:
-    case MUT_ARR_PTRS_FROZEN_DIRTY:
-        return mut_arr_ptrs_sizeW((StgMutArrPtrs*)p);
-    case SMALL_MUT_ARR_PTRS_CLEAN:
-    case SMALL_MUT_ARR_PTRS_DIRTY:
-    case SMALL_MUT_ARR_PTRS_FROZEN_CLEAN:
-    case SMALL_MUT_ARR_PTRS_FROZEN_DIRTY:
-        return small_mut_arr_ptrs_sizeW((StgSmallMutArrPtrs*)p);
-    case TSO:
-        return sizeofW(StgTSO);
-    case STACK:
-        return stack_sizeW((StgStack*)p);
-    case BCO:
-        return bco_sizeW((StgBCO *)p);
-    case TREC_CHUNK:
-        return sizeofW(StgTRecChunk);
-    default:
-        return sizeW_fromITBL(info);
-    }
-}
 
 // The definitive way to find the size, in words, of a heap-allocated closure
 EXTERN_INLINE uint32_t closure_sizeW (const StgClosure *p);
diff --git a/rts/rts.cabal.in b/rts/rts.cabal.in
index ec907cd10b..a2449a5ce5 100644
--- a/rts/rts.cabal.in
+++ b/rts/rts.cabal.in
@@ -477,6 +477,7 @@ library
                CheckUnload.c
                CloneStack.c
                ClosureFlags.c
+               ClosureSize.c
                Disassembler.c
                FileLock.c
                ForeignExports.c
diff --git a/rts/sm/NonMovingMark.c b/rts/sm/NonMovingMark.c
index 87b8f774bd..3b227bb805 100644
--- a/rts/sm/NonMovingMark.c
+++ b/rts/sm/NonMovingMark.c
@@ -687,21 +687,26 @@ STATIC_INLINE bool needs_upd_rem_set_mark(StgClosure *p)
     }
 }
 
+static void finish_upd_rem_set_mark_large(bdescr* bd) {
+    // Someone else may have already marked it.
+    ACQUIRE_LOCK(&nonmoving_large_objects_mutex);
+    if (! (bd->flags & BF_MARKED)) {
+        bd->flags |= BF_MARKED;
+        dbl_link_remove(bd, &nonmoving_large_objects);
+        dbl_link_onto(bd, &nonmoving_marked_large_objects);
+        n_nonmoving_large_blocks -= bd->blocks;
+        n_nonmoving_marked_large_blocks += bd->blocks;
+    }
+    RELEASE_LOCK(&nonmoving_large_objects_mutex);
+}
+
 /* Set the mark bit; only to be called *after* we have fully marked the closure */
 STATIC_INLINE void finish_upd_rem_set_mark(StgClosure *p)
 {
     bdescr *bd = Bdescr((StgPtr) p);
     if (bd->flags & BF_LARGE) {
-        // Someone else may have already marked it.
-        ACQUIRE_LOCK(&nonmoving_large_objects_mutex);
-        if (! (bd->flags & BF_MARKED)) {
-            bd->flags |= BF_MARKED;
-            dbl_link_remove(bd, &nonmoving_large_objects);
-            dbl_link_onto(bd, &nonmoving_marked_large_objects);
-            n_nonmoving_large_blocks -= bd->blocks;
-            n_nonmoving_marked_large_blocks += bd->blocks;
-        }
-        RELEASE_LOCK(&nonmoving_large_objects_mutex);
+        // This function is extracted so that this function can be inline
+        finish_upd_rem_set_mark_large(bd);
     } else {
         struct NonmovingSegment *seg = nonmovingGetSegment((StgPtr) p);
         nonmoving_block_idx block_idx = nonmovingGetBlockIdx((StgPtr) p);
@@ -746,6 +751,13 @@ void updateRemembSetPushStack(Capability *cap, StgStack *stack)
     }
 }
 
+void updateRemembSetPushMessageThrowTo(Capability *cap, MessageThrowTo *m) {
+    updateRemembSetPushClosure(cap, (StgClosure *) m->link);
+    updateRemembSetPushClosure(cap, (StgClosure *) m->source);
+    updateRemembSetPushClosure(cap, (StgClosure *) m->target);
+    updateRemembSetPushClosure(cap, (StgClosure *) m->exception);
+}
+
 /*********************************************************
  * Pushing to the mark queue
  *********************************************************/
diff --git a/rts/sm/NonMovingMark.h b/rts/sm/NonMovingMark.h
index 0938e2775a..1cfcc860bf 100644
--- a/rts/sm/NonMovingMark.h
+++ b/rts/sm/NonMovingMark.h
@@ -143,6 +143,7 @@ void updateRemembSetPushClosure(Capability *cap, StgClosure *p);
 void updateRemembSetPushThunk(Capability *cap, StgThunk *p);
 void updateRemembSetPushTSO(Capability *cap, StgTSO *tso);
 void updateRemembSetPushStack(Capability *cap, StgStack *stack);
+void updateRemembSetPushMessageThrowTo(Capability *cap, MessageThrowTo *m);
 
 #if defined(THREADED_RTS)
 void nonmovingFlushCapUpdRemSetBlocks(Capability *cap);
diff --git a/rts/sm/Storage.c b/rts/sm/Storage.c
index c592595737..1e801b8dfc 100644
--- a/rts/sm/Storage.c
+++ b/rts/sm/Storage.c
@@ -109,6 +109,7 @@ Mutex sm_mutex;
 
 static void allocNurseries (uint32_t from, uint32_t to);
 static void assignNurseriesToCapabilities (uint32_t from, uint32_t to);
+static StgInd * lockCAF (StgRegTable *reg, StgIndStatic *caf);
 
 void
 initGeneration (generation *gen, int g)
@@ -510,7 +511,7 @@ void listAllBlocks (ListBlocksCb cb, void *user)
 
    -------------------------------------------------------------------------- */
 
-STATIC_INLINE StgInd *
+static StgInd *
 lockCAF (StgRegTable *reg, StgIndStatic *caf)
 {
     const StgInfoTable *orig_info;