NonMoving: Fuse sweep preparation into mark prep

3 files changed, 25 insertions, 45 deletions

diff --git a/rts/sm/NonMoving.c b/rts/sm/NonMoving.c
index 60416cb0e3..bffc0c744e 100644
--- a/rts/sm/NonMoving.c
+++ b/rts/sm/NonMoving.c
@@ -502,6 +502,9 @@ static void nonmovingPrepareMark(void)
     static_flag =
         static_flag == STATIC_FLAG_A ? STATIC_FLAG_B : STATIC_FLAG_A;
 
+    // Should have been cleared by the last sweep
+    ASSERT(nonmovingHeap.sweep_list == NULL);
+
     nonmovingBumpEpoch();
     for (int alloca_idx = 0; alloca_idx < NONMOVING_ALLOCA_CNT; ++alloca_idx) {
         struct NonmovingAllocator *alloca = nonmovingHeap.allocators[alloca_idx];
@@ -512,14 +515,28 @@ static void nonmovingPrepareMark(void)
             seg->next_free_snap = seg->next_free;
         }
 
-        // Update filled segments' snapshot pointers
-        struct NonmovingSegment *seg = alloca->filled;
-        while (seg) {
-            prefetchForRead(seg->link);
-            prefetchForWrite(seg->link->bitmap);
-            nonmovingClearBitmap(seg);
-            seg->next_free_snap = seg->next_free;
-            seg = seg->link;
+        // Update filled segments' snapshot pointers and move to sweep_list
+        uint32_t n_filled = 0;
+        struct NonmovingSegment *const filled = alloca->filled;
+        alloca->filled = NULL;
+        if (filled) {
+            struct NonmovingSegment *seg = filled;
+            while (true) {
+                n_filled++;
+                prefetchForRead(seg->link);
+                // Clear bitmap
+                prefetchForWrite(seg->link->bitmap);
+                nonmovingClearBitmap(seg);
+                // Set snapshot
+                seg->next_free_snap = seg->next_free;
+                if (seg->link)
+                    seg = seg->link;
+                else
+                    break;
+            }
+            // add filled segments to sweep_list
+            seg->link = nonmovingHeap.sweep_list;
+            nonmovingHeap.sweep_list = filled;
         }
 
         // N.B. It's not necessary to update snapshot pointers of active segments;
@@ -596,7 +613,6 @@ void nonmovingCollect(StgWeak **dead_weaks, StgTSO **resurrected_threads)
     resizeGenerations();
 
     nonmovingPrepareMark();
-    nonmovingPrepareSweep();
 
     // N.B. These should have been cleared at the end of the last sweep.
     ASSERT(nonmoving_marked_large_objects == NULL);
diff --git a/rts/sm/NonMovingSweep.c b/rts/sm/NonMovingSweep.c
index b67bba0ce5..0236ef82e9 100644
--- a/rts/sm/NonMovingSweep.c
+++ b/rts/sm/NonMovingSweep.c
@@ -17,38 +17,6 @@
 #include "Trace.h"
 #include "StableName.h"
 
-static struct NonmovingSegment *pop_all_filled_segments(struct NonmovingAllocator *alloc)
-{
-    while (true) {
-        struct NonmovingSegment *head = alloc->filled;
-        if (cas((StgVolatilePtr) &alloc->filled, (StgWord) head, (StgWord) NULL) == (StgWord) head)
-            return head;
-    }
-}
-
-void nonmovingPrepareSweep()
-{
-    ASSERT(nonmovingHeap.sweep_list == NULL);
-
-    // Move blocks in the allocators' filled lists into sweep_list
-    for (unsigned int alloc_idx = 0; alloc_idx < NONMOVING_ALLOCA_CNT; alloc_idx++)
-    {
-        struct NonmovingAllocator *alloc = nonmovingHeap.allocators[alloc_idx];
-        struct NonmovingSegment *filled = pop_all_filled_segments(alloc);
-
-        // Link filled to sweep_list
-        if (filled) {
-            struct NonmovingSegment *filled_head = filled;
-            // Find end of filled list
-            while (filled->link) {
-                filled = filled->link;
-            }
-            filled->link = nonmovingHeap.sweep_list;
-            nonmovingHeap.sweep_list = filled_head;
-        }
-    }
-}
-
 // On which list should a particular segment be placed?
 enum SweepResult {
     SEGMENT_FREE,     // segment is empty: place on free list
diff --git a/rts/sm/NonMovingSweep.h b/rts/sm/NonMovingSweep.h
index f21936004f..5ae5b687e3 100644
--- a/rts/sm/NonMovingSweep.h
+++ b/rts/sm/NonMovingSweep.h
@@ -22,10 +22,6 @@ void nonmovingSweepLargeObjects(void);
 // Remove dead entries in the stable name table
 void nonmovingSweepStableNameTable(void);
 
-// Collect the set of segments to be collected during a major GC into
-// nonmovingHeap.sweep_list.
-void nonmovingPrepareSweep(void);
-
 #if defined(DEBUG)
 // The non-moving equivalent of the moving collector's gcCAFs.
 void nonmovingGcCafs(void);