nonmoving: Don't do two passes over large and compact object lists

Previously we would first move the new objects to their appropriate
non-moving GC list, then do another pass over that list to clear their
mark bits. This is needlessly expensive. First clear the mark bits of
the existing objects, then add the newly evacuated objects and, at the
same time, clear their mark bits.

This cuts the preparatory GC time in half for the Pusher benchmark with
a large queue size.

1 files changed, 14 insertions, 10 deletions

diff --git a/rts/sm/NonMoving.c b/rts/sm/NonMoving.c
index 518b06bd19..50cf784aab 100644
--- a/rts/sm/NonMoving.c
+++ b/rts/sm/NonMoving.c
@@ -737,11 +737,18 @@ static void nonmovingPrepareMark(void)
         // since.
     }
 
-    ASSERT(oldest_gen->scavenged_large_objects == NULL);
+    // Clear large object bits of existing large objects
+    for (bdescr *bd = nonmoving_large_objects; bd; bd = bd->link) {
+        bd->flags &= ~BF_MARKED;
+    }
+
+    // Add newly promoted large objects and clear mark bits
     bdescr *next;
+    ASSERT(oldest_gen->scavenged_large_objects == NULL);
     for (bdescr *bd = oldest_gen->large_objects; bd; bd = next) {
         next = bd->link;
         bd->flags |= BF_NONMOVING_SWEEPING;
+        bd->flags &= ~BF_MARKED;
         dbl_link_onto(bd, &nonmoving_large_objects);
     }
     n_nonmoving_large_blocks += oldest_gen->n_large_blocks;
@@ -750,10 +757,16 @@ static void nonmovingPrepareMark(void)
     oldest_gen->n_large_blocks = 0;
     nonmoving_live_words = 0;
 
+    // Clear compact object mark bits
+    for (bdescr *bd = nonmoving_compact_objects; bd; bd = bd->link) {
+        bd->flags &= ~BF_MARKED;
+    }
+
     // Move new compact objects from younger generations to nonmoving_compact_objects
     for (bdescr *bd = oldest_gen->compact_objects; bd; bd = next) {
         next = bd->link;
         bd->flags |= BF_NONMOVING_SWEEPING;
+        bd->flags &= ~BF_MARKED;
         dbl_link_onto(bd, &nonmoving_compact_objects);
     }
     n_nonmoving_compact_blocks += oldest_gen->n_compact_blocks;
@@ -761,15 +774,6 @@ static void nonmovingPrepareMark(void)
     oldest_gen->compact_objects = NULL;
     // TODO (osa): what about "in import" stuff??
 
-    // Clear compact object mark bits
-    for (bdescr *bd = nonmoving_compact_objects; bd; bd = bd->link) {
-        bd->flags &= ~BF_MARKED;
-    }
-
-    // Clear large object bits
-    for (bdescr *bd = nonmoving_large_objects; bd; bd = bd->link) {
-        bd->flags &= ~BF_MARKED;
-    }
 
 
 #if defined(DEBUG)