NonMoving: Move next_free_snap to block descriptor

6 files changed, 18 insertions, 12 deletions

diff --git a/includes/rts/storage/Block.h b/includes/rts/storage/Block.h
index 3887ac2980..4afc3689cb 100644
--- a/includes/rts/storage/Block.h
+++ b/includes/rts/storage/Block.h
@@ -102,6 +102,7 @@ typedef struct bdescr_ {
                                    // Unused by the non-moving allocator.
         struct NonmovingSegmentInfo {
             StgWord8 log_block_size;
+            StgWord16 next_free_snap;
         } nonmoving_segment;
     };
 
diff --git a/rts/sm/NonMoving.c b/rts/sm/NonMoving.c
index 0cad2f4fce..f451e38f66 100644
--- a/rts/sm/NonMoving.c
+++ b/rts/sm/NonMoving.c
@@ -209,9 +209,9 @@ static void nonmovingInitSegment(struct NonmovingSegment *seg, uint8_t log_block
     seg->link = NULL;
     seg->todo_link = NULL;
     seg->next_free = 0;
-    seg->next_free_snap = 0;
     nonmovingClearBitmap(seg);
     bd->nonmoving_segment.log_block_size = log_block_size;
+    bd->nonmoving_segment.next_free_snap = 0;
     bd->u.scan = nonmovingSegmentGetBlock(seg, 0);
 }
 
@@ -387,7 +387,7 @@ void *nonmovingAllocate(Capability *cap, StgWord sz)
 
         // Update live data estimate.
         // See Note [Live data accounting in nonmoving collector].
-        unsigned int new_blocks = block_count - current->next_free_snap;
+        unsigned int new_blocks = block_count - nonmovingSegmentInfo(current)->next_free_snap;
         unsigned int block_size = 1 << log_block_size;
         atomic_inc(&oldest_gen->live_estimate, new_blocks * block_size / sizeof(W_));
 
@@ -508,7 +508,7 @@ static void nonmovingPrepareMark(void)
         // Update current segments' snapshot pointers
         for (uint32_t cap_n = 0; cap_n < n_capabilities; ++cap_n) {
             struct NonmovingSegment *seg = alloca->current[cap_n];
-            seg->next_free_snap = seg->next_free;
+            nonmovingSegmentInfo(seg)->next_free_snap = seg->next_free;
         }
 
         // Update filled segments' snapshot pointers and move to sweep_list
@@ -524,7 +524,7 @@ static void nonmovingPrepareMark(void)
                 prefetchForWrite(seg->link->bitmap);
                 nonmovingClearBitmap(seg);
                 // Set snapshot
-                seg->next_free_snap = seg->next_free;
+                nonmovingSegmentInfo(seg)->next_free_snap = seg->next_free;
                 if (seg->link)
                     seg = seg->link;
                 else
diff --git a/rts/sm/NonMoving.h b/rts/sm/NonMoving.h
index c51bb64700..b6fd37e508 100644
--- a/rts/sm/NonMoving.h
+++ b/rts/sm/NonMoving.h
@@ -38,7 +38,6 @@ struct NonmovingSegment {
     struct NonmovingSegment *link;     // for linking together segments into lists
     struct NonmovingSegment *todo_link; // NULL when not in todo list
     nonmoving_block_idx next_free;      // index of the next unallocated block
-    nonmoving_block_idx next_free_snap; // snapshot of next_free
     uint8_t bitmap[];                   // liveness bitmap
     // After the liveness bitmap comes the data blocks. Note that we need to
     // ensure that the size of this struct (including the bitmap) is a multiple
@@ -49,6 +48,11 @@ struct NonmovingSegment {
     // NonmovingBlockInfo stored in the segment's block descriptor. Namely:
     //
     //  * the block size can be found in nonmovingBlockInfo(seg)->log_block_size.
+    //  * the next_free snapshot can be found in
+    //    nonmovingBlockInfo(seg)->next_free_snap.
+    //
+    // This allows us to mark a nonmoving closure without bringing the
+    // NonmovingSegment header into cache.
 };
 
 // This is how we mark end of todo lists. Not NULL because todo_link == NULL
@@ -279,8 +283,9 @@ INLINE_HEADER bool nonmovingClosureMarked(StgPtr p)
 // segment is in the set of segments that will be swept this collection cycle.
 INLINE_HEADER bool nonmovingSegmentBeingSwept(struct NonmovingSegment *seg)
 {
-    unsigned int n = nonmovingSegmentBlockCount(seg);
-    return seg->next_free_snap >= n;
+    struct NonmovingSegmentInfo *seginfo = nonmovingSegmentInfo(seg);
+    unsigned int n = nonmovingBlockCountFromSize(seginfo->log_block_size);
+    return seginfo->next_free_snap >= n;
 }
 
 // Can be called during a major collection to determine whether a particular
diff --git a/rts/sm/NonMovingMark.c b/rts/sm/NonMovingMark.c
index c31939e9a4..4d450ace20 100644
--- a/rts/sm/NonMovingMark.c
+++ b/rts/sm/NonMovingMark.c
@@ -1214,7 +1214,7 @@ mark_closure (MarkQueue *queue, const StgClosure *p0, StgClosure **origin)
                 goto done;
 
             StgClosure *snapshot_loc =
-              (StgClosure *) nonmovingSegmentGetBlock(seg, seg->next_free_snap);
+              (StgClosure *) nonmovingSegmentGetBlock(seg, nonmovingSegmentInfo(seg)->next_free_snap);
             if (p >= snapshot_loc && mark == 0) {
                 /*
                  * In this case we are looking at a block that wasn't allocated
@@ -1646,7 +1646,7 @@ bool nonmovingIsAlive (StgClosure *p)
         struct NonmovingSegment *seg = nonmovingGetSegment((StgPtr) p);
         nonmoving_block_idx i = nonmovingGetBlockIdx((StgPtr) p);
         uint8_t mark =  nonmovingGetMark(seg, i);
-        if (i >= seg->next_free_snap) {
+        if (i >= nonmovingSegmentInfo(seg)->next_free_snap) {
             // If the object is allocated after next_free_snap then one of the
             // following must be true:
             //
diff --git a/rts/sm/NonMovingSweep.c b/rts/sm/NonMovingSweep.c
index 7f13f7d88a..e70933f8a5 100644
--- a/rts/sm/NonMovingSweep.c
+++ b/rts/sm/NonMovingSweep.c
@@ -41,7 +41,7 @@ nonmovingSweepSegment(struct NonmovingSegment *seg)
         } else if (!found_free) {
             found_free = true;
             seg->next_free = i;
-            seg->next_free_snap = i;
+            nonmovingSegmentInfo(seg)->next_free_snap = i;
             Bdescr((P_)seg)->u.scan = (P_)nonmovingSegmentGetBlock(seg, i);
             seg->bitmap[i] = 0;
         } else {
@@ -63,7 +63,7 @@ nonmovingSweepSegment(struct NonmovingSegment *seg)
         return SEGMENT_FILLED;
     } else {
         ASSERT(seg->next_free == 0);
-        ASSERT(seg->next_free_snap == 0);
+        ASSERT(nonmovingSegmentInfo(seg)->next_free_snap == 0);
         return SEGMENT_FREE;
     }
 }
diff --git a/rts/sm/Sanity.c b/rts/sm/Sanity.c
index 631e4a6a1d..b14d865e80 100644
--- a/rts/sm/Sanity.c
+++ b/rts/sm/Sanity.c
@@ -500,7 +500,7 @@ static void checkNonmovingSegments (struct NonmovingSegment *seg)
             if (seg->bitmap[i] == nonmovingMarkEpoch) {
                 StgPtr p = nonmovingSegmentGetBlock(seg, i);
                 checkClosure((StgClosure *) p);
-            } else if (i < seg->next_free_snap){
+            } else if (i < nonmovingSegmentInfo(seg)->next_free_snap){
                 seg->bitmap[i] = 0;
             }
         }