nonmoving: Ensure that evacuated large objects are marked

See Note [Non-moving GC: Marking evacuated objects].

(cherry picked from commit b416189e4004506b89f06f147be37e76f4cd507f)

2 files changed, 66 insertions, 8 deletions

diff --git a/rts/sm/Evac.c b/rts/sm/Evac.c
index 049520b694..a4cbd6e6b3 100644
--- a/rts/sm/Evac.c
+++ b/rts/sm/Evac.c
@@ -109,6 +109,8 @@ alloc_for_copy (uint32_t size, uint32_t gen_no)
             //
             // However, if we are in a deadlock detection GC then we disable aging
             // so there is no need.
+            //
+            // See Note [Non-moving GC: Marking evacuated objects].
             if (major_gc && !deadlock_detect_gc)
                 markQueuePushClosureGC(&gct->cap->upd_rem_set.queue, (StgClosure *) to);
             return to;
@@ -134,7 +136,58 @@ alloc_for_copy (uint32_t size, uint32_t gen_no)
    The evacuate() code
    -------------------------------------------------------------------------- */
 
-/* size is in words */
+/*
+ * Note [Non-moving GC: Marking evacuated objects]
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ *
+ * When the non-moving collector is in use we must be careful to ensure that any
+ * references to objects in the non-moving generation from younger generations
+ * are pushed to the mark queue.
+ *
+ * In particular we need to ensure that we handle newly-promoted objects are
+ * correctly marked. For instance, consider this case:
+ *
+ *     generation 0                          generation 1
+ *    ──────────────                        ──────────────
+ *
+ *                                            ┌───────┐
+ *      ┌───────┐                             │   A   │
+ *      │   B   │ ◁────────────────────────── │       │
+ *      │       │ ──┬─────────────────┐       └───────┘
+ *      └───────┘   ┆        after GC │
+ *                  ┆                 │
+ *      ┌───────┐   ┆ before GC       │       ┌───────┐
+ *      │   C   │ ◁┄┘                 └─────▷ │   C'  │
+ *      │       │                             │       │
+ *      └───────┘                             └───────┘
+ *
+ *
+ * In this case object C started off in generation 0 and was evacuated into
+ * generation 1 during the preparatory GC. However, the only reference to C'
+ * is from B, which lives in the generation 0 (via aging); this reference will
+ * not be visible to the concurrent non-moving collector (which can only
+ * traverse the generation 1 heap). Consequently, upon evacuating C we need to
+ * ensure that C' is added to the update remembered set as we know that it will
+ * continue to be reachable via B (which is assumed to be reachable as it lives
+ * in a younger generation).
+ *
+ * Where this happens depends upon the type of the object (e.g. C'):
+ *
+ *  - In the case of "normal" small heap-allocated objects this happens in
+ *    alloc_for_copy.
+ *  - In the case of compact region this happens in evacuate_compact.
+ *  - In the case of large objects this happens in evacuate_large.
+ *
+ * See also Note [Aging under the non-moving collector] in NonMoving.c.
+ *
+ */
+
+/* size is in words
+
+   We want to *always* inline this as often the size of the closure is static,
+   which allows unrolling of the copy loop.
+
+ */
 STATIC_INLINE GNUC_ATTR_HOT void
 copy_tag(StgClosure **p, const StgInfoTable *info,
          StgClosure *src, uint32_t size, uint32_t gen_no, StgWord tag)
@@ -351,6 +404,9 @@ evacuate_large(StgPtr p)
   __atomic_fetch_or(&bd->flags, BF_EVACUATED, __ATOMIC_ACQ_REL);
   if (RTS_UNLIKELY(RtsFlags.GcFlags.useNonmoving && new_gen == oldest_gen)) {
       __atomic_fetch_or(&bd->flags, BF_NONMOVING, __ATOMIC_ACQ_REL);
+
+      // See Note [Non-moving GC: Marking evacuated objects].
+      markQueuePushClosureGC(&gct->cap->upd_rem_set.queue, (StgClosure *) p);
   }
   initBdescr(bd, new_gen, new_gen->to);
 
@@ -505,6 +561,9 @@ evacuate_compact (StgPtr p)
     bd->flags |= BF_EVACUATED;
     if (RTS_UNLIKELY(RtsFlags.GcFlags.useNonmoving && new_gen == oldest_gen)) {
       __atomic_fetch_or(&bd->flags, BF_NONMOVING, __ATOMIC_RELAXED);
+
+      // See Note [Non-moving GC: Marking evacuated objects].
+      markQueuePushClosureGC(&gct->cap->upd_rem_set.queue, (StgClosure *) str);
     }
     initBdescr(bd, new_gen, new_gen->to);
 
@@ -690,13 +749,6 @@ loop:
        */
       if (flags & BF_LARGE) {
           evacuate_large((P_)q);
-
-          // We may have evacuated the block to the nonmoving generation. If so
-          // we need to make sure it is added to the mark queue since the only
-          // reference to it may be from the moving heap.
-          if (major_gc && flags & BF_NONMOVING && !deadlock_detect_gc) {
-              markQueuePushClosureGC(&gct->cap->upd_rem_set.queue, q);
-          }
           return;
       }
 
diff --git a/rts/sm/NonMoving.c b/rts/sm/NonMoving.c
index 3adfc0892b..99fd9c1ece 100644
--- a/rts/sm/NonMoving.c
+++ b/rts/sm/NonMoving.c
@@ -204,6 +204,10 @@ Mutex concurrent_coll_finished_lock;
  *  - Note [Aging under the non-moving collector] (NonMoving.c) describes how
  *    we accomodate aging
  *
+ *  - Note [Non-moving GC: Marking evacuated objects] (Evac.c) describes how
+ *    non-moving objects reached by evacuate() are marked, which is necessary
+ *    due to aging.
+ *
  *  - Note [Large objects in the non-moving collector] (NonMovingMark.c)
  *    describes how we track large objects.
  *
@@ -317,6 +321,8 @@ Mutex concurrent_coll_finished_lock;
  *
  *     The non-moving collector will come to C in the mark queue and mark it.
  *
+ * The implementation details of this are described in Note [Non-moving GC:
+ * Marking evacuated objects] in Evac.c.
  *
  * Note [Deadlock detection under the non-moving collector]
  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~