Don't cleanup until we've stopped the collector

This requires that we break nonmovingExit into two pieces since we need
to first stop the collector to relinquish any capabilities, then we need
to shutdown the scheduler, then we need to free the nonmoving
allocators.

4 files changed, 21 insertions, 3 deletions

diff --git a/rts/Schedule.c b/rts/Schedule.c
index 8d82daf381..f867d7f359 100644
--- a/rts/Schedule.c
+++ b/rts/Schedule.c
@@ -2714,7 +2714,7 @@ exitScheduler (bool wait_foreign USED_IF_THREADS)
     // If we haven't killed all the threads yet, do it now.
     if (sched_state < SCHED_SHUTTING_DOWN) {
         sched_state = SCHED_INTERRUPTING;
-        nonmovingExit();
+        nonmovingStop();
         Capability *cap = task->cap;
         waitForCapability(&cap,task);
         scheduleDoGC(&cap,task,true);
diff --git a/rts/sm/NonMoving.c b/rts/sm/NonMoving.c
index 6bccf7f100..24464a43f9 100644
--- a/rts/sm/NonMoving.c
+++ b/rts/sm/NonMoving.c
@@ -291,6 +291,11 @@ static struct NonmovingAllocator *alloc_nonmoving_allocator(uint32_t n_caps)
     return alloc;
 }
 
+static void free_nonmoving_allocator(struct NonmovingAllocator *alloc)
+{
+    stgFree(alloc);
+}
+
 void nonmovingInit(void)
 {
     if (! RtsFlags.GcFlags.useNonmoving) return;
@@ -305,7 +310,8 @@ void nonmovingInit(void)
     nonmovingMarkInitUpdRemSet();
 }
 
-void nonmovingExit(void)
+// Stop any nonmoving collection in preparation for RTS shutdown.
+void nonmovingStop(void)
 {
     if (! RtsFlags.GcFlags.useNonmoving) return;
 #if defined(THREADED_RTS)
@@ -315,14 +321,24 @@ void nonmovingExit(void)
         ACQUIRE_LOCK(&concurrent_coll_finished_lock);
         waitCondition(&concurrent_coll_finished, &concurrent_coll_finished_lock);
     }
+#endif
+}
 
+void nonmovingExit(void)
+{
+    if (! RtsFlags.GcFlags.useNonmoving) return;
+
+    // First make sure collector is stopped before we tear things down.
+    nonmovingStop();
+
+#if defined(THREADED_RTS)
     closeMutex(&concurrent_coll_finished_lock);
     closeCondition(&concurrent_coll_finished);
     closeMutex(&nonmoving_collection_mutex);
 #endif
 
     for (unsigned int i = 0; i < NONMOVING_ALLOCA_CNT; i++) {
-        stgFree(nonmovingHeap.allocators[i]);
+        free_nonmoving_allocator(nonmovingHeap.allocators[i]);
     }
 }
 
diff --git a/rts/sm/NonMoving.h b/rts/sm/NonMoving.h
index 21c69b1ca1..0f9cfa0e48 100644
--- a/rts/sm/NonMoving.h
+++ b/rts/sm/NonMoving.h
@@ -94,6 +94,7 @@ extern struct NonmovingHeap nonmovingHeap;
 extern memcount nonmoving_live_words;
 
 void nonmovingInit(void);
+void nonmovingStop(void);
 void nonmovingExit(void);
 
 
diff --git a/rts/sm/Storage.c b/rts/sm/Storage.c
index 0199bc58f3..7ed4896301 100644
--- a/rts/sm/Storage.c
+++ b/rts/sm/Storage.c
@@ -300,6 +300,7 @@ void storageAddCapabilities (uint32_t from, uint32_t to)
 void
 exitStorage (void)
 {
+    nonmovingExit();
     updateNurseriesStats();
     stat_exit();
 }