GC: move static object processinng into thread-local storage

10 files changed, 78 insertions, 69 deletions

diff --git a/includes/Storage.h b/includes/Storage.h
index 28225d7545..c51f51b4ba 100644
--- a/includes/Storage.h
+++ b/includes/Storage.h
@@ -576,7 +576,6 @@ extern void newDynCAF(StgClosure *);
 extern void move_TSO(StgTSO *src, StgTSO *dest);
 extern StgTSO *relocate_stack(StgTSO *dest, ptrdiff_t diff);
 
-extern StgClosure * RTS_VAR(scavenged_static_objects);
 extern StgWeak    * RTS_VAR(old_weak_ptr_list);
 extern StgWeak    * RTS_VAR(weak_ptr_list);
 extern StgClosure * RTS_VAR(caf_list);
diff --git a/rts/RetainerProfile.c b/rts/RetainerProfile.c
index e96356734d..dec886aba6 100644
--- a/rts/RetainerProfile.c
+++ b/rts/RetainerProfile.c
@@ -1913,7 +1913,7 @@ computeRetainerSet( void )
  *    they are not taken into consideration in computing retainer sets.
  * -------------------------------------------------------------------------- */
 void
-resetStaticObjectForRetainerProfiling( void )
+resetStaticObjectForRetainerProfiling( StgClosure *static_objects )
 {
 #ifdef DEBUG_RETAINER
     nat count;
@@ -1923,7 +1923,7 @@ resetStaticObjectForRetainerProfiling( void )
 #ifdef DEBUG_RETAINER
     count = 0;
 #endif
-    p = scavenged_static_objects;
+    p = static_objects;
     while (p != END_OF_STATIC_LIST) {
 #ifdef DEBUG_RETAINER
 	count++;
diff --git a/rts/RetainerProfile.h b/rts/RetainerProfile.h
index 827daa8ef4..f33c079f04 100644
--- a/rts/RetainerProfile.h
+++ b/rts/RetainerProfile.h
@@ -18,7 +18,7 @@ extern void  initRetainerProfiling ( void );
 extern void  endRetainerProfiling  ( void );
 extern void  printRetainer         ( FILE *, retainer );
 extern void  retainerProfile       ( void );
-extern void  resetStaticObjectForRetainerProfiling ( void );
+extern void resetStaticObjectForRetainerProfiling( StgClosure *static_objects );
 
 extern StgWord RTS_VAR(flip);
 
diff --git a/rts/Stats.c b/rts/Stats.c
index 73479181f9..9a9accad6c 100644
--- a/rts/Stats.c
+++ b/rts/Stats.c
@@ -644,7 +644,6 @@ stat_exit(int alloc)
             
             statsPrintf("recordMutableGen_sync: %"FMT_Word64"\n", recordMutableGen_sync.spin);
             statsPrintf("gc_alloc_block_sync: %"FMT_Word64"\n", gc_alloc_block_sync.spin);
-            statsPrintf("static_objects_sync: %"FMT_Word64"\n", static_objects_sync.spin);
             statsPrintf("whitehole_spin: %"FMT_Word64"\n", whitehole_spin);
             for (g = 0; g < RtsFlags.GcFlags.generations; g++) {
                 for (s = 0; s < generations[g].n_steps; s++) {
diff --git a/rts/sm/Compact.c b/rts/sm/Compact.c
index b8a40d47e9..44b5242023 100644
--- a/rts/sm/Compact.c
+++ b/rts/sm/Compact.c
@@ -999,7 +999,7 @@ compact(void)
     }
 
     // the static objects
-    thread_static(scavenged_static_objects);
+    thread_static(gct->scavenged_static_objects /* ToDo: ok? */);
 
     // the stable pointer table
     threadStablePtrTable((evac_fn)thread);
diff --git a/rts/sm/Evac.c-inc b/rts/sm/Evac.c-inc
index 752fe92f84..e6514180d5 100644
--- a/rts/sm/Evac.c-inc
+++ b/rts/sm/Evac.c-inc
@@ -217,26 +217,39 @@ loop:
       switch (info->type) {
 
       case THUNK_STATIC:
-	  if (info->srt_bitmap != 0 &&
-	      *THUNK_STATIC_LINK((StgClosure *)q) == NULL) {
-	      ACQUIRE_SPIN_LOCK(&static_objects_sync);
+	  if (info->srt_bitmap != 0) {
 	      if (*THUNK_STATIC_LINK((StgClosure *)q) == NULL) {
-		  *THUNK_STATIC_LINK((StgClosure *)q) = static_objects;
-		  static_objects = (StgClosure *)q;
+#ifndef THREADED_RTS
+		  *THUNK_STATIC_LINK((StgClosure *)q) = gct->static_objects;
+		  gct->static_objects = (StgClosure *)q;
+#else
+                  StgPtr link;
+                  link = (StgPtr)cas((StgPtr)THUNK_STATIC_LINK((StgClosure *)q),
+                                     (StgWord)NULL,
+                                     (StgWord)gct->static_objects);
+                  if (link == NULL) {
+                      gct->static_objects = (StgClosure *)q;
+                  }
+#endif
 	      }
-	      RELEASE_SPIN_LOCK(&static_objects_sync);
 	  }
 	  return;
-	  
+
       case FUN_STATIC:
 	  if (info->srt_bitmap != 0 &&
 	      *FUN_STATIC_LINK((StgClosure *)q) == NULL) {
-	      ACQUIRE_SPIN_LOCK(&static_objects_sync);
-	      if (*FUN_STATIC_LINK((StgClosure *)q) == NULL) {
-		  *FUN_STATIC_LINK((StgClosure *)q) = static_objects;
-		  static_objects = (StgClosure *)q;
-	      }
-	      RELEASE_SPIN_LOCK(&static_objects_sync);
+#ifndef THREADED_RTS
+              *FUN_STATIC_LINK((StgClosure *)q) = gct->static_objects;
+              gct->static_objects = (StgClosure *)q;
+#else
+              StgPtr link;
+              link = (StgPtr)cas((StgPtr)FUN_STATIC_LINK((StgClosure *)q),
+                                 (StgWord)NULL,
+                                 (StgWord)gct->static_objects);
+              if (link == NULL) {
+                  gct->static_objects = (StgClosure *)q;
+              }
+#endif
 	  }
 	  return;
 	  
@@ -246,27 +259,40 @@ loop:
 	   * scavenge it later).
 	   */
 	  if (((StgIndStatic *)q)->saved_info == NULL) {
-	      ACQUIRE_SPIN_LOCK(&static_objects_sync);
 	      if (*IND_STATIC_LINK((StgClosure *)q) == NULL) {
-		  *IND_STATIC_LINK((StgClosure *)q) = static_objects;
-		  static_objects = (StgClosure *)q;
+#ifndef THREADED_RTS
+		  *IND_STATIC_LINK((StgClosure *)q) = gct->static_objects;
+		  gct->static_objects = (StgClosure *)q;
+#else
+                  StgPtr link;
+                  link = (StgPtr)cas((StgPtr)IND_STATIC_LINK((StgClosure *)q),
+                                     (StgWord)NULL,
+                                     (StgWord)gct->static_objects);
+                  if (link == NULL) {
+                      gct->static_objects = (StgClosure *)q;
+                  }
+#endif
 	      }
-	      RELEASE_SPIN_LOCK(&static_objects_sync);
 	  }
 	  return;
 	  
       case CONSTR_STATIC:
 	  if (*STATIC_LINK(info,(StgClosure *)q) == NULL) {
-	      ACQUIRE_SPIN_LOCK(&static_objects_sync);
-	      // re-test, after acquiring lock
-	      if (*STATIC_LINK(info,(StgClosure *)q) == NULL) {
-		  *STATIC_LINK(info,(StgClosure *)q) = static_objects;
-		  static_objects = (StgClosure *)q;
-	      }
-	      RELEASE_SPIN_LOCK(&static_objects_sync);
-	        /* I am assuming that static_objects pointers are not
-		 * written to other objects, and thus, no need to retag. */
-	  }
+#ifndef THREADED_RTS
+              *STATIC_LINK(info,(StgClosure *)q) = gct->static_objects;
+              gct->static_objects = (StgClosure *)q;
+#else
+              StgPtr link;
+              link = (StgPtr)cas((StgPtr)STATIC_LINK(info,(StgClosure *)q),
+                                 (StgWord)NULL,
+                                 (StgWord)gct->static_objects);
+              if (link == NULL) {
+                  gct->static_objects = (StgClosure *)q;
+              }
+#endif
+          }
+          /* I am assuming that static_objects pointers are not
+           * written to other objects, and thus, no need to retag. */
           return;
 	  
       case CONSTR_NOCAF_STATIC:
diff --git a/rts/sm/GC.c b/rts/sm/GC.c
index 6c19a6aff2..5b166943d7 100644
--- a/rts/sm/GC.c
+++ b/rts/sm/GC.c
@@ -90,11 +90,6 @@
  * We build up a static object list while collecting generations 0..N,
  * which is then appended to the static object list of generation N+1.
  */
-StgClosure* static_objects;      // live static objects
-StgClosure* scavenged_static_objects;   // static objects scavenged so far
-#ifdef THREADED_RTS
-SpinLock static_objects_sync;
-#endif
 
 /* N is the oldest generation being collected, where the generations
  * are numbered starting at 0.  A major GC (indicated by the major_gc
@@ -269,11 +264,6 @@ GarbageCollect ( rtsBool force_major_gc )
   // check stack sanity *before* GC (ToDo: check all threads) 
   IF_DEBUG(sanity, checkFreeListSanity());
 
-  /* Initialise the static object lists
-   */
-  static_objects = END_OF_STATIC_LIST;
-  scavenged_static_objects = END_OF_STATIC_LIST;
-
   // Initialise all the generations/steps that we're collecting.
   for (g = 0; g <= N; g++) {
       init_collected_gen(g,n_gc_threads);
@@ -629,12 +619,19 @@ GarbageCollect ( rtsBool force_major_gc )
 #ifdef PROFILING
   // resetStaticObjectForRetainerProfiling() must be called before
   // zeroing below.
-  resetStaticObjectForRetainerProfiling();
+  if (n_gc_threads > 1) {
+      barf("profiling is currently broken with multi-threaded GC");
+      // ToDo: fix the gct->scavenged_static_objects below
+  }
+  resetStaticObjectForRetainerProfiling(gct->scavenged_static_objects);
 #endif
 
   // zero the scavenged static object list 
   if (major_gc) {
-    zero_static_object_list(scavenged_static_objects);
+      nat i;
+      for (i = 0; i < n_gc_threads; i++) {
+          zero_static_object_list(gc_threads[i]->scavenged_static_objects);
+      }
   }
 
   // Reset the nursery
@@ -1357,6 +1354,8 @@ init_uncollected_gen (nat g, nat threads)
 static void
 init_gc_thread (gc_thread *t)
 {
+    t->static_objects = END_OF_STATIC_LIST;
+    t->scavenged_static_objects = END_OF_STATIC_LIST;
     t->evac_step = 0;
     t->failed_to_evac = rtsFalse;
     t->eager_promotion = rtsTrue;
@@ -1366,6 +1365,7 @@ init_gc_thread (gc_thread *t)
     t->no_work = 0;
     t->scav_global_work = 0;
     t->scav_local_work = 0;
+
 }
 
 /* -----------------------------------------------------------------------------
diff --git a/rts/sm/GC.h b/rts/sm/GC.h
index 58aa46f373..01c6024266 100644
--- a/rts/sm/GC.h
+++ b/rts/sm/GC.h
@@ -120,6 +120,9 @@ typedef struct gc_thread_ {
                                    //  during GC without accessing the block
                                    //   allocators spin lock. 
 
+    StgClosure* static_objects;      // live static objects
+    StgClosure* scavenged_static_objects;   // static objects scavenged so far
+
     lnat gc_count;                 // number of gc's this thread has done
 
     // --------------------
@@ -174,9 +177,6 @@ extern gc_thread **gc_threads;
 register gc_thread *gct __asm__("%rbx");
 // extern gc_thread *gct;  // this thread's gct TODO: make thread-local
 
-extern StgClosure* static_objects;
-extern StgClosure* scavenged_static_objects;
-
 extern bdescr *mark_stack_bdescr;
 extern StgPtr *mark_stack;
 extern StgPtr *mark_sp;
@@ -188,10 +188,6 @@ extern StgPtr  oldgen_scan;
 
 extern long copied;
 
-#ifdef THREADED_RTS
-extern SpinLock static_objects_sync;
-#endif
-
 #ifdef DEBUG
 extern nat mutlist_MUTVARS, mutlist_MUTARRS, mutlist_MVARS, mutlist_OTHERS;
 #endif
diff --git a/rts/sm/Scav.c b/rts/sm/Scav.c
index c284dee9cc..fc63a8578f 100644
--- a/rts/sm/Scav.c
+++ b/rts/sm/Scav.c
@@ -1078,15 +1078,12 @@ scavenge_static(void)
 
   while (1) {
       
-    ACQUIRE_SPIN_LOCK(&static_objects_sync);
-    
     /* get the next static object from the list.  Remember, there might
      * be more stuff on this list after each evacuation...
      * (static_objects is a global)
      */
-    p = static_objects;
+    p = gct->static_objects;
     if (p == END_OF_STATIC_LIST) {
-    	  RELEASE_SPIN_LOCK(&static_objects_sync);
     	  break;
     }
     
@@ -1101,11 +1098,9 @@ scavenge_static(void)
     /* Take this object *off* the static_objects list,
      * and put it on the scavenged_static_objects list.
      */
-    static_objects = *STATIC_LINK(info,p);
-    *STATIC_LINK(info,p) = scavenged_static_objects;
-    scavenged_static_objects = p;
-    
-    RELEASE_SPIN_LOCK(&static_objects_sync);
+    gct->static_objects = *STATIC_LINK(info,p);
+    *STATIC_LINK(info,p) = gct->scavenged_static_objects;
+    gct->scavenged_static_objects = p;
     
     switch (info -> type) {
       
@@ -1528,8 +1523,8 @@ loop:
     work_to_do = rtsFalse;
 
     // scavenge static objects 
-    if (major_gc && static_objects != END_OF_STATIC_LIST) {
-	IF_DEBUG(sanity, checkStaticObjects(static_objects));
+    if (major_gc && gct->static_objects != END_OF_STATIC_LIST) {
+	IF_DEBUG(sanity, checkStaticObjects(gct->static_objects));
 	scavenge_static();
     }
     
@@ -1561,11 +1556,6 @@ any_work (void)
 
     write_barrier();
 
-    // scavenge static objects 
-    if (major_gc && static_objects != END_OF_STATIC_LIST) {
-	return rtsTrue;
-    }
-    
     // scavenge objects in compacted generation
     if (mark_stack_overflowed || oldgen_scan_bd != NULL ||
 	(mark_stack_bdescr != NULL && !mark_stack_empty())) {
diff --git a/rts/sm/Storage.c b/rts/sm/Storage.c
index 7f4c723934..6b16cc409e 100644
--- a/rts/sm/Storage.c
+++ b/rts/sm/Storage.c
@@ -259,7 +259,6 @@ initStorage( void )
 
 #ifdef THREADED_RTS
   initSpinLock(&gc_alloc_block_sync);
-  initSpinLock(&static_objects_sync);
   initSpinLock(&recordMutableGen_sync);
   whitehole_spin = 0;
 #endif