Specialise evac/scav for single-threaded, not minor, GC

So we can parallelise minor collections too.  Sometimes it's worth it.

6 files changed, 26 insertions, 49 deletions

diff --git a/rts/sm/Evac.c b/rts/sm/Evac.c
index 4c386f75df..f117526f3a 100644
--- a/rts/sm/Evac.c
+++ b/rts/sm/Evac.c
@@ -75,10 +75,10 @@ alloc_for_copy (nat size, step *stp)
    The evacuate() code
    -------------------------------------------------------------------------- */
 
-#define MINOR_GC
+#define PARALLEL_GC
 #include "Evac.c-inc"
 
-#undef MINOR_GC
+#undef PARALLEL_GC
 #include "Evac.c-inc"
 
 /* -----------------------------------------------------------------------------
diff --git a/rts/sm/Evac.c-inc b/rts/sm/Evac.c-inc
index e6514180d5..6f629f0a6b 100644
--- a/rts/sm/Evac.c-inc
+++ b/rts/sm/Evac.c-inc
@@ -10,11 +10,11 @@
 // non-minor, parallel, GC.  This file contains the code for both,
 // controllled by the CPP symbol MINOR_GC.
 
-#ifdef MINOR_GC
-#define copy(a,b,c,d) copy0(a,b,c,d)
-#define copy_tag(a,b,c,d,e) copy_tag0(a,b,c,d,e)
-#define copyPart(a,b,c,d,e) copyPart0(a,b,c,d,e)
-#define evacuate(a) evacuate0(a)
+#ifndef PARALLEL_GC
+#define copy(a,b,c,d) copy1(a,b,c,d)
+#define copy_tag(a,b,c,d,e) copy_tag1(a,b,c,d,e)
+#define copyPart(a,b,c,d,e) copyPart1(a,b,c,d,e)
+#define evacuate(a) evacuate1(a)
 #else
 #undef copy
 #undef copy_tag
@@ -29,7 +29,7 @@ copy_tag(StgClosure **p, StgClosure *src, nat size, step *stp, StgWord tag)
     nat i;
     StgWord info;
 
-#if !defined(MINOR_GC) && defined(THREADED_RTS)
+#if defined(PARALLEL_GC) && defined(THREADED_RTS)
 spin:
 	info = xchg((StgPtr)&src->header.info, (W_)&stg_WHITEHOLE_info);
 	// so..  what is it?
@@ -68,7 +68,7 @@ spin:
 //  }
 
     ((StgEvacuated*)from)->evacuee = (StgClosure *)tagged_to;
-#if !defined(MINOR_GC) && defined(THREADED_RTS)
+#if defined(PARALLEL_GC) && defined(THREADED_RTS)
     write_barrier();
     ((StgEvacuated*)from)->header.info = &stg_EVACUATED_info;
 #endif
@@ -92,7 +92,7 @@ copyPart(StgClosure **p, StgClosure *src, nat size_to_reserve, nat size_to_copy,
     nat i;
     StgWord info;
     
-#if !defined(MINOR_GC) && defined(THREADED_RTS)
+#if defined(PARALLEL_GC) && defined(THREADED_RTS)
 spin:
 	info = xchg((StgPtr)&src->header.info, (W_)&stg_WHITEHOLE_info);
 	if (info == (W_)&stg_WHITEHOLE_info) {
@@ -123,7 +123,7 @@ spin:
     }
     
     ((StgEvacuated*)from)->evacuee = (StgClosure *)to;
-#if !defined(MINOR_GC) && defined(THREADED_RTS)
+#if defined(PARALLEL_GC) && defined(THREADED_RTS)
     write_barrier();
     ((StgEvacuated*)from)->header.info = &stg_EVACUATED_info;
 #endif
@@ -208,9 +208,6 @@ loop:
 
   if (!HEAP_ALLOCED(q)) {
 
-#ifdef MINOR_GC
-      return;
-#endif
       if (!major_gc) return;
 
       info = get_itbl(q);
diff --git a/rts/sm/Evac.h b/rts/sm/Evac.h
index 1bce74a336..893f79e37e 100644
--- a/rts/sm/Evac.h
+++ b/rts/sm/Evac.h
@@ -28,7 +28,7 @@
 #endif
 
 REGPARM1 void evacuate  (StgClosure **p);
-REGPARM1 void evacuate0 (StgClosure **p);
+REGPARM1 void evacuate1 (StgClosure **p);
 
 extern lnat thunk_selector_depth;
 
diff --git a/rts/sm/GC.h b/rts/sm/GC.h
index 01c6024266..612da64bf2 100644
--- a/rts/sm/GC.h
+++ b/rts/sm/GC.h
@@ -172,6 +172,7 @@ typedef struct gc_thread_ {
 
 extern nat N;
 extern rtsBool major_gc;
+extern nat n_gc_threads;
 
 extern gc_thread **gc_threads;
 register gc_thread *gct __asm__("%rbx");
diff --git a/rts/sm/Scav.c b/rts/sm/Scav.c
index d8d158a377..e08572a19f 100644
--- a/rts/sm/Scav.c
+++ b/rts/sm/Scav.c
@@ -1383,9 +1383,9 @@ scavenge_large (step_workspace *ws)
    Scavenge a block
    ------------------------------------------------------------------------- */
 
-#define MINOR_GC
+#define PARALLEL_GC
 #include "Scav.c-inc"
-#undef MINOR_GC
+#undef PARALLEL_GC
 #include "Scav.c-inc"
 
 /* ----------------------------------------------------------------------------
@@ -1421,8 +1421,8 @@ scavenge_find_global_work (void)
             // to scavenge the whole thing and then push it on
             // our scavd list.  This saves pushing out the
             // scan_bd block, which might be partial.
-            if (N == 0) {
-                scavenge_block0(bd, bd->start);
+            if (n_gc_threads == 1) {
+                scavenge_block1(bd, bd->start);
             } else {
                 scavenge_block(bd, bd->start);
             }
@@ -1480,8 +1480,8 @@ scavenge_find_local_work (void)
         // scavenge everything up to the free pointer.
         if (ws->scan != NULL && ws->scan < ws->scan_bd->free)
         {
-            if (N == 0) {
-                scavenge_block0(ws->scan_bd, ws->scan);
+            if (n_gc_threads == 1) {
+                scavenge_block1(ws->scan_bd, ws->scan);
             } else {
                 scavenge_block(ws->scan_bd, ws->scan);
             }
diff --git a/rts/sm/Scav.c-inc b/rts/sm/Scav.c-inc
index 28aa80d128..d694887a6e 100644
--- a/rts/sm/Scav.c-inc
+++ b/rts/sm/Scav.c-inc
@@ -11,12 +11,12 @@
  *
  * ---------------------------------------------------------------------------*/
 
-// This file is #included into Scav.c, twice: firstly with MINOR_GC
+// This file is #included into Scav.c, twice: firstly with PARALLEL_GC
 // defined, the second time without.
 
-#ifdef MINOR_GC
-#define scavenge_block(a,b) scavenge_block0(a,b)
-#define evacuate(a) evacuate0(a)
+#ifndef PARALLEL_GC
+#define scavenge_block(a,b) scavenge_block1(a,b)
+#define evacuate(a) evacuate1(a)
 #else
 #undef scavenge_block
 #undef evacuate
@@ -91,18 +91,14 @@ scavenge_block (bdescr *bd, StgPtr scan)
     }
 
     case FUN_2_0:
-#ifndef MINOR_GC
 	scavenge_fun_srt(info);
-#endif
 	evacuate(&((StgClosure *)p)->payload[1]);
 	evacuate(&((StgClosure *)p)->payload[0]);
 	p += sizeofW(StgHeader) + 2;
 	break;
 
     case THUNK_2_0:
-#ifndef MINOR_GC
 	scavenge_thunk_srt(info);
-#endif
 	evacuate(&((StgThunk *)p)->payload[1]);
 	evacuate(&((StgThunk *)p)->payload[0]);
 	p += sizeofW(StgThunk) + 2;
@@ -115,82 +111,62 @@ scavenge_block (bdescr *bd, StgPtr scan)
 	break;
 	
     case THUNK_1_0:
-#ifndef MINOR_GC
 	scavenge_thunk_srt(info);
-#endif
 	evacuate(&((StgThunk *)p)->payload[0]);
 	p += sizeofW(StgThunk) + 1;
 	break;
 	
     case FUN_1_0:
-#ifndef MINOR_GC
 	scavenge_fun_srt(info);
-#endif
     case CONSTR_1_0:
 	evacuate(&((StgClosure *)p)->payload[0]);
 	p += sizeofW(StgHeader) + 1;
 	break;
 	
     case THUNK_0_1:
-#ifndef MINOR_GC
 	scavenge_thunk_srt(info);
-#endif
 	p += sizeofW(StgThunk) + 1;
 	break;
 	
     case FUN_0_1:
-#ifndef MINOR_GC
 	scavenge_fun_srt(info);
-#endif
     case CONSTR_0_1:
 	p += sizeofW(StgHeader) + 1;
 	break;
 	
     case THUNK_0_2:
-#ifndef MINOR_GC
 	scavenge_thunk_srt(info);
-#endif
 	p += sizeofW(StgThunk) + 2;
 	break;
 	
     case FUN_0_2:
-#ifndef MINOR_GC
 	scavenge_fun_srt(info);
-#endif
     case CONSTR_0_2:
 	p += sizeofW(StgHeader) + 2;
 	break;
 	
     case THUNK_1_1:
-#ifndef MINOR_GC
 	scavenge_thunk_srt(info);
-#endif
 	evacuate(&((StgThunk *)p)->payload[0]);
 	p += sizeofW(StgThunk) + 2;
 	break;
 
     case FUN_1_1:
-#ifndef MINOR_GC
 	scavenge_fun_srt(info);
-#endif
     case CONSTR_1_1:
 	evacuate(&((StgClosure *)p)->payload[0]);
 	p += sizeofW(StgHeader) + 2;
 	break;
 	
     case FUN:
-#ifndef MINOR_GC
 	scavenge_fun_srt(info);
-#endif
 	goto gen_obj;
 
     case THUNK:
     {
 	StgPtr end;
 
-#ifndef MINOR_GC
 	scavenge_thunk_srt(info);
-#endif
 	end = (P_)((StgThunk *)p)->payload + info->layout.payload.ptrs;
 	for (p = (P_)((StgThunk *)p)->payload; p < end; p++) {
 	    evacuate((StgClosure **)p);
@@ -473,3 +449,6 @@ scavenge_block (bdescr *bd, StgPtr scan)
   debugTrace(DEBUG_gc, "   scavenged %ld bytes",
              (unsigned long)((bd->free - scan) * sizeof(W_)));
 }
+
+#undef scavenge_block
+#undef evacuate