More principled treatment of acquire fences

4 files changed, 23 insertions, 9 deletions

diff --git a/rts/StgMiscClosures.cmm b/rts/StgMiscClosures.cmm
index efe71bb20b..f0f2cab6fc 100644
--- a/rts/StgMiscClosures.cmm
+++ b/rts/StgMiscClosures.cmm
@@ -521,7 +521,7 @@ INFO_TABLE(stg_IND,1,0,IND,"IND","IND")
     (P_ node)
 {
     TICK_ENT_DYN_IND(); /* tick */
-    ACQUIRE_FENCE;
+    ACQUIRE_FENCE_ON(node + OFFSET_StgHeader_info);
     node = UNTAG(StgInd_indirectee(node));
     TICK_ENT_VIA_NODE();
     jump %GET_ENTRY(node) (node);
@@ -530,7 +530,7 @@ INFO_TABLE(stg_IND,1,0,IND,"IND","IND")
     /* explicit stack */
 {
     TICK_ENT_DYN_IND(); /* tick */
-    ACQUIRE_FENCE;
+    ACQUIRE_FENCE_ON(node + OFFSET_StgHeader_info);
     R1 = UNTAG(StgInd_indirectee(R1));
     TICK_ENT_VIA_NODE();
     jump %GET_ENTRY(R1) [R1];
@@ -541,7 +541,7 @@ INFO_TABLE(stg_IND_STATIC,1,0,IND_STATIC,"IND_STATIC","IND_STATIC")
     /* explicit stack */
 {
     TICK_ENT_STATIC_IND(); /* tick */
-    ACQUIRE_FENCE;
+    ACQUIRE_FENCE_ON(node + OFFSET_StgHeader_info);
     R1 = UNTAG(StgInd_indirectee(R1));
     TICK_ENT_VIA_NODE();
     jump %GET_ENTRY(R1) [R1];
@@ -565,13 +565,9 @@ INFO_TABLE(stg_BLACKHOLE,1,0,BLACKHOLE,"BLACKHOLE","BLACKHOLE")
     TICK_ENT_DYN_IND(); /* tick */
 
 retry:
-#if defined(TSAN_ENABLED)
-    // See Note [ThreadSanitizer and fences]
-    W_ unused; unused = %acquire GET_INFO(node);
-#endif
     // Synchronizes with the release-store in updateWithIndirection.
     // See Note [Heap memory barriers] in SMP.h.
-    ACQUIRE_FENCE;
+    ACQUIRE_FENCE_ON(node + OFFSET_StgHeader_info);
     p = %relaxed StgInd_indirectee(node);
     if (GETTAG(p) != 0) {
         return (p);
diff --git a/rts/include/Cmm.h b/rts/include/Cmm.h
index a1ea4efe92..0d5fd3ece1 100644
--- a/rts/include/Cmm.h
+++ b/rts/include/Cmm.h
@@ -684,12 +684,20 @@
 #define RELEASE_FENCE prim %write_barrier()
 #define ACQUIRE_FENCE prim %read_barrier()
 
+// TODO
+#if 1
+#define ACQUIRE_FENCE_ON(x) if (1) { W_ tmp; (tmp) = prim %load_acquire64(x); }
+#else
+#define ACQUIRE_FENCE_ON(x) ACQUIRE_FENCE
+#endif
+
 #else
 
 #define prim_read_barrier /* nothing */
 #define prim_write_barrier /* nothing */
 #define RELEASE_FENCE /* nothing */
 #define ACQUIRE_FENCE /* nothing */
+#define ACQUIRE_FENCE_ON(x) /* nothing */
 #endif /* THREADED_RTS */
 
 /* -----------------------------------------------------------------------------
diff --git a/rts/include/stg/SMP.h b/rts/include/stg/SMP.h
index c8facda14a..56bbfedf3f 100644
--- a/rts/include/stg/SMP.h
+++ b/rts/include/stg/SMP.h
@@ -592,6 +592,14 @@ load_load_barrier(void) {
 #define RELEASE_FENCE() __atomic_thread_fence(__ATOMIC_RELEASE)
 #define SEQ_CST_FENCE() __atomic_thread_fence(__ATOMIC_SEQ_CST)
 
+#if defined(TSAN_ENABLED)
+#define ACQUIRE_FENCE_ON(x) ACQUIRE_LOAD(x)
+#define RELEASE_FENCE_ON(x) RELEASE_STORE()
+#else
+#define ACQUIRE_FENCE_ON(x) __atomic_thread_fence(__ATOMIC_ACQUIRE)
+#define RELEASE_FENCE_ON(x) __atomic_thread_fence(__ATOMIC_RELEASE)
+#endif
+
 /* ---------------------------------------------------------------------- */
 #else /* !THREADED_RTS */
 
@@ -626,6 +634,8 @@ EXTERN_INLINE void load_load_barrier () {} /* nothing */
 #define ACQUIRE_FENCE()
 #define RELEASE_FENCE()
 #define SEQ_CST_FENCE()
+#define ACQUIRE_FENCE_ON(x)
+#define RELEASE_FENCE_ON(x)
 
 #if !IN_STG_CODE || IN_STGCRUN
 INLINE_HEADER StgWord
diff --git a/rts/sm/NonMovingMark.c b/rts/sm/NonMovingMark.c
index 60ada48193..89a12362ba 100644
--- a/rts/sm/NonMovingMark.c
+++ b/rts/sm/NonMovingMark.c
@@ -1590,7 +1590,7 @@ mark_closure (MarkQueue *queue, const StgClosure *p0, StgClosure **origin)
         // Synchronizes with the release-store in updateWithIndirection.
         // See Note [Heap memory barriers] in SMP.h.
         StgInd *ind = (StgInd *) p;
-        ACQUIRE_FENCE();
+        ACQUIRE_FENCE_ON(&p->header.info);
         StgClosure *indirectee = RELAXED_LOAD(&ind->indirectee);
         markQueuePushClosure(queue, indirectee, &ind->indirectee);
         if (GET_CLOSURE_TAG(indirectee) == 0 || origin == NULL) {