summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBen Gamari <ben@smart-cactus.org>2023-03-24 20:52:57 -0400
committerBen Gamari <ben@smart-cactus.org>2023-04-24 06:03:49 -0400
commit48512cb5166dc1885ccce8ba4d59a10426a5aba6 (patch)
tree56b84c0b1c3d00ad443806c8a18a3cf8e0bcddcf
parentc3b59aec3be9e673c106e9717ae47080b98d8a5c (diff)
downloadhaskell-48512cb5166dc1885ccce8ba4d59a10426a5aba6.tar.gz
More principled treatment of acquire fences
-rw-r--r--rts/StgMiscClosures.cmm12
-rw-r--r--rts/include/Cmm.h8
-rw-r--r--rts/include/stg/SMP.h10
-rw-r--r--rts/sm/NonMovingMark.c2
4 files changed, 23 insertions, 9 deletions
diff --git a/rts/StgMiscClosures.cmm b/rts/StgMiscClosures.cmm
index efe71bb20b..f0f2cab6fc 100644
--- a/rts/StgMiscClosures.cmm
+++ b/rts/StgMiscClosures.cmm
@@ -521,7 +521,7 @@ INFO_TABLE(stg_IND,1,0,IND,"IND","IND")
(P_ node)
{
TICK_ENT_DYN_IND(); /* tick */
- ACQUIRE_FENCE;
+ ACQUIRE_FENCE_ON(node + OFFSET_StgHeader_info);
node = UNTAG(StgInd_indirectee(node));
TICK_ENT_VIA_NODE();
jump %GET_ENTRY(node) (node);
@@ -530,7 +530,7 @@ INFO_TABLE(stg_IND,1,0,IND,"IND","IND")
/* explicit stack */
{
TICK_ENT_DYN_IND(); /* tick */
- ACQUIRE_FENCE;
+ ACQUIRE_FENCE_ON(node + OFFSET_StgHeader_info);
R1 = UNTAG(StgInd_indirectee(R1));
TICK_ENT_VIA_NODE();
jump %GET_ENTRY(R1) [R1];
@@ -541,7 +541,7 @@ INFO_TABLE(stg_IND_STATIC,1,0,IND_STATIC,"IND_STATIC","IND_STATIC")
/* explicit stack */
{
TICK_ENT_STATIC_IND(); /* tick */
- ACQUIRE_FENCE;
+ ACQUIRE_FENCE_ON(node + OFFSET_StgHeader_info);
R1 = UNTAG(StgInd_indirectee(R1));
TICK_ENT_VIA_NODE();
jump %GET_ENTRY(R1) [R1];
@@ -565,13 +565,9 @@ INFO_TABLE(stg_BLACKHOLE,1,0,BLACKHOLE,"BLACKHOLE","BLACKHOLE")
TICK_ENT_DYN_IND(); /* tick */
retry:
-#if defined(TSAN_ENABLED)
- // See Note [ThreadSanitizer and fences]
- W_ unused; unused = %acquire GET_INFO(node);
-#endif
// Synchronizes with the release-store in updateWithIndirection.
// See Note [Heap memory barriers] in SMP.h.
- ACQUIRE_FENCE;
+ ACQUIRE_FENCE_ON(node + OFFSET_StgHeader_info);
p = %relaxed StgInd_indirectee(node);
if (GETTAG(p) != 0) {
return (p);
diff --git a/rts/include/Cmm.h b/rts/include/Cmm.h
index a1ea4efe92..0d5fd3ece1 100644
--- a/rts/include/Cmm.h
+++ b/rts/include/Cmm.h
@@ -684,12 +684,20 @@
#define RELEASE_FENCE prim %write_barrier()
#define ACQUIRE_FENCE prim %read_barrier()
+// TODO
+#if 1
+#define ACQUIRE_FENCE_ON(x) if (1) { W_ tmp; (tmp) = prim %load_acquire64(x); }
+#else
+#define ACQUIRE_FENCE_ON(x) ACQUIRE_FENCE
+#endif
+
#else
#define prim_read_barrier /* nothing */
#define prim_write_barrier /* nothing */
#define RELEASE_FENCE /* nothing */
#define ACQUIRE_FENCE /* nothing */
+#define ACQUIRE_FENCE_ON(x) /* nothing */
#endif /* THREADED_RTS */
/* -----------------------------------------------------------------------------
diff --git a/rts/include/stg/SMP.h b/rts/include/stg/SMP.h
index c8facda14a..56bbfedf3f 100644
--- a/rts/include/stg/SMP.h
+++ b/rts/include/stg/SMP.h
@@ -592,6 +592,14 @@ load_load_barrier(void) {
#define RELEASE_FENCE() __atomic_thread_fence(__ATOMIC_RELEASE)
#define SEQ_CST_FENCE() __atomic_thread_fence(__ATOMIC_SEQ_CST)
+#if defined(TSAN_ENABLED)
+#define ACQUIRE_FENCE_ON(x) ACQUIRE_LOAD(x)
+#define RELEASE_FENCE_ON(x) RELEASE_STORE()
+#else
+#define ACQUIRE_FENCE_ON(x) __atomic_thread_fence(__ATOMIC_ACQUIRE)
+#define RELEASE_FENCE_ON(x) __atomic_thread_fence(__ATOMIC_RELEASE)
+#endif
+
/* ---------------------------------------------------------------------- */
#else /* !THREADED_RTS */
@@ -626,6 +634,8 @@ EXTERN_INLINE void load_load_barrier () {} /* nothing */
#define ACQUIRE_FENCE()
#define RELEASE_FENCE()
#define SEQ_CST_FENCE()
+#define ACQUIRE_FENCE_ON(x)
+#define RELEASE_FENCE_ON(x)
#if !IN_STG_CODE || IN_STGCRUN
INLINE_HEADER StgWord
diff --git a/rts/sm/NonMovingMark.c b/rts/sm/NonMovingMark.c
index 60ada48193..89a12362ba 100644
--- a/rts/sm/NonMovingMark.c
+++ b/rts/sm/NonMovingMark.c
@@ -1590,7 +1590,7 @@ mark_closure (MarkQueue *queue, const StgClosure *p0, StgClosure **origin)
// Synchronizes with the release-store in updateWithIndirection.
// See Note [Heap memory barriers] in SMP.h.
StgInd *ind = (StgInd *) p;
- ACQUIRE_FENCE();
+ ACQUIRE_FENCE_ON(&p->header.info);
StgClosure *indirectee = RELAXED_LOAD(&ind->indirectee);
markQueuePushClosure(queue, indirectee, &ind->indirectee);
if (GET_CLOSURE_TAG(indirectee) == 0 || origin == NULL) {