Avoid AO_stack_t to cross CPU cache line boundary

(a cherry-pick of commit ed712f7dc from 'release-7_6')

Issue #45 (libatomic_ops).

Enforce proper alignment of AO_stack_t.AO_ptr to avoid the structure
value to cross the CPU cache line boundary.  A workaround for
almost-lock-free push/pop test failures on aarch64, at least.

* src/atomic_ops_stack.h [AO_USE_ALMOST_LOCK_FREE
&& !AO_STACK_ATTR_ALLIGNED] (AO_STACK_ATTR_ALLIGNED): Define.
* src/atomic_ops_stack.h [AO_USE_ALMOST_LOCK_FREE]
(AO_stack_t.AO_ptr): Add AO_STACK_ATTR_ALLIGNED attribute.

1 files changed, 31 insertions, 1 deletions

diff --git a/src/atomic_ops_stack.h b/src/atomic_ops_stack.h
index dd19d35..886bbe1 100644
--- a/src/atomic_ops_stack.h
+++ b/src/atomic_ops_stack.h
@@ -89,6 +89,36 @@
 #  error AO_BL_SIZE too big
 #endif
 
+#ifndef AO_STACK_ATTR_ALLIGNED
+  /* Enforce proper alignment of AO_stack_t.AO_ptr to avoid the         */
+  /* structure value to cross the CPU cache line boundary.              */
+  /* A workaround for almost-lock-free push/pop test failures           */
+  /* on aarch64, at least.                                              */
+# if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1)
+#   define AO_STACK_LOG_BL_SZP1 \
+        (AO_BL_SIZE > 7 ? 4 : AO_BL_SIZE > 3 ? 3 : AO_BL_SIZE > 1 ? 2 : 1)
+#   define AO_STACK_ATTR_ALLIGNED \
+        __attribute__((__aligned__(sizeof(AO_t) << AO_STACK_LOG_BL_SZP1)))
+# elif defined(_MSC_VER) && _MSC_VER >= 1400 /* Visual Studio 2005+ */
+    /* MS compiler accepts only a literal number in align, not expression.  */
+    /* AO_STACK_ALLIGN_N is 1 << (AO_N_BITS + AO_STACK_LOG_BL_SZP1).        */
+#   if AO_N_BITS > 2 && AO_BL_SIZE > 7
+#     define AO_STACK_ALLIGN_N 128
+#   elif (AO_N_BITS > 2 && AO_BL_SIZE > 3) || AO_BL_SIZE > 7
+#     define AO_STACK_ALLIGN_N 64
+#   elif (AO_N_BITS > 2 && AO_BL_SIZE > 1) || AO_BL_SIZE > 3
+#     define AO_STACK_ALLIGN_N 32
+#   elif AO_N_BITS > 2 || AO_BL_SIZE > 1
+#     define AO_STACK_ALLIGN_N 16
+#   else
+#     define AO_STACK_ALLIGN_N 8
+#   endif
+#   define AO_STACK_ATTR_ALLIGNED __declspec(align(AO_STACK_ALLIGN_N))
+# else
+#   define AO_STACK_ATTR_ALLIGNED /* TODO: alignment is not enforced */
+# endif
+#endif /* !AO_STACK_ATTR_ALLIGNED */
+
 typedef struct AO__stack_aux {
   volatile AO_t AO_stack_bl[AO_BL_SIZE];
 } AO_stack_aux;
@@ -114,7 +144,7 @@ AO_stack_pop_explicit_aux_acquire(volatile AO_t *list, AO_stack_aux *);
 /* And now AO_stack_t for the real interface:                           */
 
 typedef struct AO__stack {
-  volatile AO_t AO_ptr;
+  AO_STACK_ATTR_ALLIGNED volatile AO_t AO_ptr;
   AO_stack_aux AO_aux;
 } AO_stack_t;