diff options
author | Ivan Maidanski <ivmai@mail.ru> | 2022-02-20 16:57:28 +0300 |
---|---|---|
committer | Ivan Maidanski <ivmai@mail.ru> | 2022-08-12 00:43:51 +0300 |
commit | ed712f7dc6effa68e1d02bc7c21a2dc0504e8cfe (patch) | |
tree | 5a9077197e6dd659db0e4854b56df1e6788ca85e /src/atomic_ops_stack.h | |
parent | ae0f8e7e4081c10b1febfa9c481925a666dd8e23 (diff) | |
download | libatomic_ops-ed712f7dc6effa68e1d02bc7c21a2dc0504e8cfe.tar.gz |
Avoid AO_stack_t to cross CPU cache line boundary
(a cherry-pick of commit c664a6113 from 'master')
Issue #45 (libatomic_ops).
Enforce proper alignment of AO_stack_t.AO_ptr to avoid the structure
value to cross the CPU cache line boundary. A workaround for
almost-lock-free push/pop test failures on aarch64, at least.
* src/atomic_ops_stack.h [AO_USE_ALMOST_LOCK_FREE
&& !AO_STACK_ATTR_ALLIGNED] (AO_STACK_ATTR_ALLIGNED): Define.
* src/atomic_ops_stack.h [AO_USE_ALMOST_LOCK_FREE]
(AO_stack_t.AO_ptr): Add AO_STACK_ATTR_ALLIGNED attribute.
Diffstat (limited to 'src/atomic_ops_stack.h')
-rw-r--r-- | src/atomic_ops_stack.h | 32 |
1 files changed, 31 insertions, 1 deletions
diff --git a/src/atomic_ops_stack.h b/src/atomic_ops_stack.h index 16e5614..e03c186 100644 --- a/src/atomic_ops_stack.h +++ b/src/atomic_ops_stack.h @@ -96,6 +96,36 @@ # error AO_BL_SIZE too big #endif +#ifndef AO_STACK_ATTR_ALLIGNED + /* Enforce proper alignment of AO_stack_t.AO_ptr to avoid the */ + /* structure value to cross the CPU cache line boundary. */ + /* A workaround for almost-lock-free push/pop test failures */ + /* on aarch64, at least. */ +# if AO_GNUC_PREREQ(3, 1) +# define AO_STACK_LOG_BL_SZP1 \ + (AO_BL_SIZE > 7 ? 4 : AO_BL_SIZE > 3 ? 3 : AO_BL_SIZE > 1 ? 2 : 1) +# define AO_STACK_ATTR_ALLIGNED \ + __attribute__((__aligned__(sizeof(AO_t) << AO_STACK_LOG_BL_SZP1))) +# elif defined(_MSC_VER) && _MSC_VER >= 1400 /* Visual Studio 2005+ */ + /* MS compiler accepts only a literal number in align, not expression. */ + /* AO_STACK_ALLIGN_N is 1 << (AO_N_BITS + AO_STACK_LOG_BL_SZP1). */ +# if AO_N_BITS > 2 && AO_BL_SIZE > 7 +# define AO_STACK_ALLIGN_N 128 +# elif (AO_N_BITS > 2 && AO_BL_SIZE > 3) || AO_BL_SIZE > 7 +# define AO_STACK_ALLIGN_N 64 +# elif (AO_N_BITS > 2 && AO_BL_SIZE > 1) || AO_BL_SIZE > 3 +# define AO_STACK_ALLIGN_N 32 +# elif AO_N_BITS > 2 || AO_BL_SIZE > 1 +# define AO_STACK_ALLIGN_N 16 +# else +# define AO_STACK_ALLIGN_N 8 +# endif +# define AO_STACK_ATTR_ALLIGNED __declspec(align(AO_STACK_ALLIGN_N)) +# else +# define AO_STACK_ATTR_ALLIGNED /* TODO: alignment is not enforced */ +# endif +#endif /* !AO_STACK_ATTR_ALLIGNED */ + typedef struct AO__stack_aux { volatile AO_t AO_stack_bl[AO_BL_SIZE]; } AO_stack_aux; @@ -121,7 +151,7 @@ AO_stack_pop_explicit_aux_acquire(volatile AO_t *list, AO_stack_aux *); /* And now AO_stack_t for the real interface: */ typedef struct AO__stack { - volatile AO_t AO_ptr; + AO_STACK_ATTR_ALLIGNED volatile AO_t AO_ptr; AO_stack_aux AO_aux; } AO_stack_t; |