1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
|
/*
* The implementation of the routines described here is covered by the GPL.
* This header file is covered by the following license:
*/
/*
* Copyright (c) 2005 Hewlett-Packard Development Company, L.P.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/* Almost lock-free LIFO linked lists (linked stacks). */
#ifndef AO_STACK_H
#define AO_STACK_H
#include "atomic_ops.h"
#ifdef __cplusplus
extern "C" {
#endif
#ifdef AO_USE_ALMOST_LOCK_FREE
/* Use the almost-non-blocking implementation regardless of the */
/* double-word CAS availability. */
#elif !defined(AO_HAVE_compare_double_and_swap_double) \
&& !defined(AO_HAVE_compare_double_and_swap) \
&& defined(AO_HAVE_compare_and_swap)
# define AO_USE_ALMOST_LOCK_FREE
#else
/* If we have no compare-and-swap operation defined, we assume */
/* that we will actually be using CAS emulation. If we do that, */
/* it's cheaper to use the version-based implementation. */
# define AO_STACK_IS_LOCK_FREE
#endif
/*
* These are not guaranteed to be completely lock-free.
* List insertion may spin under extremely unlikely conditions.
* It cannot deadlock due to recursive reentry unless AO_list_remove
* is called while at least AO_BL_SIZE activations of
* AO_list_remove are currently active in the same thread, i.e.
* we must have at least AO_BL_SIZE recursive signal handler
* invocations.
*
* All operations take an AO_list_aux argument. It is safe to
* share a single AO_list_aux structure among all lists, but that
* may increase contention. Any given list must always be accessed
* with the same AO_list_aux structure.
*
* We make some machine-dependent assumptions:
* - We have a compare-and-swap operation.
* - At least _AO_N_BITS low order bits in pointers are
* zero and normally unused.
* - size_t and pointers have the same size.
*
* We do use a fully lock-free implementation if double-width
* compare-and-swap operations are available.
*/
#ifdef AO_USE_ALMOST_LOCK_FREE
/* The number of low order pointer bits we can use for a small */
/* version number. */
# if defined(__LP64__) || defined(_LP64) || defined(_WIN64)
# define AO_N_BITS 3
# else
# define AO_N_BITS 2
# endif
# define AO_BIT_MASK ((1 << AO_N_BITS) - 1)
/*
* AO_stack_aux should be treated as opaque.
* It is fully defined here, so it can be allocated, and to facilitate
* debugging.
*/
#ifndef AO_BL_SIZE
# define AO_BL_SIZE 2
#endif
#if AO_BL_SIZE > (1 << AO_N_BITS)
# error AO_BL_SIZE too big
#endif
#ifndef AO_STACK_ATTR_ALLIGNED
/* Enforce proper alignment of AO_stack_t.AO_ptr to avoid the */
/* structure value to cross the CPU cache line boundary. */
/* A workaround for almost-lock-free push/pop test failures */
/* on aarch64, at least. */
# if AO_GNUC_PREREQ(3, 1)
# define AO_STACK_LOG_BL_SZP1 \
(AO_BL_SIZE > 7 ? 4 : AO_BL_SIZE > 3 ? 3 : AO_BL_SIZE > 1 ? 2 : 1)
# define AO_STACK_ATTR_ALLIGNED \
__attribute__((__aligned__(sizeof(AO_t) << AO_STACK_LOG_BL_SZP1)))
# elif defined(_MSC_VER) && _MSC_VER >= 1400 /* Visual Studio 2005+ */
/* MS compiler accepts only a literal number in align, not expression. */
/* AO_STACK_ALLIGN_N is 1 << (AO_N_BITS + AO_STACK_LOG_BL_SZP1). */
# if AO_N_BITS > 2 && AO_BL_SIZE > 7
# define AO_STACK_ALLIGN_N 128
# elif (AO_N_BITS > 2 && AO_BL_SIZE > 3) || AO_BL_SIZE > 7
# define AO_STACK_ALLIGN_N 64
# elif (AO_N_BITS > 2 && AO_BL_SIZE > 1) || AO_BL_SIZE > 3
# define AO_STACK_ALLIGN_N 32
# elif AO_N_BITS > 2 || AO_BL_SIZE > 1
# define AO_STACK_ALLIGN_N 16
# else
# define AO_STACK_ALLIGN_N 8
# endif
# define AO_STACK_ATTR_ALLIGNED __declspec(align(AO_STACK_ALLIGN_N))
# else
# define AO_STACK_ATTR_ALLIGNED /* TODO: alignment is not enforced */
# endif
#endif /* !AO_STACK_ATTR_ALLIGNED */
typedef struct AO__stack_aux {
volatile AO_t AO_stack_bl[AO_BL_SIZE];
} AO_stack_aux;
/* The stack implementation knows only about the location of */
/* link fields in nodes, and nothing about the rest of the */
/* stack elements. Link fields hold an AO_t, which is not */
/* necessarily a real pointer. This converts the AO_t to a */
/* real (AO_t *) which is either NULL, or points at the link */
/* field in the next node. */
#define AO_REAL_NEXT_PTR(x) (AO_t *)((x) & ~AO_BIT_MASK)
/* The following two routines should not normally be used directly. */
/* We make them visible here for the rare cases in which it makes sense */
/* to share the AO_stack_aux between stacks. */
void
AO_stack_push_explicit_aux_release(volatile AO_t *list, AO_t *x,
AO_stack_aux *);
AO_t *
AO_stack_pop_explicit_aux_acquire(volatile AO_t *list, AO_stack_aux *);
/* And now AO_stack_t for the real interface: */
typedef struct AO__stack {
AO_STACK_ATTR_ALLIGNED volatile AO_t AO_ptr;
AO_stack_aux AO_aux;
} AO_stack_t;
#define AO_STACK_INITIALIZER {0,{{0}}}
AO_INLINE void AO_stack_init(AO_stack_t *list)
{
# if AO_BL_SIZE == 2
list -> AO_aux.AO_stack_bl[0] = 0;
list -> AO_aux.AO_stack_bl[1] = 0;
# else
int i;
for (i = 0; i < AO_BL_SIZE; ++i)
list -> AO_aux.AO_stack_bl[i] = 0;
# endif
list -> AO_ptr = 0;
}
/* Convert an AO_stack_t to a pointer to the link field in */
/* the first element. */
#define AO_REAL_HEAD_PTR(x) AO_REAL_NEXT_PTR((x).AO_ptr)
#define AO_stack_push_release(l, e) \
AO_stack_push_explicit_aux_release(&((l)->AO_ptr), e, &((l)->AO_aux))
#define AO_HAVE_stack_push_release
#define AO_stack_pop_acquire(l) \
AO_stack_pop_explicit_aux_acquire(&((l)->AO_ptr), &((l)->AO_aux))
#define AO_HAVE_stack_pop_acquire
# else /* Use fully non-blocking data structure, wide CAS */
#ifndef AO_HAVE_double_t
/* Can happen if we're using CAS emulation, since we don't want to */
/* force that here, in case other atomic_ops clients don't want it. */
# ifdef __cplusplus
} /* extern "C" */
# endif
# include "atomic_ops/sysdeps/standard_ao_double_t.h"
# ifdef __cplusplus
extern "C" {
# endif
#endif
typedef volatile AO_double_t AO_stack_t;
/* AO_val1 is version, AO_val2 is pointer. */
/* Note: AO_stack_t variables are not intended to be local ones, */
/* otherwise it is the client responsibility to ensure they have */
/* double-word alignment. */
#define AO_STACK_INITIALIZER AO_DOUBLE_T_INITIALIZER
AO_INLINE void AO_stack_init(AO_stack_t *list)
{
list -> AO_val1 = 0;
list -> AO_val2 = 0;
}
#define AO_REAL_HEAD_PTR(x) (AO_t *)((x).AO_val2)
#define AO_REAL_NEXT_PTR(x) (AO_t *)(x)
void AO_stack_push_release(AO_stack_t *list, AO_t *new_element);
#define AO_HAVE_stack_push_release
AO_t * AO_stack_pop_acquire(AO_stack_t *list);
#define AO_HAVE_stack_pop_acquire
#endif /* Wide CAS case */
#if defined(AO_HAVE_stack_push_release) && !defined(AO_HAVE_stack_push)
# define AO_stack_push(l, e) AO_stack_push_release(l, e)
# define AO_HAVE_stack_push
#endif
#if defined(AO_HAVE_stack_pop_acquire) && !defined(AO_HAVE_stack_pop)
# define AO_stack_pop(l) AO_stack_pop_acquire(l)
# define AO_HAVE_stack_pop
#endif
#ifdef __cplusplus
} /* extern "C" */
#endif
#endif /* !AO_STACK_H */
|
