diff options
| -rw-r--r-- | src/atomic_ops.c | 4 | ||||
| -rw-r--r-- | src/atomic_ops.h | 7 | ||||
| -rw-r--r-- | src/atomic_ops/generalize-small.h | 27 | ||||
| -rw-r--r-- | src/atomic_ops/generalize-small.template | 9 | ||||
| -rw-r--r-- | src/atomic_ops/generalize.h | 14 | ||||
| -rw-r--r-- | src/atomic_ops/sysdeps/gcc/arm.h | 2 | ||||
| -rw-r--r-- | src/atomic_ops_stack.c | 15 |
7 files changed, 49 insertions, 29 deletions
diff --git a/src/atomic_ops.c b/src/atomic_ops.c index 79ae28f..c432f2a 100644 --- a/src/atomic_ops.c +++ b/src/atomic_ops.c @@ -150,7 +150,7 @@ static void lock_ool(volatile AO_TS_t *l) AO_INLINE void lock(volatile AO_TS_t *l) { - if (AO_test_and_set_acquire(l) == AO_TS_SET) + if (AO_EXPECT_FALSE(AO_test_and_set_acquire(l) == AO_TS_SET)) lock_ool(l); } @@ -166,7 +166,7 @@ AO_INLINE void unlock(volatile AO_TS_t *l) AO_INLINE void block_all_signals(sigset_t *old_sigs_ptr) { - if (!AO_load_acquire(&initialized)) + if (AO_EXPECT_FALSE(!AO_load_acquire(&initialized))) { lock(&init_lock); if (!initialized) diff --git a/src/atomic_ops.h b/src/atomic_ops.h index 99d927a..962060e 100644 --- a/src/atomic_ops.h +++ b/src/atomic_ops.h @@ -169,6 +169,13 @@ # define AO_INLINE static #endif +#if __GNUC__ >= 3 && !defined(LINT2) +# define AO_EXPECT_FALSE(expr) __builtin_expect(expr, 0) + /* Equivalent to (expr) but predict that usually (expr) == 0. */ +#else +# define AO_EXPECT_FALSE(expr) (expr) +#endif /* !__GNUC__ */ + #if defined(__GNUC__) && !defined(__INTEL_COMPILER) # define AO_compiler_barrier() __asm__ __volatile__("" : : : "memory") #elif defined(_MSC_VER) || defined(__DMC__) || defined(__BORLANDC__) \ diff --git a/src/atomic_ops/generalize-small.h b/src/atomic_ops/generalize-small.h index 26c6e1e..f30ebac 100644 --- a/src/atomic_ops/generalize-small.h +++ b/src/atomic_ops/generalize-small.h @@ -203,7 +203,8 @@ { old = *addr; } - while (!AO_char_compare_and_swap_full(addr, old, old+incr)); + while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old, + old + incr))); return old; } # define AO_HAVE_char_fetch_and_add_full @@ -220,7 +221,8 @@ { old = *addr; } - while (!AO_char_compare_and_swap_acquire(addr, old, old+incr)); + while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_acquire(addr, old, + old + incr))); return old; } # define AO_HAVE_char_fetch_and_add_acquire @@ -237,7 +239,8 @@ { old = *addr; } - while (!AO_char_compare_and_swap_release(addr, old, old+incr)); + while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_release(addr, old, + old + incr))); return old; } # define AO_HAVE_char_fetch_and_add_release @@ -823,7 +826,8 @@ { old = *addr; } - while (!AO_short_compare_and_swap_full(addr, old, old+incr)); + while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old, + old + incr))); return old; } # define AO_HAVE_short_fetch_and_add_full @@ -840,7 +844,8 @@ { old = *addr; } - while (!AO_short_compare_and_swap_acquire(addr, old, old+incr)); + while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_acquire(addr, old, + old + incr))); return old; } # define AO_HAVE_short_fetch_and_add_acquire @@ -857,7 +862,8 @@ { old = *addr; } - while (!AO_short_compare_and_swap_release(addr, old, old+incr)); + while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_release(addr, old, + old + incr))); return old; } # define AO_HAVE_short_fetch_and_add_release @@ -1443,7 +1449,8 @@ { old = *addr; } - while (!AO_int_compare_and_swap_full(addr, old, old+incr)); + while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old, + old + incr))); return old; } # define AO_HAVE_int_fetch_and_add_full @@ -1460,7 +1467,8 @@ { old = *addr; } - while (!AO_int_compare_and_swap_acquire(addr, old, old+incr)); + while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_acquire(addr, old, + old + incr))); return old; } # define AO_HAVE_int_fetch_and_add_acquire @@ -1477,7 +1485,8 @@ { old = *addr; } - while (!AO_int_compare_and_swap_release(addr, old, old+incr)); + while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_release(addr, old, + old + incr))); return old; } # define AO_HAVE_int_fetch_and_add_release diff --git a/src/atomic_ops/generalize-small.template b/src/atomic_ops/generalize-small.template index 8b5ee34..b11aa10 100644 --- a/src/atomic_ops/generalize-small.template +++ b/src/atomic_ops/generalize-small.template @@ -203,7 +203,8 @@ { old = *addr; } - while (!AO_XSIZE_compare_and_swap_full(addr, old, old+incr)); + while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old, + old + incr))); return old; } # define AO_HAVE_XSIZE_fetch_and_add_full @@ -220,7 +221,8 @@ { old = *addr; } - while (!AO_XSIZE_compare_and_swap_acquire(addr, old, old+incr)); + while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_acquire(addr, old, + old + incr))); return old; } # define AO_HAVE_XSIZE_fetch_and_add_acquire @@ -237,7 +239,8 @@ { old = *addr; } - while (!AO_XSIZE_compare_and_swap_release(addr, old, old+incr)); + while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_release(addr, old, + old + incr))); return old; } # define AO_HAVE_XSIZE_fetch_and_add_release diff --git a/src/atomic_ops/generalize.h b/src/atomic_ops/generalize.h index f1a82ec..6f72e22 100644 --- a/src/atomic_ops/generalize.h +++ b/src/atomic_ops/generalize.h @@ -371,7 +371,7 @@ { old = *addr; } - while (!AO_compare_and_swap_full(addr, old, old+incr)); + while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old, old+incr))); return old; } # define AO_HAVE_fetch_and_add_full @@ -387,7 +387,7 @@ { old = *addr; } - while (!AO_compare_and_swap_acquire(addr, old, old+incr)); + while (AO_EXPECT_FALSE(!AO_compare_and_swap_acquire(addr, old, old+incr))); return old; } # define AO_HAVE_fetch_and_add_acquire @@ -403,7 +403,7 @@ { old = *addr; } - while (!AO_compare_and_swap_release(addr, old, old+incr)); + while (AO_EXPECT_FALSE(!AO_compare_and_swap_release(addr, old, old+incr))); return old; } # define AO_HAVE_fetch_and_add_release @@ -418,7 +418,7 @@ { old = *addr; } - while (!AO_compare_and_swap(addr, old, old+incr)); + while (AO_EXPECT_FALSE(!AO_compare_and_swap(addr, old, old+incr))); return old; } # define AO_HAVE_fetch_and_add @@ -763,7 +763,7 @@ { old = *addr; } - while (!AO_compare_and_swap_full(addr, old, old & value)); + while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old, old & value))); } # define AO_HAVE_and_full #endif @@ -837,7 +837,7 @@ { old = *addr; } - while (!AO_compare_and_swap_full(addr, old, old | value)); + while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old, old | value))); } # define AO_HAVE_or_full #endif @@ -911,7 +911,7 @@ { old = *addr; } - while (!AO_compare_and_swap_full(addr, old, old ^ value)); + while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old, old ^ value))); } # define AO_HAVE_xor_full #endif diff --git a/src/atomic_ops/sysdeps/gcc/arm.h b/src/atomic_ops/sysdeps/gcc/arm.h index 4811e1a..8f364eb 100644 --- a/src/atomic_ops/sysdeps/gcc/arm.h +++ b/src/atomic_ops/sysdeps/gcc/arm.h @@ -340,7 +340,7 @@ AO_fetch_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val) : "=&r"(result), "+m"(*addr) : "r"(new_val), "r"(addr) : "cc"); - } while (result); + } while (AO_EXPECT_FALSE(result)); return !result; /* if succeded, return 1 else 0 */ } # define AO_HAVE_compare_double_and_swap_double diff --git a/src/atomic_ops_stack.c b/src/atomic_ops_stack.c index b3ae513..7475903 100644 --- a/src/atomic_ops_stack.c +++ b/src/atomic_ops_stack.c @@ -137,7 +137,7 @@ AO_stack_push_explicit_aux_release(volatile AO_t *list, AO_t *x, next = AO_load(list); *x = next; } - while(!AO_compare_and_swap_release(list, next, x_bits)); + while (AO_EXPECT_FALSE(!AO_compare_and_swap_release(list, next, x_bits))); } /* @@ -192,13 +192,13 @@ AO_stack_pop_explicit_aux_acquire(volatile AO_t *list, AO_stack_aux * a) /* We need to make sure that first is still the first entry on the */ /* list. Otherwise it's possible that a reinsertion of it was */ /* already started before we added the black list entry. */ - if (first != AO_load(list)) { + if (AO_EXPECT_FALSE(first != AO_load(list))) { AO_store_release(a->AO_stack_bl+i, 0); goto retry; } first_ptr = AO_REAL_NEXT_PTR(first); next = AO_load(first_ptr); - if (!AO_compare_and_swap_release(list, first, next)) { + if (AO_EXPECT_FALSE(!AO_compare_and_swap_release(list, first, next))) { AO_store_release(a->AO_stack_bl+i, 0); goto retry; } @@ -231,8 +231,8 @@ void AO_stack_push_release(AO_stack_t *list, AO_t *element) do { next = AO_load(&(list -> ptr)); *element = next; - } while (!AO_compare_and_swap_release - ( &(list -> ptr), next, (AO_t) element)); + } while (AO_EXPECT_FALSE(!AO_compare_and_swap_release(&(list -> ptr), + next, (AO_t)element))); /* This uses a narrow CAS here, an old optimization suggested */ /* by Treiber. Pop is still safe, since we run into the ABA */ /* problem only if there were both intervening "pop"s and "push"es. */ @@ -257,8 +257,9 @@ AO_t *AO_stack_pop_acquire(AO_stack_t *list) cptr = (AO_t *)AO_load(&(list -> ptr)); if (cptr == 0) return 0; next = *cptr; - } while (!AO_compare_double_and_swap_double_release - (list, cversion, (AO_t) cptr, cversion+1, (AO_t) next)); + } while (AO_EXPECT_FALSE(!AO_compare_double_and_swap_double_release(list, + cversion, (AO_t)cptr, + cversion+1, (AO_t)next))); return cptr; } |