diff options
Diffstat (limited to 'lib/tsan/rtl/tsan_interface_atomic.cc')
-rw-r--r-- | lib/tsan/rtl/tsan_interface_atomic.cc | 193 |
1 files changed, 145 insertions, 48 deletions
diff --git a/lib/tsan/rtl/tsan_interface_atomic.cc b/lib/tsan/rtl/tsan_interface_atomic.cc index 7fbc9c67d..8c25c167a 100644 --- a/lib/tsan/rtl/tsan_interface_atomic.cc +++ b/lib/tsan/rtl/tsan_interface_atomic.cc @@ -27,24 +27,12 @@ using namespace __tsan; // NOLINT -#define SCOPED_ATOMIC(func, ...) \ - const uptr callpc = (uptr)__builtin_return_address(0); \ - uptr pc = __sanitizer::StackTrace::GetCurrentPc(); \ - mo = flags()->force_seq_cst_atomics ? (morder)mo_seq_cst : mo; \ - ThreadState *const thr = cur_thread(); \ - if (thr->ignore_interceptors) \ - return NoTsanAtomic##func(__VA_ARGS__); \ - AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \ - ScopedAtomic sa(thr, callpc, a, mo, __func__); \ - return Atomic##func(thr, pc, __VA_ARGS__); \ -/**/ - // These should match declarations from public tsan_interface_atomic.h header. typedef unsigned char a8; typedef unsigned short a16; // NOLINT typedef unsigned int a32; typedef unsigned long long a64; // NOLINT -#if defined(__SIZEOF_INT128__) \ +#if !defined(TSAN_GO) && defined(__SIZEOF_INT128__) \ || (__clang_major__ * 100 + __clang_minor__ >= 302) __extension__ typedef __int128 a128; # define __TSAN_HAS_INT128 1 @@ -52,8 +40,10 @@ __extension__ typedef __int128 a128; # define __TSAN_HAS_INT128 0 #endif +#ifndef TSAN_GO // Protects emulation of 128-bit atomic operations. static StaticSpinMutex mutex128; +#endif // Part of ABI, do not change. // http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/atomic?view=markup @@ -66,38 +56,6 @@ typedef enum { mo_seq_cst } morder; -class ScopedAtomic { - public: - ScopedAtomic(ThreadState *thr, uptr pc, const volatile void *a, - morder mo, const char *func) - : thr_(thr) { - FuncEntry(thr_, pc); - DPrintf("#%d: %s(%p, %d)\n", thr_->tid, func, a, mo); - } - ~ScopedAtomic() { - ProcessPendingSignals(thr_); - FuncExit(thr_); - } - private: - ThreadState *thr_; -}; - -static void AtomicStatInc(ThreadState *thr, uptr size, morder mo, StatType t) { - StatInc(thr, StatAtomic); - StatInc(thr, t); - StatInc(thr, size == 1 ? StatAtomic1 - : size == 2 ? StatAtomic2 - : size == 4 ? StatAtomic4 - : size == 8 ? StatAtomic8 - : StatAtomic16); - StatInc(thr, mo == mo_relaxed ? StatAtomicRelaxed - : mo == mo_consume ? StatAtomicConsume - : mo == mo_acquire ? StatAtomicAcquire - : mo == mo_release ? StatAtomicRelease - : mo == mo_acq_rel ? StatAtomicAcq_Rel - : StatAtomicSeq_Cst); -} - static bool IsLoadOrder(morder mo) { return mo == mo_relaxed || mo == mo_consume || mo == mo_acquire || mo == mo_seq_cst; @@ -167,7 +125,7 @@ template<typename T> T func_cas(volatile T *v, T cmp, T xch) { // Atomic ops are executed under tsan internal mutex, // here we assume that the atomic variables are not accessed // from non-instrumented code. -#ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 +#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !defined(TSAN_GO) a128 func_xchg(volatile a128 *v, a128 op) { SpinMutexLock lock(&mutex128); a128 cmp = *v; @@ -240,6 +198,7 @@ static int SizeLog() { // this leads to false negatives only in very obscure cases. } +#ifndef TSAN_GO static atomic_uint8_t *to_atomic(const volatile a8 *a) { return (atomic_uint8_t*)a; } @@ -247,6 +206,7 @@ static atomic_uint8_t *to_atomic(const volatile a8 *a) { static atomic_uint16_t *to_atomic(const volatile a16 *a) { return (atomic_uint16_t*)a; } +#endif static atomic_uint32_t *to_atomic(const volatile a32 *a) { return (atomic_uint32_t*)a; @@ -451,8 +411,9 @@ static bool NoTsanAtomicCAS(volatile a128 *a, a128 *c, a128 v, #endif template<typename T> -static bool NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) { - return NoTsanAtomicCAS(a, &c, v, mo, fmo); +static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) { + NoTsanAtomicCAS(a, &c, v, mo, fmo); + return c; } template<typename T> @@ -495,6 +456,7 @@ static T AtomicCAS(ThreadState *thr, uptr pc, return c; } +#ifndef TSAN_GO static void NoTsanAtomicFence(morder mo) { __sync_synchronize(); } @@ -503,6 +465,56 @@ static void AtomicFence(ThreadState *thr, uptr pc, morder mo) { // FIXME(dvyukov): not implemented. __sync_synchronize(); } +#endif + +// Interface functions follow. +#ifndef TSAN_GO + +// C/C++ + +#define SCOPED_ATOMIC(func, ...) \ + const uptr callpc = (uptr)__builtin_return_address(0); \ + uptr pc = __sanitizer::StackTrace::GetCurrentPc(); \ + mo = flags()->force_seq_cst_atomics ? (morder)mo_seq_cst : mo; \ + ThreadState *const thr = cur_thread(); \ + if (thr->ignore_interceptors) \ + return NoTsanAtomic##func(__VA_ARGS__); \ + AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \ + ScopedAtomic sa(thr, callpc, a, mo, __func__); \ + return Atomic##func(thr, pc, __VA_ARGS__); \ +/**/ + +class ScopedAtomic { + public: + ScopedAtomic(ThreadState *thr, uptr pc, const volatile void *a, + morder mo, const char *func) + : thr_(thr) { + FuncEntry(thr_, pc); + DPrintf("#%d: %s(%p, %d)\n", thr_->tid, func, a, mo); + } + ~ScopedAtomic() { + ProcessPendingSignals(thr_); + FuncExit(thr_); + } + private: + ThreadState *thr_; +}; + +static void AtomicStatInc(ThreadState *thr, uptr size, morder mo, StatType t) { + StatInc(thr, StatAtomic); + StatInc(thr, t); + StatInc(thr, size == 1 ? StatAtomic1 + : size == 2 ? StatAtomic2 + : size == 4 ? StatAtomic4 + : size == 8 ? StatAtomic8 + : StatAtomic16); + StatInc(thr, mo == mo_relaxed ? StatAtomicRelaxed + : mo == mo_consume ? StatAtomicConsume + : mo == mo_acquire ? StatAtomicAcquire + : mo == mo_release ? StatAtomicRelease + : mo == mo_acq_rel ? StatAtomicAcq_Rel + : StatAtomicSeq_Cst); +} extern "C" { SANITIZER_INTERFACE_ATTRIBUTE @@ -854,3 +866,88 @@ SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic_signal_fence(morder mo) { } } // extern "C" + +#else // #ifndef TSAN_GO + +// Go + +#define ATOMIC(func, ...) \ + if (thr->ignore_sync) { \ + NoTsanAtomic##func(__VA_ARGS__); \ + } else { \ + FuncEntry(thr, cpc); \ + Atomic##func(thr, pc, __VA_ARGS__); \ + FuncExit(thr); \ + } \ +/**/ + +#define ATOMIC_RET(func, ret, ...) \ + if (thr->ignore_sync) { \ + (ret) = NoTsanAtomic##func(__VA_ARGS__); \ + } else { \ + FuncEntry(thr, cpc); \ + (ret) = Atomic##func(thr, pc, __VA_ARGS__); \ + FuncExit(thr); \ + } \ +/**/ + +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC_RET(Load, *(a32*)(a+8), *(a32**)a, mo_acquire); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC_RET(Load, *(a64*)(a+8), *(a64**)a, mo_acquire); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC(Store, *(a32**)a, *(a32*)(a+8), mo_release); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC(Store, *(a64**)a, *(a64*)(a+8), mo_release); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC_RET(FetchAdd, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC_RET(FetchAdd, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC_RET(Exchange, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC_RET(Exchange, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_compare_exchange( + ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + a32 cur = 0; + a32 cmp = *(a32*)(a+8); + ATOMIC_RET(CAS, cur, *(a32**)a, cmp, *(a32*)(a+12), mo_acq_rel, mo_acquire); + *(bool*)(a+16) = (cur == cmp); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_compare_exchange( + ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + a64 cur = 0; + a64 cmp = *(a64*)(a+8); + ATOMIC_RET(CAS, cur, *(a64**)a, cmp, *(a64*)(a+16), mo_acq_rel, mo_acquire); + *(bool*)(a+24) = (cur == cmp); +} +} // extern "C" +#endif // #ifndef TSAN_GO |