diff options
| -rw-r--r-- | storage/innobase/include/srw_lock.h | 9 | ||||
| -rw-r--r-- | storage/innobase/sync/srw_lock.cc | 42 |
2 files changed, 29 insertions, 22 deletions
diff --git a/storage/innobase/include/srw_lock.h b/storage/innobase/include/srw_lock.h index fdac659d494..fa3368d5c1b 100644 --- a/storage/innobase/include/srw_lock.h +++ b/storage/innobase/include/srw_lock.h @@ -36,7 +36,8 @@ public: /** Futex-based mutex */ class srw_mutex final { - /** The lock word, containing HOLDER and a count of waiters */ + /** The lock word, containing HOLDER + 1 if the lock is being held, + plus the number of waiters */ std::atomic<uint32_t> lock; /** Identifies that the lock is being held */ static constexpr uint32_t HOLDER= 1U << 31; @@ -62,7 +63,7 @@ public: bool wr_lock_try() { uint32_t lk= 0; - return lock.compare_exchange_strong(lk, HOLDER, + return lock.compare_exchange_strong(lk, HOLDER + 1, std::memory_order_acquire, std::memory_order_relaxed); } @@ -70,8 +71,8 @@ public: void wr_lock() { if (!wr_lock_try()) wait_and_lock(); } void wr_unlock() { - const uint32_t lk= lock.fetch_and(~HOLDER, std::memory_order_release); - if (lk != HOLDER) + const uint32_t lk= lock.fetch_sub(HOLDER + 1, std::memory_order_release); + if (lk != HOLDER + 1) { DBUG_ASSERT(lk & HOLDER); wake(); diff --git a/storage/innobase/sync/srw_lock.cc b/storage/innobase/sync/srw_lock.cc index b76194c89e3..780add3f705 100644 --- a/storage/innobase/sync/srw_lock.cc +++ b/storage/innobase/sync/srw_lock.cc @@ -233,33 +233,39 @@ void ssux_lock_low::wake() { SRW_FUTEX(&readers, WAKE, 1); } void srw_mutex::wait_and_lock() { uint32_t lk= 1 + lock.fetch_add(1, std::memory_order_relaxed); - for (auto spin= srv_n_spin_wait_rounds; spin; spin--) + for (auto spin= srv_n_spin_wait_rounds;;) { - lk&= ~HOLDER; - DBUG_ASSERT(lk); - while (!lock.compare_exchange_weak(lk, HOLDER | (lk - 1), - std::memory_order_acquire, - std::memory_order_relaxed)) - if (lk & HOLDER) - goto occupied; - return; -occupied: + DBUG_ASSERT(~HOLDER & lk); + if (lk & HOLDER) + lk= lock.load(std::memory_order_relaxed); + else + { + lk= lock.fetch_or(HOLDER, std::memory_order_relaxed); + if (!(lk & HOLDER)) + goto acquired; + } ut_delay(srv_spin_wait_delay); + if (!--spin) + break; } - for (;;) + for (;; wait(lk)) { - lk= lock.load(std::memory_order_relaxed); - while (!(lk & HOLDER)) + if (lk & HOLDER) { + lk= lock.load(std::memory_order_relaxed); + if (lk & HOLDER) + continue; + } + lk= lock.fetch_or(HOLDER, std::memory_order_relaxed); + if (!(lk & HOLDER)) + { +acquired: DBUG_ASSERT(lk); - if (lock.compare_exchange_weak(lk, HOLDER | (lk - 1), - std::memory_order_acquire, - std::memory_order_relaxed)) - return; + std::atomic_thread_fence(std::memory_order_acquire); + return; } DBUG_ASSERT(lk > HOLDER); - wait(lk); } } |