MDEV-26779: work around lock_sys.wait_mutex contention by spinloopbb-10.6-MDEV-26779

This should be equivalent to pull request #1923
when CMAKE_CXX_FLAGS includes -DSPIN_MUTEX_INLINE.

All invocations of spin_mutex_lock() where we are holding other latches
are marked with comments. My hypothesis is that some of those
spin_mutex_lock() invocations should be reverted to improve performance.

3 files changed, 52 insertions, 9 deletions

diff --git a/storage/innobase/include/srw_lock.h b/storage/innobase/include/srw_lock.h
index 54d042419ca..897d7d855a4 100644
--- a/storage/innobase/include/srw_lock.h
+++ b/storage/innobase/include/srw_lock.h
@@ -20,6 +20,33 @@ this program; if not, write to the Free Software Foundation, Inc.,
 #include "univ.i"
 #include "rw_lock.h"
 
+#ifdef SPIN_MUTEX_INLINE
+# include "my_cpu.h"
+extern ulong srv_n_spin_wait_rounds;
+extern uint srv_spin_wait_delay;
+/** Acquire a mutex in a spinloop. */
+inline void spin_for_mutex_lock(mysql_mutex_t *mutex)
+{
+  for (auto spin= srv_n_spin_wait_rounds; spin; spin--)
+  {
+    ut_delay(srv_spin_wait_delay);
+    if (!mysql_mutex_trylock(mutex))
+      return;
+  }
+  mysql_mutex_lock(mutex);
+}
+#else
+/** Acquire a mutex in a spinloop. */
+void spin_for_mutex_lock(mysql_mutex_t *mutex);
+#endif
+
+/** Acquire a mutex in a spinloop. */
+inline void spin_mutex_lock(mysql_mutex_t *mutex)
+{
+  if (mysql_mutex_trylock(mutex))
+    spin_for_mutex_lock(mutex);
+}
+
 #ifdef SUX_LOCK_GENERIC
 /** An exclusive-only variant of srw_lock */
 template<bool spinloop>
diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc
index 234d215d1df..4508e1be21d 100644
--- a/storage/innobase/lock/lock0lock.cc
+++ b/storage/innobase/lock/lock0lock.cc
@@ -1795,7 +1795,7 @@ dberr_t lock_wait(que_thr_t *thr)
                  ? THD_WAIT_TABLE_LOCK : THD_WAIT_ROW_LOCK);
   dberr_t error_state= DB_SUCCESS;
 
-  mysql_mutex_lock(&lock_sys.wait_mutex);
+  spin_mutex_lock(&lock_sys.wait_mutex);
   if (trx->lock.wait_lock)
   {
     if (Deadlock::check_and_resolve(trx))
@@ -1927,7 +1927,7 @@ one can now be granted! */
 static void lock_rec_cancel(lock_t *lock)
 {
   trx_t *trx= lock->trx;
-  mysql_mutex_lock(&lock_sys.wait_mutex);
+  spin_mutex_lock(&lock_sys.wait_mutex);
   trx->mutex_lock();
 
   ut_d(lock_sys.hash_get(lock->type_mode).
@@ -1990,7 +1990,7 @@ static void lock_rec_dequeue_from_page(lock_t *in_lock, bool owns_wait_mutex)
 		}
 
 		if (!owns_wait_mutex) {
-			mysql_mutex_lock(&lock_sys.wait_mutex);
+			spin_mutex_lock(&lock_sys.wait_mutex);//holding lock_sys.latch
 			acquired = owns_wait_mutex = true;
 		}
 
@@ -3598,7 +3598,7 @@ static void lock_table_dequeue(lock_t *in_lock, bool owns_wait_mutex)
 		}
 
 		if (!owns_wait_mutex) {
-			mysql_mutex_lock(&lock_sys.wait_mutex);
+			spin_mutex_lock(&lock_sys.wait_mutex);//holding trx->mutex
 			acquired = owns_wait_mutex = true;
 		}
 
@@ -3766,7 +3766,7 @@ released:
 		if (!lock->is_waiting()) {
 			continue;
 		}
-		mysql_mutex_lock(&lock_sys.wait_mutex);
+		spin_mutex_lock(&lock_sys.wait_mutex);// holding dict_sys.latch
 		ut_ad(lock->trx->lock.wait_trx);
 		ut_ad(lock->trx->lock.wait_lock);
 
@@ -3921,7 +3921,7 @@ restart:
 released:
   if (UNIV_UNLIKELY(Deadlock::to_be_checked))
   {
-    mysql_mutex_lock(&lock_sys.wait_mutex);
+    spin_mutex_lock(&lock_sys.wait_mutex);
     lock_sys.deadlock_check();
     mysql_mutex_unlock(&lock_sys.wait_mutex);
   }
@@ -5678,7 +5678,7 @@ dberr_t lock_sys_t::cancel(trx_t *trx, lock_t *lock)
     {
       mysql_mutex_unlock(&lock_sys.wait_mutex);
       lock_sys.rd_lock(SRW_LOCK_CALL);
-      mysql_mutex_lock(&lock_sys.wait_mutex);
+      spin_mutex_lock(&lock_sys.wait_mutex); // holding lock_sys.latch
       lock= trx->lock.wait_lock;
       if (!lock);
       else if (check_victim && trx->lock.was_chosen_as_deadlock_victim)
@@ -5759,7 +5759,7 @@ resolve_record_lock:
 /** Cancel a waiting lock request (if any) when killing a transaction */
 void lock_sys_t::cancel(trx_t *trx)
 {
-  mysql_mutex_lock(&lock_sys.wait_mutex);
+  spin_mutex_lock(&lock_sys.wait_mutex);
   if (lock_t *lock= trx->lock.wait_lock)
   {
     /* Dictionary transactions must be immune to KILL, because they
@@ -5817,7 +5817,7 @@ dberr_t lock_trx_handle_wait(trx_t *trx)
   if (!trx->lock.wait_lock)
     return DB_SUCCESS;
   dberr_t err= DB_SUCCESS;
-  mysql_mutex_lock(&lock_sys.wait_mutex);
+  spin_mutex_lock(&lock_sys.wait_mutex); // holding buffer page latches
   if (trx->lock.was_chosen_as_deadlock_victim)
     err= DB_DEADLOCK;
   else if (lock_t *wait_lock= trx->lock.wait_lock)
diff --git a/storage/innobase/sync/srw_lock.cc b/storage/innobase/sync/srw_lock.cc
index b54191d91b0..f3eda8e3402 100644
--- a/storage/innobase/sync/srw_lock.cc
+++ b/storage/innobase/sync/srw_lock.cc
@@ -102,6 +102,22 @@ static inline void srw_pause(unsigned delay)
   HMT_medium();
 }
 
+#ifndef SPIN_MUTEX_INLINE
+/** Acquire a mutex in a spinloop. */
+ATTRIBUTE_NOINLINE void spin_for_mutex_lock(mysql_mutex_t *mutex)
+{
+  const unsigned delay= srw_pause_delay();
+
+  for (auto spin= srv_n_spin_wait_rounds; spin; spin--)
+  {
+    srw_pause(delay);
+    if (!mysql_mutex_trylock(mutex))
+      return;
+  }
+  mysql_mutex_lock(mutex);
+}
+#endif
+
 #ifdef SUX_LOCK_GENERIC
 template<> void srw_mutex_impl<true>::wr_wait()
 {