MDEV-30551 InnoDB recovery hangs when buffer pool ran out of memory

- During non-last batch of multi-batch recovery, InnoDB holds
log_sys.mutex and preallocates the block which may intiate
page flush, which may initiate log flush, which requires
log_sys.mutex to acquire again. This leads to assert failure.
So InnoDB recovery should release log_sys.mutex before
preallocating the block.

2 files changed, 26 insertions, 1 deletions

diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc
index f7a3543c493..6d83d448bd6 100644
--- a/storage/innobase/buf/buf0lru.cc
+++ b/storage/innobase/buf/buf0lru.cc
@@ -408,6 +408,11 @@ buf_block_t *buf_LRU_get_free_block(bool have_mutex)
 		mysql_mutex_assert_owner(&buf_pool.mutex);
 		goto got_mutex;
 	}
+	DBUG_EXECUTE_IF("recv_ran_out_of_buffer",
+			if (recv_recovery_is_on()
+			    && recv_sys.apply_log_recs) {
+				goto flush_lru;
+			});
 	mysql_mutex_lock(&buf_pool.mutex);
 got_mutex:
 	buf_LRU_check_size_of_non_data_objects();
@@ -493,7 +498,9 @@ not_found:
 	removing the block from buf_pool.page_hash and buf_pool.LRU is fairly
 	involved (particularly in case of ROW_FORMAT=COMPRESSED pages). We
 	can do that in a separate patch sometime in future. */
-
+#ifndef DBUG_OFF
+flush_lru:
+#endif
 	if (!buf_flush_LRU(innodb_lru_flush_size)) {
 		MONITOR_INC(MONITOR_LRU_SINGLE_FLUSH_FAILURE_COUNT);
 		++flush_failures;
diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc
index 4a6527962f6..eb2931adf0e 100644
--- a/storage/innobase/log/log0recv.cc
+++ b/storage/innobase/log/log0recv.cc
@@ -2692,8 +2692,23 @@ void recv_sys_t::apply(bool last_batch)
 
     fil_system.extend_to_recv_size();
 
+    /* Release the log_sys mutex in non-last batches of multi-batch
+    recovery mode and recv_sys.mutex before preallocating the
+    block because while preallocating the block which may initiate
+    log flush which requires log_sys mutex to acquire again, which
+    should be acquired before recv_sys.mutex in order to avoid
+    deadlocks. */
+    mutex_exit(&mutex);
+    if (!last_batch)
+      mysql_mutex_unlock(&log_sys.mutex);
+
+    mysql_mutex_assert_not_owner(&log_sys.mutex);
     buf_block_t *free_block= buf_LRU_get_free_block(false);
 
+    if (!last_batch)
+      mysql_mutex_lock(&log_sys.mutex);
+    mutex_enter(&mutex);
+
     for (map::iterator p= pages.begin(); p != pages.end(); )
     {
       const page_id_t page_id= p->first;
@@ -2708,7 +2723,10 @@ void recv_sys_t::apply(bool last_batch)
         if (UNIV_LIKELY(!!recover_low(page_id, p, mtr, free_block)))
         {
           mutex_exit(&mutex);
+          if (!last_batch) mysql_mutex_unlock(&log_sys.mutex);
+          mysql_mutex_assert_not_owner(&log_sys.mutex);
           free_block= buf_LRU_get_free_block(false);
+          if (!last_batch) mysql_mutex_lock(&log_sys.mutex);
           mutex_enter(&mutex);
           break;
         }