MDEV-21452: Replace all direct use of os_event_t

Let us replace os_event_t with mysql_cond_t, and replace the
necessary ib_mutex_t with mysql_mutex_t so that they can be
used with condition variables.

Also, let us replace polling (os_thread_sleep() or timed waits)
with plain mysql_cond_wait() wherever possible.

Furthermore, we will use the lightweight srw_mutex for trx_t::mutex,
to hopefully reduce contention on lock_sys.mutex.

FIXME: Add test coverage of
mariabackup --backup --kill-long-queries-timeout

2 files changed, 37 insertions, 55 deletions

diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc
index 6b076110819..d176a69bee6 100644
--- a/storage/innobase/srv/srv0srv.cc
+++ b/storage/innobase/srv/srv0srv.cc
@@ -152,7 +152,7 @@ ulong	innodb_compression_algorithm;
 /** Used by SET GLOBAL innodb_master_thread_disabled_debug = X. */
 my_bool	srv_master_thread_disabled_debug;
 /** Event used to inform that master thread is disabled. */
-static os_event_t	srv_master_thread_disabled_event;
+static mysql_cond_t srv_master_thread_disabled_cond;
 #endif /* UNIV_DEBUG */
 
 /*------------------------- LOG FILES ------------------------ */
@@ -194,10 +194,6 @@ srv_printf_innodb_monitor() will request mutex acquisition
 with mutex_enter(), which will wait until it gets the mutex. */
 #define MUTEX_NOWAIT(mutex_skipped)	((mutex_skipped) < MAX_MUTEX_NOWAIT)
 
-#ifdef WITH_INNODB_DISALLOW_WRITES
-UNIV_INTERN os_event_t	srv_allow_writes_event;
-#endif /* WITH_INNODB_DISALLOW_WRITES */
-
 /** copy of innodb_buffer_pool_size */
 ulint	srv_buf_pool_size;
 const ulint	srv_buf_pool_min_size	= 5 * 1024 * 1024;
@@ -704,7 +700,7 @@ static void srv_init()
 	}
 
 	need_srv_free = true;
-	ut_d(srv_master_thread_disabled_event = os_event_create(0));
+	ut_d(mysql_cond_init(0, &srv_master_thread_disabled_cond, nullptr));
 
 	/* page_zip_stat_per_index_mutex is acquired from:
 	1. page_zip_compress()
@@ -716,15 +712,6 @@ static void srv_init()
 	mutex_create(LATCH_ID_PAGE_ZIP_STAT_PER_INDEX,
 		     &page_zip_stat_per_index_mutex);
 
-#ifdef WITH_INNODB_DISALLOW_WRITES
-	/* Writes have to be enabled on init or else we hang. Thus, we
-	always set the event here regardless of innobase_disallow_writes.
-	That flag will always be 0 at this point because it isn't settable
-	via my.cnf or command line arg. */
-	srv_allow_writes_event = os_event_create(0);
-	os_event_set(srv_allow_writes_event);
-#endif /* WITH_INNODB_DISALLOW_WRITES */
-
 	/* Initialize some INFORMATION SCHEMA internal structures */
 	trx_i_s_cache_init(trx_i_s_cache);
 
@@ -747,7 +734,7 @@ srv_free(void)
 		mutex_free(&srv_sys.tasks_mutex);
 	}
 
-	ut_d(os_event_destroy(srv_master_thread_disabled_event));
+	ut_d(mysql_cond_destroy(&srv_master_thread_disabled_cond));
 
 	trx_i_s_cache_free(trx_i_s_cache);
 	srv_thread_pool_end();
@@ -1399,7 +1386,7 @@ void srv_monitor_task(void*)
 	if (sync_array_print_long_waits(&waiter, &sema)
 	    && sema == old_sema && os_thread_eq(waiter, old_waiter)) {
 #if defined(WITH_WSREP) && defined(WITH_INNODB_DISALLOW_WRITES)
-		if (!os_event_is_set(srv_allow_writes_event)) {
+		if (UNIV_UNLIKELY(innodb_disallow_writes)) {
 			fprintf(stderr,
 				"WSREP: avoiding InnoDB self crash due to "
 				"long semaphore wait of  > %lu seconds\n"
@@ -1636,26 +1623,17 @@ srv_shutdown_print_master_pending(
 
 #ifdef UNIV_DEBUG
 /** Waits in loop as long as master thread is disabled (debug) */
-static
-void
-srv_master_do_disabled_loop(void)
+static void srv_master_do_disabled_loop()
 {
-	if (!srv_master_thread_disabled_debug) {
-		/* We return here to avoid changing op_info. */
-		return;
-	}
-
-	srv_main_thread_op_info = "disabled";
-
-	while (srv_master_thread_disabled_debug) {
-		os_event_set(srv_master_thread_disabled_event);
-		if (srv_shutdown_state != SRV_SHUTDOWN_NONE) {
-			break;
-		}
-		os_thread_sleep(100000);
-	}
-
-	srv_main_thread_op_info = "";
+  if (!srv_master_thread_disabled_debug)
+    return;
+  srv_main_thread_op_info = "disabled";
+  mysql_mutex_lock(&LOCK_global_system_variables);
+  while (srv_master_thread_disabled_debug)
+    mysql_cond_wait(&srv_master_thread_disabled_cond,
+                    &LOCK_global_system_variables);
+  mysql_mutex_unlock(&LOCK_global_system_variables);
+  srv_main_thread_op_info = "";
 }
 
 /** Disables master thread. It's used by:
@@ -1663,22 +1641,25 @@ srv_master_do_disabled_loop(void)
 @param[in]	save		immediate result from check function */
 void
 srv_master_thread_disabled_debug_update(THD*, st_mysql_sys_var*, void*,
-					const void* save)
+                                        const void* save)
 {
-	/* This method is protected by mutex, as every SET GLOBAL .. */
-	ut_ad(srv_master_thread_disabled_event != NULL);
-
-	const bool disable = *static_cast<const my_bool*>(save);
-
-	const int64_t sig_count = os_event_reset(
-		srv_master_thread_disabled_event);
-
-	srv_master_thread_disabled_debug = disable;
+  mysql_mutex_assert_owner(&LOCK_global_system_variables);
+  const bool disable= *static_cast<const my_bool*>(save);
+  srv_master_thread_disabled_debug= disable;
+  if (!disable)
+    mysql_cond_signal(&srv_master_thread_disabled_cond);
+}
 
-	if (disable) {
-		os_event_wait_low(
-			srv_master_thread_disabled_event, sig_count);
-	}
+/** Enable the master thread on shutdown. */
+void srv_master_thread_enable()
+{
+  if (srv_master_thread_disabled_debug)
+  {
+    mysql_mutex_lock(&LOCK_global_system_variables);
+    srv_master_thread_disabled_debug= FALSE;
+    mysql_cond_signal(&srv_master_thread_disabled_cond);
+    mysql_mutex_unlock(&LOCK_global_system_variables);
+  }
 }
 #endif /* UNIV_DEBUG */
 
diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc
index 5adabb53773..fa9484d8545 100644
--- a/storage/innobase/srv/srv0start.cc
+++ b/storage/innobase/srv/srv0start.cc
@@ -96,7 +96,6 @@ Created 2/16/1996 Heikki Tuuri
 #include "row0row.h"
 #include "row0mysql.h"
 #include "btr0pcur.h"
-#include "os0event.h"
 #include "zlib.h"
 #include "ut0crc32.h"
 
@@ -825,7 +824,7 @@ static void srv_shutdown_threads()
 {
 	ut_ad(!srv_undo_sources);
 	srv_shutdown_state = SRV_SHUTDOWN_EXIT_THREADS;
-
+	ut_d(srv_master_thread_enable());
 	lock_sys.timeout_timer.reset();
 	srv_master_timer.reset();
 
@@ -1488,8 +1487,8 @@ file_checked:
 
 			recv_sys.apply(true);
 
-			if (recv_sys.found_corrupt_log
-			    || recv_sys.found_corrupt_fs) {
+			if (recv_sys.is_corrupt_log()
+			    || recv_sys.is_corrupt_fs()) {
 				return(srv_init_abort(DB_CORRUPTION));
 			}
 
@@ -1922,7 +1921,7 @@ skip_monitors:
 		/* Create thread(s) that handles key rotation. This is
 		needed already here as log_preflush_pool_modified_pages
 		will flush dirty pages and that might need e.g.
-		fil_crypt_threads_event. */
+		fil_crypt_threads_cond. */
 		fil_system_enter();
 		fil_crypt_threads_init();
 		fil_system_exit();
@@ -1941,6 +1940,8 @@ void srv_shutdown_bg_undo_sources()
 {
 	srv_shutdown_state = SRV_SHUTDOWN_INITIATED;
 
+	ut_d(srv_master_thread_enable());
+
 	if (srv_undo_sources) {
 		ut_ad(!srv_read_only_mode);
 		fts_optimize_shutdown();