MDEV-9359: encryption.create_or_replace fails sporadically in buildbot: failing assertion: mutex->magic_n == MUTEX_MAGIC_N

Make sure that encryption threads mutex is initialized before starting
encryption threads.

6 files changed, 52 insertions, 20 deletions

diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc
index 100f304bf6f..3d9d6a18122 100644
--- a/storage/innobase/fil/fil0crypt.cc
+++ b/storage/innobase/fil/fil0crypt.cc
@@ -2294,6 +2294,10 @@ fil_crypt_set_thread_cnt(
 /*=====================*/
 	uint	new_cnt)	/*!< in: New key rotation thread count */
 {
+	if (!fil_crypt_threads_inited) {
+		fil_crypt_threads_init();
+	}
+
 	if (new_cnt > srv_n_fil_crypt_threads) {
 		uint add = new_cnt - srv_n_fil_crypt_threads;
 		srv_n_fil_crypt_threads = new_cnt;
@@ -2358,15 +2362,18 @@ void
 fil_crypt_threads_init()
 /*====================*/
 {
-	fil_crypt_event = os_event_create();
-	fil_crypt_threads_event = os_event_create();
-	mutex_create(fil_crypt_threads_mutex_key,
-		     &fil_crypt_threads_mutex, SYNC_NO_ORDER_CHECK);
-
-	uint cnt = srv_n_fil_crypt_threads;
-	srv_n_fil_crypt_threads = 0;
-	fil_crypt_set_thread_cnt(cnt);
-	fil_crypt_threads_inited = true;
+	ut_ad(mutex_own(&fil_system->mutex));
+	if (!fil_crypt_threads_inited) {
+		fil_crypt_event = os_event_create();
+		fil_crypt_threads_event = os_event_create();
+		mutex_create(fil_crypt_threads_mutex_key,
+			&fil_crypt_threads_mutex, SYNC_NO_ORDER_CHECK);
+
+		uint cnt = srv_n_fil_crypt_threads;
+		srv_n_fil_crypt_threads = 0;
+		fil_crypt_threads_inited = true;
+		fil_crypt_set_thread_cnt(cnt);
+	}
 }
 
 /*********************************************************************
@@ -2389,6 +2396,7 @@ fil_crypt_threads_cleanup()
 {
 	os_event_free(fil_crypt_event);
 	os_event_free(fil_crypt_threads_event);
+	fil_crypt_threads_inited = false;
 }
 
 /*********************************************************************
diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc
index 8c54d6eb083..d7b37b5bc1e 100644
--- a/storage/innobase/srv/srv0start.cc
+++ b/storage/innobase/srv/srv0start.cc
@@ -2984,7 +2984,9 @@ files_checked:
 		fts_optimize_init();
 
 		/* Create thread(s) that handles key rotation */
+		fil_system_enter();
 		fil_crypt_threads_init();
+		fil_system_exit();
 
 		/* Create the log scrub thread */
 		if (srv_scrub_log)
diff --git a/storage/innobase/sync/sync0sync.cc b/storage/innobase/sync/sync0sync.cc
index 2e1737da3ec..7eb3d0bd6f2 100644
--- a/storage/innobase/sync/sync0sync.cc
+++ b/storage/innobase/sync/sync0sync.cc
@@ -422,7 +422,13 @@ mutex_validate(
 	const ib_mutex_t*	mutex)	/*!< in: mutex */
 {
 	ut_a(mutex);
-	ut_a(mutex->magic_n == MUTEX_MAGIC_N);
+
+	if (mutex->magic_n != MUTEX_MAGIC_N) {
+		ib_logf(IB_LOG_LEVEL_ERROR,
+			"Mutex %p not initialized file %s line %lu.",
+			mutex, mutex->cfile_name, mutex->cline);
+	}
+	ut_ad(mutex->magic_n == MUTEX_MAGIC_N);
 
 	return(TRUE);
 }
diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc
index 100f304bf6f..3d9d6a18122 100644
--- a/storage/xtradb/fil/fil0crypt.cc
+++ b/storage/xtradb/fil/fil0crypt.cc
@@ -2294,6 +2294,10 @@ fil_crypt_set_thread_cnt(
 /*=====================*/
 	uint	new_cnt)	/*!< in: New key rotation thread count */
 {
+	if (!fil_crypt_threads_inited) {
+		fil_crypt_threads_init();
+	}
+
 	if (new_cnt > srv_n_fil_crypt_threads) {
 		uint add = new_cnt - srv_n_fil_crypt_threads;
 		srv_n_fil_crypt_threads = new_cnt;
@@ -2358,15 +2362,18 @@ void
 fil_crypt_threads_init()
 /*====================*/
 {
-	fil_crypt_event = os_event_create();
-	fil_crypt_threads_event = os_event_create();
-	mutex_create(fil_crypt_threads_mutex_key,
-		     &fil_crypt_threads_mutex, SYNC_NO_ORDER_CHECK);
-
-	uint cnt = srv_n_fil_crypt_threads;
-	srv_n_fil_crypt_threads = 0;
-	fil_crypt_set_thread_cnt(cnt);
-	fil_crypt_threads_inited = true;
+	ut_ad(mutex_own(&fil_system->mutex));
+	if (!fil_crypt_threads_inited) {
+		fil_crypt_event = os_event_create();
+		fil_crypt_threads_event = os_event_create();
+		mutex_create(fil_crypt_threads_mutex_key,
+			&fil_crypt_threads_mutex, SYNC_NO_ORDER_CHECK);
+
+		uint cnt = srv_n_fil_crypt_threads;
+		srv_n_fil_crypt_threads = 0;
+		fil_crypt_threads_inited = true;
+		fil_crypt_set_thread_cnt(cnt);
+	}
 }
 
 /*********************************************************************
@@ -2389,6 +2396,7 @@ fil_crypt_threads_cleanup()
 {
 	os_event_free(fil_crypt_event);
 	os_event_free(fil_crypt_threads_event);
+	fil_crypt_threads_inited = false;
 }
 
 /*********************************************************************
diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc
index 6e45725098c..e0f9bde63fa 100644
--- a/storage/xtradb/srv/srv0start.cc
+++ b/storage/xtradb/srv/srv0start.cc
@@ -3076,7 +3076,9 @@ files_checked:
 		fts_optimize_init();
 
 		/* Create thread(s) that handles key rotation */
+		fil_system_enter();
 		fil_crypt_threads_init();
+		fil_system_exit();
 
 		/* Create the log scrub thread */
 		if (srv_scrub_log)
diff --git a/storage/xtradb/sync/sync0sync.cc b/storage/xtradb/sync/sync0sync.cc
index 14b332a2185..3facb99ac0a 100644
--- a/storage/xtradb/sync/sync0sync.cc
+++ b/storage/xtradb/sync/sync0sync.cc
@@ -482,7 +482,13 @@ mutex_validate(
 	const ib_mutex_t*	mutex)	/*!< in: mutex */
 {
 	ut_a(mutex);
-	ut_a(mutex->magic_n == MUTEX_MAGIC_N);
+
+	if (mutex->magic_n != MUTEX_MAGIC_N) {
+		ib_logf(IB_LOG_LEVEL_ERROR,
+			"Mutex %p not initialized file %s line %lu.",
+			mutex, mutex->cfile_name, mutex->cline);
+	}
+	ut_ad(mutex->magic_n == MUTEX_MAGIC_N);
 
 	return(TRUE);
 }