Bug#54583 InnoDB: Assertion failure sync/sync0sync.c:1226

      
Silence the UNIV_SYNC_DEBUG assertion failure while upgrading old data files
to multiple rollback segments during server startup. Because the upgrade
takes place while InnoDB is running a single thread, we can safely ignore the
latching order checks without fearing deadlocks.
      
innobase_start_or_create_for_mysql(): Set srv_is_being_started = FALSE
only after trx_sys_create_rsegs() has completed.
      
sync_thread_add_level(): If srv_is_being_started, ignore latching order
violations for SYNC_TRX_SYS_HEADER and SYNC_IBUF_BITMAP.

Create all the non-IO threads after creating the extra rollback segments.

Patch originally from Marko with some additions by Sunny.

2 files changed, 37 insertions, 17 deletions

diff --git a/storage/innobase/srv/srv0start.c b/storage/innobase/srv/srv0start.c
index 3d4aaa9d5d4..4da836672ec 100644
--- a/storage/innobase/srv/srv0start.c
+++ b/storage/innobase/srv/srv0start.c
@@ -1703,20 +1703,6 @@ innobase_start_or_create_for_mysql(void)
 	/* fprintf(stderr, "Max allowed record size %lu\n",
 	page_get_free_space_of_empty() / 2); */
 
-	/* Create the thread which watches the timeouts for lock waits */
-	os_thread_create(&srv_lock_timeout_thread, NULL,
-			 thread_ids + 2 + SRV_MAX_N_IO_THREADS);
-
-	/* Create the thread which warns of long semaphore waits */
-	os_thread_create(&srv_error_monitor_thread, NULL,
-			 thread_ids + 3 + SRV_MAX_N_IO_THREADS);
-
-	/* Create the thread which prints InnoDB monitor info */
-	os_thread_create(&srv_monitor_thread, NULL,
-			 thread_ids + 4 + SRV_MAX_N_IO_THREADS);
-
-	srv_is_being_started = FALSE;
-
 	if (trx_doublewrite == NULL) {
 		/* Create the doublewrite buffer to a new tablespace */
 
@@ -1729,8 +1715,29 @@ innobase_start_or_create_for_mysql(void)
 	We create the new segments only if it's a new database or
 	the database was shutdown cleanly. */
 
+	/* Note: When creating the extra rollback segments during an upgrade
+	we violate the latching order, even if the change buffer is empty.
+	We make an exception in sync0sync.c and check srv_is_being_started
+	for that violation. It cannot create a deadlock because we are still
+	running in single threaded mode essentially. Only the IO threads
+	should be running at this stage. */
+
 	trx_sys_create_rsegs(TRX_SYS_N_RSEGS - 1);
 
+	/* Create the thread which watches the timeouts for lock waits */
+	os_thread_create(&srv_lock_timeout_thread, NULL,
+			 thread_ids + 2 + SRV_MAX_N_IO_THREADS);
+
+	/* Create the thread which warns of long semaphore waits */
+	os_thread_create(&srv_error_monitor_thread, NULL,
+			 thread_ids + 3 + SRV_MAX_N_IO_THREADS);
+
+	/* Create the thread which prints InnoDB monitor info */
+	os_thread_create(&srv_monitor_thread, NULL,
+			 thread_ids + 4 + SRV_MAX_N_IO_THREADS);
+
+	srv_is_being_started = FALSE;
+
 	err = dict_create_or_check_foreign_constraint_tables();
 
 	if (err != DB_SUCCESS) {
diff --git a/storage/innobase/sync/sync0sync.c b/storage/innobase/sync/sync0sync.c
index 0bc96aae9de..8062d9e902e 100644
--- a/storage/innobase/sync/sync0sync.c
+++ b/storage/innobase/sync/sync0sync.c
@@ -40,6 +40,9 @@ Created 9/5/1995 Heikki Tuuri
 #include "srv0srv.h"
 #include "buf0types.h"
 #include "os0sync.h" /* for HAVE_ATOMIC_BUILTINS */
+#ifdef UNIV_SYNC_DEBUG
+# include "srv0start.h" /* srv_is_being_started */
+#endif /* UNIV_SYNC_DEBUG */
 
 /*
 	REASONS FOR IMPLEMENTING THE SPIN LOCK MUTEX
@@ -1152,6 +1155,13 @@ sync_thread_add_level(
 	case SYNC_TREE_NODE_FROM_HASH:
 		/* Do no order checking */
 		break;
+	case SYNC_TRX_SYS_HEADER:
+		if (srv_is_being_started) {
+			/* This is violated during trx_sys_create_rsegs()
+			when creating additional rollback segments when
+			upgrading in innobase_start_or_create_for_mysql(). */
+			break;
+		}
 	case SYNC_MEM_POOL:
 	case SYNC_MEM_HASH:
 	case SYNC_RECV:
@@ -1160,7 +1170,6 @@ sync_thread_add_level(
 	case SYNC_LOG_FLUSH_ORDER:
 	case SYNC_THR_LOCAL:
 	case SYNC_ANY_LATCH:
-	case SYNC_TRX_SYS_HEADER:
 	case SYNC_FILE_FORMAT_TAG:
 	case SYNC_DOUBLEWRITE:
 	case SYNC_SEARCH_SYS:
@@ -1222,8 +1231,12 @@ sync_thread_add_level(
 			ut_a(sync_thread_levels_g(array, SYNC_IBUF_BITMAP - 1,
 						  TRUE));
 		} else {
-			ut_a(sync_thread_levels_g(array, SYNC_IBUF_BITMAP,
-						  TRUE));
+			/* This is violated during trx_sys_create_rsegs()
+			when creating additional rollback segments when
+			upgrading in innobase_start_or_create_for_mysql(). */
+			ut_a(srv_is_being_started
+			     || sync_thread_levels_g(array, SYNC_IBUF_BITMAP,
+						     TRUE));
 		}
 		break;
 	case SYNC_FSP_PAGE: