MDEV-13039 innodb_fast_shutdown=0 may fail to purge all undo log

When a slow shutdown is performed soon after spawning some work for
background threads that can create or commit transactions, it is possible
that new transactions are started or committed after the purge has finished.
This is violating the specification of innodb_fast_shutdown=0, namely that
the purge must be completed. (None of the history of the recent transactions
would be purged.)

Also, it is possible that the purge threads would exit in slow shutdown
while there exist active transactions, such as recovered incomplete
transactions that are being rolled back. Thus, the slow shutdown could
fail to purge some undo log that becomes purgeable after the transaction
commit or rollback.

srv_undo_sources: A flag that indicates if undo log can be generated
or the persistent, whether by background threads or by user SQL.
Even when this flag is clear, active transactions that already exist
in the system may be committed or rolled back.

innodb_shutdown(): Renamed from innobase_shutdown_for_mysql().
Do not return an error code; the operation never fails.
Clear the srv_undo_sources flag, and also ensure that the background
DROP TABLE queue is empty.

srv_purge_should_exit(): Do not allow the purge to exit if
srv_undo_sources are active or the background DROP TABLE queue is not
empty, or in slow shutdown, if any active transactions exist
(and are being rolled back).

srv_purge_coordinator_thread(): Remove some previous workarounds
for this bug.

innobase_start_or_create_for_mysql(): Set buf_page_cleaner_is_active
and srv_dict_stats_thread_active directly. Set srv_undo_sources before
starting the purge subsystem, to prevent immediate shutdown of the purge.
Create dict_stats_thread and fts_optimize_thread immediately
after setting srv_undo_sources, so that shutdown can use this flag to
determine if these subsystems were started.

dict_stats_shutdown(): Shut down dict_stats_thread. Backported from 10.2.

srv_shutdown_table_bg_threads(): Remove (unused).

4 files changed, 14 insertions, 16 deletions

diff --git a/storage/xtradb/include/buf0flu.h b/storage/xtradb/include/buf0flu.h
index 96035452e6e..384014939ae 100644
--- a/storage/xtradb/include/buf0flu.h
+++ b/storage/xtradb/include/buf0flu.h
@@ -34,7 +34,7 @@ Created 11/5/1995 Heikki Tuuri
 #include "buf0types.h"
 
 /** Flag indicating if the page_cleaner is in active state. */
-extern ibool buf_page_cleaner_is_active;
+extern bool buf_page_cleaner_is_active;
 
 /** Flag indicating if the lru_manager is in active state. */
 extern bool buf_lru_manager_is_active;
diff --git a/storage/xtradb/include/dict0stats_bg.h b/storage/xtradb/include/dict0stats_bg.h
index 1973380d197..2bd441d5f5e 100644
--- a/storage/xtradb/include/dict0stats_bg.h
+++ b/storage/xtradb/include/dict0stats_bg.h
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, MariaDB Corporation. All Rights Reserved.
+Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -122,6 +122,10 @@ DECLARE_THREAD(dict_stats_thread)(
 	void*	arg);	/*!< in: a dummy parameter
 			required by os_thread_create */
 
+/** Shut down the dict_stats_thread. */
+void
+dict_stats_shutdown();
+
 # ifndef UNIV_NONINL
 #  include "dict0stats_bg.ic"
 # endif
diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h
index cfd961d9c46..f27ed9857ec 100644
--- a/storage/xtradb/include/srv0srv.h
+++ b/storage/xtradb/include/srv0srv.h
@@ -499,7 +499,7 @@ extern ibool	srv_error_monitor_active;
 extern ibool	srv_buf_dump_thread_active;
 
 /* TRUE during the lifetime of the stats thread */
-extern ibool	srv_dict_stats_thread_active;
+extern bool	srv_dict_stats_thread_active;
 
 extern ulong	srv_n_spin_wait_rounds;
 extern ulong	srv_n_free_tickets_to_enter;
diff --git a/storage/xtradb/include/srv0start.h b/storage/xtradb/include/srv0start.h
index a60776a4665..b055a9d834f 100644
--- a/storage/xtradb/include/srv0start.h
+++ b/storage/xtradb/include/srv0start.h
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -75,22 +76,12 @@ are not found and the user wants.
 @return	DB_SUCCESS or error code */
 UNIV_INTERN
 dberr_t
-innobase_start_or_create_for_mysql(void);
-/*====================================*/
-/****************************************************************//**
-Shuts down the Innobase database.
-@return	DB_SUCCESS or error code */
-UNIV_INTERN
-dberr_t
-innobase_shutdown_for_mysql(void);
+innobase_start_or_create_for_mysql();
 
-/********************************************************************
-Signal all per-table background threads to shutdown, and wait for them to do
-so. */
+/** Shut down InnoDB. */
 UNIV_INTERN
 void
-srv_shutdown_table_bg_threads(void);
-/*=============================*/
+innodb_shutdown();
 
 /*************************************************************//**
 Copy the file path component of the physical file to parameter. It will
@@ -158,6 +149,9 @@ enum srv_shutdown_state {
 	SRV_SHUTDOWN_EXIT_THREADS/*!< Exit all threads */
 };
 
+/** Whether any undo log records can be generated */
+extern bool srv_undo_sources;
+
 /** At a shutdown this value climbs from SRV_SHUTDOWN_NONE to
 SRV_SHUTDOWN_CLEANUP and then to SRV_SHUTDOWN_LAST_PHASE, and so on */
 extern	enum srv_shutdown_state	srv_shutdown_state;