MDEV-15832 With innodb_fast_shutdown=3, skip the rollback of connected transactions

row_undo_step(): If innodb_fast_shutdown=3 has been requested,
abort the rollback of any non-DDL transactions. Starting with
MDEV-12323, we aborted the rollback of recovered transactions. The
transactions would be rolled back on subsequent server startup.

trx_roll_report_progress(): Renamed from trx_roll_must_shutdown(),
now that the shutdown check has been moved to the only caller.

trx_commit_low(): Allow mtr=NULL for transactions that are aborted
on rollback.

trx_rollback_finish(): Clean up aborted transactions to avoid
assertion failures and memory leaks on shutdown. This code was
previously in trx_rollback_active().

trx_rollback_to_savepoint_low(), trx_rollback_for_mysql_low():
Remove some redundant assertions.

11 files changed, 90 insertions, 85 deletions

diff --git a/mysql-test/suite/innodb/r/innodb_prefix_index_restart_server.result b/mysql-test/suite/innodb/r/innodb_prefix_index_restart_server.result
index 620157d67b1..a43649f4067 100644
--- a/mysql-test/suite/innodb/r/innodb_prefix_index_restart_server.result
+++ b/mysql-test/suite/innodb/r/innodb_prefix_index_restart_server.result
@@ -28,6 +28,7 @@ COUNT(*)
 1
 START TRANSACTION;
 connection default;
+SET GLOBAL innodb_fast_shutdown=3;
 SELECT COUNT(*) FROM worklog5743;
 COUNT(*)
 1
@@ -53,6 +54,7 @@ worklog5743;
 col_1_text = REPEAT("a", 3500)	col_2_text = REPEAT("o", 3500)
 1	1
 connection default;
+SET GLOBAL innodb_fast_shutdown=3;
 SELECT COUNT(*) FROM worklog5743;
 COUNT(*)
 1
@@ -76,6 +78,7 @@ worklog5743;
 col_1_text = REPEAT("b", 3500)	col_2_text = REPEAT("o", 3500)
 1	1
 connection default;
+SET GLOBAL innodb_fast_shutdown=3;
 SELECT COUNT(*) FROM worklog5743;
 COUNT(*)
 1
diff --git a/mysql-test/suite/innodb/t/innodb_prefix_index_restart_server.test b/mysql-test/suite/innodb/t/innodb_prefix_index_restart_server.test
index c82e67a9a62..bfb5081d336 100644
--- a/mysql-test/suite/innodb/t/innodb_prefix_index_restart_server.test
+++ b/mysql-test/suite/innodb/t/innodb_prefix_index_restart_server.test
@@ -49,6 +49,7 @@ START TRANSACTION;
 
 --connection default
 # Restart the server
+SET GLOBAL innodb_fast_shutdown=3;
 -- source include/restart_mysqld.inc
 SELECT COUNT(*) FROM worklog5743;
 SELECT col_1_text = REPEAT("a", 3500) , col_2_text  = REPEAT("o", 3500) FROM
@@ -69,6 +70,7 @@ worklog5743;
 
 --connection default
 # Restart the server
+SET GLOBAL innodb_fast_shutdown=3;
 -- source include/restart_mysqld.inc
 SELECT COUNT(*) FROM worklog5743;
 SELECT col_1_text = REPEAT("a", 3500) , col_2_text  = REPEAT("o", 3500) FROM
@@ -87,6 +89,7 @@ worklog5743;
 
 --connection default
 # Restart the server
+SET GLOBAL innodb_fast_shutdown=3;
 -- source include/restart_mysqld.inc
 SELECT COUNT(*) FROM worklog5743;
 SELECT col_1_text = REPEAT("a", 3500) , col_2_text  = REPEAT("o", 3500) FROM
diff --git a/mysql-test/suite/sys_vars/r/innodb_fast_shutdown_basic.result b/mysql-test/suite/sys_vars/r/innodb_fast_shutdown_basic.result
index 38d5365b3f3..7d0b48f2947 100644
--- a/mysql-test/suite/sys_vars/r/innodb_fast_shutdown_basic.result
+++ b/mysql-test/suite/sys_vars/r/innodb_fast_shutdown_basic.result
@@ -33,6 +33,10 @@ SET @@global.innodb_fast_shutdown = 2;
 SELECT @@global.innodb_fast_shutdown;
 @@global.innodb_fast_shutdown
 2
+SET @@global.innodb_fast_shutdown = 3;
+SELECT @@global.innodb_fast_shutdown;
+@@global.innodb_fast_shutdown
+3
 '#--------------------FN_DYNVARS_042_04-------------------------#'
 SET @@global.innodb_fast_shutdown = -1;
 Warnings:
diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result
index ab66efb13d4..5e73ee465c4 100644
--- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result
+++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result
@@ -882,9 +882,9 @@ GLOBAL_VALUE_ORIGIN	COMPILE-TIME
 DEFAULT_VALUE	1
 VARIABLE_SCOPE	GLOBAL
 VARIABLE_TYPE	INT UNSIGNED
-VARIABLE_COMMENT	Speeds up the shutdown process of the InnoDB storage engine. Possible values are 0, 1 (faster) or 2 (fastest - crash-like).
+VARIABLE_COMMENT	Speeds up the shutdown process of the InnoDB storage engine. Possible values are 0, 1 (faster), 2 (crash-like), 3 (fastest clean).
 NUMERIC_MIN_VALUE	0
-NUMERIC_MAX_VALUE	2
+NUMERIC_MAX_VALUE	3
 NUMERIC_BLOCK_SIZE	0
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
diff --git a/mysql-test/suite/sys_vars/t/innodb_fast_shutdown_basic.test b/mysql-test/suite/sys_vars/t/innodb_fast_shutdown_basic.test
index 9fe9f490aa4..21d47588796 100644
--- a/mysql-test/suite/sys_vars/t/innodb_fast_shutdown_basic.test
+++ b/mysql-test/suite/sys_vars/t/innodb_fast_shutdown_basic.test
@@ -5,7 +5,7 @@
 # Access Type: Dynamic                                                        #
 # Data Type: numeric                                                          #
 # Default Value: 1                                                            #
-# Valid Values: 0,1,2                                                           #
+# Valid Values: 0,1,2,3                                                       #
 #                                                                             #
 #                                                                             #
 # Creation Date: 2008-02-20                                                   #
@@ -81,6 +81,8 @@ SELECT @@global.innodb_fast_shutdown;
 SET @@global.innodb_fast_shutdown = 2;
 SELECT @@global.innodb_fast_shutdown;
 
+SET @@global.innodb_fast_shutdown = 3;
+SELECT @@global.innodb_fast_shutdown;
 
 --echo '#--------------------FN_DYNVARS_042_04-------------------------#'
 ###########################################################################
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index d9c3995f305..c13237ce18d 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -19424,8 +19424,8 @@ static MYSQL_SYSVAR_ULONG(sync_array_size, srv_sync_array_size,
 static MYSQL_SYSVAR_UINT(fast_shutdown, srv_fast_shutdown,
   PLUGIN_VAR_OPCMDARG,
   "Speeds up the shutdown process of the InnoDB storage engine. Possible"
-  " values are 0, 1 (faster) or 2 (fastest - crash-like).",
-  fast_shutdown_validate, NULL, 1, 0, 2, 0);
+  " values are 0, 1 (faster), 2 (crash-like), 3 (fastest clean).",
+  fast_shutdown_validate, NULL, 1, 0, 3, 0);
 
 static MYSQL_SYSVAR_BOOL(file_per_table, srv_file_per_table,
   PLUGIN_VAR_NOCMDARG,
diff --git a/storage/innobase/include/trx0roll.h b/storage/innobase/include/trx0roll.h
index ba9c901d4f7..e1718294f6e 100644
--- a/storage/innobase/include/trx0roll.h
+++ b/storage/innobase/include/trx0roll.h
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2015, 2017, MariaDB Corporation.
+Copyright (c) 2015, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -63,10 +63,8 @@ trx_undo_rec_t*
 trx_roll_pop_top_rec_of_trx(trx_t* trx, roll_ptr_t* roll_ptr, mem_heap_t* heap)
 	MY_ATTRIBUTE((nonnull, warn_unused_result));
 
-/** Report progress when rolling back a row of a recovered transaction.
-@return	whether the rollback should be aborted due to pending shutdown */
-bool
-trx_roll_must_shutdown();
+/** Report progress when rolling back a row of a recovered transaction. */
+void trx_roll_report_progress();
 /*******************************************************************//**
 Rollback or clean up any incomplete transactions which were
 encountered in crash recovery.  If the transaction already was
diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h
index 8a4a1151b46..4b7ca171740 100644
--- a/storage/innobase/include/trx0trx.h
+++ b/storage/innobase/include/trx0trx.h
@@ -203,14 +203,10 @@ trx_commit(
 /*=======*/
 	trx_t*	trx);	/*!< in/out: transaction */
 
-/****************************************************************//**
-Commits a transaction and a mini-transaction. */
-void
-trx_commit_low(
-/*===========*/
-	trx_t*	trx,	/*!< in/out: transaction */
-	mtr_t*	mtr);	/*!< in/out: mini-transaction (will be committed),
-			or NULL if trx made no modifications */
+/** Commit a transaction and a mini-transaction.
+@param[in,out]	trx	transaction
+@param[in,out]	mtr	mini-transaction (NULL if no modifications) */
+void trx_commit_low(trx_t* trx, mtr_t* mtr);
 /**********************************************************************//**
 Does the transaction commit for MySQL.
 @return DB_SUCCESS or error number */
diff --git a/storage/innobase/row/row0undo.cc b/storage/innobase/row/row0undo.cc
index 2fcfecd5c11..a85b817d5cb 100644
--- a/storage/innobase/row/row0undo.cc
+++ b/storage/innobase/row/row0undo.cc
@@ -42,6 +42,7 @@ Created 1/8/1997 Heikki Tuuri
 #include "row0upd.h"
 #include "row0mysql.h"
 #include "srv0srv.h"
+#include "srv0start.h"
 
 /* How to undo row operations?
 (1) For an insert, we have stored a prefix of the clustered index record
@@ -345,11 +346,17 @@ row_undo_step(
 
 	ut_ad(que_node_get_type(node) == QUE_NODE_UNDO);
 
-	if (UNIV_UNLIKELY(trx == trx_roll_crash_recv_trx)
-	    && trx_roll_must_shutdown()) {
+	if (UNIV_UNLIKELY(trx_get_dict_operation(trx) == TRX_DICT_OP_NONE
+			  && !srv_undo_sources
+			  && !srv_is_being_started)
+	    && (srv_fast_shutdown == 3 || trx == trx_roll_crash_recv_trx)) {
 		/* Shutdown has been initiated. */
 		trx->error_state = DB_INTERRUPTED;
-		return(NULL);
+		return NULL;
+	}
+
+	if (UNIV_UNLIKELY(trx == trx_roll_crash_recv_trx)) {
+		trx_roll_report_progress();
 	}
 
 	err = row_undo(node, thr);
diff --git a/storage/innobase/trx/trx0roll.cc b/storage/innobase/trx/trx0roll.cc
index 6874f929726..bd994dec765 100644
--- a/storage/innobase/trx/trx0roll.cc
+++ b/storage/innobase/trx/trx0roll.cc
@@ -58,13 +58,47 @@ bool			trx_rollback_is_active;
 /** In crash recovery, the current trx to be rolled back; NULL otherwise */
 const trx_t*		trx_roll_crash_recv_trx;
 
-/****************************************************************//**
-Finishes a transaction rollback. */
-static
-void
-trx_rollback_finish(
-/*================*/
-	trx_t*		trx);	/*!< in: transaction */
+/** Finish transaction rollback.
+@param[in,out]	trx	transaction
+@return	whether the rollback was completed normally
+@retval	false	if the rollback was aborted by shutdown  */
+static bool trx_rollback_finish(trx_t* trx)
+{
+	trx->mod_tables.clear();
+	bool finished = trx->error_state == DB_SUCCESS;
+	if (UNIV_LIKELY(finished)) {
+		trx_commit(trx);
+	} else {
+		ut_a(trx->error_state == DB_INTERRUPTED);
+		ut_ad(!srv_is_being_started);
+		ut_a(!srv_undo_sources);
+		ut_ad(srv_fast_shutdown);
+		ut_d(trx->in_rollback = false);
+		if (trx_undo_t*& undo = trx->rsegs.m_redo.old_insert) {
+			UT_LIST_REMOVE(trx->rsegs.m_redo.rseg->old_insert_list,
+				       undo);
+			ut_free(undo);
+			undo = NULL;
+		}
+		if (trx_undo_t*& undo = trx->rsegs.m_redo.undo) {
+			UT_LIST_REMOVE(trx->rsegs.m_redo.rseg->undo_list,
+				       undo);
+			ut_free(undo);
+			undo = NULL;
+		}
+		if (trx_undo_t*& undo = trx->rsegs.m_noredo.undo) {
+			UT_LIST_REMOVE(trx->rsegs.m_noredo.rseg->undo_list,
+				       undo);
+			ut_free(undo);
+			undo = NULL;
+		}
+		trx_commit_low(trx, NULL);
+	}
+
+	trx->lock.que_state = TRX_QUE_RUNNING;
+
+	return finished;
+}
 
 /*******************************************************************//**
 Rollback a transaction used in MySQL. */
@@ -119,6 +153,7 @@ trx_rollback_to_savepoint_low(
 		trx_rollback_finish(trx);
 		MONITOR_INC(MONITOR_TRX_ROLLBACK);
 	} else {
+		ut_a(trx->error_state == DB_SUCCESS);
 		const undo_no_t limit = savept->least_undo_no;
 		for (trx_mod_tables_t::iterator i = trx->mod_tables.begin();
 		     i != trx->mod_tables.end(); ) {
@@ -132,9 +167,6 @@ trx_rollback_to_savepoint_low(
 		MONITOR_INC(MONITOR_TRX_ROLLBACK_SAVEPOINT);
 	}
 
-	ut_a(trx->error_state == DB_SUCCESS);
-	ut_a(trx->lock.que_state == TRX_QUE_RUNNING);
-
 	mem_heap_free(heap);
 
 	/* There might be work for utility threads.*/
@@ -183,8 +215,6 @@ trx_rollback_for_mysql_low(
 
 	trx->op_info = "";
 
-	ut_a(trx->error_state == DB_SUCCESS);
-
 	return(trx->error_state);
 }
 
@@ -639,23 +669,14 @@ trx_rollback_active(
 
 	que_run_threads(roll_node->undo_thr);
 
-	if (trx->error_state != DB_SUCCESS) {
-		ut_ad(trx->error_state == DB_INTERRUPTED);
-		ut_ad(!srv_is_being_started);
-		ut_ad(!srv_undo_sources);
-		ut_ad(srv_fast_shutdown);
+	que_graph_free(
+		static_cast<que_t*>(roll_node->undo_thr->common.parent));
+
+	if (UNIV_UNLIKELY(!trx_rollback_finish(trx))) {
 		ut_ad(!dictionary_locked);
-		que_graph_free(static_cast<que_t*>(
-				       roll_node->undo_thr->common.parent));
 		goto func_exit;
 	}
 
-	trx_rollback_finish(thr_get_trx(roll_node->undo_thr));
-
-	/* Free the memory reserved by the undo graph */
-	que_graph_free(static_cast<que_t*>(
-			       roll_node->undo_thr->common.parent));
-
 	ut_a(trx->lock.que_state == TRX_QUE_RUNNING);
 
 	if (trx_get_dict_operation(trx) != TRX_DICT_OP_NONE
@@ -718,23 +739,9 @@ static my_bool trx_roll_count_callback(rw_trx_hash_element_t *element,
   return 0;
 }
 
-
-/** Report progress when rolling back a row of a recovered transaction.
-@return	whether the rollback should be aborted due to pending shutdown */
-bool
-trx_roll_must_shutdown()
+/** Report progress when rolling back a row of a recovered transaction. */
+void trx_roll_report_progress()
 {
-	const trx_t* trx = trx_roll_crash_recv_trx;
-	ut_ad(trx);
-	ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE));
-	ut_ad(trx->in_rollback);
-
-	if (trx_get_dict_operation(trx) == TRX_DICT_OP_NONE
-	    && !srv_is_being_started
-	    && !srv_undo_sources && srv_fast_shutdown) {
-		return true;
-	}
-
 	ib_time_t time = ut_time();
 	mutex_enter(&recv_sys->mutex);
 	bool report = recv_sys->report(time);
@@ -754,7 +761,6 @@ trx_roll_must_shutdown()
 		sd_notifyf(0, "STATUS=To roll back: " UINT32PF " transactions,"
 			   " " UINT64PF " rows", arg.n_trx, arg.n_rows);
 	}
-	return false;
 }
 
 
@@ -1114,19 +1120,6 @@ trx_rollback_start(
 	return(que_fork_start_command(roll_graph));
 }
 
-/****************************************************************//**
-Finishes a transaction rollback. */
-static
-void
-trx_rollback_finish(
-/*================*/
-	trx_t*		trx)	/*!< in: transaction */
-{
-	trx->mod_tables.clear();
-	trx_commit(trx);
-	trx->lock.que_state = TRX_QUE_RUNNING;
-}
-
 /*********************************************************************//**
 Creates a rollback command node struct.
 @return own: rollback node struct */
diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc
index 6690d3e7676..9b3e67a9043 100644
--- a/storage/innobase/trx/trx0trx.cc
+++ b/storage/innobase/trx/trx0trx.cc
@@ -1457,19 +1457,18 @@ trx_commit_in_memory(
 	srv_wake_purge_thread_if_not_active();
 }
 
-/****************************************************************//**
-Commits a transaction and a mini-transaction. */
-void
-trx_commit_low(
-/*===========*/
-	trx_t*	trx,	/*!< in/out: transaction */
-	mtr_t*	mtr)	/*!< in/out: mini-transaction (will be committed),
-			or NULL if trx made no modifications */
+/** Commit a transaction and a mini-transaction.
+@param[in,out]	trx	transaction
+@param[in,out]	mtr	mini-transaction (NULL if no modifications) */
+void trx_commit_low(trx_t* trx, mtr_t* mtr)
 {
 	assert_trx_nonlocking_or_in_list(trx);
 	ut_ad(!trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY));
 	ut_ad(!mtr || mtr->is_active());
-	ut_ad(!mtr == !trx->has_logged_or_recovered());
+	ut_d(bool aborted = trx->in_rollback
+	     && trx->error_state == DB_DEADLOCK);
+	ut_ad(!mtr == (aborted || !trx->has_logged_or_recovered()));
+	ut_ad(!mtr || !aborted);
 
 	/* undo_no is non-zero if we're doing the final commit. */
 	if (trx->fts_trx != NULL && trx->undo_no != 0) {