MDEV-13797 InnoDB may hang if shutdown is initiated soon after startup while rolling back recovered incomplete transactions

trx_rollback_resurrected(): If shutdown was initiated, fake all
remaining active transactions to XA PREPARE state, so that shutdown
can proceed. Also, make the parameter "all" an output that will be
assigned to FALSE in this case.

trx_rollback_or_clean_recovered(): Remove the shutdown check
(it was moved to trx_rollback_resurrected()).

trx_undo_free_prepared(): Relax assertions.

2 files changed, 102 insertions, 0 deletions

diff --git a/mysql-test/suite/innodb/r/recovery_shutdown.result b/mysql-test/suite/innodb/r/recovery_shutdown.result
new file mode 100644
index 00000000000..028a0bd6239
--- /dev/null
+++ b/mysql-test/suite/innodb/r/recovery_shutdown.result
@@ -0,0 +1,51 @@
+#
+# MDEV-13797 InnoDB may hang if shutdown is initiated soon after startup
+# while rolling back recovered incomplete transactions
+#
+CREATE TABLE t (a INT) ENGINE=InnoDB;
+BEGIN;
+COMMIT;
+CREATE TABLE t8 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t8 (a) SELECT NULL FROM t;
+UPDATE t8 SET a=a+100, b=a;
+DELETE FROM t8;
+CREATE TABLE t7 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t7 (a) SELECT NULL FROM t;
+UPDATE t7 SET a=a+100, b=a;
+DELETE FROM t7;
+CREATE TABLE t6 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t6 (a) SELECT NULL FROM t;
+UPDATE t6 SET a=a+100, b=a;
+DELETE FROM t6;
+CREATE TABLE t5 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t5 (a) SELECT NULL FROM t;
+UPDATE t5 SET a=a+100, b=a;
+DELETE FROM t5;
+CREATE TABLE t4 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t4 (a) SELECT NULL FROM t;
+UPDATE t4 SET a=a+100, b=a;
+DELETE FROM t4;
+CREATE TABLE t3 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t3 (a) SELECT NULL FROM t;
+UPDATE t3 SET a=a+100, b=a;
+DELETE FROM t3;
+CREATE TABLE t2 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t2 (a) SELECT NULL FROM t;
+UPDATE t2 SET a=a+100, b=a;
+DELETE FROM t2;
+CREATE TABLE t1 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t1 (a) SELECT NULL FROM t;
+UPDATE t1 SET a=a+100, b=a;
+DELETE FROM t1;
+SET GLOBAL innodb_flush_log_at_trx_commit=1;
+CREATE TABLE u(a SERIAL) ENGINE=INNODB;
+# Kill and restart
+DROP TABLE t,u;
diff --git a/mysql-test/suite/innodb/t/recovery_shutdown.test b/mysql-test/suite/innodb/t/recovery_shutdown.test
new file mode 100644
index 00000000000..ea38bd19a9f
--- /dev/null
+++ b/mysql-test/suite/innodb/t/recovery_shutdown.test
@@ -0,0 +1,51 @@
+--source include/have_innodb.inc
+--source include/not_embedded.inc
+
+--echo #
+--echo # MDEV-13797 InnoDB may hang if shutdown is initiated soon after startup
+--echo # while rolling back recovered incomplete transactions
+--echo #
+
+CREATE TABLE t (a INT) ENGINE=InnoDB;
+let $size = 100;
+let $trx = 8;
+let $c = $size;
+BEGIN;
+--disable_query_log
+while ($c) {
+INSERT INTO t VALUES();
+dec $c;
+}
+--enable_query_log
+COMMIT;
+
+let $c = $trx;
+while ($c)
+{
+connect (con$c,localhost,root,,);
+eval CREATE TABLE t$c (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+eval INSERT INTO t$c (a) SELECT NULL FROM t;
+eval UPDATE t$c SET a=a+$size, b=a;
+eval DELETE FROM t$c;
+dec $c;
+}
+
+--connection default
+SET GLOBAL innodb_flush_log_at_trx_commit=1;
+CREATE TABLE u(a SERIAL) ENGINE=INNODB;
+
+--source include/kill_and_restart_mysqld.inc
+--source include/restart_mysqld.inc
+
+--disable_query_log
+let $c = $trx;
+while ($c)
+{
+disconnect con$c;
+eval DROP TABLE t$c;
+dec $c;
+}
+--enable_query_log
+
+DROP TABLE t,u;