MDEV-8474: InnoDB sets per-connection data unsafely

Analysis: At check_trx_exists function InnoDB allocates
a new trx if no trx is found from thd but this newly
allocated trx is not registered to thd. This is unsafe,
because nothing prevents InnoDB plugin from being uninstalled
while there's active transaction. This can cause crashes, hang
and any other odd behavior. It may also corrupt stack, as
functions pointers are not available after dlclose.

Fix: The fix is to use thd_set_ha_data() when
manipulating per-connection handler data. It does appropriate
plugin locking.

6 files changed, 89 insertions, 0 deletions

diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl
index 566423a1c07..26b31d34d3c 100755
--- a/mysql-test/mysql-test-run.pl
+++ b/mysql-test/mysql-test-run.pl
@@ -4806,6 +4806,7 @@ sub extract_warning_lines ($$) {
      qr|Plugin 'FEEDBACK' registration as a INFORMATION SCHEMA failed|,
      qr|Failed to setup SSL|,
      qr|SSL error: Failed to set ciphers to use|,
+     qr/Plugin 'InnoDB' will be forced to shutdown/,
     );
 
   my $matched_lines= [];
diff --git a/mysql-test/r/innodb_load_xa.result b/mysql-test/r/innodb_load_xa.result
index e738ca6e3af..8b8de2032e5 100644
--- a/mysql-test/r/innodb_load_xa.result
+++ b/mysql-test/r/innodb_load_xa.result
@@ -18,3 +18,5 @@ mysqld-bin.000001	#	Query	#	#	use `test`; insert t1 values (2)
 mysqld-bin.000001	#	Query	#	#	COMMIT
 drop table t1;
 uninstall plugin innodb;
+Warnings:
+Warning	1620	Plugin is busy and will be uninstalled on shutdown
diff --git a/mysql-test/suite/innodb/r/innodb_uninstall.result b/mysql-test/suite/innodb/r/innodb_uninstall.result
new file mode 100644
index 00000000000..2064269a02e
--- /dev/null
+++ b/mysql-test/suite/innodb/r/innodb_uninstall.result
@@ -0,0 +1,22 @@
+install plugin innodb soname 'ha_innodb';
+create table t1(a int not null primary key) engine=innodb;
+begin;
+insert into t1 values(1);
+flush tables;
+uninstall plugin innodb;
+select sleep(1);
+sleep(1)
+0
+Warnings:
+Warning	1620	Plugin is busy and will be uninstalled on shutdown
+drop table t1;
+install plugin innodb soname 'ha_innodb';
+create table t2(a int not null primary key) engine=innodb;
+insert into t2 values(1);
+drop table t2;
+uninstall plugin innodb;
+select sleep(1);
+sleep(1)
+0
+Warnings:
+Warning	1620	Plugin is busy and will be uninstalled on shutdown
diff --git a/mysql-test/suite/innodb/t/innodb_uninstall.opt b/mysql-test/suite/innodb/t/innodb_uninstall.opt
new file mode 100644
index 00000000000..918855a7b01
--- /dev/null
+++ b/mysql-test/suite/innodb/t/innodb_uninstall.opt
@@ -0,0 +1,3 @@
+--ignore-builtin-innodb
+--loose-innodb
+
diff --git a/mysql-test/suite/innodb/t/innodb_uninstall.test b/mysql-test/suite/innodb/t/innodb_uninstall.test
new file mode 100644
index 00000000000..34fc8345a02
--- /dev/null
+++ b/mysql-test/suite/innodb/t/innodb_uninstall.test
@@ -0,0 +1,58 @@
+--source include/not_embedded.inc
+--source include/not_windows.inc
+
+if (!$HA_INNODB_SO) {
+  --skip Need InnoDB plugin
+}
+
+#
+# MDEV-8474: InnoDB sets per-connection data unsafely
+# Below test caused hang
+#
+install plugin innodb soname 'ha_innodb';
+create table t1(a int not null primary key) engine=innodb;
+
+connect (con1, localhost, root);
+connection con1;
+begin;
+insert into t1 values(1);
+
+connection default;
+flush tables;
+send uninstall plugin innodb;
+
+connection con1;
+select sleep(1);
+disconnect con1;
+
+connection default;
+reap;
+
+--source include/restart_mysqld.inc
+
+drop table t1;
+
+#
+# Another test that caused hang.
+#
+
+connect (con1, localhost, root);
+connection con1;
+install plugin innodb soname 'ha_innodb';
+create table t2(a int not null primary key) engine=innodb;
+insert into t2 values(1);
+drop table t2;
+
+connection default;
+send uninstall plugin innodb;
+
+connection con1;
+select sleep(1);
+disconnect con1;
+
+connection default;
+reap;
+
+--source include/restart_mysqld.inc
+
+
diff --git a/mysql-test/t/innodb_load_xa.test b/mysql-test/t/innodb_load_xa.test
index 52862151b22..9ecddde5d13 100644
--- a/mysql-test/t/innodb_load_xa.test
+++ b/mysql-test/t/innodb_load_xa.test
@@ -16,3 +16,6 @@ commit;
 --source include/show_binlog_events.inc
 drop table t1;
 uninstall plugin innodb;
+
+--source include/restart_mysqld.inc
+