BUG#43171 - Assertion failed: thd->transaction.xid_state.xid.is_null()

XA START may cause assertion failure/server crash when it is called
after unilateral roll back issued by the Resource Manager (both
in regular transaction and after XA transaction).

The problem was that rm_error variable wasn't set/reset properly.

mysql-test/r/xa.result:
  A test case for BUG#43171.
mysql-test/t/xa.test:
  A test case for BUG#43171.
sql/handler.cc:
  Setting rm_error when we're out of XA transaction has no
  special meaning. But it blocks reset of thd->transaction.xid
  structure later.
sql/sql_parse.cc:
  Reset rm_error before we enter ha_rollback(), so
  thd->transaction.xid strucure is reinitialized.

2 files changed, 87 insertions, 0 deletions

diff --git a/mysql-test/r/xa.result b/mysql-test/r/xa.result
index a597806d897..d23c8e672b0 100644
--- a/mysql-test/r/xa.result
+++ b/mysql-test/r/xa.result
@@ -89,3 +89,28 @@ xa start 'a';
 xa end 'a';
 xa prepare 'a';
 xa commit 'a';
+CREATE TABLE t1(a INT, KEY(a)) ENGINE=InnoDB;
+INSERT INTO t1 VALUES(1),(2);
+BEGIN;
+UPDATE t1 SET a=3 WHERE a=1;
+BEGIN;
+UPDATE t1 SET a=4 WHERE a=2;
+UPDATE t1 SET a=5 WHERE a=2;
+UPDATE t1 SET a=5 WHERE a=1;
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+ROLLBACK;
+ROLLBACK;
+BEGIN;
+UPDATE t1 SET a=3 WHERE a=1;
+XA START 'xid1';
+UPDATE t1 SET a=4 WHERE a=2;
+UPDATE t1 SET a=5 WHERE a=2;
+UPDATE t1 SET a=5 WHERE a=1;
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+XA END 'xid1';
+ERROR XA102: XA_RBDEADLOCK: Transaction branch was rolled back: deadlock was detected
+XA ROLLBACK 'xid1';
+XA START 'xid1';
+XA END 'xid1';
+XA ROLLBACK 'xid1';
+DROP TABLE t1;
diff --git a/mysql-test/t/xa.test b/mysql-test/t/xa.test
index 7b1c6a268d5..f84d822170f 100644
--- a/mysql-test/t/xa.test
+++ b/mysql-test/t/xa.test
@@ -149,6 +149,68 @@ xa end 'a';
 xa prepare 'a';
 xa commit 'a';
 
+#
+# BUG#43171 - Assertion failed: thd->transaction.xid_state.xid.is_null()
+#
+CREATE TABLE t1(a INT, KEY(a)) ENGINE=InnoDB;
+INSERT INTO t1 VALUES(1),(2);
+connect(con1,localhost,root,,);
+
+# Part 1: Prepare to test XA START after regular transaction deadlock
+BEGIN;
+UPDATE t1 SET a=3 WHERE a=1;
+
+connection default;
+BEGIN;
+UPDATE t1 SET a=4 WHERE a=2;
+
+connection con1;
+let $conn_id= `SELECT CONNECTION_ID()`;
+SEND UPDATE t1 SET a=5 WHERE a=2;
+
+connection default;
+let $wait_timeout= 2;
+let $wait_condition= SELECT 1 FROM INFORMATION_SCHEMA.PROCESSLIST
+WHERE ID=$conn_id AND STATE='Searching rows for update';
+--source include/wait_condition.inc
+
+--error ER_LOCK_DEADLOCK
+UPDATE t1 SET a=5 WHERE a=1;
+ROLLBACK;
+
+# Part 2: Prepare to test XA START after XA transaction deadlock
+connection con1;
+REAP;
+ROLLBACK;
+BEGIN;
+UPDATE t1 SET a=3 WHERE a=1;
+
+connection default;
+XA START 'xid1';
+UPDATE t1 SET a=4 WHERE a=2;
+
+connection con1;
+SEND UPDATE t1 SET a=5 WHERE a=2;
+
+connection default;
+let $wait_timeout= 2;
+let $wait_condition= SELECT 1 FROM INFORMATION_SCHEMA.PROCESSLIST
+WHERE ID=$conn_id AND STATE='Searching rows for update';
+--source include/wait_condition.inc
+
+--error ER_LOCK_DEADLOCK
+UPDATE t1 SET a=5 WHERE a=1;
+--error ER_XA_RBDEADLOCK
+XA END 'xid1';
+XA ROLLBACK 'xid1';
+
+XA START 'xid1';
+XA END 'xid1';
+XA ROLLBACK 'xid1';
+
+disconnect con1;
+DROP TABLE t1;
+
 # Wait till all disconnects are completed
 --source include/wait_until_count_sessions.inc