Bug#12352846 - TRANS_XA_START(THD*):

               ASSERTION THD->TRANSACTION.XID_STATE.XID.IS_NULL()
               FAILED

The triggered assert checks that the previous XA transaction has
done proper cleanup before a new XA transaction is started.
The bug that caused it to be triggered was that XA COMMIT did not
clean up error state if XA COMMIT discovered that the current XA
transaction had to be rolled back.

This patch fixes the problem by resetting the XA error state
before XA COMMIT calls ha_rollback_trans(). This allows following
XA transactions to be started without triggering the assert.

Test case added to xa.test.

1 files changed, 29 insertions, 11 deletions

diff --git a/sql/transaction.cc b/sql/transaction.cc
index cd783e44792..94a32200274 100644
--- a/sql/transaction.cc
+++ b/sql/transaction.cc
@@ -79,6 +79,33 @@ static bool xa_trans_rolled_back(XID_STATE *xid_state)
 
 
 /**
+  Rollback the active XA transaction.
+
+  @note Resets rm_error before calling ha_rollback(), so
+        the thd->transaction.xid structure gets reset
+        by ha_rollback() / THD::transaction::cleanup().
+
+  @return TRUE if the rollback failed, FALSE otherwise.
+*/
+
+static bool xa_trans_force_rollback(THD *thd)
+{
+  /*
+    We must reset rm_error before calling ha_rollback(),
+    so thd->transaction.xid structure gets reset
+    by ha_rollback()/THD::transaction::cleanup().
+  */
+  thd->transaction.xid_state.rm_error= 0;
+  if (ha_rollback_trans(thd, true))
+  {
+    my_error(ER_XAER_RMERR, MYF(0));
+    return true;
+  }
+  return false;
+}
+
+
+/**
   Begin a new transaction.
 
   @note Beginning a transaction implicitly commits any current
@@ -649,8 +676,7 @@ bool trans_xa_commit(THD *thd)
 
   if (xa_trans_rolled_back(&thd->transaction.xid_state))
   {
-    if (ha_rollback_trans(thd, TRUE))
-      my_error(ER_XAER_RMERR, MYF(0));
+    xa_trans_force_rollback(thd);
     res= thd->is_error();
   }
   else if (xa_state == XA_IDLE && thd->lex->xa_opt == XA_ONE_PHASE)
@@ -739,15 +765,7 @@ bool trans_xa_rollback(THD *thd)
     DBUG_RETURN(TRUE);
   }
 
-  /*
-    Resource Manager error is meaningless at this point, as we perform
-    explicit rollback request by user. We must reset rm_error before
-    calling ha_rollback(), so thd->transaction.xid structure gets reset
-    by ha_rollback()/THD::transaction::cleanup().
-  */
-  thd->transaction.xid_state.rm_error= 0;
-  if ((res= test(ha_rollback_trans(thd, TRUE))))
-    my_error(ER_XAER_RMERR, MYF(0));
+  res= xa_trans_force_rollback(thd);
 
   thd->variables.option_bits&= ~(OPTION_BEGIN | OPTION_KEEP_LOG);
   thd->transaction.all.modified_non_trans_table= FALSE;