Bug#28323: Server crashed in xid cache operations

The problem was that the server did not robustly handle a
unilateral roll back issued by the Resource Manager (RM)
due to a resource deadlock within the transaction branch.
By not acknowledging the roll back, the server (TM) would
eventually corrupt the XA transaction state and crash.

The solution is to mark the transaction as rollback-only
if the RM indicates that it rolled back its branch of the
transaction.

mysql-test/r/xa.result:
  Add test case result for Bug#28323
mysql-test/t/xa.test:
  Add test case for Bug#28323
sql/handler.cc:
  Reset XID only at the end of the global transaction.
sql/share/errmsg.txt:
  Add new error codes.
sql/sql_class.h:
  Remember the error reported by the Resource Manager.
sql/sql_parse.cc:
  Rollback the transaction if the Resource Manager reported
  a error and rolled back its branch of the transaction.

6 files changed, 145 insertions, 11 deletions

diff --git a/mysql-test/r/xa.result b/mysql-test/r/xa.result
index 5fb03d2378e..25d09f59247 100644
--- a/mysql-test/r/xa.result
+++ b/mysql-test/r/xa.result
@@ -55,3 +55,22 @@ select * from t1;
 a
 20
 drop table t1;
+drop table if exists t1;
+create table t1(a int, b int, c varchar(20), primary key(a)) engine = innodb;
+insert into t1 values(1, 1, 'a');
+insert into t1 values(2, 2, 'b');
+xa start 'a','b';
+update t1 set c = 'aa' where a = 1;
+xa start 'a','c';
+update t1 set c = 'bb' where a = 2;
+update t1 set c = 'bb' where a = 2;
+update t1 set c = 'aa' where a = 1;
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+select count(*) from t1;
+count(*)
+2
+xa end 'a','c';
+ERROR XA102: XA_RBDEADLOCK: Transaction branch was rolled back: deadlock was detected
+xa rollback 'a','c';
+xa start 'a','c';
+End of 5.0 tests
diff --git a/mysql-test/t/xa.test b/mysql-test/t/xa.test
index 0d564727fe3..8f408fb1eda 100644
--- a/mysql-test/t/xa.test
+++ b/mysql-test/t/xa.test
@@ -74,3 +74,47 @@ xa start 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz';
 select * from t1;
 drop table t1;
 
+disconnect con1;
+
+#
+# Bug#28323: Server crashed in xid cache operations
+#
+
+--disable_warnings
+drop table if exists t1;
+--enable_warnings
+
+create table t1(a int, b int, c varchar(20), primary key(a)) engine = innodb;
+insert into t1 values(1, 1, 'a');
+insert into t1 values(2, 2, 'b');
+
+connect (con1,localhost,root,,);
+connect (con2,localhost,root,,);
+
+--connection con1
+xa start 'a','b';
+update t1 set c = 'aa' where a = 1;
+--connection con2
+xa start 'a','c';
+update t1 set c = 'bb' where a = 2;
+--connection con1
+--send update t1 set c = 'bb' where a = 2
+--connection con2
+--sleep 1
+--error ER_LOCK_DEADLOCK
+update t1 set c = 'aa' where a = 1;
+select count(*) from t1;
+--error ER_XA_RBDEADLOCK
+xa end 'a','c';
+xa rollback 'a','c';
+--disconnect con2
+
+connect (con3,localhost,root,,);
+--connection con3
+xa start 'a','c';
+
+--disconnect con1
+--disconnect con3
+--connection default
+
+--echo End of 5.0 tests
diff --git a/sql/handler.cc b/sql/handler.cc
index 0de772e366b..67ec5f3e759 100644
--- a/sql/handler.cc
+++ b/sql/handler.cc
@@ -817,7 +817,12 @@ int ha_rollback_trans(THD *thd, bool all)
     trans->nht=0;
     trans->no_2pc=0;
     if (is_real_trans)
-      thd->transaction.xid_state.xid.null();
+    {
+      if (thd->transaction_rollback_request)
+        thd->transaction.xid_state.rm_error= thd->net.last_errno;
+      else
+        thd->transaction.xid_state.xid.null();
+    }
     if (all)
     {
       thd->variables.tx_isolation=thd->session_tx_isolation;
diff --git a/sql/share/errmsg.txt b/sql/share/errmsg.txt
index 0916ad56cef..c688ba88b7b 100644
--- a/sql/share/errmsg.txt
+++ b/sql/share/errmsg.txt
@@ -5645,3 +5645,7 @@ ER_LOAD_DATA_INVALID_COLUMN
   eng "Invalid column reference (%-.64s) in LOAD DATA"
 ER_LOG_PURGE_NO_FILE  
 	eng "Being purged log %s was not found"
+ER_XA_RBTIMEOUT XA106
+	eng "XA_RBTIMEOUT: Transaction branch was rolled back: took too long"
+ER_XA_RBDEADLOCK XA102
+	eng "XA_RBDEADLOCK: Transaction branch was rolled back: deadlock was detected"
diff --git a/sql/sql_class.h b/sql/sql_class.h
index 9fe0a7423de..c8d42d44df7 100644
--- a/sql/sql_class.h
+++ b/sql/sql_class.h
@@ -927,7 +927,7 @@ struct st_savepoint {
   uint                 length, nht;
 };
 
-enum xa_states {XA_NOTR=0, XA_ACTIVE, XA_IDLE, XA_PREPARED};
+enum xa_states {XA_NOTR=0, XA_ACTIVE, XA_IDLE, XA_PREPARED, XA_ROLLBACK_ONLY};
 extern const char *xa_state_names[];
 
 typedef struct st_xid_state {
@@ -935,6 +935,8 @@ typedef struct st_xid_state {
   XID  xid;                           // transaction identifier
   enum xa_states xa_state;            // used by external XA only
   bool in_thd;
+  /* Error reported by the Resource Manager (RM) to the Transaction Manager. */
+  uint rm_error;
 } XID_STATE;
 
 extern pthread_mutex_t LOCK_xid_cache;
diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc
index 005fdcac7f3..91c5cacc4d0 100644
--- a/sql/sql_parse.cc
+++ b/sql/sql_parse.cc
@@ -90,9 +90,57 @@ const char *command_name[]={
 };
 
 const char *xa_state_names[]={
-  "NON-EXISTING", "ACTIVE", "IDLE", "PREPARED"
+  "NON-EXISTING", "ACTIVE", "IDLE", "PREPARED", "ROLLBACK ONLY"
 };
 
+/**
+  Mark a XA transaction as rollback-only if the RM unilaterally
+  rolled back the transaction branch.
+
+  @note If a rollback was requested by the RM, this function sets
+        the appropriate rollback error code and transits the state
+        to XA_ROLLBACK_ONLY.
+
+  @return TRUE if transaction was rolled back or if the transaction
+          state is XA_ROLLBACK_ONLY. FALSE otherwise.
+*/
+static bool xa_trans_rolled_back(XID_STATE *xid_state)
+{
+  if (xid_state->rm_error)
+  {
+    switch (xid_state->rm_error) {
+    case ER_LOCK_WAIT_TIMEOUT:
+      my_error(ER_XA_RBTIMEOUT, MYF(0));
+      break;
+    case ER_LOCK_DEADLOCK:
+      my_error(ER_XA_RBDEADLOCK, MYF(0));
+      break;
+    default:
+      my_error(ER_XA_RBROLLBACK, MYF(0));
+    }
+    xid_state->xa_state= XA_ROLLBACK_ONLY;
+  }
+
+  return (xid_state->xa_state == XA_ROLLBACK_ONLY);
+}
+
+/**
+  Rollback work done on behalf of at ransaction branch.
+*/
+static bool xa_trans_rollback(THD *thd)
+{
+  bool status= test(ha_rollback(thd));
+
+  thd->options&= ~(ulong) OPTION_BEGIN;
+  thd->transaction.all.modified_non_trans_table= FALSE;
+  thd->server_status&= ~SERVER_STATUS_IN_TRANS;
+  xid_cache_delete(&thd->transaction.xid_state);
+  thd->transaction.xid_state.xa_state= XA_NOTR;
+  thd->transaction.xid_state.rm_error= 0;
+
+  return status;
+}
+
 #ifndef EMBEDDED_LIBRARY
 static bool do_command(THD *thd);
 #endif // EMBEDDED_LIBRARY
@@ -5070,6 +5118,7 @@ create_sp_error:
     }
     DBUG_ASSERT(thd->transaction.xid_state.xid.is_null());
     thd->transaction.xid_state.xa_state=XA_ACTIVE;
+    thd->transaction.xid_state.rm_error= 0;
     thd->transaction.xid_state.xid.set(thd->lex->xid);
     xid_cache_insert(&thd->transaction.xid_state);
     thd->transaction.all.modified_non_trans_table= FALSE;
@@ -5095,6 +5144,8 @@ create_sp_error:
       my_error(ER_XAER_NOTA, MYF(0));
       break;
     }
+    if (xa_trans_rolled_back(&thd->transaction.xid_state))
+      break;
     thd->transaction.xid_state.xa_state=XA_IDLE;
     send_ok(thd);
     break;
@@ -5126,6 +5177,12 @@ create_sp_error:
       XID_STATE *xs=xid_cache_search(thd->lex->xid);
       if (!xs || xs->in_thd)
         my_error(ER_XAER_NOTA, MYF(0));
+      else if (xa_trans_rolled_back(xs))
+      {
+        ha_commit_or_rollback_by_xid(thd->lex->xid, 0);
+        xid_cache_delete(xs);
+        break;
+      }
       else
       {
         ha_commit_or_rollback_by_xid(thd->lex->xid, 1);
@@ -5134,6 +5191,11 @@ create_sp_error:
       }
       break;
     }
+    if (xa_trans_rolled_back(&thd->transaction.xid_state))
+    {
+      xa_trans_rollback(thd);
+      break;
+    }
     if (thd->transaction.xid_state.xa_state == XA_IDLE &&
         thd->lex->xa_opt == XA_ONE_PHASE)
     {
@@ -5180,28 +5242,26 @@ create_sp_error:
         my_error(ER_XAER_NOTA, MYF(0));
       else
       {
+        bool ok= !xa_trans_rolled_back(xs);
         ha_commit_or_rollback_by_xid(thd->lex->xid, 0);
         xid_cache_delete(xs);
-        send_ok(thd);
+        if (ok)
+          send_ok(thd);
       }
       break;
     }
     if (thd->transaction.xid_state.xa_state != XA_IDLE &&
-        thd->transaction.xid_state.xa_state != XA_PREPARED)
+        thd->transaction.xid_state.xa_state != XA_PREPARED &&
+        thd->transaction.xid_state.xa_state != XA_ROLLBACK_ONLY)
     {
       my_error(ER_XAER_RMFAIL, MYF(0),
                xa_state_names[thd->transaction.xid_state.xa_state]);
       break;
     }
-    if (ha_rollback(thd))
+    if (xa_trans_rollback(thd))
       my_error(ER_XAER_RMERR, MYF(0));
     else
       send_ok(thd);
-    thd->options&= ~(ulong) OPTION_BEGIN;
-    thd->transaction.all.modified_non_trans_table= FALSE;
-    thd->server_status&= ~SERVER_STATUS_IN_TRANS;
-    xid_cache_delete(&thd->transaction.xid_state);
-    thd->transaction.xid_state.xa_state=XA_NOTR;
     break;
   case SQLCOM_XA_RECOVER:
     res= mysql_xa_recover(thd);