MDEV-14609 XA Transction unable to ROLLBACK TO SAVEPOINT

The function trans_rollback_to_savepoint(), unlike trans_savepoint(),
did not allow xa_state=XA_ACTIVE, so an attempt to do ROLLBCK TO SAVEPOINT
inside an XA transaction incorrectly returned an error
"...command cannot be executed ... in the ACTIVE state...".

Partially merging a MySQL patch:
  7fb5c47390311d9b1b5367f97cb8fedd4102dd05
  This is WL#7193 (Decouple THD and st_transactions)...

The currently merged part includes these changes:
- Introducing st_xid_state::check_has_uncommitted_xa()
- Reusing it in both trans_rollback_to_savepoint() and trans_savepoint(),
  so now both allow XA_ACTIVE.

4 files changed, 51 insertions, 10 deletions

diff --git a/mysql-test/r/xa.result b/mysql-test/r/xa.result
index 6c242d950ab..0bf9b07bb2f 100644
--- a/mysql-test/r/xa.result
+++ b/mysql-test/r/xa.result
@@ -200,6 +200,17 @@ a
 1
 DROP TABLE t1;
 #
+# MDEV-14609 XA Transction unable to ROLLBACK TO SAVEPOINT
+#
+CREATE TABLE t1 (c1 INT) ENGINE=INNODB;
+XA START 'xa1';
+SAVEPOINT savepoint1;
+INSERT INTO t1 (c1) VALUES (1),(2),(3),(4);
+ROLLBACK TO SAVEPOINT savepoint1;
+XA END 'xa1';
+XA ROLLBACK 'xa1';
+DROP TABLE t1;
+#
 # Bug#12352846 - TRANS_XA_START(THD*):
 #                ASSERTION THD->TRANSACTION.XID_STATE.XID.IS_NULL()
 #                FAILED 
diff --git a/mysql-test/t/xa.test b/mysql-test/t/xa.test
index 1709886eb0c..0995b8c26b2 100644
--- a/mysql-test/t/xa.test
+++ b/mysql-test/t/xa.test
@@ -328,6 +328,20 @@ DROP TABLE t1;
 
 
 --echo #
+--echo # MDEV-14609 XA Transction unable to ROLLBACK TO SAVEPOINT
+--echo #
+
+CREATE TABLE t1 (c1 INT) ENGINE=INNODB;
+XA START 'xa1';
+SAVEPOINT savepoint1;
+INSERT INTO t1 (c1) VALUES (1),(2),(3),(4);
+ROLLBACK TO SAVEPOINT savepoint1;
+XA END 'xa1';
+XA ROLLBACK 'xa1';
+DROP TABLE t1;
+
+
+--echo #
 --echo # Bug#12352846 - TRANS_XA_START(THD*):
 --echo #                ASSERTION THD->TRANSACTION.XID_STATE.XID.IS_NULL()
 --echo #                FAILED 
diff --git a/sql/sql_class.h b/sql/sql_class.h
index f4be996c9a9..e8f50f13ebc 100644
--- a/sql/sql_class.h
+++ b/sql/sql_class.h
@@ -991,6 +991,30 @@ typedef struct st_xid_state {
   bool in_thd;
   /* Error reported by the Resource Manager (RM) to the Transaction Manager. */
   uint rm_error;
+
+  /**
+    Check that XA transaction has an uncommitted work. Report an error
+    to the user in case when there is an uncommitted work for XA transaction.
+
+    @return  result of check
+      @retval  false  XA transaction is NOT in state IDLE, PREPARED
+                      or ROLLBACK_ONLY.
+      @retval  true   XA transaction is in state IDLE or PREPARED
+                      or ROLLBACK_ONLY.
+  */
+
+  bool check_has_uncommitted_xa() const
+  {
+    if (xa_state == XA_IDLE ||
+        xa_state == XA_PREPARED ||
+        xa_state == XA_ROLLBACK_ONLY)
+    {
+      my_error(ER_XAER_RMFAIL, MYF(0), xa_state_names[xa_state]);
+      return true;
+    }
+    return false;
+  }
+
 } XID_STATE;
 
 extern mysql_mutex_t LOCK_xid_cache;
diff --git a/sql/transaction.cc b/sql/transaction.cc
index ae38e920a1d..6b09bd18d4c 100644
--- a/sql/transaction.cc
+++ b/sql/transaction.cc
@@ -433,12 +433,8 @@ bool trans_savepoint(THD *thd, LEX_STRING name)
       !opt_using_transactions)
     DBUG_RETURN(FALSE);
 
-  enum xa_states xa_state= thd->transaction.xid_state.xa_state;
-  if (xa_state != XA_NOTR && xa_state != XA_ACTIVE)
-  {
-    my_error(ER_XAER_RMFAIL, MYF(0), xa_state_names[xa_state]);
+  if (thd->transaction.xid_state.check_has_uncommitted_xa())
     DBUG_RETURN(TRUE);
-  }
 
   sv= find_savepoint(thd, name);
 
@@ -513,12 +509,8 @@ bool trans_rollback_to_savepoint(THD *thd, LEX_STRING name)
     DBUG_RETURN(TRUE);
   }
 
-  enum xa_states xa_state= thd->transaction.xid_state.xa_state;
-  if (xa_state != XA_NOTR)
-  {
-    my_error(ER_XAER_RMFAIL, MYF(0), xa_state_names[xa_state]);
+  if (thd->transaction.xid_state.check_has_uncommitted_xa())
     DBUG_RETURN(TRUE);
-  }
 
   if (ha_rollback_to_savepoint(thd, sv))
     res= TRUE;