MDEV-21766 - Forbid XID with empty 'gtrid'

XA specification doesn't permit empty gtrid. It is now enforced by this
patch. This solution was agreed in favour of fixing InnoDB, which doesn't
expect empty XID since early 10.5.

Also fixed wrong assertion (and added a test cases) that didn't permit
64 bytes gtrid + 64 bytes bqual.

4 files changed, 61 insertions, 3 deletions

diff --git a/mysql-test/main/xa.result b/mysql-test/main/xa.result
index f77c0afdec5..030415a5320 100644
--- a/mysql-test/main/xa.result
+++ b/mysql-test/main/xa.result
@@ -345,3 +345,27 @@ connection default;
 XA END 'xid1';
 XA ROLLBACK 'xid1';
 DROP TABLE t1, t2, t3;
+#
+# MDEV-21766 - Forbid XID with empty 'gtrid'
+#
+CREATE TABLE t1(a INT) ENGINE=InnoDB;
+XA BEGIN '';
+ERROR XAE05: XAER_INVAL: Invalid arguments (or unsupported command)
+XA BEGIN '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x',
+'8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x';
+INSERT INTO t1 VALUES(1);
+XA END '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x',
+'8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x';
+XA PREPARE '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x',
+'8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x';
+XA ROLLBACK '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x',
+'8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x';
+SET NAMES utf8;
+XA BEGIN 'Я_упала_с_сеновала_тормозила_головой';
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '' at line 1
+XA BEGIN 'Я_упaлa_c_сеновала_тормозила_головой';
+XA END 'Я_упaлa_c_сеновала_тормозила_головой';
+XA PREPARE 'Я_упaлa_c_сеновала_тормозила_головой';
+XA ROLLBACK 'Я_упaлa_c_сеновала_тормозила_головой';
+SET NAMES default;
+DROP TABLE t1;
diff --git a/mysql-test/main/xa.test b/mysql-test/main/xa.test
index 176ef6aa760..41c1f5a8859 100644
--- a/mysql-test/main/xa.test
+++ b/mysql-test/main/xa.test
@@ -476,5 +476,34 @@ XA END 'xid1';
 XA ROLLBACK 'xid1';
 DROP TABLE t1, t2, t3;
 
---source include/wait_until_count_sessions.inc
 
+--echo #
+--echo # MDEV-21766 - Forbid XID with empty 'gtrid'
+--echo #
+CREATE TABLE t1(a INT) ENGINE=InnoDB;
+
+--error ER_XAER_INVAL
+XA BEGIN '';
+
+XA BEGIN '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x',
+         '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x';
+INSERT INTO t1 VALUES(1);
+XA END '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x',
+       '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x';
+XA PREPARE '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x',
+           '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x';
+XA ROLLBACK '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x',
+            '8bytes1x8bytes2x8bytes3x8bytes4x8bytes5x8bytes6x8bytes7x8bytes8x';
+
+SET NAMES utf8;
+--error ER_PARSE_ERROR
+XA BEGIN 'Я_упала_с_сеновала_тормозила_головой'; # 36 characters, 67 bytes
+XA BEGIN 'Я_упaлa_c_сеновала_тормозила_головой'; # 36 characters, 64 bytes
+XA END 'Я_упaлa_c_сеновала_тормозила_головой';
+XA PREPARE 'Я_упaлa_c_сеновала_тормозила_головой';
+XA ROLLBACK 'Я_упaлa_c_сеновала_тормозила_головой';
+SET NAMES default;
+
+DROP TABLE t1;
+
+--source include/wait_until_count_sessions.inc
diff --git a/sql/xa.cc b/sql/xa.cc
index 3ead73fe1e1..9b1545a6bd5 100644
--- a/sql/xa.cc
+++ b/sql/xa.cc
@@ -432,6 +432,8 @@ bool trans_xa_start(THD *thd)
   /* TODO: JOIN is not supported yet. */
   if (thd->lex->xa_opt != XA_NONE)
     my_error(ER_XAER_INVAL, MYF(0));
+  else if (!thd->lex->xid->gtrid_length)
+    my_error(ER_XAER_INVAL, MYF(0));
   else if (thd->transaction.xid_state.is_explicit_XA())
     thd->transaction.xid_state.er_xaer_rmfail();
   else if (thd->locked_tables_mode || thd->in_active_multi_stmt_transaction())
diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc
index 17b50fca56c..4b4d71611ed 100644
--- a/storage/innobase/trx/trx0undo.cc
+++ b/storage/innobase/trx/trx0undo.cc
@@ -497,9 +497,12 @@ static uint16_t trx_undo_header_create(buf_block_t *undo_page, trx_id_t trx_id,
 static void trx_undo_write_xid(buf_block_t *block, uint16_t offset,
                                const XID &xid, mtr_t *mtr)
 {
-  DBUG_ASSERT(xid.gtrid_length >= 0);
+  DBUG_ASSERT(xid.gtrid_length > 0);
   DBUG_ASSERT(xid.bqual_length >= 0);
-  DBUG_ASSERT(xid.gtrid_length + xid.bqual_length < XIDDATASIZE);
+  DBUG_ASSERT(xid.gtrid_length <= MAXGTRIDSIZE);
+  DBUG_ASSERT(xid.bqual_length <= MAXBQUALSIZE);
+  static_assert(MAXGTRIDSIZE + MAXBQUALSIZE == XIDDATASIZE,
+                "gtrid and bqual don't fit xid data");
   DBUG_ASSERT(mach_read_from_2(TRX_UNDO_SEG_HDR + TRX_UNDO_LAST_LOG +
                                block->frame) == offset);