MDEV-28709 unexpected X lock on Supremum in READ COMMITTED

The lock is created during page splitting after moving records and
locks(lock_move_rec_list_(start|end)()) to the new page, and inheriting
the locks to the supremum of left page from the successor of the infimum
on right page.

There is no need in such inheritance for READ COMMITTED isolation level
and not-gap locks, so the fix is to add the corresponding condition in
gap lock inheritance function.

One more fix is to forbid gap lock inheritance if XA was prepared. Use the
most significant bit of trx_t::n_ref to indicate that gap lock inheritance
is forbidden. This fix is based on
mysql/mysql-server@b063e52a8367dc9d5ed418e7f6d96400867e9f43

2 files changed, 110 insertions, 0 deletions

diff --git a/mysql-test/suite/innodb/r/lock_update_split_rc.result b/mysql-test/suite/innodb/r/lock_update_split_rc.result
new file mode 100644
index 00000000000..6bdce124c5b
--- /dev/null
+++ b/mysql-test/suite/innodb/r/lock_update_split_rc.result
@@ -0,0 +1,34 @@
+CREATE TABLE t (
+`a` INT NOT NULL,
+`b` INT NOT NULL,
+PRIMARY KEY (`a`)
+) ENGINE=InnoDB;
+SET GLOBAL innodb_limit_optimistic_insert_debug = 3;
+INSERT INTO t VALUES(10, 0);
+INSERT INTO t VALUES(20, 0);
+INSERT INTO t VALUES(30, 0);
+SET TRANSACTION ISOLATION LEVEL READ COMMITTED;
+XA START '1';
+REPLACE INTO t VALUES(10, 1);
+REPLACE INTO t VALUES(20, 1);
+SET DEBUG_SYNC= 'ib_after_row_insert SIGNAL inserted WAIT_FOR cont';
+REPLACE INTO t VALUES(30, 1);
+connect  con1,localhost,root;
+SET TRANSACTION ISOLATION LEVEL READ COMMITTED;
+XA START '2';
+SET DEBUG_SYNC= 'now WAIT_FOR inserted';
+INSERT INTO t VALUES(40, 2);
+SET DEBUG_SYNC= 'now SIGNAL cont';
+connection default;
+XA END '1';
+XA PREPARE '1';
+connection default;
+XA COMMIT '1';
+connection con1;
+XA END '2';
+XA PREPARE '2';
+XA COMMIT '2';
+disconnect con1;
+connection default;
+SET DEBUG_SYNC= "RESET";
+DROP TABLE t;
diff --git a/mysql-test/suite/innodb/t/lock_update_split_rc.test b/mysql-test/suite/innodb/t/lock_update_split_rc.test
new file mode 100644
index 00000000000..38910e53ef3
--- /dev/null
+++ b/mysql-test/suite/innodb/t/lock_update_split_rc.test
@@ -0,0 +1,76 @@
+--source include/have_innodb.inc
+--source include/have_debug.inc
+--source include/count_sessions.inc
+
+CREATE TABLE t (
+  `a` INT NOT NULL,
+  `b` INT NOT NULL,
+  PRIMARY KEY (`a`)
+) ENGINE=InnoDB;
+
+--disable_query_log
+SET @old_innodb_limit_optimistic_insert_debug = @@innodb_limit_optimistic_insert_debug;
+--enable_query_log
+
+SET GLOBAL innodb_limit_optimistic_insert_debug = 3;
+
+INSERT INTO t VALUES(10, 0);
+INSERT INTO t VALUES(20, 0);
+INSERT INTO t VALUES(30, 0);
+
+SET TRANSACTION ISOLATION LEVEL READ COMMITTED;
+XA START '1';
+REPLACE INTO t VALUES(10, 1);
+REPLACE INTO t VALUES(20, 1);
+
+# We need the following sync point because mysql_insert() resets
+# trx->duplicates with the following condition:
+#
+#  if (duplic == DUP_REPLACE &&
+#      (!table->triggers || !table->triggers->has_delete_triggers()))
+#    table->file->extra(HA_EXTRA_WRITE_CANNOT_REPLACE);
+#
+# and ha_innobase::extra() resets trx_t::duplicates, but we need
+# lock_update_split_right() to be invoked when trx->duplicates is set to
+# repeat the bug. So the transaction will hang just after
+# row_insert_for_mysql() call until another transaction inserts new row and
+# splits the page.
+SET DEBUG_SYNC= 'ib_after_row_insert SIGNAL inserted WAIT_FOR cont';
+--send REPLACE INTO t VALUES(30, 1)
+
+connect (con1,localhost,root);
+SET TRANSACTION ISOLATION LEVEL READ COMMITTED;
+XA START '2';
+SET DEBUG_SYNC= 'now WAIT_FOR inserted';
+# The following statement will cause page split and (20, ...) will be split
+# record. As the previous REPLACE set non-gap X-lock on it,
+# lock_update_split_right() and lock_rec_inherit_to_gap() will 'inherit' the
+# lock from the very first (20, ...) new right page record to the supremum of
+# the old left page, what should not be for READ COMMITTED isolation level
+INSERT INTO t VALUES(40, 2);
+SET DEBUG_SYNC= 'now SIGNAL cont';
+
+--connection default
+--reap
+XA END '1';
+# This will cause the assertion failure, because the supremum of the left page
+# has X-lock.
+XA PREPARE '1';
+--connection default
+XA COMMIT '1';
+
+--connection con1
+XA END '2';
+XA PREPARE '2';
+XA COMMIT '2';
+--disconnect con1
+
+--connection default
+SET DEBUG_SYNC= "RESET";
+DROP TABLE t;
+
+--disable_query_log
+SET GLOBAL innodb_limit_optimistic_insert_debug = @old_innodb_limit_optimistic_insert_debug;
+--enable_query_log
+
+--source include/wait_until_count_sessions.inc