MDEV-25872 InnoDB: Assertion failure in row_merge_read_clustered_index upon ALTER on table with indexed virtual columnsbb-10.6-MDEV-14180

- InnoDB fails to check DB_COMPUTE_VALUE_FAILED error in
row_merge_read_clustered_index() and wrongly asserts that
the buffer shouldn't be ran out of memory.

4 files changed, 52 insertions, 12 deletions

diff --git a/mysql-test/suite/gcol/r/innodb_virtual_index.result b/mysql-test/suite/gcol/r/innodb_virtual_index.result
index 70c9d10a68b..8fe0591bd8d 100644
--- a/mysql-test/suite/gcol/r/innodb_virtual_index.result
+++ b/mysql-test/suite/gcol/r/innodb_virtual_index.result
@@ -262,3 +262,23 @@ CHECK TABLE t1;
 Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
 DROP TABLE t1;
+#
+# MDEV-25872 InnoDB: Assertion failure in row_merge_read_clustered_index
+#       upon ALTER on table with indexed virtual columns
+#
+CREATE TABLE t1 (
+id BIGINT AUTO_INCREMENT PRIMARY KEY,
+a INT,
+va INT ZEROFILL AS (a) VIRTUAL,
+b TIMESTAMP,
+c CHAR(204),
+vc CHAR(8),
+KEY(vc,c(64),b,va)
+) ENGINE=InnoDB CHARACTER SET utf32;
+INSERT INTO t1 (id) SELECT NULL FROM seq_1_to_75;
+INSERT IGNORE INTO t1 (id, a) VALUES (NULL, -1);
+Warnings:
+Warning	1264	Out of range value for column 'va' at row 1
+ALTER TABLE t1 FORCE;
+ERROR 22003: Out of range value for column 'va' at row 1
+DROP TABLE t1;
diff --git a/mysql-test/suite/gcol/t/innodb_virtual_index.opt b/mysql-test/suite/gcol/t/innodb_virtual_index.opt
new file mode 100644
index 00000000000..c3f4a891cca
--- /dev/null
+++ b/mysql-test/suite/gcol/t/innodb_virtual_index.opt
@@ -0,0 +1 @@
+--innodb_sort_buffer_size=64k
diff --git a/mysql-test/suite/gcol/t/innodb_virtual_index.test b/mysql-test/suite/gcol/t/innodb_virtual_index.test
index 9886378ced1..3d06283d35a 100644
--- a/mysql-test/suite/gcol/t/innodb_virtual_index.test
+++ b/mysql-test/suite/gcol/t/innodb_virtual_index.test
@@ -1,4 +1,5 @@
 --source include/have_innodb.inc
+--source include/have_sequence.inc
 
 # Ensure that the history list length will actually be decremented by purge.
 SET @saved_frequency = @@GLOBAL.innodb_purge_rseg_truncate_frequency;
@@ -288,3 +289,22 @@ ROLLBACK;
 SELECT * FROM t1;
 CHECK TABLE t1;
 DROP TABLE t1;
+--echo #
+--echo # MDEV-25872 InnoDB: Assertion failure in row_merge_read_clustered_index
+--echo #       upon ALTER on table with indexed virtual columns
+--echo #
+
+CREATE TABLE t1 (
+  id BIGINT AUTO_INCREMENT PRIMARY KEY,
+  a INT,
+  va INT ZEROFILL AS (a) VIRTUAL,
+  b TIMESTAMP,
+  c CHAR(204),
+  vc CHAR(8),
+  KEY(vc,c(64),b,va)
+) ENGINE=InnoDB CHARACTER SET utf32;
+INSERT INTO t1 (id) SELECT NULL FROM seq_1_to_75;
+INSERT IGNORE INTO t1 (id, a) VALUES (NULL, -1);
+--error ER_WARN_DATA_OUT_OF_RANGE
+ALTER TABLE t1 FORCE;
+DROP TABLE t1;
diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc
index 715a49cd3ae..7ca24a9b6d8 100644
--- a/storage/innobase/row/row0merge.cc
+++ b/storage/innobase/row/row0merge.cc
@@ -515,7 +515,9 @@ row_merge_buf_add(
 	DBUG_ENTER("row_merge_buf_add");
 
 	if (buf->n_tuples >= buf->max_tuples) {
-		DBUG_RETURN(0);
+error:
+		n_row_added = 0;
+		goto end;
 	}
 
 	DBUG_EXECUTE_IF(
@@ -838,11 +840,6 @@ end:
         if (vcol_storage.innobase_record)
 		innobase_free_row_for_vcol(&vcol_storage);
 	DBUG_RETURN(n_row_added);
-
-error:
-        if (vcol_storage.innobase_record)
-		innobase_free_row_for_vcol(&vcol_storage);
-        DBUG_RETURN(0);
 }
 
 /*************************************************************//**
@@ -2687,14 +2684,16 @@ write_buffers:
 						&err, &v_heap, eval_table, trx)))) {
 					/* An empty buffer should have enough
 					room for at least one record. */
-					ut_error;
-				}
-
-				if (err != DB_SUCCESS) {
-					break;
+					ut_ad(err == DB_COMPUTE_VALUE_FAILED
+					      || err == DB_OUT_OF_MEMORY
+					      || err == DB_TOO_BIG_RECORD);
+				} else if (err == DB_SUCCESS) {
+					file->n_rec += rows_added;
+					continue;
 				}
 
-				file->n_rec += rows_added;
+				trx->error_key_num = i;
+				break;
 			}
 		}