MDEV-26203 CREATE INDEX may enforce incorrect maximum column length

ha_innobase::prepare_inplace_alter_table(): Unless the table is
being rebuilt, determine the maximum column length based on the
current ROW_FORMAT of the table. When TABLE_SHARE (and the .frm file)
contains no explicit ROW_FORMAT, InnoDB table creation or rebuild
will use innodb_default_row_format.

Based on mysql/mysql-server@3287d33acdc4260806a2a407ca15e9d1e04dddcb

3 files changed, 50 insertions, 9 deletions

diff --git a/mysql-test/suite/innodb/r/default_row_format_alter.result b/mysql-test/suite/innodb/r/default_row_format_alter.result
index fd88fb5a3fe..1c31a287fcd 100644
--- a/mysql-test/suite/innodb/r/default_row_format_alter.result
+++ b/mysql-test/suite/innodb/r/default_row_format_alter.result
@@ -23,6 +23,7 @@ INSERT INTO t1 VALUES (1, 'abc');
 SHOW TABLE STATUS LIKE 't1';
 Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment
 t1	InnoDB	#	Compact	#	#	#	#	#	#	NULL	#	#	NULL	latin1_swedish_ci	NULL		
+CREATE TABLE t2 (b VARCHAR(255) CHARACTER SET utf8mb4 NOT NULL) ENGINE=InnoDB;
 SET GLOBAL innodb_default_row_format = DYNAMIC;
 ALTER TABLE t1 DROP PRIMARY KEY, ADD COLUMN c INT PRIMARY KEY;
 # Here we expect DYNAMIC because there is no explicit ROW_FORMAT and the
@@ -31,6 +32,10 @@ SHOW TABLE STATUS LIKE 't1';
 Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment
 t1	InnoDB	#	Dynamic	#	#	#	#	#	#	NULL	#	#	NULL	latin1_swedish_ci	NULL		
 DROP TABLE t1;
+ALTER TABLE t2 ADD INDEX(b);
+ERROR HY000: Index column size too large. The maximum column size is 767 bytes
+ALTER TABLE t2 FORCE, ADD INDEX(b);
+DROP TABLE t2;
 ####################################
 # Check the row_format effect on ALTER, ALGORITHM=COPY
 SET GLOBAL innodb_default_row_format = REDUNDANT;
@@ -39,6 +44,7 @@ INSERT INTO t1 VALUES (1, REPEAT('abc',1000));
 SHOW TABLE STATUS LIKE 't1';
 Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment
 t1	InnoDB	#	Redundant	#	#	#	#	#	#	NULL	#	#	NULL	latin1_swedish_ci	NULL		
+CREATE TABLE t2 (b VARCHAR(255) CHARACTER SET utf8mb4 NOT NULL) ENGINE=InnoDB;
 SET GLOBAL innoDB_default_row_format = COMPACT;
 ALTER TABLE t1 ADD COLUMN c2 BLOB, ALGORITHM=COPY;
 # Because of ALGORITHM=COPY, there is TABLE REBUILD and the table isn't
@@ -47,9 +53,18 @@ SHOW TABLE STATUS LIKE 't1';
 Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment
 t1	InnoDB	#	Compact	#	#	#	#	#	#	NULL	#	#	NULL	latin1_swedish_ci	NULL		
 DROP TABLE t1;
+ALTER TABLE t2 ADD INDEX(b);
+ERROR HY000: Index column size too large. The maximum column size is 767 bytes
+ALTER TABLE t2 FORCE, ADD INDEX(b);
+ERROR HY000: Index column size too large. The maximum column size is 767 bytes
+SET GLOBAL innodb_default_row_format = DYNAMIC;
+ALTER TABLE t2 ADD INDEX(b);
+ERROR HY000: Index column size too large. The maximum column size is 767 bytes
+ALTER TABLE t2 FORCE, ADD INDEX(b);
+DROP TABLE t2;
 
 ###################################
-#  Check the row_format effect on ALTER, ALGORITH=COPY on
+#  Check the row_format effect on ALTER, ALGORITHM=COPY on
 # create table with explicit row_format
 CREATE TABLE t1 (a INT PRIMARY KEY, b TEXT) ROW_FORMAT=REDUNDANT ENGINE=INNODB;
 INSERT INTO t1 VALUES (1, REPEAT('abc',1000));
diff --git a/mysql-test/suite/innodb/t/default_row_format_alter.test b/mysql-test/suite/innodb/t/default_row_format_alter.test
index 8f7217bcf0c..56ad70e78ca 100644
--- a/mysql-test/suite/innodb/t/default_row_format_alter.test
+++ b/mysql-test/suite/innodb/t/default_row_format_alter.test
@@ -5,7 +5,7 @@ SET @row_format = @@GLOBAL.innodb_default_row_format;
 --echo ####################################
 --echo # Check if table rebuilding alter isn't affect if table is created
 --echo # with explicit row_format
-eval CREATE TABLE t1 (a INT PRIMARY KEY, b TEXT) ROW_FORMAT=COMPACT ENGINE=INNODB;
+CREATE TABLE t1 (a INT PRIMARY KEY, b TEXT) ROW_FORMAT=COMPACT ENGINE=INNODB;
 INSERT INTO t1 VALUES (1, 'abc');
 --replace_column 3 # 5 # 6 # 7 # 8 # 9 # 10 # 12 # 13 #
 SHOW TABLE STATUS LIKE 't1';
@@ -22,12 +22,14 @@ DROP TABLE t1;
 --echo # Check if table rebuilding alter is affected when there is no
 --echo # row_format specified at CREATE TABLE.
 SET GLOBAL innodb_default_row_format = COMPACT;
-eval CREATE TABLE t1 (a INT PRIMARY KEY, b TEXT) ENGINE=INNODB;
+CREATE TABLE t1 (a INT PRIMARY KEY, b TEXT) ENGINE=INNODB;
 INSERT INTO t1 VALUES (1, 'abc');
 
 --replace_column 3 # 5 # 6 # 7 # 8 # 9 # 10 # 12 # 13 #
 SHOW TABLE STATUS LIKE 't1';
 
+CREATE TABLE t2 (b VARCHAR(255) CHARACTER SET utf8mb4 NOT NULL) ENGINE=InnoDB;
+
 SET GLOBAL innodb_default_row_format = DYNAMIC;
 ALTER TABLE t1 DROP PRIMARY KEY, ADD COLUMN c INT PRIMARY KEY;
 
@@ -37,15 +39,22 @@ ALTER TABLE t1 DROP PRIMARY KEY, ADD COLUMN c INT PRIMARY KEY;
 SHOW TABLE STATUS LIKE 't1';
 DROP TABLE t1;
 
+--error ER_INDEX_COLUMN_TOO_LONG
+ALTER TABLE t2 ADD INDEX(b);
+ALTER TABLE t2 FORCE, ADD INDEX(b);
+DROP TABLE t2;
+
 --echo ####################################
 --echo # Check the row_format effect on ALTER, ALGORITHM=COPY
 SET GLOBAL innodb_default_row_format = REDUNDANT;
-eval CREATE TABLE t1 (a INT PRIMARY KEY, b TEXT) ENGINE=INNODB;
+CREATE TABLE t1 (a INT PRIMARY KEY, b TEXT) ENGINE=INNODB;
 INSERT INTO t1 VALUES (1, REPEAT('abc',1000));
 
 --replace_column 3 # 5 # 6 # 7 # 8 # 9 # 10 # 12 # 13 #
 SHOW TABLE STATUS LIKE 't1';
 
+CREATE TABLE t2 (b VARCHAR(255) CHARACTER SET utf8mb4 NOT NULL) ENGINE=InnoDB;
+
 SET GLOBAL innoDB_default_row_format = COMPACT;
 ALTER TABLE t1 ADD COLUMN c2 BLOB, ALGORITHM=COPY;
 
@@ -55,11 +64,23 @@ ALTER TABLE t1 ADD COLUMN c2 BLOB, ALGORITHM=COPY;
 SHOW TABLE STATUS LIKE 't1';
 DROP TABLE t1;
 
+--error ER_INDEX_COLUMN_TOO_LONG
+ALTER TABLE t2 ADD INDEX(b);
+--error ER_INDEX_COLUMN_TOO_LONG
+ALTER TABLE t2 FORCE, ADD INDEX(b);
+
+SET GLOBAL innodb_default_row_format = DYNAMIC;
+--error ER_INDEX_COLUMN_TOO_LONG
+ALTER TABLE t2 ADD INDEX(b);
+ALTER TABLE t2 FORCE, ADD INDEX(b);
+
+DROP TABLE t2;
+
 --echo
 --echo ###################################
---echo #  Check the row_format effect on ALTER, ALGORITH=COPY on
+--echo #  Check the row_format effect on ALTER, ALGORITHM=COPY on
 --echo # create table with explicit row_format
-eval CREATE TABLE t1 (a INT PRIMARY KEY, b TEXT) ROW_FORMAT=REDUNDANT ENGINE=INNODB;
+CREATE TABLE t1 (a INT PRIMARY KEY, b TEXT) ROW_FORMAT=REDUNDANT ENGINE=INNODB;
 INSERT INTO t1 VALUES (1, REPEAT('abc',1000));
 
 --replace_column 3 # 5 # 6 # 7 # 8 # 9 # 10 # 12 # 13 #
@@ -80,7 +101,7 @@ DROP TABLE t1;
 --echo # Check row_format on ALTER ALGORITHM=INPLACE
 SET GLOBAL innodb_default_row_format=COMPACT;
 
-eval CREATE TABLE t1 (a INT PRIMARY KEY, b TEXT, KEY k1(b(10))) ENGINE=INNODB;
+CREATE TABLE t1 (a INT PRIMARY KEY, b TEXT, KEY k1(b(10))) ENGINE=INNODB;
 INSERT INTO t1 VALUES (1, REPEAT('abc',1000));
 
 --replace_column 3 # 5 # 6 # 7 # 8 # 9 # 10 # 12 # 13 #
diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc
index 3ac4f4fe4c1..24a7c4a74ad 100644
--- a/storage/innobase/handler/handler0alter.cc
+++ b/storage/innobase/handler/handler0alter.cc
@@ -5387,7 +5387,6 @@ ha_innobase::prepare_inplace_alter_table(
 	mem_heap_t*	heap;
 	const char**	col_names;
 	int		error;
-	ulint		max_col_len;
 	ulint		add_autoinc_col_no	= ULINT_UNDEFINED;
 	ulonglong	autoinc_col_max_value	= 0;
 	ulint		fts_doc_col_no		= ULINT_UNDEFINED;
@@ -5601,7 +5600,13 @@ check_if_ok_to_rename:
 			       & 1U << DICT_TF_POS_DATA_DIR);
 	}
 
-	max_col_len = DICT_MAX_FIELD_LEN_BY_FORMAT_FLAG(info.flags());
+
+	/* ALGORITHM=INPLACE without rebuild (10.3+ ALGORITHM=NOCOPY)
+	must use the current ROW_FORMAT of the table. */
+	const ulint max_col_len = DICT_MAX_FIELD_LEN_BY_FORMAT_FLAG(
+		innobase_need_rebuild(ha_alter_info, this->table)
+		? info.flags()
+		: m_prebuilt->table->flags);
 
 	/* Check each index's column length to make sure they do not
 	exceed limit */