MDEV-18719 Assertion (c.prtype ^ o->prtype) & ... failed on ALTER TABLEbb-10.4-elenst-no-mdev371

The prtype & DATA_LONG_TRUE_VARCHAR flag only plays a role when
converting between InnoDB internal format and the MariaDB SQL layer
row format. Ideally this flag would never have been persisted in the
InnoDB data dictionary.

There were bogus assertion failures when an instant ADD, DROP, or
column reordering was combined with a change of extending a VARCHAR
from less than 256 bytes to more than 255 bytes. Such changes are
allowed starting with MDEV-15563 in MariaDB 10.4.3.

dict_table_t::instant_column(), dict_col_t::same_format(): Ignore
the DATA_LONG_TRUE_VARCHAR flag, because it does not affect the
persistent storage format.

4 files changed, 40 insertions, 3 deletions

diff --git a/mysql-test/suite/innodb/r/instant_alter.result b/mysql-test/suite/innodb/r/instant_alter.result
index c180e107b70..0eb66fa22b5 100644
--- a/mysql-test/suite/innodb/r/instant_alter.result
+++ b/mysql-test/suite/innodb/r/instant_alter.result
@@ -869,6 +869,15 @@ a	b	c
 2	1	2
 3	2	4
 DROP TABLE t1;
+CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(1)) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+INSERT INTO t1 VALUES(1,'a');
+ALTER TABLE t1 MODIFY b VARCHAR(256), ADD COLUMN c INT;
+INSERT INTO t1 VALUES(2,'bah',3);
+SELECT * FROM t1;
+a	b	c
+1	a	NULL
+2	bah	3
+DROP TABLE t1;
 CREATE TABLE t1
 (id INT PRIMARY KEY, c2 INT UNIQUE,
 c3 POINT NOT NULL DEFAULT ST_GeomFromText('POINT(3 4)'),
@@ -1684,6 +1693,15 @@ a	b	c
 2	1	2
 3	2	4
 DROP TABLE t1;
+CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(1)) ENGINE=InnoDB ROW_FORMAT=COMPACT;
+INSERT INTO t1 VALUES(1,'a');
+ALTER TABLE t1 MODIFY b VARCHAR(256), ADD COLUMN c INT;
+INSERT INTO t1 VALUES(2,'bah',3);
+SELECT * FROM t1;
+a	b	c
+1	a	NULL
+2	bah	3
+DROP TABLE t1;
 CREATE TABLE t1
 (id INT PRIMARY KEY, c2 INT UNIQUE,
 c3 POINT NOT NULL DEFAULT ST_GeomFromText('POINT(3 4)'),
@@ -2499,10 +2517,19 @@ a	b	c
 2	1	2
 3	2	4
 DROP TABLE t1;
+CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(1)) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
+INSERT INTO t1 VALUES(1,'a');
+ALTER TABLE t1 MODIFY b VARCHAR(256), ADD COLUMN c INT;
+INSERT INTO t1 VALUES(2,'bah',3);
+SELECT * FROM t1;
+a	b	c
+1	a	NULL
+2	bah	3
+DROP TABLE t1;
 disconnect analyze;
 SELECT variable_value-@old_instant instants
 FROM information_schema.global_status
 WHERE variable_name = 'innodb_instant_alter_column';
 instants
-178
+181
 SET GLOBAL innodb_purge_rseg_truncate_frequency= @saved_frequency;
diff --git a/mysql-test/suite/innodb/t/instant_alter.test b/mysql-test/suite/innodb/t/instant_alter.test
index fbee8b79fa3..b6be9137532 100644
--- a/mysql-test/suite/innodb/t/instant_alter.test
+++ b/mysql-test/suite/innodb/t/instant_alter.test
@@ -737,6 +737,14 @@ UPDATE t1 SET c=c+1;
 SELECT * FROM t1;
 DROP TABLE t1;
 
+# MDEV-18719 Assertion (c.prtype ^ o->prtype) & ... failed on ALTER TABLE
+eval CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(1)) $engine;
+INSERT INTO t1 VALUES(1,'a');
+ALTER TABLE t1 MODIFY b VARCHAR(256), ADD COLUMN c INT;
+INSERT INTO t1 VALUES(2,'bah',3);
+SELECT * FROM t1;
+DROP TABLE t1;
+
 dec $format;
 }
 disconnect analyze;
diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc
index 23e7eda8b66..8102060414b 100644
--- a/storage/innobase/handler/handler0alter.cc
+++ b/storage/innobase/handler/handler0alter.cc
@@ -536,7 +536,8 @@ inline bool dict_table_t::instant_column(const dict_table_t& table,
 		if (const dict_col_t* o = find(old_cols, col_map, n_cols, i)) {
 			c.def_val = o->def_val;
 			DBUG_ASSERT(!((c.prtype ^ o->prtype)
-				      & ~(DATA_NOT_NULL | DATA_VERSIONED)));
+				      & ~(DATA_NOT_NULL | DATA_VERSIONED
+					  | DATA_LONG_TRUE_VARCHAR)));
 			DBUG_ASSERT(c.mtype == o->mtype);
 			DBUG_ASSERT(c.len >= o->len);
 
diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h
index 11e150c4f78..9f76728e6cf 100644
--- a/storage/innobase/include/dict0mem.h
+++ b/storage/innobase/include/dict0mem.h
@@ -693,7 +693,8 @@ public:
 			&& mbminlen == other.mbminlen
 			&& mbmaxlen == other.mbmaxlen
 			&& !((prtype ^ other.prtype)
-			     & ~(DATA_NOT_NULL | DATA_VERSIONED));
+			     & ~(DATA_NOT_NULL | DATA_VERSIONED
+				 | DATA_LONG_TRUE_VARCHAR));
 	}
 };