MDEV-17589: Stack-buffer-overflow with indexed varchar (utf8) fieldbb-10.0-varun

Create a new constant MAX_DATA_LENGTH_FOR_KEY.
Replace the value of MAX_KEY_LENGTH to also include the LENGTH and NULL BYTES
of a field.

1 files changed, 26 insertions, 0 deletions

diff --git a/mysql-test/t/func_group_innodb.test b/mysql-test/t/func_group_innodb.test
index c62d3d08496..a65d2326d0f 100644
--- a/mysql-test/t/func_group_innodb.test
+++ b/mysql-test/t/func_group_innodb.test
@@ -192,4 +192,30 @@ EXPLAIN SELECT MIN(c) FROM t1 GROUP BY b;
 
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-17589: Stack-buffer-overflow with indexed varchar (utf8) field
+--echo #
+
+set @save_innodb_file_format= @@innodb_file_format;
+set @save_innodb_large_prefix= @@innodb_large_prefix;
+set global innodb_file_format = BARRACUDA;
+set global innodb_large_prefix = ON;
+
+CREATE TABLE t1 (v1 varchar(1020), v2 varchar(2), v3 varchar(2),
+                KEY k1 (v3,v2,v1)) ENGINE=InnoDB CHARACTER SET=utf8 ROW_FORMAT=DYNAMIC;
+INSERT INTO t1 VALUES ('king', 'qu','qu'), ('bad','go','go');
+explain
+SELECT MIN(t1.v1) FROM t1 where t1.v2='qu' and t1.v3='qu';
+SELECT MIN(t1.v1) FROM t1 where t1.v2='qu' and t1.v3='qu';
+drop table t1;
+
+CREATE TABLE t1 (v1 varchar(1024) CHARACTER SET utf8, KEY v1 (v1)) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
+INSERT INTO t1 VALUES ('king'), ('bad');
+explain
+SELECT MIN(x.v1) FROM (SELECT t1.* FROM t1 WHERE t1.v1 >= 'p') x;
+SELECT MIN(x.v1) FROM (SELECT t1.* FROM t1 WHERE t1.v1 >= 'p') x;
+drop table t1;
+set global innodb_file_format = @save_innodb_file_format;
+set global innodb_large_prefix = @save_innodb_large_prefix;
+
 --echo End of 5.5 tests