Backported MDEV-11196(e4d10e09cf31) and MDEV-10360(8a8ba1949bf4) to 10.0

3 files changed, 244 insertions, 5 deletions

diff --git a/mysql-test/r/innodb_ext_key.result b/mysql-test/r/innodb_ext_key.result
index 2b3b98eb26a..600269ba433 100644
--- a/mysql-test/r/innodb_ext_key.result
+++ b/mysql-test/r/innodb_ext_key.result
@@ -1068,5 +1068,100 @@ a
 1
 drop table t1, t2;
 set optimizer_switch=@save_optimizer_switch;
+# 
+# MDEV-10360: Extended keys: index properties depend on index order
+# 
+create table t0 (a int);
+insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
+create table t1 (
+index_id bigint(20) unsigned NOT NULL,
+index_class varchar(265) COLLATE latin1_general_ci DEFAULT NULL ,
+index_object_id int(10) unsigned NOT NULL DEFAULT '0' ,
+index_date_updated int(10) unsigned DEFAULT NULL ,
+PRIMARY KEY (index_id),
+KEY object (index_class(181),index_object_id),
+KEY index_date_updated (index_date_updated)
+) engine=innodb;
+create table t2 (
+index_id bigint(20) unsigned NOT NULL,
+index_class varchar(265) COLLATE latin1_general_ci DEFAULT NULL ,
+index_object_id int(10) unsigned NOT NULL DEFAULT '0' ,
+index_date_updated int(10) unsigned DEFAULT NULL ,
+PRIMARY KEY (index_id),
+KEY index_date_updated (index_date_updated),
+KEY object (index_class(181),index_object_id)
+) engine=innodb;
+insert into t1 select 
+@a:=A.a + 10*B.a + 100*C.a,
+concat('val-', @a),
+123456,
+A.a + 10*B.a
+from 
+t0 A, t0 B, t0 C;
+insert into t2 select * from t1;
+# This must have the same query plan as the query below it:
+# type=range, key=index_date_updated, key_len=13
+explain 
+select * from t1 force index(index_date_updated) 
+where index_date_updated= 10 and index_id < 800;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	range	index_date_updated	index_date_updated	13	NULL	#	Using index condition
+# This used to work from the start:
+explain 
+select * from t2 force index(index_date_updated) 
+where index_date_updated= 10 and index_id < 800;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t2	range	index_date_updated	index_date_updated	13	NULL	#	Using index condition
+drop table t0,t1,t2;
+#
+# MDEV-11196: Error:Run-Time Check Failure #2 - Stack around the variable 'key_buff'
+# was corrupted, server crashes in opt_sum_query
+set @save_innodb_file_format= @@innodb_file_format;
+set @save_innodb_large_prefix= @@innodb_large_prefix;
+set global innodb_file_format = BARRACUDA;
+set global innodb_large_prefix = ON;
+CREATE TABLE t1 (
+pk INT,
+f1 VARCHAR(3),
+f2 VARCHAR(1024),
+PRIMARY KEY (pk),
+KEY(f2)
+) ENGINE=InnoDB CHARSET utf8 ROW_FORMAT= DYNAMIC;
+INSERT INTO t1 VALUES (1,'foo','abc'),(2,'bar','def');
+SELECT MAX(t2.pk) FROM t1 t2 INNER JOIN t1 t3 ON t2.f1 = t3.f1 WHERE t2.pk <= 4;
+MAX(t2.pk)
+2
+drop table t1;
+CREATE TABLE t1 (
+pk1 INT,
+pk2 INT,
+f1 VARCHAR(3),
+f2 VARCHAR(1021),
+PRIMARY KEY (pk1,pk2),
+KEY(f2)
+) ENGINE=InnoDB CHARSET utf8 ROW_FORMAT= DYNAMIC;
+INSERT INTO t1 VALUES (1,2,'2','abc'),(2,3,'3','def');
+explain
+select * from t1 force index(f2)  where pk1 <= 5 and pk2 <=5 and f2 = 'abc' and f1 <= '3';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	range	f2	f2	3070	NULL	1	Using index condition; Using where
+drop table t1;
+CREATE TABLE t1 (
+f2 INT,
+pk2 INT,
+f1 VARCHAR(3),
+pk1 VARCHAR(1000),
+PRIMARY KEY (pk1,pk2),
+KEY k1(pk1,f2)
+) ENGINE=InnoDB CHARSET utf8 ROW_FORMAT= DYNAMIC;
+INSERT INTO t1 VALUES (1,2,'2','abc'),(2,3,'3','def');
+explain
+select * from t1 force index(k1)  where f2 <= 5 and pk2 <=5 and pk1 = 'abc' and f1 <= '3';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	range	k1	k1	3011	NULL	1	Using index condition; Using where
+drop table t1;
+set optimizer_switch=@save_ext_key_optimizer_switch;
+set global innodb_file_format = @save_innodb_file_format;
+set global innodb_large_prefix = @save_innodb_large_prefix;
 set optimizer_switch=@save_ext_key_optimizer_switch;
 SET SESSION STORAGE_ENGINE=DEFAULT;
diff --git a/mysql-test/t/innodb_ext_key.test b/mysql-test/t/innodb_ext_key.test
index 9f3a89ff948..d53deb46348 100644
--- a/mysql-test/t/innodb_ext_key.test
+++ b/mysql-test/t/innodb_ext_key.test
@@ -693,5 +693,111 @@ drop table t1, t2;
 
 set optimizer_switch=@save_optimizer_switch;
 
+--echo # 
+--echo # MDEV-10360: Extended keys: index properties depend on index order
+--echo # 
+create table t0 (a int);
+insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
+
+create table t1 (
+  index_id bigint(20) unsigned NOT NULL,
+  index_class varchar(265) COLLATE latin1_general_ci DEFAULT NULL ,
+  index_object_id int(10) unsigned NOT NULL DEFAULT '0' ,
+  index_date_updated int(10) unsigned DEFAULT NULL ,
+ 
+  PRIMARY KEY (index_id),
+  KEY object (index_class(181),index_object_id),
+  KEY index_date_updated (index_date_updated)
+) engine=innodb;
+
+create table t2 (
+  index_id bigint(20) unsigned NOT NULL,
+  index_class varchar(265) COLLATE latin1_general_ci DEFAULT NULL ,
+  index_object_id int(10) unsigned NOT NULL DEFAULT '0' ,
+  index_date_updated int(10) unsigned DEFAULT NULL ,
+ 
+  PRIMARY KEY (index_id),
+  KEY index_date_updated (index_date_updated),
+  KEY object (index_class(181),index_object_id)
+) engine=innodb;
+
+insert into t1 select 
+  @a:=A.a + 10*B.a + 100*C.a,
+  concat('val-', @a),
+  123456,
+  A.a + 10*B.a
+from 
+  t0 A, t0 B, t0 C;
+
+insert into t2 select * from t1;
+
+--echo # This must have the same query plan as the query below it:
+--echo # type=range, key=index_date_updated, key_len=13
+--replace_column 9 #
+explain 
+select * from t1 force index(index_date_updated) 
+where index_date_updated= 10 and index_id < 800;
+
+--echo # This used to work from the start:
+--replace_column 9 #
+explain 
+select * from t2 force index(index_date_updated) 
+where index_date_updated= 10 and index_id < 800;
+
+drop table t0,t1,t2;
+
+
+--echo #
+--echo # MDEV-11196: Error:Run-Time Check Failure #2 - Stack around the variable 'key_buff'
+--echo # was corrupted, server crashes in opt_sum_query
+
+set @save_innodb_file_format= @@innodb_file_format;
+set @save_innodb_large_prefix= @@innodb_large_prefix;
+set global innodb_file_format = BARRACUDA;
+set global innodb_large_prefix = ON;
+
+CREATE TABLE t1 (
+  pk INT,
+  f1 VARCHAR(3),
+  f2 VARCHAR(1024),
+  PRIMARY KEY (pk),
+  KEY(f2)
+) ENGINE=InnoDB CHARSET utf8 ROW_FORMAT= DYNAMIC;
+
+INSERT INTO t1 VALUES (1,'foo','abc'),(2,'bar','def');
+SELECT MAX(t2.pk) FROM t1 t2 INNER JOIN t1 t3 ON t2.f1 = t3.f1 WHERE t2.pk <= 4;
+drop table t1;
+
+CREATE TABLE t1 (
+  pk1 INT,
+  pk2 INT,
+  f1 VARCHAR(3),
+  f2 VARCHAR(1021),
+  PRIMARY KEY (pk1,pk2),
+  KEY(f2)
+) ENGINE=InnoDB CHARSET utf8 ROW_FORMAT= DYNAMIC;
+
+INSERT INTO t1 VALUES (1,2,'2','abc'),(2,3,'3','def');
+explain
+select * from t1 force index(f2)  where pk1 <= 5 and pk2 <=5 and f2 = 'abc' and f1 <= '3';
+drop table t1;
+
+CREATE TABLE t1 (
+f2 INT,
+pk2 INT,
+f1 VARCHAR(3),
+pk1 VARCHAR(1000),
+PRIMARY KEY (pk1,pk2),
+KEY k1(pk1,f2)
+) ENGINE=InnoDB CHARSET utf8 ROW_FORMAT= DYNAMIC;
+INSERT INTO t1 VALUES (1,2,'2','abc'),(2,3,'3','def');
+explain
+select * from t1 force index(k1)  where f2 <= 5 and pk2 <=5 and pk1 = 'abc' and f1 <= '3';
+drop table t1;
+
+set optimizer_switch=@save_ext_key_optimizer_switch;
+set global innodb_file_format = @save_innodb_file_format;
+set global innodb_large_prefix = @save_innodb_large_prefix;
+
 set optimizer_switch=@save_ext_key_optimizer_switch;
 SET SESSION STORAGE_ENGINE=DEFAULT;
diff --git a/sql/table.cc b/sql/table.cc
index 98bf6d8b4dd..5a34d47367a 100644
--- a/sql/table.cc
+++ b/sql/table.cc
@@ -1690,6 +1690,7 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
     keyinfo= share->key_info;
     uint primary_key= my_strcasecmp(system_charset_info, share->keynames.type_names[0],
                                     primary_key_name) ? MAX_KEY : 0;
+    KEY* key_first_info= NULL;
 
     if (primary_key >= MAX_KEY && keyinfo->flags & HA_NOSAME)
     {
@@ -1769,34 +1770,71 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
                keyinfo->name_length+1);
       }
 
+      if (!key)
+        key_first_info= keyinfo;
+
       if (ext_key_parts > share->key_parts && key)
       {
         KEY_PART_INFO *new_key_part= (keyinfo-1)->key_part +
                                      (keyinfo-1)->ext_key_parts;
+        uint add_keyparts_for_this_key= add_first_key_parts;
+        uint length_bytes= 0, len_null_byte= 0, ext_key_length= 0;
+        Field *field;
 
         /* 
           Do not extend the key that contains a component
           defined over the beginning of a field.
 	*/ 
         for (i= 0; i < keyinfo->user_defined_key_parts; i++)
-	{
+        {
           uint fieldnr= keyinfo->key_part[i].fieldnr;
+          field= share->field[keyinfo->key_part[i].fieldnr-1];
+
+          if (field->null_ptr)
+            len_null_byte= HA_KEY_NULL_LENGTH;
+
+          if (field->type() == MYSQL_TYPE_BLOB ||
+             field->real_type() == MYSQL_TYPE_VARCHAR ||
+             field->type() == MYSQL_TYPE_GEOMETRY)
+          {
+            length_bytes= HA_KEY_BLOB_LENGTH;
+          }
+          ext_key_length+= keyinfo->key_part[i].length + len_null_byte
+                            + length_bytes;
           if (share->field[fieldnr-1]->key_length() !=
               keyinfo->key_part[i].length)
 	  {
-            add_first_key_parts= 0;
+            add_keyparts_for_this_key= 0;
             break;
           }
         }
 
-        if (add_first_key_parts < keyinfo->ext_key_parts-keyinfo->user_defined_key_parts)
-	{
+        if (add_keyparts_for_this_key)
+        {
+          for (i= 0; i < add_keyparts_for_this_key; i++)
+          {
+            uint pk_part_length= key_first_info->key_part[i].store_length;
+            if (keyinfo->ext_key_part_map & 1<<i)
+            {
+              if (ext_key_length + pk_part_length > MAX_KEY_LENGTH)
+              {
+                add_keyparts_for_this_key= i;
+                break;
+              }
+              ext_key_length+= pk_part_length;
+            }
+          }
+        }
+
+        if (add_keyparts_for_this_key < keyinfo->ext_key_parts -
+                                        keyinfo->user_defined_key_parts)
+        {
           share->ext_key_parts-= keyinfo->ext_key_parts;
           key_part_map ext_key_part_map= keyinfo->ext_key_part_map;
           keyinfo->ext_key_parts= keyinfo->user_defined_key_parts;
           keyinfo->ext_key_flags= keyinfo->flags;
 	  keyinfo->ext_key_part_map= 0; 
-          for (i= 0; i < add_first_key_parts; i++)
+          for (i= 0; i <  add_keyparts_for_this_key; i++)
 	  {
             if (ext_key_part_map & 1<<i)
 	    {