MDEV-8779: mysqld got signal 11 in sql/opt_range_mrr.cc:100(step_down_to)

The crash was caused by range optimizer using RANGE_OPT_PARAM::min_key
(and max_key) to store keys. Buffer size was a good upper bound for
range analysis and partition pruning, but not for EITS selectivity
calculations.

Fixed by making these buffers variable-size. The sizes are calculated
from [pseudo]indexes used for range analysis.

4 files changed, 73 insertions, 4 deletions

diff --git a/mysql-test/r/selectivity_no_engine.result b/mysql-test/r/selectivity_no_engine.result
index a14832f9c0a..31037e90a84 100644
--- a/mysql-test/r/selectivity_no_engine.result
+++ b/mysql-test/r/selectivity_no_engine.result
@@ -276,6 +276,23 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 Warnings:
 Note	1003	select `test`.`ta`.`a` AS `a`,`test`.`tb`.`a` AS `a` from `test`.`t1` `ta` join `test`.`t2` `tb` where ((`test`.`tb`.`a` = `test`.`ta`.`a`) and (`test`.`ta`.`a` < 40) and (`test`.`ta`.`a` < 100))
 drop table t0,t1,t2;
+#
+# MDEV-8779: mysqld got signal 11 in sql/opt_range_mrr.cc:100(step_down_to)
+#
+set @tmp_mdev8779=@@optimizer_use_condition_selectivity;
+set optimizer_use_condition_selectivity=5;
+CREATE TABLE t1 (
+i int(10) unsigned NOT NULL AUTO_INCREMENT,
+n varchar(2048) NOT NULL,
+d tinyint(1) unsigned NOT NULL,
+p int(10) unsigned NOT NULL,
+PRIMARY KEY (i)
+) DEFAULT CHARSET=utf8;
+insert into t1 values (1,'aaa',1,1), (2,'bbb',2,2);
+SELECT * FROM t1 WHERE t1.d = 0 AND t1.p = '1' AND t1.i != '-1' AND t1.n = 'some text';
+i	n	d	p
+set optimizer_use_condition_selectivity= @tmp_mdev8779;
+DROP TABLE t1;
 # 
 # End of the test file
 # 
diff --git a/mysql-test/t/selectivity_no_engine.test b/mysql-test/t/selectivity_no_engine.test
index fc425daff6d..2a31c01ed97 100644
--- a/mysql-test/t/selectivity_no_engine.test
+++ b/mysql-test/t/selectivity_no_engine.test
@@ -210,6 +210,23 @@ explain extended select * from t1 ta, t2 tb where ta.a < 40 and tb.a < 100 and t
 
 drop table t0,t1,t2;
 
+--echo #
+--echo # MDEV-8779: mysqld got signal 11 in sql/opt_range_mrr.cc:100(step_down_to)
+--echo #
+set @tmp_mdev8779=@@optimizer_use_condition_selectivity;
+set optimizer_use_condition_selectivity=5;
+CREATE TABLE t1 (
+  i int(10) unsigned NOT NULL AUTO_INCREMENT,
+  n varchar(2048) NOT NULL,
+  d tinyint(1) unsigned NOT NULL,
+  p int(10) unsigned NOT NULL,
+  PRIMARY KEY (i)
+) DEFAULT CHARSET=utf8;
+insert into t1 values (1,'aaa',1,1), (2,'bbb',2,2);
+SELECT * FROM t1 WHERE t1.d = 0 AND t1.p = '1' AND t1.i != '-1' AND t1.n = 'some text';
+set optimizer_use_condition_selectivity= @tmp_mdev8779;
+DROP TABLE t1;
+
 --echo # 
 --echo # End of the test file
 --echo # 
diff --git a/sql/opt_range.cc b/sql/opt_range.cc
index e17d19ff9f8..b8d53f8ef0f 100644
--- a/sql/opt_range.cc
+++ b/sql/opt_range.cc
@@ -2469,13 +2469,13 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
       DBUG_RETURN(0);				// Can't use range
     }
     key_parts= param.key_parts;
-    thd->mem_root= &alloc;
 
     /*
       Make an array with description of all key parts of all table keys.
       This is used in get_mm_parts function.
     */
     key_info= head->key_info;
+    uint max_key_len= 0;
     for (idx=0 ; idx < head->s->keys ; idx++, key_info++)
     {
       KEY_PART_INFO *key_part_info;
@@ -2488,6 +2488,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
 
       param.key[param.keys]=key_parts;
       key_part_info= key_info->key_part;
+      uint cur_key_len= 0;
       for (uint part= 0 ; part < n_key_parts ; 
            part++, key_parts++, key_part_info++)
      {
@@ -2495,6 +2496,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
 	key_parts->part=	 part;
 	key_parts->length=       key_part_info->length;
 	key_parts->store_length= key_part_info->store_length;
+        cur_key_len += key_part_info->store_length;
 	key_parts->field=	 key_part_info->field;
 	key_parts->null_bit=	 key_part_info->null_bit;
         key_parts->image_type =
@@ -2503,10 +2505,21 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
         key_parts->flag=         (uint8) key_part_info->key_part_flag;
       }
       param.real_keynr[param.keys++]=idx;
+      if (cur_key_len > max_key_len)
+        max_key_len= cur_key_len;
     }
     param.key_parts_end=key_parts;
     param.alloced_sel_args= 0;
 
+    if (!(param.min_key= (uchar*)alloc_root(&alloc,max_key_len)) ||
+        !(param.max_key= (uchar*)alloc_root(&alloc,max_key_len)))
+    {
+      thd->no_errors=0;
+      free_root(&alloc,MYF(0));			// Return memory & allocator
+      DBUG_RETURN(0);				// Can't use range
+    }
+
+    thd->mem_root= &alloc;
     /* Calculate cost of full index read for the shortest covering index */
     if (!force_quick_range && !head->covering_keys.is_clear_all())
     {
@@ -2730,7 +2743,7 @@ bool create_key_parts_for_pseudo_indexes(RANGE_OPT_PARAM *param,
     return TRUE;
 
   param->key_parts= key_part;
-
+  uint max_key_len= 0;
   for (field_ptr= table->field; *field_ptr; field_ptr++)
   {
     if (bitmap_is_set(used_fields, (*field_ptr)->field_index))
@@ -2745,6 +2758,8 @@ bool create_key_parts_for_pseudo_indexes(RANGE_OPT_PARAM *param,
         store_length+= HA_KEY_NULL_LENGTH;
       if (field->real_type() == MYSQL_TYPE_VARCHAR)
         store_length+= HA_KEY_BLOB_LENGTH;
+      if (max_key_len < store_length)
+        max_key_len= store_length;
       key_part->store_length= store_length; 
       key_part->field= field; 
       key_part->image_type= Field::itRAW;
@@ -2754,6 +2769,12 @@ bool create_key_parts_for_pseudo_indexes(RANGE_OPT_PARAM *param,
       key_part++;
     }
   }
+
+  if (!(param->min_key= (uchar*)alloc_root(param->mem_root, max_key_len)) ||
+      !(param->max_key= (uchar*)alloc_root(param->mem_root, max_key_len)))
+  {
+    return true;
+  }
   param->keys= keys;
   param->key_parts_end= key_part;
 
@@ -4306,12 +4327,15 @@ static bool create_partition_index_description(PART_PRUNE_PARAM *ppar)
   Field **field= (ppar->part_fields)? part_info->part_field_array :
                                            part_info->subpart_field_array;
   bool in_subpart_fields= FALSE;
+  uint max_key_len= 0;
+  uint cur_key_len;
   for (uint part= 0; part < total_parts; part++, key_part++)
   {
     key_part->key=          0;
     key_part->part=	    part;
     key_part->length= (uint16)(*field)->key_length();
     key_part->store_length= (uint16)get_partition_field_store_length(*field);
+    cur_key_len += key_part->store_length;
 
     DBUG_PRINT("info", ("part %u length %u store_length %u", part,
                          key_part->length, key_part->store_length));
@@ -4337,10 +4361,21 @@ static bool create_partition_index_description(PART_PRUNE_PARAM *ppar)
     {
       field= part_info->subpart_field_array;
       in_subpart_fields= TRUE;
+      max_key_len= cur_key_len;
+      cur_key_len= 0;
     }
   }
   range_par->key_parts_end= key_part;
 
+  if (cur_key_len > max_key_len)
+    max_key_len= cur_key_len;
+
+  if (!(range_par->min_key= (uchar*)alloc_root(alloc,max_key_len)) ||
+      !(range_par->max_key= (uchar*)alloc_root(alloc,max_key_len)))
+  {
+    return true;
+  }
+
   DBUG_EXECUTE("info", print_partitioning_index(range_par->key_parts,
                                                 range_par->key_parts_end););
   return FALSE;
diff --git a/sql/opt_range.h b/sql/opt_range.h
index d47fe765184..0c495639db6 100644
--- a/sql/opt_range.h
+++ b/sql/opt_range.h
@@ -643,8 +643,8 @@ public:
     Used to store 'current key tuples', in both range analysis and
     partitioning (list) analysis
   */
-  uchar min_key[MAX_KEY_LENGTH+MAX_FIELD_WIDTH],
-    max_key[MAX_KEY_LENGTH+MAX_FIELD_WIDTH];
+  uchar *min_key;
+  uchar *max_key;
 
   /* Number of SEL_ARG objects allocated by SEL_ARG::clone_tree operations */
   uint alloced_sel_args;