MDEV-17783: AddressSanitizer: stack-buffer-overflow in table_cond_selectivity

A less-intrusive fix: don't have table_cond_selectivity() assume that
there are less than MAX_REF_PARTS hash-join KEYUSEs.

If there are more than that, switch to using an array. Allocate the array
on the heap: we can't allocate it on MEM_ROOT as table_cond_selectivity()
is called many times during the optimization.

(Variant 2, with review input addressed)

3 files changed, 92 insertions, 2 deletions

diff --git a/mysql-test/r/selectivity_innodb.result b/mysql-test/r/selectivity_innodb.result
index 5452919aa6d..23e0fcc9387 100644
--- a/mysql-test/r/selectivity_innodb.result
+++ b/mysql-test/r/selectivity_innodb.result
@@ -2103,6 +2103,35 @@ drop view v1;
 #
 # End of 10.1 tests
 #
+#
+# MDEV-17783: AddressSanitizer: stack-buffer-overflow in table_cond_selectivity
+#
+set
+@tmp_jcl=@@join_cache_level,
+@tmp_sel=@@optimizer_use_condition_selectivity;
+set
+join_cache_level=3,
+optimizer_use_condition_selectivity=2;
+CREATE TABLE t1 (
+c1 int, c2 int, c3 int, c4 int, c5 int, c6 int, c7 int, c8 int, c9 int, c10 int,
+c11 int, c12 int, c13 int, c14 int, c15 int, c16 int, c17 int, c18 int, c19 int,
+c20 int, c21 int, c22 int, c23 int, c24 int, c25 int, c26 int, c27 int, c28 int,
+c29 int, c30 int, c31 int, c32 int, c33 int, c34 int
+) ENGINE=InnoDB;
+SELECT * FROM t1
+WHERE
+(c1, c2, c3, c4, c5, c6, c7, c8, c9, c10,
+c11, c12, c13, c14, c15, c16, c17, c18, c19,
+c20, c21, c22, c23, c24, c25, c26, c27, c28, c29,
+c30, c31, c32, c33, c34) IN (SELECT * FROM t1) ;
+c1	c2	c3	c4	c5	c6	c7	c8	c9	c10	c11	c12	c13	c14	c15	c16	c17	c18	c19	c20	c21	c22	c23	c24	c25	c26	c27	c28	c29	c30	c31	c32	c33	c34
+set
+join_cache_level=@tmp_jcl,
+optimizer_use_condition_selectivity=@tmp_sel;
+drop table t1;
+#
+# End of 10.1 tests
+#
 set use_stat_tables= @tmp_ust;
 set optimizer_use_condition_selectivity= @tmp_oucs;
 set @@global.histogram_size=@save_histogram_size;
diff --git a/mysql-test/t/selectivity_innodb.test b/mysql-test/t/selectivity_innodb.test
index 6c457e2848b..eb05091e43a 100644
--- a/mysql-test/t/selectivity_innodb.test
+++ b/mysql-test/t/selectivity_innodb.test
@@ -174,6 +174,40 @@ drop view v1;
 --echo # End of 10.1 tests
 --echo #
 
+--echo #
+--echo # MDEV-17783: AddressSanitizer: stack-buffer-overflow in table_cond_selectivity
+--echo #
+
+set
+  @tmp_jcl=@@join_cache_level,
+  @tmp_sel=@@optimizer_use_condition_selectivity;
+set
+  join_cache_level=3,
+  optimizer_use_condition_selectivity=2;
+
+CREATE TABLE t1 (
+  c1 int, c2 int, c3 int, c4 int, c5 int, c6 int, c7 int, c8 int, c9 int, c10 int,
+  c11 int, c12 int, c13 int, c14 int, c15 int, c16 int, c17 int, c18 int, c19 int,
+  c20 int, c21 int, c22 int, c23 int, c24 int, c25 int, c26 int, c27 int, c28 int,
+  c29 int, c30 int, c31 int, c32 int, c33 int, c34 int
+) ENGINE=InnoDB;
+
+SELECT * FROM t1
+WHERE
+  (c1, c2, c3, c4, c5, c6, c7, c8, c9, c10,
+   c11, c12, c13, c14, c15, c16, c17, c18, c19,
+   c20, c21, c22, c23, c24, c25, c26, c27, c28, c29,
+   c30, c31, c32, c33, c34) IN (SELECT * FROM t1) ;
+
+set
+  join_cache_level=@tmp_jcl,
+  optimizer_use_condition_selectivity=@tmp_sel;
+drop table t1;
+
+--echo #
+--echo # End of 10.1 tests
+--echo #
+
 set use_stat_tables= @tmp_ust;
 set optimizer_use_condition_selectivity= @tmp_oucs;
 set @@global.histogram_size=@save_histogram_size;
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index fcdb671941d..362adc9853f 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -7943,7 +7943,9 @@ static
 double table_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s,
                               table_map rem_tables)
 {
-  uint16 ref_keyuse_steps[MAX_REF_PARTS - 1];
+  uint16 ref_keyuse_steps_buf[MAX_REF_PARTS];
+  uint   ref_keyuse_size= MAX_REF_PARTS;
+  uint16 *ref_keyuse_steps= ref_keyuse_steps_buf;
   Field *field;
   TABLE *table= s->table;
   MY_BITMAP *read_set= table->read_set;
@@ -8090,6 +8092,29 @@ double table_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s,
             }
             if (keyparts > 1)
 	    {
+              /*
+                Prepare to set ref_keyuse_steps[keyparts-2]: resize the array
+                if it is not large enough
+              */
+              if (keyparts - 2 >= ref_keyuse_size)
+              {
+                uint new_size= MY_MAX(ref_keyuse_size*2, keyparts);
+                void *new_buf;
+                if (!(new_buf= my_malloc(sizeof(*ref_keyuse_steps)*new_size,
+                                         MYF(0))))
+                {
+                  sel= 1.0; // As if no selectivity was computed
+                  goto exit;
+                }
+                memcpy(new_buf, ref_keyuse_steps,
+                       sizeof(*ref_keyuse_steps)*ref_keyuse_size);
+                if (ref_keyuse_steps != ref_keyuse_steps_buf)
+                  my_free(ref_keyuse_steps);
+
+                ref_keyuse_steps= (uint16*)new_buf;
+                ref_keyuse_size= new_size;
+              }
+
               ref_keyuse_steps[keyparts-2]= (uint16)(keyuse - prev_ref_keyuse);
               prev_ref_keyuse= keyuse;
             }
@@ -8144,7 +8169,9 @@ double table_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s,
 
   sel*= table_multi_eq_cond_selectivity(join, idx, s, rem_tables,
                                         keyparts, ref_keyuse_steps);
-
+exit:
+  if (ref_keyuse_steps != ref_keyuse_steps_buf)
+    my_free(ref_keyuse_steps);
   return sel;
 }