MDEV-9505: Valgrind failure in SEL_ARG::store_min,find_used_partitions

create_partition_index_description() had wrong logic to calculate
length of the key value buffer that is used by the range optimizer.

For some reason it used MAX(partitioning_columns_len,
subpartitioning_columns_len) while it should use SUM of these values.

3 files changed, 42 insertions, 11 deletions

diff --git a/mysql-test/r/partition_pruning.result b/mysql-test/r/partition_pruning.result
index e52c2c7d886..65b788a7d1b 100644
--- a/mysql-test/r/partition_pruning.result
+++ b/mysql-test/r/partition_pruning.result
@@ -3467,3 +3467,21 @@ id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t2	p_1000	ref	PRIMARY	PRIMARY	8	const	2	Using index
 1	PRIMARY	t1	p_1000	ALL	PRIMARY	NULL	NULL	NULL	6	Using where; Using join buffer (flat, BNL join)
 drop table t1,t2;
+#
+# MDEV-9505: Valgrind failure in SEL_ARG::store_min,find_used_partitions,...
+#
+create table t1 (a int, b char(10), c varchar(5), d int)
+partition by range columns(a,b,c)
+subpartition by key (c,d)
+subpartitions 3
+( partition p0 values less than (1,'abc','abc'),
+partition p1 values less than (2,'abc','abc'),
+partition p2 values less than (3,'abc','abc'),
+partition p3 values less than (4,'abc','abc')
+);
+insert into t1 values (1,'a','b',1),(2,'a','b',2),(3,'a','b',3);
+select * from t1 where (a = 1 AND b < 'd' AND (c = 'b' OR (c = 'c' AND d = 1)) OR
+(a = 1 AND b >= 'a' AND (c = 'c' OR (c = 'd' AND d = 2))));
+a	b	c	d
+1	a	b	1
+drop table t1;
diff --git a/mysql-test/t/partition_pruning.test b/mysql-test/t/partition_pruning.test
index 06ef99e1e70..9d72e9c0d01 100644
--- a/mysql-test/t/partition_pruning.test
+++ b/mysql-test/t/partition_pruning.test
@@ -1517,3 +1517,22 @@ and dept_id in (select dept_id from t2 where COMPANY_ID = 1000);
 
 drop table t1,t2;
 
+--echo #
+--echo # MDEV-9505: Valgrind failure in SEL_ARG::store_min,find_used_partitions,...
+--echo #
+create table t1 (a int, b char(10), c varchar(5), d int)
+partition by range columns(a,b,c)
+subpartition by key (c,d)
+subpartitions 3
+( partition p0 values less than (1,'abc','abc'),
+  partition p1 values less than (2,'abc','abc'),
+  partition p2 values less than (3,'abc','abc'),
+  partition p3 values less than (4,'abc','abc')
+);
+
+insert into t1 values (1,'a','b',1),(2,'a','b',2),(3,'a','b',3);
+select * from t1 where (a = 1 AND b < 'd' AND (c = 'b' OR (c = 'c' AND d = 1)) OR
+                       (a = 1 AND b >= 'a' AND (c = 'c' OR (c = 'd' AND d = 2))));
+drop table t1;
+
+
diff --git a/sql/opt_range.cc b/sql/opt_range.cc
index 6b72f9c336b..f051ed07a7e 100644
--- a/sql/opt_range.cc
+++ b/sql/opt_range.cc
@@ -4931,15 +4931,14 @@ static bool create_partition_index_description(PART_PRUNE_PARAM *ppar)
   Field **field= (ppar->part_fields)? part_info->part_field_array :
                                            part_info->subpart_field_array;
   bool in_subpart_fields= FALSE;
-  uint max_key_len= 0;
-  uint cur_key_len= 0;
+  uint total_key_len= 0;
   for (uint part= 0; part < total_parts; part++, key_part++)
   {
     key_part->key=          0;
     key_part->part=	    part;
     key_part->length= (uint16)(*field)->key_length();
     key_part->store_length= (uint16)get_partition_field_store_length(*field);
-    cur_key_len += key_part->store_length;
+    total_key_len += key_part->store_length;
 
     DBUG_PRINT("info", ("part %u length %u store_length %u", part,
                          key_part->length, key_part->store_length));
@@ -4965,18 +4964,13 @@ static bool create_partition_index_description(PART_PRUNE_PARAM *ppar)
     {
       field= part_info->subpart_field_array;
       in_subpart_fields= TRUE;
-      max_key_len= cur_key_len;
-      cur_key_len= 0;
     }
   }
   range_par->key_parts_end= key_part;
 
-  if (cur_key_len > max_key_len)
-    max_key_len= cur_key_len;
-
-  max_key_len++; /* Take into account the "+1" in QUICK_RANGE::QUICK_RANGE */
-  if (!(range_par->min_key= (uchar*)alloc_root(alloc,max_key_len)) ||
-      !(range_par->max_key= (uchar*)alloc_root(alloc,max_key_len)))
+  total_key_len++; /* Take into account the "+1" in QUICK_RANGE::QUICK_RANGE */
+  if (!(range_par->min_key= (uchar*)alloc_root(alloc,total_key_len)) ||
+      !(range_par->max_key= (uchar*)alloc_root(alloc,total_key_len)))
   {
     return true;
   }