MDEV-10324: Server crash in get_sel_arg_for_keypart or Assertion

The crash was caused by this problem:
get_best_group_min_max() tries to construct query plans for keys that
are not processed by the range optimizer. This wasn't a problem as long
as SEL_TREE::keys was an array of MAX_KEY elements.
However, now it is a Mem_root_array and only has elements for the used
keys, and get_best_group_min_max attempts to address beyond the end of
the array.

The obvious way to fix the crash was to port (and improve) a part of
96fcfcbd7b5120e8f64fd45985001eca8d36fbfb from mysql-5.7. This makes
get_best_group_min_max not to consider indexes that Mem_root_arrays
have no element for.

After that, I got non-sensical query plans (see MDEV-10325 for details).
Fixed that by making get_best_group_min_max to check if the index is in
table->keys_in_use_for_group_by bitmap.

1 files changed, 36 insertions, 0 deletions

diff --git a/mysql-test/t/group_by.test b/mysql-test/t/group_by.test
index 5d8b79b9fca..4aa5c10ece8 100644
--- a/mysql-test/t/group_by.test
+++ b/mysql-test/t/group_by.test
@@ -1803,3 +1803,39 @@ select distinct a from t1 group by 'a';
 insert into t1 values("2001-02-02"),("2001-02-03");
 select distinct a from t1 group by 'a';
 drop table t1;
+
+--echo #
+--echo # MDEV-10324: Server crash in get_sel_arg_for_keypart or Assertion `n < size()' failed in Mem_root_array
+--echo #
+CREATE TABLE t1 (
+  job_id int(10) unsigned NOT NULL AUTO_INCREMENT,
+  job_cmd varbinary(60) NOT NULL DEFAULT '',
+  job_namespace int(11) NOT NULL,
+  job_title varbinary(255) NOT NULL,
+  job_params blob NOT NULL,
+  job_timestamp varbinary(14) DEFAULT NULL,
+  job_random int(10) unsigned NOT NULL DEFAULT '0',
+  job_token varbinary(32) NOT NULL DEFAULT '',
+  job_token_timestamp varbinary(14) DEFAULT NULL,
+  job_sha1 varbinary(32) NOT NULL DEFAULT '',
+  job_attempts int(10) unsigned NOT NULL DEFAULT '0',
+  PRIMARY KEY (job_id),
+  KEY job_cmd (job_cmd,job_namespace,job_title,job_params(128)),
+  KEY job_timestamp (job_timestamp),
+  KEY job_sha1 (job_sha1),
+  KEY job_cmd_token (job_cmd,job_token,job_random),
+  KEY job_cmd_token_id (job_cmd,job_token,job_id)
+);
+ 
+INSERT INTO t1 VALUES 
+    (NULL, 'foo', 1, 'foo', 'foo', 'foo', 1, 'foo', 'foo', 'foo', 1),
+    (NULL, 'bar', 2, 'bar', 'bar', 'bar', 2, 'bar', 'bar', 'bar', 2);
+    
+SELECT DISTINCT job_cmd  FROM t1 WHERE job_cmd IN ('foobar','null');
+drop table t1;
+
+CREATE TABLE t1 (f1 INT NOT NULL, f2 VARCHAR(3) NOT NULL, KEY(f1), KEY(f2, f1));
+INSERT INTO t1 VALUES (0,'foo'),(1,'bar');
+SELECT 1 IN ( SELECT COUNT( DISTINCT f2 ) FROM t1 WHERE f1 <= 4 );
+drop table t1;
+