Fixed MDEV-5424: SELECT using ORDER BY DESC and LIMIT produces unexpected results (InnoDB/XtraDB)

This only happend when using an ORDER BY on a primary key part, where all other key parts where constant.
Remove of duplicated expressions in ORDER BY (as the old code did this in some strange cases)


mysql-test/r/group_by.result:
  Fixed results to take into account that duplicate order by parts are now deleted
mysql-test/r/group_by_innodb.result:
  Ensure extended keys are on
mysql-test/r/innodb_ext_key.result:
  More tests
mysql-test/r/order_by.result:
  More tests
mysql-test/t/group_by.test:
  Fixed results to take into account that duplicate order by parts are now deleted
mysql-test/t/group_by_innodb.test:
  Ensure extended keys are on
mysql-test/t/innodb_ext_key.test:
  More tests
mysql-test/t/order_by.test:
  More tests
sql/sql_select.cc:
  Fixed bug where we looked at extended key parts when we shouldn't
  Remove of duplicated expressions in ORDER BY
sql/table.cc:
  Indentation fixes

4 files changed, 72 insertions, 11 deletions

diff --git a/mysql-test/r/group_by.result b/mysql-test/r/group_by.result
index 9bb22eb8169..553607dc03f 100644
--- a/mysql-test/r/group_by.result
+++ b/mysql-test/r/group_by.result
@@ -1957,12 +1957,12 @@ UNIQUE INDEX idx (col1));
 INSERT INTO t1 VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10),
 (11),(12),(13),(14),(15),(16),(17),(18),(19),(20);
 EXPLAIN SELECT col1 AS field1, col1 AS field2
-FROM t1 GROUP BY field1, field2;;
+FROM t1 GROUP BY field1, field2+0;;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	index	NULL	idx	5	NULL	20	Using index; Using temporary; Using filesort
 FLUSH STATUS;
 SELECT col1 AS field1, col1 AS field2
-FROM t1 GROUP BY field1, field2;;
+FROM t1 GROUP BY field1, field2+0;;
 field1	field2
 1	1
 2	2
@@ -2054,8 +2054,12 @@ field1	field2
 explain
 select col1 f1, col1 f2 from t1 order by f2, f1;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	NULL	idx	5	NULL	20	Using index
+explain
+select col1 f1, col1 f2 from t1 order by f2, f1+0;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	index	NULL	idx	5	NULL	20	Using index; Using filesort
-select col1 f1, col1 f2 from t1 order by f2, f1;
+select col1 f1, col1 f2 from t1 order by f2, f1+0;
 f1	f2
 1	1
 2	2
@@ -2080,7 +2084,7 @@ f1	f2
 explain
 select col1 f1, col1 f2 from t1 group by f2 order by f2, f1;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	range	NULL	idx	5	NULL	7	Using index for group-by; Using temporary; Using filesort
+1	SIMPLE	t1	range	NULL	idx	5	NULL	7	Using index for group-by
 select col1 f1, col1 f2 from t1 group by f2 order by f2, f1;
 f1	f2
 1	1
@@ -2106,7 +2110,7 @@ f1	f2
 explain
 select col1 f1, col1 f2 from t1 group by f1, f2 order by f2, f1;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	index	NULL	idx	5	NULL	20	Using index; Using temporary; Using filesort
+1	SIMPLE	t1	index	NULL	idx	5	NULL	20	Using index
 select col1 f1, col1 f2 from t1 group by f1, f2 order by f2, f1;
 f1	f2
 1	1
@@ -2137,10 +2141,10 @@ INSERT INTO t2(col1, col2) VALUES
 (1,20),(2,19),(3,18),(4,17),(5,16),(6,15),(7,14),(8,13),(9,12),(10,11),
 (11,10),(12,9),(13,8),(14,7),(15,6),(16,5),(17,4),(18,3),(19,2),(20,1);
 explain
-select col1 f1, col2 f2, col1 f3 from t2 group by f1, f2, f3;
+select col1 f1, col2 f2, col1 f3 from t2 group by f1, f2, f3+0;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t2	index	NULL	idx	10	NULL	20	Using index; Using temporary; Using filesort
-select col1 f1, col2 f2, col1 f3 from t2 group by f1, f2, f3;
+select col1 f1, col2 f2, col1 f3 from t2 group by f1, f2, f3+0;
 f1	f2	f3
 1	20	1
 2	19	2
@@ -2163,10 +2167,10 @@ f1	f2	f3
 19	2	19
 20	1	20
 explain
-select col1 f1, col2 f2, col1 f3 from t2 order by f1, f2, f3;
+select col1 f1, col2 f2, col1 f3 from t2 order by f1, f2, f3+0;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t2	index	NULL	idx	10	NULL	20	Using index; Using filesort
-select col1 f1, col2 f2, col1 f3 from t2 order by f1, f2, f3;
+select col1 f1, col2 f2, col1 f3 from t2 order by f1, f2, f3+0;
 f1	f2	f3
 1	20	1
 2	19	2
diff --git a/mysql-test/r/group_by_innodb.result b/mysql-test/r/group_by_innodb.result
index d165834cbe3..86d431b00d2 100644
--- a/mysql-test/r/group_by_innodb.result
+++ b/mysql-test/r/group_by_innodb.result
@@ -1,3 +1,4 @@
+set @save_ext_key_optimizer_switch=@@optimizer_switch;
 #
 # MDEV-3992 Server crash or valgrind errors in test_if_skip_sort_order/test_if_cheaper_ordering
 # on GROUP BY with indexes on InnoDB table
@@ -7,13 +8,14 @@ pk INT PRIMARY KEY,
 a VARCHAR(1) NOT NULL,
 KEY (pk)
 ) ENGINE=InnoDB;
+set optimizer_switch='extended_keys=on';
 INSERT INTO t1 VALUES (1,'a'),(2,'b');
 EXPLAIN
 SELECT COUNT(*), pk field1, pk AS field2
 FROM t1 WHERE a = 'r' OR pk = 183 
 GROUP BY field1, field2;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	index	PRIMARY,pk	pk	4	NULL	2	Using where
+1	SIMPLE	t1	index	PRIMARY,pk	PRIMARY	4	NULL	2	Using where
 SELECT COUNT(*), pk field1, pk AS field2
 FROM t1 WHERE a = 'r' OR pk = 183 
 GROUP BY field1, field2;
@@ -22,9 +24,10 @@ EXPLAIN
 SELECT COUNT(*), pk field1 FROM t1
 WHERE a = 'r' OR pk = 183 GROUP BY field1, field1;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	index	PRIMARY,pk	pk	4	NULL	2	Using where
+1	SIMPLE	t1	index	PRIMARY,pk	PRIMARY	4	NULL	2	Using where
 SELECT COUNT(*), pk field1 FROM t1
 WHERE a = 'r' OR pk = 183 GROUP BY field1, field1;
 COUNT(*)	field1
 drop table t1;
 End of 5.5 tests
+set optimizer_switch=@save_ext_key_optimizer_switch;
diff --git a/mysql-test/r/innodb_ext_key.result b/mysql-test/r/innodb_ext_key.result
index 4a6b902e869..df681f21f3b 100644
--- a/mysql-test/r/innodb_ext_key.result
+++ b/mysql-test/r/innodb_ext_key.result
@@ -987,6 +987,54 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t2	ref	page_timestamp	page_timestamp	4	const	10	Using where
 1	SIMPLE	t3	eq_ref	PRIMARY	PRIMARY	4	test.t2.rev_text_id	1	
 DROP TABLE t1,t2,t3;
+#
+# MDEV-5424 SELECT using ORDER BY DESC and LIMIT produces unexpected
+# results (InnoDB/XtraDB)
+#
+create table t1 (a bigint not null unique auto_increment, b varchar(10), primary key (a), key (b(2))) engine = myisam default character set utf8;
+create table t2 (a bigint not null unique auto_increment, b varchar(10), primary key (a), key (b(2))) engine = innodb default character set utf8;
+insert into t1 (b) values (null), (null), (null);
+insert into t2 (b) values (null), (null), (null);
+set optimizer_switch='extended_keys=on';
+explain select a from t1 where b is null order by a desc limit 2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ref	b	b	9	const	2	Using where; Using filesort
+select a from t1 where b is null order by a desc limit 2;
+a
+3
+2
+explain select a from t2 where b is null order by a desc limit 2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t2	range	b	b	9	NULL	3	Using where; Using filesort
+select a from t2 where b is null order by a desc limit 2;
+a
+3
+2
+set optimizer_switch='extended_keys=off';
+explain select a from t2 where b is null order by a desc limit 2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t2	range	b	b	9	NULL	3	Using where; Using filesort
+select a from t2 where b is null order by a desc limit 2;
+a
+3
+2
+explain select a from t2 where b is null order by a desc;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t2	index	b	PRIMARY	8	NULL	3	Using where
+select a from t2 where b is null order by a desc;
+a
+3
+2
+1
+explain select a from t2 where b is null order by a desc,a,a;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t2	index	b	PRIMARY	8	NULL	3	Using where
+select a from t2 where b is null order by a desc,a,a;
+a
+3
+2
+1
+drop table t1, t2;
 set optimizer_switch=@save_optimizer_switch;
 set optimizer_switch=@save_ext_key_optimizer_switch;
 SET SESSION STORAGE_ENGINE=DEFAULT;
diff --git a/mysql-test/r/order_by.result b/mysql-test/r/order_by.result
index e37d64d6d44..0487f9222e0 100644
--- a/mysql-test/r/order_by.result
+++ b/mysql-test/r/order_by.result
@@ -357,6 +357,12 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 explain select * from t1 where a = 1 order by b desc;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ref	a	a	4	const	5	Using where; Using index
+explain select * from t1 where a = 2 and b > 0 order by a desc,b desc,b,a;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	range	a	a	9	NULL	5	Using where; Using index
+explain select * from t1 where a = 2 and b < 2 order by a desc,a,b desc,a,b;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	range	a	a	9	NULL	2	Using where; Using index
 select * from t1 where a = 1 order by b desc;
 a	b	c
 1	3	b