MDEV-12575: Server crash in AGGR_OP::put_record or in JOIN_CACHE::free or Invalid write in JOIN::make_aggr_tables_info

During the optimize state of a query, we come know that the result set
would atmost contain one row, then for such a query we don't need
to compute GROUP BY, ORDER BY and DISTINCT.

3 files changed, 86 insertions, 0 deletions

diff --git a/mysql-test/t/distinct.test b/mysql-test/t/distinct.test
index c11f8b501bc..d41340c29fd 100644
--- a/mysql-test/t/distinct.test
+++ b/mysql-test/t/distinct.test
@@ -798,4 +798,22 @@ CREATE TABLE t1 (b1 BIT, b2 BIT, b3 BIT, b4 BIT , b5 BIT, b6 BIT);
 INSERT INTO t1 VALUES (1,0,0,1,0,1),(0,1,0,0,1,0);
 SELECT DISTINCT b1+'0', b2+'0', b3+'0', b4+'0', b5+'0', b6 +'0'   FROM t1;
 DROP TABLE t1;
+
+--echo #
+--echo # MDEV-12575: Server crash in AGGR_OP::put_record or in JOIN_CACHE::free
+--echo # or Invalid write in JOIN::make_aggr_tables_info
+--echo #
+
+CREATE TABLE t1 (pk INT PRIMARY KEY);
+INSERT INTO t1 VALUES (1),(2);
+explain
+( SELECT DISTINCT 1 FROM t1 ORDER BY BENCHMARK(1, MIN(pk)) )
+UNION
+( SELECT DISTINCT 1 FROM t1 ORDER BY BENCHMARK(1, MIN(pk)) );
+
+( SELECT DISTINCT 1 FROM t1 ORDER BY BENCHMARK(1, MIN(pk)) )
+UNION
+( SELECT DISTINCT 1 FROM t1 ORDER BY BENCHMARK(1, MIN(pk)) );
+drop table t1;
+
 --echo End of 5.5 tests
diff --git a/mysql-test/t/group_by.test b/mysql-test/t/group_by.test
index 275939df5c5..61be676bb11 100644
--- a/mysql-test/t/group_by.test
+++ b/mysql-test/t/group_by.test
@@ -1948,3 +1948,15 @@ insert into t1 values(-126,7),(1,1),(0,0),(-1,1),(351,65534);
 select distinct 1 from t1 group by a,b with rollup limit 1;
 drop table t1;
 
+#
+# MDEV-12575: Server crash in AGGR_OP::put_record or in JOIN_CACHE::free
+# or Invalid write in JOIN::make_aggr_tables_info
+#
+
+CREATE TABLE t1 ( pk int, i1 int, v1 varchar(1));
+explain
+SELECT 1 FROM t1
+GROUP BY REPEAT((BINARY pk), v1), AES_DECRYPT((@A := i1), 20852) WITH ROLLUP HAVING LOAD_FILE('a') ;
+SELECT 1 FROM t1
+GROUP BY REPEAT((BINARY pk), v1), AES_DECRYPT((@A := i1), 20852) WITH ROLLUP HAVING LOAD_FILE('a') ;
+drop table t1;
diff --git a/mysql-test/t/win.test b/mysql-test/t/win.test
index e9c8ee05773..7dda2b6215f 100644
--- a/mysql-test/t/win.test
+++ b/mysql-test/t/win.test
@@ -2119,3 +2119,59 @@ SELECT name, test, score,
 FROM t1
 ORDER BY test, name;
 drop table t1;
+
+--echo #
+--echo # MDEV-12575: Server crash in AGGR_OP::put_record or in JOIN_CACHE::free
+--echo # or Invalid write in JOIN::make_aggr_tables_info
+--echo #
+
+SELECT DISTINCT BIT_OR(100) OVER () FROM dual
+GROUP BY LEFT('2018-08-24', 100)  order by 1+2;
+
+CREATE TABLE t1 (i INT);
+INSERT INTO t1 VALUES (1),(2);
+
+SELECT * FROM (
+  SELECT
+    ROW_NUMBER() OVER(), i, sum(i)
+    FROM t1
+    WHERE 1=0
+    limit 0
+) AS sq;
+
+SELECT * FROM (
+  SELECT
+    ROW_NUMBER() OVER(), i, sum(i)
+    FROM t1
+    WHERE 1=0
+    GROUP BY i
+) AS sq;
+drop table t1;
+
+create table t1 (a int);
+explain
+select distinct 1, row_number() over (order by 1) from t1 where a=0 group by a with rollup;
+select distinct 1, row_number() over (order by 1) from t1 where a=0 group by a with rollup;
+drop table t1;
+
+explain
+SELECT DISTINCT BIT_OR(100) OVER () FROM dual
+GROUP BY LEFT('2018-08-24', 100) WITH ROLLUP
+HAVING @A := 'qwerty';
+SELECT DISTINCT BIT_OR(100) OVER () FROM dual
+GROUP BY LEFT('2018-08-24', 100) WITH ROLLUP
+HAVING @A := 'qwerty';
+
+explain
+SELECT DISTINCT BIT_OR(100) OVER () FROM dual
+GROUP BY LEFT('2018-08-24', 100)
+HAVING @A := 'qwerty';
+SELECT DISTINCT BIT_OR(100) OVER () FROM dual
+GROUP BY LEFT('2018-08-24', 100)
+HAVING @A := 'qwerty';
+
+create table t1 (a int);
+explain
+SELECT DISTINCT BIT_OR(100) OVER () FROM t1
+GROUP BY LEFT('2018-08-24', 100) having 1=1 limit 0;
+drop table t1;