From 26c52659c9fb2ddbdabb661f0a089705d16a46c1 Mon Sep 17 00:00:00 2001
From: Tor Didriksen <tor.didriksen@oracle.com>
Date: Fri, 27 Jan 2012 11:13:13 +0100
Subject: Bug#13580775 ASSERTION FAILED: RECORD_LENGTH == M_RECORD_LENGTH
 Bug#13011410 CRASH IN FILESORT CODE WITH GROUP BY/ROLLUP

The assert in 13580775 is visible in 5.6 only,
but shows that all versions are vulnerable.
13011410 crashes in all versions.

filesort tries to re-use the sort buffer between invocations in order to save
malloc/free overhead.
The fix for Bug 11748783 - 37359: FILESORT CAN BE MORE EFFICIENT.
added an assert that buffer properties (num_records, record_length) are
consistent between invocations. Indeed, they are not necessarily consistent.

Fix: re-allocate the sort buffer if properties change.


mysql-test/r/partition.result:
  New tests.
mysql-test/t/partition.test:
  New tests.
sql/filesort.cc:
  If we already have allocated a sort buffer in a previous execution,
  then verify that it is big enough for the current one.
sql/table.h:
  Add sort_keys_size; Number of bytes allocated for the sort_keys buffer.
---
 mysql-test/t/partition.test | 45 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

(limited to 'mysql-test/t/partition.test')

diff --git a/mysql-test/t/partition.test b/mysql-test/t/partition.test
index 341e8780b2b..134bbd0527d 100644
--- a/mysql-test/t/partition.test
+++ b/mysql-test/t/partition.test
@@ -2218,4 +2218,49 @@ INSERT INTO t1 VALUES(0);
 DROP TABLE t1;
 SET GLOBAL myisam_use_mmap=default;
 
+--echo #
+--echo # Bug#13580775 ASSERTION FAILED: RECORD_LENGTH == M_RECORD_LENGTH,
+--echo # FILE FILESORT_UTILS.CC
+--echo #
+
+CREATE TABLE t1 (
+  a INT PRIMARY KEY,
+  b INT,
+  c CHAR(1),
+  d INT,
+  KEY (c,d)
+) PARTITION BY KEY () PARTITIONS 1;
+
+INSERT INTO t1 VALUES (1,1,'a',1), (2,2,'a',1);
+
+SELECT 1 FROM t1 WHERE 1 IN
+(SELECT  group_concat(b)
+ FROM t1
+ WHERE c > geomfromtext('point(1 1)')
+ GROUP BY b
+);
+
+DROP TABLE t1;
+
+--echo #
+--echo # Bug#13011410 CRASH IN FILESORT CODE WITH GROUP BY/ROLLUP
+--echo #
+
+CREATE TABLE t1 (
+  a INT,
+  b MEDIUMINT,
+  c VARCHAR(300) CHARACTER SET hp8 COLLATE hp8_bin,
+  PRIMARY KEY (a,c(299))) 
+ENGINE=myisam
+PARTITION BY LINEAR KEY () PARTITIONS 2;
+
+INSERT INTO t1 VALUES (1,2,'test'), (2,3,'hi'), (4,5,'bye');
+SELECT 1 FROM t1 WHERE b < SOME
+( SELECT 1 FROM t1 WHERE a >= 1
+  GROUP BY b WITH ROLLUP
+  HAVING b > geomfromtext("")
+);
+
+DROP TABLE t1;
+
 --echo End of 5.1 tests
-- 
cgit v1.2.1