Bug#11766300 59387: FAILING ASSERTION: CURSOR->POS_STATE == 1997660512 (BTR_PCUR_IS_POSITIONE

Bug#13639204 64111: CRASH ON SELECT SUBQUERY WITH NON UNIQUE INDEX
The crash happened due to wrong calculation
of key length during creation of reference for
sort order index. The problem is that
keyuse->used_tables can have OUTER_REF_TABLE_BIT enabled
but used_tables parameter(create_ref_for_key() func) does
not have it. So key parts which have OUTER_REF_TABLE_BIT
are ommited and it could lead to incorrect key length
calculation(zero key length).


mysql-test/r/subselect_innodb.result:
  test result
mysql-test/t/subselect_innodb.test:
  test case
sql/sql_select.cc:
  added OUTER_REF_TABLE_BIT to the used_tables parameter
  for create_ref_for_key() function.
storage/innobase/handler/ha_innodb.cc:
  added assertion, request from Inno team
storage/innodb_plugin/handler/ha_innodb.cc:
  added assertion, request from Inno team

1 files changed, 56 insertions, 0 deletions

diff --git a/mysql-test/t/subselect_innodb.test b/mysql-test/t/subselect_innodb.test
index 573fe0c1810..ce070b454c3 100644
--- a/mysql-test/t/subselect_innodb.test
+++ b/mysql-test/t/subselect_innodb.test
@@ -238,3 +238,59 @@ call p1();
 call p1();
 drop procedure p1;
 drop tables t1,t2,t3;
+
+--echo #
+--echo # Bug #11766300  59387: FAILING ASSERTION: CURSOR->POS_STATE == 1997660512 (BTR_PCUR_IS_POSITIONE
+--echo #
+
+CREATE TABLE t1 (a INT) ENGINE=INNODB;
+INSERT INTO t1 VALUES (0);
+CREATE TABLE t2 (d BINARY(2), PRIMARY KEY (d(1)), UNIQUE KEY (d)) ENGINE=INNODB;
+
+SELECT 1 FROM t1 WHERE NOT EXISTS
+(SELECT 1 FROM t2 WHERE d = (SELECT d FROM t2 WHERE a >= 1) ORDER BY d);
+
+EXPLAIN SELECT 1 FROM t1 WHERE NOT EXISTS
+(SELECT 1 FROM t2 WHERE d = (SELECT d FROM t2 WHERE a >= 1) ORDER BY d);
+
+DROP TABLE t2;
+
+CREATE TABLE t2 (b INT, c INT, UNIQUE KEY (b), UNIQUE KEY (b, c )) ENGINE=INNODB;
+INSERT INTO t2 VALUES (1, 1);
+
+SELECT 1 FROM t1
+WHERE a != (SELECT 1 FROM t2 WHERE a <=> b OR a > '' AND 6 = 7 ORDER BY b, c);
+
+DROP TABLE t1, t2;
+
+--echo #
+--echo # Bug #13639204 64111: CRASH ON SELECT SUBQUERY WITH NON UNIQUE
+--echo # INDEX
+--echo #
+CREATE TABLE t1 (
+id int
+) ENGINE=InnoDB;
+INSERT INTO t1 (id) VALUES (11);
+
+CREATE TABLE t2 (
+t1_id int,
+position int,
+KEY t1_id (t1_id),
+KEY t1_id_position (t1_id,position)
+) ENGINE=InnoDB;
+
+let $query=SELECT
+(SELECT position FROM t2
+WHERE t2.t1_id = t1.id
+ORDER BY t2.t1_id , t2.position
+LIMIT 10,1
+) AS maxkey
+FROM t1
+LIMIT 1;
+
+eval EXPLAIN $query;
+eval $query;
+
+DROP TABLE t1,t2;
+
+--echo End of 5.1 tests