LP BUG#823169 fix.

For ANY subqueries NULLs should be ignored (if there is other values) when finding max min.
For ALL subqueries NULLs should be saved if they found.

Optimisation for ALL suqbueries if NULL is possible in the SELECT list with max/min aggregate function switched off.

Some test changed where NULL is not used but optimization with max/min aggregate function important so NOT NULL added.

mysql-test/r/explain.result:
  Forced old optimization.
mysql-test/r/subselect.result:
  Forced old optimization.
  
  New test suite.
mysql-test/t/explain.test:
  Forced old optimization.
mysql-test/t/subselect.test:
  Forced old optimization.
  
  New test suite.
sql/item_subselect.cc:
  Store converted subquery type.
  
  Switch off aggregate function optimisation for ALL and nulls.
sql/sql_class.cc:
  Fixed NULL comparison.
sql/sql_class.h:
  Store converted subquery type.

1 files changed, 114 insertions, 1 deletions

diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test
index 1c5767d2acf..08790104082 100644
--- a/mysql-test/t/subselect.test
+++ b/mysql-test/t/subselect.test
@@ -488,6 +488,54 @@ SELECT t1.a, t1.a in (select t2.a from t2,t3 where t3.a=t2.a) FROM t1;
 explain extended SELECT t1.a, t1.a in (select t2.a from t2,t3 where t3.a=t2.a) FROM t1;
 drop table t1,t2,t3;
 
+--echo # check correct NULL Processing for normal IN/ALL/ANY
+--echo # and 2 ways of max/min optimization
+create table t1 (a int);
+insert into t1 values (1), (100), (NULL), (1000);
+create table t2 (a int not null);
+
+--echo # subselect returns empty set (for NULL and non-NULL left part)
+select a, a in (select * from t2) from t1;
+select a, a > any (select * from t2) from t1;
+select a, a > all (select * from t2) from t1;
+select a from t1 where a in (select * from t2);
+select a from t1 where a > any (select * from t2);
+select a from t1 where a > all (select * from t2);
+select a from t1 where a in (select * from t2 group by a);
+select a from t1 where a > any (select * from t2 group by a);
+select a from t1 where a > all (select * from t2 group by a);
+
+insert into t2 values (1),(200);
+
+--echo # sebselect returns non-empty set without NULLs
+select a, a in (select * from t2) from t1;
+select a, a > any (select * from t2) from t1;
+select a, a > all (select * from t2) from t1;
+select a from t1 where a in (select * from t2);
+select a from t1 where a > any (select * from t2);
+select a from t1 where a > all (select * from t2);
+select a from t1 where a in (select * from t2 group by a);
+select a from t1 where a > any (select * from t2 group by a);
+select a from t1 where a > all (select * from t2 group by a);
+
+drop table t2;
+create table t2 (a int);
+insert into t2 values (1),(NULL),(200);
+
+--echo # sebselect returns non-empty set with NULLs
+select a, a in (select * from t2) from t1;
+select a, a > any (select * from t2) from t1;
+select a, a > all (select * from t2) from t1;
+select a from t1 where a in (select * from t2);
+select a from t1 where a > any (select * from t2);
+select a from t1 where a > all (select * from t2);
+select a from t1 where a in (select * from t2 group by a);
+select a from t1 where a > any (select * from t2 group by a);
+select a from t1 where a > all (select * from t2 group by a);
+
+
+drop table t1, t2;
+
 #LIMIT is not supported now
 create table t1 (a float);
 -- error ER_NOT_SUPPORTED_YET
@@ -924,7 +972,7 @@ drop table t1,t2;
 #
 # correct ALL optimisation
 #
-create table t2 (a int, b int);
+create table t2 (a int, b int not null);
 create table t3 (a int);
 insert into t3 values (6),(7),(3);
 select * from t3 where a >= all (select b from t2);
@@ -3797,3 +3845,68 @@ SELECT 1 as foo FROM t1 WHERE a < SOME
   ); 
 
 DROP TABLE t1;
+
+#
+# LP BUG#823169 NULLs with ALL/ANY and maxmin optimization
+#
+CREATE TABLE t1 (a int(11), b varchar(1));
+INSERT INTO t1 VALUES (2,NULL),(5,'d'),(7,'g');
+
+SELECT a FROM t1 WHERE b < ANY ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b < ANY ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b > ANY ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b > ANY ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b <= ANY ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b <= ANY ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b >= ANY ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b >= ANY ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b = ANY ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b = ANY ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b <> ANY ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b <> ANY ( SELECT b FROM t1 GROUP BY b );
+
+SELECT a FROM t1 WHERE b < ALL ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b < ALL ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b > ALL ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b > ALL ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b <= ALL ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b <= ALL ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b >= ALL ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b >= ALL ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b = ALL ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b = ALL ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b <> ALL ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b <> ALL ( SELECT b FROM t1 GROUP BY b );
+
+delete from t1;
+INSERT INTO t1 VALUES (2,NULL),(5,'d'),(7,'g');
+
+SELECT a FROM t1 WHERE b < ANY ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b < ANY ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b > ANY ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b > ANY ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b <= ANY ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b <= ANY ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b >= ANY ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b >= ANY ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b = ANY ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b = ANY ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b <> ANY ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b <> ANY ( SELECT b FROM t1 GROUP BY b );
+
+SELECT a FROM t1 WHERE b < ALL ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b < ALL ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b > ALL ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b > ALL ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b <= ALL ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b <= ALL ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b >= ALL ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b >= ALL ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b = ALL ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b = ALL ( SELECT b FROM t1 GROUP BY b );
+SELECT a FROM t1 WHERE b <> ALL ( SELECT b FROM t1 );
+SELECT a FROM t1 WHERE b <> ALL ( SELECT b FROM t1 GROUP BY b );
+
+drop table t1;
+
+--echo End of 5.2 tests