Merge fix for Bug#11765713 from 5.1

5 files changed, 74 insertions, 14 deletions

diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result
index b516585fc37..b1af35c6bbf 100644
--- a/mysql-test/r/subselect.result
+++ b/mysql-test/r/subselect.result
@@ -5058,6 +5058,24 @@ i
 DROP TABLE t1,t1s,t2s;
 End of 5.1 tests
 #
+# Bug #11765713 58705:
+# OPTIMIZER LET ENGINE DEPEND ON UNINITIALIZED VALUES
+# CREATED BY OPT_SUM_QUERY
+#
+CREATE TABLE t1(a INT NOT NULL, KEY (a));
+INSERT INTO t1 VALUES (0), (1);
+SELECT 1 as foo FROM t1 WHERE a < SOME
+(SELECT a FROM t1 WHERE a <=>
+(SELECT a FROM t1)
+);
+ERROR 21000: Subquery returns more than 1 row
+SELECT 1 as foo FROM t1 WHERE a < SOME
+(SELECT a FROM t1 WHERE a <=>
+(SELECT a FROM t1 where a is null)
+);
+foo
+DROP TABLE t1;
+#
 # Bug #57704: Cleanup code dies with void TABLE::set_keyread(bool): 
 #             Assertion `file' failed.
 #
diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test
index f60ee247ba7..e910ea09fb2 100644
--- a/mysql-test/t/subselect.test
+++ b/mysql-test/t/subselect.test
@@ -3997,6 +3997,28 @@ DROP TABLE t1,t1s,t2s;
 --echo End of 5.1 tests
 
 --echo #
+--echo # Bug #11765713 58705:
+--echo # OPTIMIZER LET ENGINE DEPEND ON UNINITIALIZED VALUES
+--echo # CREATED BY OPT_SUM_QUERY
+--echo #
+
+CREATE TABLE t1(a INT NOT NULL, KEY (a));
+INSERT INTO t1 VALUES (0), (1);
+
+--error ER_SUBQUERY_NO_1_ROW
+SELECT 1 as foo FROM t1 WHERE a < SOME
+  (SELECT a FROM t1 WHERE a <=>
+    (SELECT a FROM t1)
+  );
+
+SELECT 1 as foo FROM t1 WHERE a < SOME
+  (SELECT a FROM t1 WHERE a <=>
+    (SELECT a FROM t1 where a is null)
+  ); 
+
+DROP TABLE t1;
+
+--echo #
 --echo # Bug #57704: Cleanup code dies with void TABLE::set_keyread(bool): 
 --echo #             Assertion `file' failed.
 --echo #
diff --git a/sql/opt_sum.cc b/sql/opt_sum.cc
index a2f8b9c4447..6b8cdfaca60 100644
--- a/sql/opt_sum.cc
+++ b/sql/opt_sum.cc
@@ -212,6 +212,7 @@ static int get_index_max_value(TABLE *table, TABLE_REF *ref, uint range_fl)
 /**
   Substitutes constants for some COUNT(), MIN() and MAX() functions.
 
+  @param thd                   thread handler
   @param tables                list of leaves of join table tree
   @param all_fields            All fields to be returned
   @param conds                 WHERE clause
@@ -229,9 +230,12 @@ static int get_index_max_value(TABLE *table, TABLE_REF *ref, uint range_fl)
     HA_ERR_KEY_NOT_FOUND on impossible conditions
   @retval
     HA_ERR_... if a deadlock or a lock wait timeout happens, for example
+  @retval
+    ER_...     e.g. ER_SUBQUERY_NO_1_ROW
 */
 
-int opt_sum_query(TABLE_LIST *tables, List<Item> &all_fields,COND *conds)
+int opt_sum_query(THD *thd,
+                  TABLE_LIST *tables, List<Item> &all_fields, COND *conds)
 {
   List_iterator_fast<Item> it(all_fields);
   int const_result= 1;
@@ -243,6 +247,8 @@ int opt_sum_query(TABLE_LIST *tables, List<Item> &all_fields,COND *conds)
   Item *item;
   int error;
 
+  DBUG_ENTER("opt_sum_query");
+
   if (conds)
     where_tables= conds->used_tables();
 
@@ -270,7 +276,7 @@ int opt_sum_query(TABLE_LIST *tables, List<Item> &all_fields,COND *conds)
           WHERE t2.field IS NULL;
       */
       if (tl->table->map & where_tables)
-        return 0;
+        DBUG_RETURN(0);
     }
     else
       used_tables|= tl->table->map;
@@ -297,7 +303,7 @@ int opt_sum_query(TABLE_LIST *tables, List<Item> &all_fields,COND *conds)
       if(error)
       {
         tl->table->file->print_error(error, MYF(ME_FATALERROR));
-        return error;
+        DBUG_RETURN(error);
       }
       count*= tl->table->file->stats.records;
     }
@@ -390,10 +396,10 @@ int opt_sum_query(TABLE_LIST *tables, List<Item> &all_fields,COND *conds)
           if (error)
 	  {
 	    if (error == HA_ERR_KEY_NOT_FOUND || error == HA_ERR_END_OF_FILE)
-	      return HA_ERR_KEY_NOT_FOUND;	      // No rows matching WHERE
+	      DBUG_RETURN(HA_ERR_KEY_NOT_FOUND); // No rows matching WHERE
 	    /* HA_ERR_LOCK_DEADLOCK or some other error */
  	    table->file->print_error(error, MYF(0));
-            return(error);
+            DBUG_RETURN(error);
 	  }
           removed_tables|= table->map;
         }
@@ -442,6 +448,10 @@ int opt_sum_query(TABLE_LIST *tables, List<Item> &all_fields,COND *conds)
         const_result= 0;
     }
   }
+
+  if (thd->is_error())
+    DBUG_RETURN(thd->stmt_da->sql_errno());
+
   /*
     If we have a where clause, we can only ignore searching in the
     tables if MIN/MAX optimisation replaced all used tables
@@ -451,7 +461,7 @@ int opt_sum_query(TABLE_LIST *tables, List<Item> &all_fields,COND *conds)
   */
   if (removed_tables && used_tables != removed_tables)
     const_result= 0;                            // We didn't remove all tables
-  return const_result;
+  DBUG_RETURN(const_result);
 }
 
 
@@ -737,6 +747,12 @@ static bool matching_cond(bool max_fl, TABLE_REF *ref, KEY *keyinfo,
 
     if (is_null || (is_null_safe_eq && args[1]->is_null()))
     {
+      /*
+        If we have a non-nullable index, we cannot use it,
+        since set_null will be ignored, and we will compare uninitialized data.
+      */
+      if (!part->field->real_maybe_null())
+        DBUG_RETURN(false);
       part->field->set_null();
       *key_ptr= (uchar) 1;
     }
@@ -807,8 +823,9 @@ static bool matching_cond(bool max_fl, TABLE_REF *ref, KEY *keyinfo,
   @param[out]    prefix_len  Length of prefix for the search range
 
   @note
-    This function may set table->key_read to 1, which must be reset after
-    index is used! (This can only happen when function returns 1)
+    This function may set field->table->key_read to true,
+    which must be reset after index is used!
+    (This can only happen when function returns 1)
 
   @retval
     0   Index can not be used to optimize MIN(field)/MAX(field)
@@ -823,7 +840,9 @@ static bool find_key_for_maxmin(bool max_fl, TABLE_REF *ref,
                                 uint *range_fl, uint *prefix_len)
 {
   if (!(field->flags & PART_KEY_FLAG))
-    return 0;                                        // Not key field
+    return false;                               // Not key field
+
+  DBUG_ENTER("find_key_for_maxmin");
 
   TABLE *table= field->table;
   uint idx= 0;
@@ -848,7 +867,7 @@ static bool find_key_for_maxmin(bool max_fl, TABLE_REF *ref,
          part++, jdx++, key_part_to_use= (key_part_to_use << 1) | 1)
     {
       if (!(table->file->index_flags(idx, jdx, 0) & HA_READ_ORDER))
-        return 0;
+        DBUG_RETURN(false);
 
       /* Check whether the index component is partial */
       Field *part_field= table->field[part->fieldnr-1];
@@ -897,12 +916,12 @@ static bool find_key_for_maxmin(bool max_fl, TABLE_REF *ref,
           */
           if (field->part_of_key.is_set(idx))
             table->set_keyread(TRUE);
-          return 1;
+          DBUG_RETURN(true);
         }
       }
     }
   }
-  return 0;
+  DBUG_RETURN(false);
 }
 
 
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 43d5a34bcb8..7334b03f076 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -991,7 +991,7 @@ JOIN::optimize()
       If all items were resolved by opt_sum_query, there is no need to
       open any tables.
     */
-    if ((res=opt_sum_query(select_lex->leaf_tables, all_fields, conds)))
+    if ((res=opt_sum_query(thd, select_lex->leaf_tables, all_fields, conds)))
     {
       if (res == HA_ERR_KEY_NOT_FOUND)
       {
diff --git a/sql/sql_select.h b/sql/sql_select.h
index bdb9d1a315c..ea4a22527d3 100644
--- a/sql/sql_select.h
+++ b/sql/sql_select.h
@@ -619,7 +619,8 @@ bool is_indexed_agg_distinct(JOIN *join, List<Item_field> *out_args);
 
 /* functions from opt_sum.cc */
 bool simple_pred(Item_func *func_item, Item **args, bool *inv_order);
-int opt_sum_query(TABLE_LIST *tables, List<Item> &all_fields,COND *conds);
+int opt_sum_query(THD* thd,
+                  TABLE_LIST *tables, List<Item> &all_fields, COND *conds);
 
 /* from sql_delete.cc, used by opt_range.cc */
 extern "C" int refpos_order_cmp(void* arg, const void *a,const void *b);