Merge fix for Bug#11765713 from 5.1

1 files changed, 31 insertions, 12 deletions

diff --git a/sql/opt_sum.cc b/sql/opt_sum.cc
index a2f8b9c4447..6b8cdfaca60 100644
--- a/sql/opt_sum.cc
+++ b/sql/opt_sum.cc
@@ -212,6 +212,7 @@ static int get_index_max_value(TABLE *table, TABLE_REF *ref, uint range_fl)
 /**
   Substitutes constants for some COUNT(), MIN() and MAX() functions.
 
+  @param thd                   thread handler
   @param tables                list of leaves of join table tree
   @param all_fields            All fields to be returned
   @param conds                 WHERE clause
@@ -229,9 +230,12 @@ static int get_index_max_value(TABLE *table, TABLE_REF *ref, uint range_fl)
     HA_ERR_KEY_NOT_FOUND on impossible conditions
   @retval
     HA_ERR_... if a deadlock or a lock wait timeout happens, for example
+  @retval
+    ER_...     e.g. ER_SUBQUERY_NO_1_ROW
 */
 
-int opt_sum_query(TABLE_LIST *tables, List<Item> &all_fields,COND *conds)
+int opt_sum_query(THD *thd,
+                  TABLE_LIST *tables, List<Item> &all_fields, COND *conds)
 {
   List_iterator_fast<Item> it(all_fields);
   int const_result= 1;
@@ -243,6 +247,8 @@ int opt_sum_query(TABLE_LIST *tables, List<Item> &all_fields,COND *conds)
   Item *item;
   int error;
 
+  DBUG_ENTER("opt_sum_query");
+
   if (conds)
     where_tables= conds->used_tables();
 
@@ -270,7 +276,7 @@ int opt_sum_query(TABLE_LIST *tables, List<Item> &all_fields,COND *conds)
           WHERE t2.field IS NULL;
       */
       if (tl->table->map & where_tables)
-        return 0;
+        DBUG_RETURN(0);
     }
     else
       used_tables|= tl->table->map;
@@ -297,7 +303,7 @@ int opt_sum_query(TABLE_LIST *tables, List<Item> &all_fields,COND *conds)
       if(error)
       {
         tl->table->file->print_error(error, MYF(ME_FATALERROR));
-        return error;
+        DBUG_RETURN(error);
       }
       count*= tl->table->file->stats.records;
     }
@@ -390,10 +396,10 @@ int opt_sum_query(TABLE_LIST *tables, List<Item> &all_fields,COND *conds)
           if (error)
 	  {
 	    if (error == HA_ERR_KEY_NOT_FOUND || error == HA_ERR_END_OF_FILE)
-	      return HA_ERR_KEY_NOT_FOUND;	      // No rows matching WHERE
+	      DBUG_RETURN(HA_ERR_KEY_NOT_FOUND); // No rows matching WHERE
 	    /* HA_ERR_LOCK_DEADLOCK or some other error */
  	    table->file->print_error(error, MYF(0));
-            return(error);
+            DBUG_RETURN(error);
 	  }
           removed_tables|= table->map;
         }
@@ -442,6 +448,10 @@ int opt_sum_query(TABLE_LIST *tables, List<Item> &all_fields,COND *conds)
         const_result= 0;
     }
   }
+
+  if (thd->is_error())
+    DBUG_RETURN(thd->stmt_da->sql_errno());
+
   /*
     If we have a where clause, we can only ignore searching in the
     tables if MIN/MAX optimisation replaced all used tables
@@ -451,7 +461,7 @@ int opt_sum_query(TABLE_LIST *tables, List<Item> &all_fields,COND *conds)
   */
   if (removed_tables && used_tables != removed_tables)
     const_result= 0;                            // We didn't remove all tables
-  return const_result;
+  DBUG_RETURN(const_result);
 }
 
 
@@ -737,6 +747,12 @@ static bool matching_cond(bool max_fl, TABLE_REF *ref, KEY *keyinfo,
 
     if (is_null || (is_null_safe_eq && args[1]->is_null()))
     {
+      /*
+        If we have a non-nullable index, we cannot use it,
+        since set_null will be ignored, and we will compare uninitialized data.
+      */
+      if (!part->field->real_maybe_null())
+        DBUG_RETURN(false);
       part->field->set_null();
       *key_ptr= (uchar) 1;
     }
@@ -807,8 +823,9 @@ static bool matching_cond(bool max_fl, TABLE_REF *ref, KEY *keyinfo,
   @param[out]    prefix_len  Length of prefix for the search range
 
   @note
-    This function may set table->key_read to 1, which must be reset after
-    index is used! (This can only happen when function returns 1)
+    This function may set field->table->key_read to true,
+    which must be reset after index is used!
+    (This can only happen when function returns 1)
 
   @retval
     0   Index can not be used to optimize MIN(field)/MAX(field)
@@ -823,7 +840,9 @@ static bool find_key_for_maxmin(bool max_fl, TABLE_REF *ref,
                                 uint *range_fl, uint *prefix_len)
 {
   if (!(field->flags & PART_KEY_FLAG))
-    return 0;                                        // Not key field
+    return false;                               // Not key field
+
+  DBUG_ENTER("find_key_for_maxmin");
 
   TABLE *table= field->table;
   uint idx= 0;
@@ -848,7 +867,7 @@ static bool find_key_for_maxmin(bool max_fl, TABLE_REF *ref,
          part++, jdx++, key_part_to_use= (key_part_to_use << 1) | 1)
     {
       if (!(table->file->index_flags(idx, jdx, 0) & HA_READ_ORDER))
-        return 0;
+        DBUG_RETURN(false);
 
       /* Check whether the index component is partial */
       Field *part_field= table->field[part->fieldnr-1];
@@ -897,12 +916,12 @@ static bool find_key_for_maxmin(bool max_fl, TABLE_REF *ref,
           */
           if (field->part_of_key.is_set(idx))
             table->set_keyread(TRUE);
-          return 1;
+          DBUG_RETURN(true);
         }
       }
     }
   }
-  return 0;
+  DBUG_RETURN(false);
 }