reporting empty result added in case of max/min optimisation of ALL/ANY/SOME subqueries

fixed null processing in NOT operation used in ALL subquery (Bug #6247)


mysql-test/r/subselect.result:
  new tests of ALL/ANY wiews
mysql-test/t/subselect.test:
  new tests of ALL/ANY wiews
sql/item_cmpfunc.cc:
  fixed special NOT ALL processing
  fixed processing max/min optimized subqueries with empty results (added methods to detect empty results) and special NOP operation to process them for SOME/ANY sobqueries
sql/item_cmpfunc.h:
  fixed processing max/min optimized subqueries with empty results (added methods to detect empty results) and special NOP operation to process them for SOME/ANY sobqueries
sql/item_subselect.cc:
  reporting empty result added for max/min subqueries
sql/item_subselect.h:
  reporting empty result added for max/min subqueries
sql/item_sum.cc:
  reporting empty result added fox max/min aggregate functions
sql/item_sum.h:
  reporting empty result added fox max/min aggregate functions
sql/sql_class.cc:
  reporting empty result added for max/min subqueries
sql/sql_parse.cc:
  reporting empty result added for max/min subqueries
sql/sql_union.cc:
  reporting empty result added for max/min subqueries

9 files changed, 132 insertions, 24 deletions

diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc
index c36f2d191c7..d3c9cfc2c58 100644
--- a/sql/item_cmpfunc.cc
+++ b/sql/item_cmpfunc.cc
@@ -106,7 +106,7 @@ longlong Item_func_not::val_int()
   DBUG_ASSERT(fixed == 1);
   double value=args[0]->val();
   null_value=args[0]->null_value;
-  return !null_value && value == 0 ? 1 : 0;
+  return ((!null_value && value == 0) ? 1 : 0);
 }
 
 /*
@@ -117,13 +117,23 @@ longlong Item_func_not_all::val_int()
 {
   DBUG_ASSERT(fixed == 1);
   double value= args[0]->val();
-  if (abort_on_null)
-  {
-    null_value= 0;
-    return (args[0]->null_value || value == 0) ? 1 : 0;
-  }
+
+  /*
+    return TRUE if there was records in underlaying select in max/min
+    optimisation
+  */
+  if (empty_underlying_subquery())
+    return 1;
+
   null_value= args[0]->null_value;
-  return (!null_value && value == 0) ? 1 : 0;
+  return ((!null_value && value == 0) ? 1 : 0);
+}
+
+
+bool Item_func_not_all::empty_underlying_subquery()
+{
+  return ((test_sum_item && !test_sum_item->any_value()) ||
+          (test_sub_item && !test_sub_item->any_value()));
 }
 
 void Item_func_not_all::print(String *str)
@@ -134,6 +144,28 @@ void Item_func_not_all::print(String *str)
     args[0]->print(str);
 }
 
+
+/*
+  special NOP for ALL subquery
+*/
+
+longlong Item_func_nop_all::val_int()
+{
+  DBUG_ASSERT(fixed == 1);
+  double value= args[0]->val();
+
+  /*
+    return TRUE if there was records in underlaying select in max/min
+    optimisation
+  */
+  if (empty_underlying_subquery())
+    return 1;
+
+  null_value= args[0]->null_value;
+  return (null_value || value == 0) ? 0 : 1;
+}
+
+
 /*
   Convert a constant expression or string to an integer.
   This is done when comparing DATE's of different formats and
diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h
index 69528099aa1..6834799688d 100644
--- a/sql/item_cmpfunc.h
+++ b/sql/item_cmpfunc.h
@@ -229,21 +229,43 @@ public:
   Item *neg_transformer(THD *thd);
 };
 
+class Item_maxmin_subselect;
 class Item_func_not_all :public Item_func_not
 {
+  /* allow to check presence od values in max/min optimisation */
+  Item_sum_hybrid *test_sum_item;
+  Item_maxmin_subselect *test_sub_item;
+
   bool abort_on_null;
 public:
   bool show;
 
-  Item_func_not_all(Item *a) :Item_func_not(a), abort_on_null(0), show(0) {}
+  Item_func_not_all(Item *a)
+    :Item_func_not(a), test_sum_item(0), test_sub_item(0), abort_on_null(0),
+     show(0)
+    {}
   virtual void top_level_item() { abort_on_null= 1; }
   bool top_level() { return abort_on_null; }
   longlong val_int();
   enum Functype functype() const { return NOT_ALL_FUNC; }
   const char *func_name() const { return "<not>"; }
   void print(String *str);
+  void set_sum_test(Item_sum_hybrid *item) { test_sum_item= item; };
+  void set_sub_test(Item_maxmin_subselect *item) { test_sub_item= item; };
+  bool empty_underlying_subquery();
 };
 
+
+class Item_func_nop_all :public Item_func_not_all
+{
+public:
+
+  Item_func_nop_all(Item *a) :Item_func_not_all(a) {}
+  longlong val_int();
+  const char *func_name() const { return "<nop>"; }
+};
+
+
 class Item_func_eq :public Item_bool_rowready_func2
 {
 public:
diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc
index 62cd016b0df..b263b06c91f 100644
--- a/sql/item_subselect.cc
+++ b/sql/item_subselect.cc
@@ -271,7 +271,7 @@ Item_singlerow_subselect::Item_singlerow_subselect(st_select_lex *select_lex)
 Item_maxmin_subselect::Item_maxmin_subselect(Item_subselect *parent,
 					     st_select_lex *select_lex,
 					     bool max_arg)
-  :Item_singlerow_subselect()
+  :Item_singlerow_subselect(), was_values(TRUE)
 {
   DBUG_ENTER("Item_maxmin_subselect::Item_maxmin_subselect");
   max= max_arg;
@@ -290,12 +290,26 @@ Item_maxmin_subselect::Item_maxmin_subselect(Item_subselect *parent,
   DBUG_VOID_RETURN;
 }
 
+void Item_maxmin_subselect::cleanup()
+{
+  /*
+    By default is is TRUE to avoid TRUE reporting by
+    Item_func_not_all/Item_func_nop_all if this item was never called.
+
+    Engine exec() set it to FALSE by reset_value_registration() call.
+  */
+
+  was_values= TRUE;
+}
+
+
 void Item_maxmin_subselect::print(String *str)
 {
   str->append(max?"<max>":"<min>", 5);
   Item_singlerow_subselect::print(str);
 }
 
+
 void Item_singlerow_subselect::reset()
 {
   null_value= 1;
@@ -303,6 +317,7 @@ void Item_singlerow_subselect::reset()
     value->null_value= 1;
 }
 
+
 Item_subselect::trans_res
 Item_singlerow_subselect::select_transformer(JOIN *join)
 {
@@ -519,7 +534,7 @@ bool Item_in_subselect::test_limit(SELECT_LEX_UNIT *unit)
 
 Item_in_subselect::Item_in_subselect(Item * left_exp,
 				     st_select_lex *select_lex):
-  Item_exists_subselect(), transformed(0), upper_not(0)
+  Item_exists_subselect(), transformed(0), upper_item(0)
 {
   DBUG_ENTER("Item_in_subselect::Item_in_subselect");
   left_expr= left_exp;
@@ -680,7 +695,7 @@ Item_in_subselect::single_value_transformer(JOIN *join,
     NULL/IS NOT NULL functions). If so, we rewrite ALL/ANY with NOT EXISTS
     later in this method.
   */
-  if ((abort_on_null || (upper_not && upper_not->top_level())) &&
+  if ((abort_on_null || (upper_item && upper_item->top_level())) &&
       !select_lex->master_unit()->uncacheable && !func->eqne_op())
   {
     if (substitution)
@@ -694,7 +709,7 @@ Item_in_subselect::single_value_transformer(JOIN *join,
 	!select_lex->with_sum_func &&
 	!(select_lex->next_select()))
     {
-      Item *item;
+      Item_sum_hybrid *item;
       if (func->l_op())
       {
 	/*
@@ -711,6 +726,8 @@ Item_in_subselect::single_value_transformer(JOIN *join,
 	*/
 	item= new Item_sum_min(*select_lex->ref_pointer_array);
       }
+      if (upper_item)
+        upper_item->set_sum_test(item);
       *select_lex->ref_pointer_array= item;
       {
 	List_iterator<Item> it(select_lex->item_list);
@@ -731,10 +748,13 @@ Item_in_subselect::single_value_transformer(JOIN *join,
     }
     else
     {
+      Item_maxmin_subselect *item;
       // remove LIMIT placed  by ALL/ANY subquery
       select_lex->master_unit()->global_parameters->select_limit=
 	HA_POS_ERROR;
-      subs= new Item_maxmin_subselect(this, select_lex, func->l_op());
+      subs= item= new Item_maxmin_subselect(this, select_lex, func->l_op());
+      if (upper_item)
+        upper_item->set_sub_test(item);
     }
     // left expression belong to outer select
     SELECT_LEX *current= thd->lex->current_select, *up;
@@ -1041,8 +1061,8 @@ Item_subselect::trans_res
 Item_allany_subselect::select_transformer(JOIN *join)
 {
   transformed= 1;
-  if (upper_not)
-    upper_not->show= 1;
+  if (upper_item)
+    upper_item->show= 1;
   return single_value_transformer(join, func);
 }
 
@@ -1247,6 +1267,7 @@ int subselect_single_select_engine::exec()
   }
   if (!executed)
   {
+    item->reset_value_registration();
     join->exec();
     executed= 1;
     join->thd->where= save_where;
diff --git a/sql/item_subselect.h b/sql/item_subselect.h
index 764c41f33b4..bd6ede49255 100644
--- a/sql/item_subselect.h
+++ b/sql/item_subselect.h
@@ -93,7 +93,7 @@ public:
     return null_value;
   }
   bool fix_fields(THD *thd, TABLE_LIST *tables, Item **ref);
-  bool exec();
+  virtual bool exec();
   virtual void fix_length_and_dec();
   table_map used_tables() const;
   bool const_item() const;
@@ -109,6 +109,11 @@ public:
     engine_changed= 1;
     return eng == 0;
   }
+  /*
+    Used by max/min subquery to initialize value presence registration
+    mechanism. Engine call this method before rexecution query.
+  */
+  virtual void reset_value_registration() {}
 
   friend class select_subselect;
   friend class Item_in_optimizer;
@@ -150,13 +155,20 @@ public:
 };
 
 /* used in static ALL/ANY optimisation */
+class select_max_min_finder_subselect;
 class Item_maxmin_subselect :public Item_singlerow_subselect
 {
+protected:
   bool max;
+  bool was_values;  // was checked at least some values
 public:
   Item_maxmin_subselect(Item_subselect *parent,
 			st_select_lex *select_lex, bool max);
   void print(String *str);
+  void cleanup();
+  bool any_value() { return was_values; }
+  void register_value() { was_values= TRUE; }
+  void reset_value_registration() { was_values= FALSE; }
 };
 
 /* exists subselect */
@@ -204,11 +216,11 @@ protected:
   bool abort_on_null;
   bool transformed;
 public:
-  Item_func_not_all *upper_not; // point on NOT before ALL subquery
+  Item_func_not_all *upper_item; // point on NOT/NOP before ALL/SOME subquery
 
   Item_in_subselect(Item * left_expr, st_select_lex *select_lex);
   Item_in_subselect()
-    :Item_exists_subselect(), abort_on_null(0), transformed(0), upper_not(0)
+    :Item_exists_subselect(), abort_on_null(0), transformed(0), upper_item(0)
      
   {}
 
@@ -249,7 +261,7 @@ public:
 		     st_select_lex *select_lex, bool all);
 
   // only ALL subquery has upper not
-  subs_type substype() { return upper_not?ALL_SUBS:ANY_SUBS; }
+  subs_type substype() { return all?ALL_SUBS:ANY_SUBS; }
   trans_res select_transformer(JOIN *join);
   void print(String *str);
 };
diff --git a/sql/item_sum.cc b/sql/item_sum.cc
index 3b3a6083725..cf07e136034 100644
--- a/sql/item_sum.cc
+++ b/sql/item_sum.cc
@@ -540,9 +540,22 @@ void Item_sum_hybrid::cleanup()
   DBUG_ENTER("Item_sum_hybrid::cleanup");
   Item_sum::cleanup();
   used_table_cache= ~(table_map) 0;
+  /*
+    by default is is TRUE to avoid TRUE reporting by
+    Item_func_not_all/Item_func_nop_all if this item was never called.
+
+    no_rows_in_result() set it to FALSE if was not results found.
+  */
+  was_values= TRUE;
   DBUG_VOID_RETURN;
 }
 
+void Item_sum_hybrid::no_rows_in_result()
+{
+  Item_sum::no_rows_in_result();
+  was_values= FALSE;
+}
+
 
 Item *Item_sum_min::copy_or_same(THD* thd)
 {
diff --git a/sql/item_sum.h b/sql/item_sum.h
index 5aa0d37190b..9993ce1bb12 100644
--- a/sql/item_sum.h
+++ b/sql/item_sum.h
@@ -403,19 +403,22 @@ class Item_sum_hybrid :public Item_sum
   int cmp_sign;
   table_map used_table_cache;
   CHARSET_INFO *cmp_charset;
+  bool was_values;  // was checked at least some values (for max/min only)
 
   public:
   Item_sum_hybrid(Item *item_par,int sign)
     :Item_sum(item_par), sum(0.0), sum_int(0),
     hybrid_type(INT_RESULT), hybrid_field_type(FIELD_TYPE_LONGLONG),
     cmp_sign(sign), used_table_cache(~(table_map) 0),
-    cmp_charset(&my_charset_bin)
+    cmp_charset(&my_charset_bin), was_values(TRUE)
   {}
   Item_sum_hybrid(THD *thd, Item_sum_hybrid *item):
     Item_sum(thd, item), value(item->value),
     sum(item->sum), sum_int(item->sum_int), hybrid_type(item->hybrid_type),
     hybrid_field_type(item->hybrid_field_type),cmp_sign(item->cmp_sign), 
-    used_table_cache(item->used_table_cache), cmp_charset(item->cmp_charset) {}
+    used_table_cache(item->used_table_cache), cmp_charset(item->cmp_charset),
+    was_values(TRUE)
+  {}
   bool fix_fields(THD *, TABLE_LIST *, Item **);
   table_map used_tables() const { return used_table_cache; }
   bool const_item() const { return !used_table_cache; }
@@ -434,6 +437,8 @@ class Item_sum_hybrid :public Item_sum
   void min_max_update_real_field();
   void min_max_update_int_field();
   void cleanup();
+  bool any_value() { return was_values; }
+  void no_rows_in_result();
 };
 
 
diff --git a/sql/sql_class.cc b/sql/sql_class.cc
index eda60b5cfdb..e99e7ccaaeb 100644
--- a/sql/sql_class.cc
+++ b/sql/sql_class.cc
@@ -1241,9 +1241,10 @@ bool select_singlerow_subselect::send_data(List<Item> &items)
 bool select_max_min_finder_subselect::send_data(List<Item> &items)
 {
   DBUG_ENTER("select_max_min_finder_subselect::send_data");
-  Item_singlerow_subselect *it= (Item_singlerow_subselect *)item;
+  Item_maxmin_subselect *it= (Item_maxmin_subselect *)item;
   List_iterator_fast<Item> li(items);
   Item *val_item= li++;
+  it->register_value();
   if (it->assigned())
   {
     cache->store(val_item);
diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc
index 7420f9de100..a760956d9de 100644
--- a/sql/sql_parse.cc
+++ b/sql/sql_parse.cc
@@ -5114,9 +5114,9 @@ Item * all_any_subquery_creator(Item *left_expr,
   Item_allany_subselect *it=
     new Item_allany_subselect(left_expr, (*cmp)(all), select_lex, all);
   if (all)
-    return it->upper_not= new Item_func_not_all(it);	/* ALL */
+    return it->upper_item= new Item_func_not_all(it);	/* ALL */
 
-  return it;						/* ANY/SOME */
+  return it->upper_item= new Item_func_nop_all(it);      /* ANY/SOME */
 }
 
 
diff --git a/sql/sql_union.cc b/sql/sql_union.cc
index e0e8f8d42c5..b35209faeb2 100644
--- a/sql/sql_union.cc
+++ b/sql/sql_union.cc
@@ -394,6 +394,8 @@ int st_select_lex_unit::exec()
   
   if (uncacheable || !item || !item->assigned() || describe)
   {
+    if (item)
+      item->reset_value_registration();
     if (optimized && item)
     {
       if (item->assigned())