BUG#29582: huge memory consumption with union, subselect, joins:

- Don't call mysql_select() several times for the select that enumerates
  a temporary table with the results of the UNION. Making this call for
  every subquery execution caused O(#enumerated-rows-in-the-outer-query) 
  memory allocations.
- Instead, call join->reinit() and join->exec(), and
  = disable constant table detection for such joins,  
  = provide special handling for table-less constant subqueries.


sql/sql_select.cc:
  BUG#29582: huge memory consumption with union, subselect, joins:
   - Don't mark tables as constant if JOIN::no_const_tables flag is set
sql/sql_select.h:
  BUG#29582: huge memory consumption with union, subselect, joins:
   - Don't mark tables as constant if JOIN::no_const_tables flag is set
sql/sql_union.cc:
  BUG#29582: huge memory consumption with union, subselect, joins:
  - Don't call mysql_select() several times for the select that enumerates
    a temporary table with UNION results. 
  - Instead, call join->reinit() and join->exec().
  - Provide special handling for table-less constant subqueries.

1 files changed, 40 insertions, 17 deletions

diff --git a/sql/sql_union.cc b/sql/sql_union.cc
index 373b03d45e6..16c4eeab4bb 100644
--- a/sql/sql_union.cc
+++ b/sql/sql_union.cc
@@ -545,6 +545,10 @@ bool st_select_lex_unit::exec()
 	/*
 	  allocate JOIN for fake select only once (prevent
 	  mysql_select automatic allocation)
+          TODO: The above is nonsense. mysql_select() will not allocate the
+          join if one already exists. There must be some other reason why we
+          don't let it allocate the join. Perhaps this is because we need
+          some special parameter values passed to join constructor?
 	*/
 	if (!(fake_select_lex->join= new JOIN(thd, item_list,
 					      fake_select_lex->options, result)))
@@ -552,33 +556,52 @@ bool st_select_lex_unit::exec()
 	  fake_select_lex->table_list.empty();
 	  DBUG_RETURN(TRUE);
 	}
+        fake_select_lex->join->no_const_tables= TRUE;
 
 	/*
 	  Fake st_select_lex should have item list for correctref_array
 	  allocation.
 	*/
 	fake_select_lex->item_list= item_list;
+        saved_error= mysql_select(thd, &fake_select_lex->ref_pointer_array,
+                              &result_table_list,
+                              0, item_list, NULL,
+                              global_parameters->order_list.elements,
+                              (ORDER*)global_parameters->order_list.first,
+                              (ORDER*) NULL, NULL, (ORDER*) NULL,
+                              fake_select_lex->options | SELECT_NO_UNLOCK,
+                              result, this, fake_select_lex);
       }
       else
       {
-	JOIN_TAB *tab,*end;
-	for (tab=join->join_tab, end=tab+join->tables ;
-	     tab && tab != end ;
-	     tab++)
-	{
-	  delete tab->select;
-	  delete tab->quick;
-	}
-	join->init(thd, item_list, fake_select_lex->options, result);
+        if (describe)
+        {
+          /*
+            In EXPLAIN command, constant subqueries that do not use any
+            tables are executed two times:
+             - 1st time is a real evaluation to get the subquery value
+             - 2nd time is to produce EXPLAIN output rows.
+            1st execution sets certain members (e.g. select_result) to perform
+            subquery execution rather than EXPLAIN line production. In order 
+            to reset them back, we re-do all of the actions (yes it is ugly):
+          */
+	  join->init(thd, item_list, fake_select_lex->options, result);
+          saved_error= mysql_select(thd, &fake_select_lex->ref_pointer_array,
+                                &result_table_list,
+                                0, item_list, NULL,
+                                global_parameters->order_list.elements,
+                                (ORDER*)global_parameters->order_list.first,
+                                (ORDER*) NULL, NULL, (ORDER*) NULL,
+                                fake_select_lex->options | SELECT_NO_UNLOCK,
+                                result, this, fake_select_lex);
+        }
+        else
+        {
+          join->examined_rows= 0;
+          join->reinit();
+          saved_error= join->exec();
+        }
       }
-      saved_error= mysql_select(thd, &fake_select_lex->ref_pointer_array,
-                            &result_table_list,
-                            0, item_list, NULL,
-                            global_parameters->order_list.elements,
-                            (ORDER*)global_parameters->order_list.first,
-                            (ORDER*) NULL, NULL, (ORDER*) NULL,
-                            fake_select_lex->options | SELECT_NO_UNLOCK,
-                            result, this, fake_select_lex);
 
       fake_select_lex->table_list.empty();
       if (!saved_error)