BUG#29582: huge memory consumption with union, subselect, joins:

- Don't call mysql_select() several times for the select that enumerates
  a temporary table with the results of the UNION. Making this call for
  every subquery execution caused O(#enumerated-rows-in-the-outer-query) 
  memory allocations.
- Instead, call join->reinit() and join->exec(), and
  = disable constant table detection for such joins,  
  = provide special handling for table-less constant subqueries.


sql/sql_select.cc:
  BUG#29582: huge memory consumption with union, subselect, joins:
   - Don't mark tables as constant if JOIN::no_const_tables flag is set
sql/sql_select.h:
  BUG#29582: huge memory consumption with union, subselect, joins:
   - Don't mark tables as constant if JOIN::no_const_tables flag is set
sql/sql_union.cc:
  BUG#29582: huge memory consumption with union, subselect, joins:
  - Don't call mysql_select() several times for the select that enumerates
    a temporary table with UNION results. 
  - Instead, call join->reinit() and join->exec().
  - Provide special handling for table-less constant subqueries.

1 files changed, 10 insertions, 0 deletions

diff --git a/sql/sql_select.h b/sql/sql_select.h
index 4f9f6e9ed48..5be4f111bef 100644
--- a/sql/sql_select.h
+++ b/sql/sql_select.h
@@ -277,6 +277,14 @@ public:
   SELECT_LEX_UNIT *unit;
   // select that processed
   SELECT_LEX *select_lex;
+  /* 
+    TRUE <=> optimizer must not mark any table as a constant table.
+    This is needed for subqueries in form "a IN (SELECT .. UNION SELECT ..):
+    when we optimize the select that reads the results of the union from a
+    temporary table, we must not mark the temp. table as constant because
+    the number of rows in it may vary from one subquery execution to another.
+  */
+  bool no_const_tables; 
   
   JOIN *tmp_join; // copy of this JOIN to be used with temporary tables
   ROLLUP rollup;				// Used with rollup
@@ -397,6 +405,8 @@ public:
     tmp_table_param.init();
     tmp_table_param.end_write_records= HA_POS_ERROR;
     rollup.state= ROLLUP::STATE_NONE;
+
+    no_const_tables= FALSE;
   }
 
   int prepare(Item ***rref_pointer_array, TABLE_LIST *tables, uint wind_num,