MDEV-16697: Fix difference between 32bit/windows and 64bit systems in allowed select nest level

14 files changed, 70 insertions, 29 deletions

diff --git a/include/my_global.h b/include/my_global.h
index 7904eed3dfc..2922b74192a 100644
--- a/include/my_global.h
+++ b/include/my_global.h
@@ -986,7 +986,6 @@ typedef unsigned long my_off_t;
   TODO Convert these to use Bitmap class.
  */
 typedef ulonglong table_map;          /* Used for table bits in join */
-typedef ulong nesting_map;  /* Used for flags of nesting constructs */
 
 /* often used type names - opaque declarations */
 typedef const struct charset_info_st CHARSET_INFO;
diff --git a/mysql-test/main/parser.result b/mysql-test/main/parser.result
index 2394c958b47..32055538700 100644
--- a/mysql-test/main/parser.result
+++ b/mysql-test/main/parser.result
@@ -1680,3 +1680,24 @@ SELECT without VERSIONING FROM t1
 SELECT without WITHOUT FROM t1
 DROP PROCEDURE p2;
 DROP PROCEDURE p1;
+#
+# MDEV-16697: Fix difference between 32bit/windows and 64bit
+# systems in allowed select nest level
+#
+SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+(SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+(SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+(SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+(SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+(SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+(SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+(SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+1
+)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+(SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+(SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+(SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+(SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+(SELECT
+1
+End of 10.3 tests
diff --git a/mysql-test/main/parser.test b/mysql-test/main/parser.test
index 8faab613a0c..65db9c3cda3 100644
--- a/mysql-test/main/parser.test
+++ b/mysql-test/main/parser.test
@@ -1443,3 +1443,20 @@ CALL p2('without');
 
 DROP PROCEDURE p2;
 DROP PROCEDURE p1;
+
+
+--echo #
+--echo # MDEV-16697: Fix difference between 32bit/windows and 64bit
+--echo # systems in allowed select nest level
+--echo #
+ SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+(SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+(SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+(SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+(SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+(SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+(SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+(SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT (SELECT
+1
+)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+--echo End of 10.3 tests
diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc
index 1947a45186a..207aa9a25c9 100644
--- a/sql/item_subselect.cc
+++ b/sql/item_subselect.cc
@@ -1983,8 +1983,7 @@ bool Item_allany_subselect::transform_into_max_min(JOIN *join)
                  print_where(item, "rewrite with MIN/MAX", QT_ORDINARY););
 
     save_allow_sum_func= thd->lex->allow_sum_func;
-    thd->lex->allow_sum_func|=
-        (nesting_map)1 << thd->lex->current_select->nest_level;
+    thd->lex->allow_sum_func.set_bit(thd->lex->current_select->nest_level);
     /*
       Item_sum_(max|min) can't substitute other item => we can use 0 as
       reference, also Item_sum_(max|min) can't be fixed after creation, so
diff --git a/sql/item_sum.cc b/sql/item_sum.cc
index 3163fb9ea2e..44e5a86c863 100644
--- a/sql/item_sum.cc
+++ b/sql/item_sum.cc
@@ -72,14 +72,15 @@ size_t Item_sum::ram_limitation(THD *thd)
 bool Item_sum::init_sum_func_check(THD *thd)
 {
   SELECT_LEX *curr_sel= thd->lex->current_select;
-  if (curr_sel && !curr_sel->name_visibility_map)
+  if (curr_sel && curr_sel->name_visibility_map.is_clear_all())
   {
     for (SELECT_LEX *sl= curr_sel; sl; sl= sl->context.outer_select())
     {
-      curr_sel->name_visibility_map|= (1 << sl-> nest_level);
+      curr_sel->name_visibility_map.set_bit(sl->nest_level);
     }
   }
-  if (!curr_sel || !(thd->lex->allow_sum_func & curr_sel->name_visibility_map))
+  if (!curr_sel ||
+      !(thd->lex->allow_sum_func.is_overlapping(curr_sel->name_visibility_map)))
   {
     my_message(ER_INVALID_GROUP_FUNC_USE, ER_THD(thd, ER_INVALID_GROUP_FUNC_USE),
                MYF(0));
@@ -155,10 +156,11 @@ bool Item_sum::init_sum_func_check(THD *thd)
 bool Item_sum::check_sum_func(THD *thd, Item **ref)
 {
   SELECT_LEX *curr_sel= thd->lex->current_select;
-  nesting_map allow_sum_func= (thd->lex->allow_sum_func &
-                               curr_sel->name_visibility_map);
+  nesting_map allow_sum_func(thd->lex->allow_sum_func);
+  allow_sum_func.intersect(curr_sel->name_visibility_map);
   bool invalid= FALSE;
-  DBUG_ASSERT(curr_sel->name_visibility_map); // should be set already
+  // should be set already
+  DBUG_ASSERT(!curr_sel->name_visibility_map.is_clear_all());
 
   /*
      Window functions can not be used as arguments to sum functions.
@@ -189,10 +191,10 @@ bool Item_sum::check_sum_func(THD *thd, Item **ref)
       If it is there under a construct where it is not allowed 
       we report an error. 
     */ 
-    invalid= !(allow_sum_func & ((nesting_map)1 << max_arg_level));
+    invalid= !(allow_sum_func.is_set(max_arg_level));
   }
   else if (max_arg_level >= 0 ||
-           !(allow_sum_func & ((nesting_map)1 << nest_level)))
+           !(allow_sum_func.is_set(nest_level)))
   {
     /*
       The set function can be aggregated only in outer subqueries.
@@ -202,7 +204,7 @@ bool Item_sum::check_sum_func(THD *thd, Item **ref)
     if (register_sum_func(thd, ref))
       return TRUE;
     invalid= aggr_level < 0 &&
-             !(allow_sum_func & ((nesting_map)1 << nest_level));
+             !(allow_sum_func.is_set(nest_level));
     if (!invalid && thd->variables.sql_mode & MODE_ANSI)
       invalid= aggr_level < 0 && max_arg_level < nest_level;
   }
@@ -354,14 +356,14 @@ bool Item_sum::register_sum_func(THD *thd, Item **ref)
        sl= sl->context.outer_select())
   {
     if (aggr_level < 0 &&
-        (allow_sum_func & ((nesting_map)1 << sl->nest_level)))
+        (allow_sum_func.is_set(sl->nest_level)))
     {
       /* Found the most nested subquery where the function can be aggregated */
       aggr_level= sl->nest_level;
       aggr_sel= sl;
     }
   }
-  if (sl && (allow_sum_func & ((nesting_map)1 << sl->nest_level)))
+  if (sl && (allow_sum_func.is_set(sl->nest_level)))
   {
     /* 
       We reached the subquery of level max_arg_level and checked
diff --git a/sql/sql_base.cc b/sql/sql_base.cc
index 52259e4b9e2..cf9a15dec68 100644
--- a/sql/sql_base.cc
+++ b/sql/sql_base.cc
@@ -7372,8 +7372,7 @@ bool setup_fields(THD *thd, Ref_ptr_array ref_pointer_array,
   thd->column_usage= column_usage;
   DBUG_PRINT("info", ("thd->column_usage: %d", thd->column_usage));
   if (allow_sum_func)
-    thd->lex->allow_sum_func|=
-      (nesting_map)1 << thd->lex->current_select->nest_level;
+    thd->lex->allow_sum_func.set_bit(thd->lex->current_select->nest_level);
   thd->where= THD::DEFAULT_WHERE;
   save_is_item_list_lookup= thd->lex->current_select->is_item_list_lookup;
   thd->lex->current_select->is_item_list_lookup= 0;
diff --git a/sql/sql_const.h b/sql/sql_const.h
index e28a0649f04..be26de872df 100644
--- a/sql/sql_const.h
+++ b/sql/sql_const.h
@@ -84,7 +84,7 @@
 #define MAX_FIELDS	4096			/* Limit in the .frm file */
 #define MAX_PARTITIONS  8192
 
-#define MAX_SELECT_NESTING (sizeof(nesting_map)*8-1)
+#define MAX_SELECT_NESTING (SELECT_NESTING_MAP_SIZE - 1)
 
 #define MAX_SORT_MEMORY 2048*1024
 #define MIN_SORT_MEMORY 1024
diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc
index fc400252397..6cf89ed09fc 100644
--- a/sql/sql_delete.cc
+++ b/sql/sql_delete.cc
@@ -936,7 +936,7 @@ int mysql_prepare_delete(THD *thd, TABLE_LIST *table_list,
   List<Item> all_fields;
 
   *delete_while_scanning= true;
-  thd->lex->allow_sum_func= 0;
+  thd->lex->allow_sum_func.clear_all();
   if (setup_tables_and_check_access(thd, &thd->lex->select_lex.context,
                                     &thd->lex->select_lex.top_join_list,
                                     table_list, 
diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc
index 11bb50a4231..e5bfccfed29 100644
--- a/sql/sql_lex.cc
+++ b/sql/sql_lex.cc
@@ -734,7 +734,7 @@ void LEX::start(THD *thd_arg)
   profile_options= PROFILE_NONE;
   nest_level=0 ;
   select_lex.nest_level_base= &unit;
-  allow_sum_func= 0;
+  allow_sum_func.clear_all();
   in_sum_func= NULL;
 
   used_tables= 0;
@@ -2335,7 +2335,7 @@ void st_select_lex::init_select()
   m_non_agg_field_used= false;
   m_agg_func_used= false;
   m_custom_agg_func_used= false;
-  name_visibility_map= 0;
+  name_visibility_map.clear_all();
   with_dep= 0;
   join= 0;
   lock_type= TL_READ_DEFAULT;
diff --git a/sql/sql_lex.h b/sql/sql_lex.h
index 48363fdd670..ea2f61c3992 100644
--- a/sql/sql_lex.h
+++ b/sql/sql_lex.h
@@ -33,6 +33,10 @@
 #include "sql_tvc.h"
 #include "item.h"
 
+/* Used for flags of nesting constructs */
+#define SELECT_NESTING_MAP_SIZE 64
+typedef Bitmap<SELECT_NESTING_MAP_SIZE> nesting_map;
+
 /* YACC and LEX Definitions */
 
 
diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc
index 151b94d1187..9e6c333d3c9 100644
--- a/sql/sql_partition.cc
+++ b/sql/sql_partition.cc
@@ -857,7 +857,7 @@ static bool fix_fields_part_func(THD *thd, Item* func_expr, TABLE *table,
     const bool save_agg_field= thd->lex->current_select->non_agg_field_used();
     const bool save_agg_func=  thd->lex->current_select->agg_func_used();
     const nesting_map saved_allow_sum_func= thd->lex->allow_sum_func;
-    thd->lex->allow_sum_func= 0;
+    thd->lex->allow_sum_func.clear_all();
 
     if (likely(!(error= func_expr->fix_fields_if_needed(thd, (Item**)&func_expr))))
       func_expr->walk(&Item::post_fix_fields_part_expr_processor, 0, NULL);
diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc
index c4871bdcc80..5324b2dd305 100644
--- a/sql/sql_prepare.cc
+++ b/sql/sql_prepare.cc
@@ -3041,7 +3041,7 @@ void reinit_stmt_before_use(THD *thd, LEX *lex)
     lex->result->cleanup();
     lex->result->set_thd(thd);
   }
-  lex->allow_sum_func= 0;
+  lex->allow_sum_func.clear_all();
   lex->in_sum_func= NULL;
   DBUG_VOID_RETURN;
 }
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index f1230666026..f3e69ba2ead 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -645,7 +645,7 @@ setup_without_group(THD *thd, Ref_ptr_array ref_pointer_array,
   const bool saved_non_agg_field_used= select->non_agg_field_used();
   DBUG_ENTER("setup_without_group");
 
-  thd->lex->allow_sum_func&= ~((nesting_map)1 << select->nest_level);
+  thd->lex->allow_sum_func.clear_bit(select->nest_level);
   res= setup_conds(thd, tables, leaves, conds);
   if (thd->lex->current_select->first_cond_optimization)
   {
@@ -658,18 +658,18 @@ setup_without_group(THD *thd, Ref_ptr_array ref_pointer_array,
   /* it's not wrong to have non-aggregated columns in a WHERE */
   select->set_non_agg_field_used(saved_non_agg_field_used);
 
-  thd->lex->allow_sum_func|= (nesting_map)1 << select->nest_level;
+  thd->lex->allow_sum_func.set_bit(select->nest_level);
   
   save_place= thd->lex->current_select->context_analysis_place;
   thd->lex->current_select->context_analysis_place= IN_ORDER_BY;
   res= res || setup_order(thd, ref_pointer_array, tables, fields, all_fields,
                           order);
-  thd->lex->allow_sum_func&= ~((nesting_map)1 << select->nest_level);
+  thd->lex->allow_sum_func.clear_bit(select->nest_level);
   thd->lex->current_select->context_analysis_place= IN_GROUP_BY;
   res= res || setup_group(thd, ref_pointer_array, tables, fields, all_fields,
                           group, hidden_group_fields);
   thd->lex->current_select->context_analysis_place= save_place;
-  thd->lex->allow_sum_func|= (nesting_map)1 << select->nest_level;
+  thd->lex->allow_sum_func.set_bit(select->nest_level);
   res= res || setup_windows(thd, ref_pointer_array, tables, fields, all_fields,
                             win_specs, win_funcs);
   thd->lex->allow_sum_func= save_allow_sum_func;
@@ -1117,7 +1117,7 @@ JOIN::prepare(TABLE_LIST *tables_init,
                        select_lex->master_unit()->global_parameters())
   {
     nesting_map save_allow_sum_func= thd->lex->allow_sum_func;
-    thd->lex->allow_sum_func|= (nesting_map)1 << select_lex->nest_level;
+    thd->lex->allow_sum_func.set_bit(select_lex->nest_level);
     thd->where= "order clause";
     for (ORDER *order= select_lex->order_list.first; order; order= order->next)
     {
@@ -1135,7 +1135,7 @@ JOIN::prepare(TABLE_LIST *tables_init,
   {
     nesting_map save_allow_sum_func= thd->lex->allow_sum_func;
     thd->where="having clause";
-    thd->lex->allow_sum_func|= (nesting_map)1 << select_lex_arg->nest_level;
+    thd->lex->allow_sum_func.set_bit(select_lex_arg->nest_level);
     select_lex->having_fix_field= 1;
     /*
       Wrap alone field in HAVING clause in case it will be outer field
diff --git a/sql/sql_update.cc b/sql/sql_update.cc
index 7a496172aed..1651faf7c87 100644
--- a/sql/sql_update.cc
+++ b/sql/sql_update.cc
@@ -1255,7 +1255,7 @@ bool mysql_prepare_update(THD *thd, TABLE_LIST *table_list,
   table_list->register_want_access(SELECT_ACL);
 #endif
 
-  thd->lex->allow_sum_func= 0;
+  thd->lex->allow_sum_func.clear_all();
 
   /*
     We do not call DT_MERGE_FOR_INSERT because it has no sense for simple