MDEV-11917 enum/set command-line options aren't respecting max-*

settings.

7 files changed, 155 insertions, 56 deletions

diff --git a/mysql-test/suite/sys_vars/r/maximum_basic.result b/mysql-test/suite/sys_vars/r/maximum_basic.result
index 20b6bbc962e..becd6da91f4 100644
--- a/mysql-test/suite/sys_vars/r/maximum_basic.result
+++ b/mysql-test/suite/sys_vars/r/maximum_basic.result
@@ -16,3 +16,33 @@ Warning	1292	Truncated incorrect max_join_size value: '40960'
 SELECT @@session.max_join_size;
 @@session.max_join_size
 8192
+SET @@session.use_stat_tables= COMPLEMENTARY;
+SELECT @@session.use_stat_tables;
+@@session.use_stat_tables
+COMPLEMENTARY
+SET @@session.use_stat_tables= PREFERABLY;
+Warnings:
+Warning	1292	Truncated incorrect use_stat_tables value: 'PREFERABLY'
+SELECT @@session.use_stat_tables;
+@@session.use_stat_tables
+COMPLEMENTARY
+SET @@session.use_stat_tables= 2;
+Warnings:
+Warning	1292	Truncated incorrect use_stat_tables value: '2'
+SELECT @@session.use_stat_tables;
+@@session.use_stat_tables
+COMPLEMENTARY
+SET @@session.sql_mode= 'REAL_AS_FLOAT';
+SELECT @@session.sql_mode;
+@@session.sql_mode
+REAL_AS_FLOAT
+SET @@session.sql_mode= 'REAL_AS_FLOAT,ANSI_QUOTES';
+SELECT @@session.sql_mode;
+@@session.sql_mode
+REAL_AS_FLOAT,ANSI_QUOTES
+SET @@session.sql_mode= 'ANSI_QUOTES,IGNORE_SPACE';
+Warnings:
+Warning	1292	Truncated incorrect sql_mode value: 'ANSI_QUOTES,IGNORE_SPACE'
+SELECT @@session.sql_mode;
+@@session.sql_mode
+ANSI_QUOTES
diff --git a/mysql-test/suite/sys_vars/t/maximum_basic-master.opt b/mysql-test/suite/sys_vars/t/maximum_basic-master.opt
index b6e5666f4fb..16e365d491c 100644
--- a/mysql-test/suite/sys_vars/t/maximum_basic-master.opt
+++ b/mysql-test/suite/sys_vars/t/maximum_basic-master.opt
@@ -1,3 +1,5 @@
 --maximum-auto-increment-increment=8192
 --maximum-tmp-table-size=8192
 --maximum-max-join-size=8192
+--maximum-use-stat-tables=COMPLEMENTARY
+--maximum-sql-mode='REAL_AS_FLOAT,ANSI_QUOTES'
diff --git a/mysql-test/suite/sys_vars/t/maximum_basic.test b/mysql-test/suite/sys_vars/t/maximum_basic.test
index 9961f65883a..3153d62d562 100644
--- a/mysql-test/suite/sys_vars/t/maximum_basic.test
+++ b/mysql-test/suite/sys_vars/t/maximum_basic.test
@@ -18,3 +18,22 @@ SELECT @@session.tmp_table_size;
 SET @@session.max_join_size=40960;
 SELECT @@session.max_join_size;
 
+#
+# enum
+#
+SET @@session.use_stat_tables= COMPLEMENTARY;
+SELECT @@session.use_stat_tables;
+SET @@session.use_stat_tables= PREFERABLY;
+SELECT @@session.use_stat_tables;
+SET @@session.use_stat_tables= 2;
+SELECT @@session.use_stat_tables;
+
+#
+# set
+#
+SET @@session.sql_mode= 'REAL_AS_FLOAT';
+SELECT @@session.sql_mode;
+SET @@session.sql_mode= 'REAL_AS_FLOAT,ANSI_QUOTES';
+SELECT @@session.sql_mode;
+SET @@session.sql_mode= 'ANSI_QUOTES,IGNORE_SPACE';
+SELECT @@session.sql_mode;
diff --git a/sql/set_var.cc b/sql/set_var.cc
index e96e636e3d3..77036fd0f5a 100644
--- a/sql/set_var.cc
+++ b/sql/set_var.cc
@@ -452,6 +452,22 @@ void sys_var::do_deprecated_warning(THD *thd)
 
   @retval         true on error, false otherwise (warning or ok)
  */
+
+
+bool throw_bounds_warning(THD *thd, const char *name,const char *v)
+{
+  if (thd->variables.sql_mode & MODE_STRICT_ALL_TABLES)
+  {
+    my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), name, v);
+    return true;
+  }
+  push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
+                      ER_TRUNCATED_WRONG_VALUE,
+                      ER_THD(thd, ER_TRUNCATED_WRONG_VALUE), name, v);
+  return false;
+}
+
+
 bool throw_bounds_warning(THD *thd, const char *name,
                           bool fixed, bool is_unsigned, longlong v)
 {
@@ -469,9 +485,7 @@ bool throw_bounds_warning(THD *thd, const char *name,
       my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), name, buf);
       return true;
     }
-    push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
-                        ER_TRUNCATED_WRONG_VALUE,
-                        ER_THD(thd, ER_TRUNCATED_WRONG_VALUE), name, buf);
+    return throw_bounds_warning(thd, name, buf);
   }
   return false;
 }
@@ -489,9 +503,7 @@ bool throw_bounds_warning(THD *thd, const char *name, bool fixed, double v)
       my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), name, buf);
       return true;
     }
-    push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
-                        ER_TRUNCATED_WRONG_VALUE,
-                        ER_THD(thd, ER_TRUNCATED_WRONG_VALUE), name, buf);
+    return throw_bounds_warning(thd, name, buf);
   }
   return false;
 }
diff --git a/sql/set_var.h b/sql/set_var.h
index 7a48e319466..8dd2cb073b7 100644
--- a/sql/set_var.h
+++ b/sql/set_var.h
@@ -243,6 +243,12 @@ protected:
   uchar *global_var_ptr()
   { return ((uchar*)&global_system_variables) + offset; }
 
+  void *max_var_ptr()
+  {
+    return scope() == SESSION ? (((uchar*)&max_system_variables) + offset) :
+                                0;
+  }
+
   friend class Session_sysvars_tracker;
   friend class Session_tracker;
 };
diff --git a/sql/sys_vars.ic b/sql/sys_vars.ic
index bc913b1adbd..b92867f98e1 100644
--- a/sql/sys_vars.ic
+++ b/sql/sys_vars.ic
@@ -142,9 +142,10 @@ public:
     option.min_value= min_val;
     option.max_value= max_val;
     option.block_size= block_size;
-    option.u_max_value= (uchar**)max_var_ptr();
-    if (max_var_ptr())
-      *max_var_ptr()= max_val;
+    if ((option.u_max_value= (uchar**) max_var_ptr()))
+    {
+      *((T*) option.u_max_value)= max_val;
+    }
 
     global_var(T)= def_val;
     SYSVAR_ASSERT(size == sizeof(T));
@@ -176,8 +177,8 @@ public:
       var->save_result.ulonglong_value=
         getopt_ull_limit_value(uv, &option, &unused);
 
-      if (max_var_ptr() && (T)var->save_result.ulonglong_value > *max_var_ptr())
-        var->save_result.ulonglong_value= *max_var_ptr();
+      if (max_var_ptr() && (T)var->save_result.ulonglong_value > get_max_var())
+        var->save_result.ulonglong_value= get_max_var();
 
       fixed= fixed || var->save_result.ulonglong_value != uv;
     }
@@ -193,8 +194,8 @@ public:
       var->save_result.longlong_value=
         getopt_ll_limit_value(v, &option, &unused);
 
-      if (max_var_ptr() && (T)var->save_result.longlong_value > *max_var_ptr())
-        var->save_result.longlong_value= *max_var_ptr();
+      if (max_var_ptr() && (T)var->save_result.longlong_value > get_max_var())
+        var->save_result.longlong_value= get_max_var();
 
       fixed= fixed || var->save_result.longlong_value != v;
     }
@@ -216,11 +217,7 @@ public:
   void global_save_default(THD *thd, set_var *var)
   { var->save_result.ulonglong_value= option.def_value; }
   private:
-  T *max_var_ptr()
-  {
-    return scope() == SESSION ? (T*)(((uchar*)&max_system_variables) + offset)
-                              : 0;
-  }
+  T get_max_var() { return *((T*) max_var_ptr()); }
   uchar *default_value_ptr(THD *thd) { return (uchar*) &option.def_value; }
 };
 
@@ -264,6 +261,9 @@ class Sys_var_typelib: public sys_var
 {
 protected:
   TYPELIB typelib;
+  virtual bool check_maximum(THD *thd, set_var *var,
+                             const char *c_val, longlong i_val)
+    { return FALSE; }
 public:
   Sys_var_typelib(const char *name_arg,
           const char *comment, int flag_args, ptrdiff_t off,
@@ -299,17 +299,14 @@ public:
         return true;
       else
         var->save_result.ulonglong_value--;
-    }
-    else
-    {
-      longlong tmp=var->value->val_int();
-      if (tmp < 0 || tmp >= typelib.count)
-        return true;
-      else
-        var->save_result.ulonglong_value= tmp;
+      return check_maximum(thd, var, res->ptr(), 0);
     }
 
-    return false;
+    longlong tmp=var->value->val_int();
+    if (tmp < 0 || tmp >= typelib.count)
+      return true;
+    var->save_result.ulonglong_value= tmp;
+    return check_maximum(thd, var, 0, tmp);
   }
 };
 
@@ -345,9 +342,25 @@ public:
   {
     option.var_type|= GET_ENUM;
     global_var(ulong)= def_val;
+    if ((option.u_max_value= (uchar**)max_var_ptr()))
+    {
+      *((ulong *) option.u_max_value)= ULONG_MAX;
+    }
     SYSVAR_ASSERT(def_val < typelib.count);
     SYSVAR_ASSERT(size == sizeof(ulong));
   }
+  bool check_maximum(THD *thd, set_var *var,
+                     const char *c_val, longlong i_val)
+  {
+    if (!max_var_ptr() ||
+        var->save_result.ulonglong_value <= get_max_var())
+      return FALSE;
+    var->save_result.ulonglong_value= get_max_var();
+
+    return c_val ? throw_bounds_warning(thd, name.str, c_val) :
+                   throw_bounds_warning(thd, name.str, TRUE,
+                                        var->value->unsigned_flag, i_val);
+  }
   bool session_update(THD *thd, set_var *var)
   {
     session_var(thd, ulong)= static_cast<ulong>(var->save_result.ulonglong_value);
@@ -370,6 +383,8 @@ public:
   { return valptr(thd, global_var(ulong)); }
   uchar *default_value_ptr(THD *thd)
   { return valptr(thd, (ulong)option.def_value); }
+
+  ulong get_max_var() { return *((ulong *) max_var_ptr()); }
 };
 
 /**
@@ -1335,11 +1350,27 @@ public:
   {
     option.var_type|= GET_SET;
     global_var(ulonglong)= def_val;
+    if ((option.u_max_value= (uchar**)max_var_ptr()))
+    {
+      *((ulonglong*) option.u_max_value)= ~0ULL;
+    }
     SYSVAR_ASSERT(typelib.count > 0);
     SYSVAR_ASSERT(typelib.count <= 64);
     SYSVAR_ASSERT(def_val <= my_set_bits(typelib.count));
     SYSVAR_ASSERT(size == sizeof(ulonglong));
   }
+  bool check_maximum(THD *thd, set_var *var,
+                     const char *c_val, longlong i_val)
+  {
+    if (!max_var_ptr() ||
+        (var->save_result.ulonglong_value & ~(get_max_var())) == 0)
+      return FALSE;
+    var->save_result.ulonglong_value&= get_max_var();
+
+    return c_val ? throw_bounds_warning(thd, name.str, c_val) :
+                   throw_bounds_warning(thd, name.str, TRUE,
+                                        var->value->unsigned_flag, i_val);
+  }
   bool do_check(THD *thd, set_var *var)
   {
     char buff[STRING_BUFFER_USUAL_SIZE];
@@ -1347,41 +1378,37 @@ public:
 
     if (var->value->result_type() == STRING_RESULT)
     {
+      char *error;
+      uint error_len;
+      bool not_used;
+
       if (!(res=var->value->val_str_ascii(&str)))
         return true;
-      else
-      {
-        char *error;
-        uint error_len;
-        bool not_used;
 
-        var->save_result.ulonglong_value=
-              find_set(&typelib, res->ptr(), res->length(), NULL,
-                      &error, &error_len, &not_used);
-        /*
-          note, we only issue an error if error_len > 0.
-          That is even while empty (zero-length) values are considered
-          errors by find_set(), these errors are ignored here
-        */
-        if (error_len)
-        {
-          ErrConvString err(error, error_len, res->charset());
-          my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), name.str, err.ptr());
-          return true;
-        }
-      }
-    }
-    else
-    {
-      longlong tmp=var->value->val_int();
-      if ((tmp < 0 && ! var->value->unsigned_flag)
-          || (ulonglong)tmp > my_set_bits(typelib.count))
+      var->save_result.ulonglong_value=
+            find_set(&typelib, res->ptr(), res->length(), NULL,
+                    &error, &error_len, &not_used);
+      /*
+        note, we only issue an error if error_len > 0.
+        That is even while empty (zero-length) values are considered
+        errors by find_set(), these errors are ignored here
+      */
+      if (error_len)
+      {
+        ErrConvString err(error, error_len, res->charset());
+        my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), name.str, err.ptr());
         return true;
-      else
-        var->save_result.ulonglong_value= tmp;
+      }
+      return check_maximum(thd, var, res->ptr(), 0);
     }
 
-    return false;
+    longlong tmp=var->value->val_int();
+    if ((tmp < 0 && ! var->value->unsigned_flag)
+        || (ulonglong)tmp > my_set_bits(typelib.count))
+      return true;
+
+    var->save_result.ulonglong_value= tmp;
+    return check_maximum(thd, var, 0, tmp);
   }
   bool session_update(THD *thd, set_var *var)
   {
@@ -1405,6 +1432,8 @@ public:
   { return valptr(thd, global_var(ulonglong)); }
   uchar *default_value_ptr(THD *thd)
   { return valptr(thd, option.def_value); }
+
+  ulonglong get_max_var() { return *((ulonglong*) max_var_ptr()); }
 };
 
 /**
diff --git a/sql/sys_vars_shared.h b/sql/sys_vars_shared.h
index ff050f63064..dfc020a187c 100644
--- a/sql/sys_vars_shared.h
+++ b/sql/sys_vars_shared.h
@@ -28,6 +28,7 @@
 #include <sql_priv.h>
 #include "set_var.h"
 
+extern bool throw_bounds_warning(THD *thd, const char *name,const char *v);
 extern bool throw_bounds_warning(THD *thd, const char *name,
                                  bool fixed, bool is_unsigned, longlong v);
 extern bool throw_bounds_warning(THD *thd, const char *name, bool fixed,