Bug #4797 - 32 bit and 64 bit builds behave differently on int32 overflow

include/my_global.h:
  uint_max constants moved from sql_analyse.cc
sql/sql_analyse.cc:
  cleanup

5 files changed, 47 insertions, 16 deletions

diff --git a/include/my_global.h b/include/my_global.h
index 284cfdc1f97..33ae35d2308 100644
--- a/include/my_global.h
+++ b/include/my_global.h
@@ -642,21 +642,27 @@ extern double		my_atof(const char*);
 #endif /* defined (HAVE_LONG_LONG) && !defined(ULONGLONG_MAX)*/
 
 #if SIZEOF_LONG == 4
-#define INT_MIN32	(long) 0x80000000L
-#define INT_MAX32	(long) 0x7FFFFFFFL
-#define INT_MIN24	((long) 0xff800000L)
-#define INT_MAX24	0x007fffffL
-#define INT_MIN16	((short int) 0x8000)
-#define INT_MAX16	0x7FFF
-#define INT_MIN8	((char) 0x80)
-#define INT_MAX8	((char) 0x7F)
+#define INT_MIN32       ((long) 0x80000000L)
+#define INT_MAX32       ((long) 0x7FFFFFFFL)
+#define UINT_MAX32      ((long) 0xFFFFFFFFL)
+#define INT_MIN24       ((long) 0xFF800000L)
+#define INT_MAX24       0x007FFFFFL
+#define UINT_MAX24      0x00FFFFFFL
+#define INT_MIN16       ((short int) 0x8000)
+#define INT_MAX16       0x7FFF
+#define UINT_MAX16      0xFFFF
+#define INT_MIN8        ((char) 0x80)
+#define INT_MAX8        ((char) 0x7F)
 #else  /* Probably Alpha */
 #define INT_MIN32	((long) (int) 0x80000000)
 #define INT_MAX32	((long) (int) 0x7FFFFFFF)
-#define INT_MIN24	((long) (int) 0xff800000)
-#define INT_MAX24	((long) (int) 0x007fffff)
-#define INT_MIN16	((short int) 0xffff8000)
+#define UINT_MAX32	((long) (int) 0xFFFFFFFF)
+#define INT_MIN24	((long) (int) 0xFF800000)
+#define INT_MAX24	((long) (int) 0x007FFFFF)
+#define UINT_MAX24	((long) (int) 0x00FFFFFF)
+#define INT_MIN16	((short int) 0xFFFF8000)
 #define INT_MAX16	((short int) 0x00007FFF)
+#define UINT_MAX16      ((short int) 0x0000FFFF)
 #endif
 
 /* From limits.h instead */
diff --git a/mysql-test/r/type_uint.result b/mysql-test/r/type_uint.result
index 1acfc700d3a..0b7452b566b 100644
--- a/mysql-test/r/type_uint.result
+++ b/mysql-test/r/type_uint.result
@@ -2,8 +2,10 @@ drop table if exists t1;
 create table t1 (this int unsigned);
 insert into t1 values (1);
 insert into t1 values (-1);
+insert into t1 values ('5000000000');
 select * from t1;
 this
 1
 0
+4294967295
 drop table t1;
diff --git a/mysql-test/t/type_uint.test b/mysql-test/t/type_uint.test
index 32bcd61ecdb..7eb48ae21ac 100644
--- a/mysql-test/t/type_uint.test
+++ b/mysql-test/t/type_uint.test
@@ -6,5 +6,6 @@ drop table if exists t1;
 create table t1 (this int unsigned);
 insert into t1 values (1);
 insert into t1 values (-1);
+insert into t1 values ('5000000000');
 select * from t1;
 drop table t1;
diff --git a/sql/field.cc b/sql/field.cc
index e3bdf78e718..946f5ed8621 100644
--- a/sql/field.cc
+++ b/sql/field.cc
@@ -1504,7 +1504,7 @@ void Field_long::store(const char *from,uint len)
   {
     len--; from++;
   }
-  long tmp;
+  long tmp, cuted_fields=0;
   String tmp_str(from,len);
   from= tmp_str.c_ptr();			// Add end null if needed
   errno=0;
@@ -1520,9 +1520,34 @@ void Field_long::store(const char *from,uint len)
   }
   else
     tmp=strtol(from, &end, 10);
-  if (errno || 
+  if (errno ||
       (from+len != end && current_thd->count_cuted_fields &&
        !test_if_int(from,len)))
+    cuted_fields=1;
+#if SIZEOF_LONG > 4
+  if (unsigned_flag)
+  {
+    if (tmp > UINT_MAX32)
+    {
+      tmp= UINT_MAX32;
+      cuted_fields=1;
+    }
+  }
+  else
+  {
+    if (tmp > INT_MAX32)
+    {
+      tmp= INT_MAX32;
+      cuted_fields=1;
+    }
+    else if (tmp < INT_MIN32)
+    {
+      tmp= INT_MIN32;
+      cuted_fields=1;
+    }
+  }
+#endif
+  if (cuted_fields)
     current_thd->cuted_fields++;
 #ifdef WORDS_BIGENDIAN
   if (table->db_low_byte_first)
diff --git a/sql/sql_analyse.cc b/sql/sql_analyse.cc
index bd8c0e5ba87..3847849d6a7 100644
--- a/sql/sql_analyse.cc
+++ b/sql/sql_analyse.cc
@@ -34,9 +34,6 @@
 
 #define MAX_TREEMEM	  8192
 #define MAX_TREE_ELEMENTS 256
-#define UINT_MAX16	  0xffff
-#define UINT_MAX24	  0xffffff
-#define UINT_MAX32	  0xffffffff
 
 int sortcmp2(void* cmp_arg __attribute__((unused)),
 	     const String *a,const String *b)