MDEV-5905: Creating tmp. memory table kills the server

The reason was that a couple of variables that hold number of rows that was used to calculate buffers was uint and caused an overflow.

Fixed by changing variables that could hold number of rows from uint to ulong and also added a cast for this test.

include/heap.h:
  Reorder to get better alignment. Changed variables that could hold number of rows from uint to ulong
mysql-test/suite/heap/heap.result:
  Added test case
mysql-test/suite/heap/heap.test:
  Added test case
mysql-test/suite/plugins/t/server_audit.test:
  Added sleep as we want to have disconnect logged before we try a new connect
storage/heap/ha_heap.cc:
  Changed variables that could hold number of rows from uint to ulong
  Limit number of rows to 4G  (as most of the variables that holds rows are ulong anyway)
  reset records_changed when key_stat_version is changed to not cause increments for every row changed
storage/heap/ha_heap.h:
  changed records_changed to ulong as this can get big
storage/heap/hp_create.c:
  Changed variables that could hold number of rows from uint to ulong
  Added cast (fixed the original bug)
storage/heap/hp_delete.c:
  Changed variables that could hold number of rows from uint to ulong
storage/heap/hp_open.c:
  Removed not needed cast
storage/heap/hp_write.c:
  Changed variables that could hold number of rows from uint to ulong
support-files/compiler_warnings.supp:
  Removed extra : from supression

6 files changed, 13 insertions, 10 deletions

diff --git a/storage/heap/ha_heap.cc b/storage/heap/ha_heap.cc
index 305a8cae720..ec219c1590d 100644
--- a/storage/heap/ha_heap.cc
+++ b/storage/heap/ha_heap.cc
@@ -225,7 +225,7 @@ void ha_heap::update_key_stats()
       else
       {
         ha_rows hash_buckets= file->s->keydef[i].hash_buckets;
-        uint no_records= hash_buckets ? (uint) (file->s->records/hash_buckets) : 2;
+        ha_rows no_records= hash_buckets ? (file->s->records/hash_buckets) : 2;
         if (no_records < 2)
           no_records= 2;
         key->rec_per_key[key->key_parts-1]= no_records;
@@ -256,6 +256,7 @@ int ha_heap::write_row(uchar * buf)
        We can perform this safely since only one writer at the time is
        allowed on the table.
     */
+    records_changed= 0;
     file->s->key_stat_version++;
   }
   return res;
@@ -274,6 +275,7 @@ int ha_heap::update_row(const uchar * old_data, uchar * new_data)
        We can perform this safely since only one writer at the time is
        allowed on the table.
     */
+    records_changed= 0;
     file->s->key_stat_version++;
   }
   return res;
@@ -290,6 +292,7 @@ int ha_heap::delete_row(const uchar * buf)
        We can perform this safely since only one writer at the time is
        allowed on the table.
     */
+    records_changed= 0;
     file->s->key_stat_version++;
   }
   return res;
@@ -740,8 +743,8 @@ heap_prepare_hp_create_info(TABLE *table_arg, bool internal_table,
   if (share->max_rows && share->max_rows < max_rows)
     max_rows= share->max_rows;
 
-  hp_create_info->max_records= (ulong) max_rows;
-  hp_create_info->min_records= (ulong) share->min_rows;
+  hp_create_info->max_records= (ulong) min(max_rows, ULONG_MAX);
+  hp_create_info->min_records= (ulong) min(share->min_rows, ULONG_MAX);
   hp_create_info->keys= share->keys;
   hp_create_info->reclength= share->reclength;
   hp_create_info->keydef= keydef;
diff --git a/storage/heap/ha_heap.h b/storage/heap/ha_heap.h
index c5b8a09b216..9216e2e45b4 100644
--- a/storage/heap/ha_heap.h
+++ b/storage/heap/ha_heap.h
@@ -31,7 +31,7 @@ class ha_heap: public handler
   HP_SHARE *internal_share;
   key_map btree_keys;
   /* number of records changed since last statistics update */
-  uint    records_changed;
+  ulong   records_changed;
   uint    key_stat_version;
   my_bool internal_table;
 public:
diff --git a/storage/heap/hp_create.c b/storage/heap/hp_create.c
index d170d1abc65..5e0caf2fa24 100644
--- a/storage/heap/hp_create.c
+++ b/storage/heap/hp_create.c
@@ -243,7 +243,7 @@ static int keys_compare(heap_rb_param *param, uchar *key1, uchar *key2)
 static void init_block(HP_BLOCK *block, uint reclength, ulong min_records,
 		       ulong max_records)
 {
-  uint i,recbuffer,records_in_block;
+  ulong i,recbuffer,records_in_block;
 
   /*
     If not min_records and max_records are given, optimize for 1000 rows
@@ -271,7 +271,7 @@ static void init_block(HP_BLOCK *block, uint reclength, ulong min_records,
     The + 1 is there to ensure that we get at least 1 row per level (for
     the exceptional case of very long rows)
   */
-  if (records_in_block*recbuffer >
+  if ((ulonglong) records_in_block*recbuffer >
       (my_default_record_cache_size-sizeof(HP_PTRS)*HP_MAX_LEVELS))
     records_in_block= (my_default_record_cache_size - sizeof(HP_PTRS) *
                        HP_MAX_LEVELS) / recbuffer + 1;
diff --git a/storage/heap/hp_delete.c b/storage/heap/hp_delete.c
index fe71b14a30a..1cbfe7408d4 100644
--- a/storage/heap/hp_delete.c
+++ b/storage/heap/hp_delete.c
@@ -68,7 +68,7 @@ int hp_rb_delete_key(HP_INFO *info, register HP_KEYDEF *keyinfo,
 		   const uchar *record, uchar *recpos, int flag)
 {
   heap_rb_param custom_arg;
-  uint old_allocated;
+  ulong old_allocated;
   int res;
 
   if (flag) 
diff --git a/storage/heap/hp_open.c b/storage/heap/hp_open.c
index c456dbdfc84..854a8b05698 100644
--- a/storage/heap/hp_open.c
+++ b/storage/heap/hp_open.c
@@ -30,7 +30,7 @@ HP_INFO *heap_open_from_share(HP_SHARE *share, int mode)
   HP_INFO *info;
   DBUG_ENTER("heap_open_from_share");
 
-  if (!(info= (HP_INFO*) my_malloc((uint) sizeof(HP_INFO) +
+  if (!(info= (HP_INFO*) my_malloc(sizeof(HP_INFO) +
 				  2 * share->max_key_length,
 				  MYF(MY_ZEROFILL))))
   {
@@ -47,7 +47,7 @@ HP_INFO *heap_open_from_share(HP_SHARE *share, int mode)
 #ifndef DBUG_OFF
   info->opt_flag= READ_CHECK_USED;		/* Check when changing */
 #endif
-  DBUG_PRINT("exit",("heap: 0x%lx  reclength: %d  records_in_block: %d",
+  DBUG_PRINT("exit",("heap: 0x%lx  reclength: %d  records_in_block: %lu",
 		     (long) info, share->reclength,
                      share->block.records_in_block));
   DBUG_RETURN(info);
diff --git a/storage/heap/hp_write.c b/storage/heap/hp_write.c
index 8bfecc7747c..a7543ea5768 100644
--- a/storage/heap/hp_write.c
+++ b/storage/heap/hp_write.c
@@ -400,7 +400,7 @@ int hp_write_key(HP_INFO *info, HP_KEYDEF *keyinfo,
 static HASH_INFO *hp_find_free_hash(HP_SHARE *info,
 				     HP_BLOCK *block, ulong records)
 {
-  uint block_pos;
+  ulong block_pos;
   size_t length;
 
   if (records < block->last_allocated)