Fixed MDEV-5724 "Server crashes on SQL select containing more group by and left join statements using innodb tables"

The problem was that a big record was allocated on the stack, which casued stack to run out.

Fixed by using my_safe_alloca() instead of my_alloca() when allocating records.
Now only records <= 16384 are allocated on the stack.

mysql-test/r/stack-crash.result:
  Added test case
mysql-test/t/stack-crash.test:
  Added test case
storage/maria/ma_blockrec.c:
  Use my_safe_alloca() instead of my_alloca()
storage/maria/ma_dynrec.c:
  Use my_safe_alloca() instead of my_alloca()
storage/maria/maria_def.h:
  Added MARIA_MAX_RECORD_ON_STACK
storage/maria/maria_pack.c:
  Use my_safe_alloca() instead of my_alloca()

4 files changed, 33 insertions, 21 deletions

diff --git a/storage/maria/ma_blockrec.c b/storage/maria/ma_blockrec.c
index c6371287063..7e996d0ced8 100644
--- a/storage/maria/ma_blockrec.c
+++ b/storage/maria/ma_blockrec.c
@@ -5144,7 +5144,12 @@ my_bool _ma_cmp_block_unique(MARIA_HA *info, MARIA_UNIQUEDEF *def,
   int error;
   DBUG_ENTER("_ma_cmp_block_unique");
 
-  if (!(old_record= my_alloca(info->s->base.reclength)))
+  /*
+    Don't allocate more than 16K on the stack to ensure we don't get
+    stack overflow.
+  */
+  if (!(old_record= my_safe_alloca(info->s->base.reclength,
+                                   MARIA_MAX_RECORD_ON_STACK)))
     DBUG_RETURN(1);
 
   /* Don't let the compare destroy blobs that may be in use */
@@ -5166,7 +5171,8 @@ my_bool _ma_cmp_block_unique(MARIA_HA *info, MARIA_UNIQUEDEF *def,
     info->rec_buff_size= org_rec_buff_size;
   }
   DBUG_PRINT("exit", ("result: %d", error));
-  my_afree(old_record);
+  my_safe_afree(old_record, info->s->base.reclength,
+                MARIA_MAX_RECORD_ON_STACK);
   DBUG_RETURN(error != 0);
 }
 
diff --git a/storage/maria/ma_dynrec.c b/storage/maria/ma_dynrec.c
index b0280f5278b..a18ea157954 100644
--- a/storage/maria/ma_dynrec.c
+++ b/storage/maria/ma_dynrec.c
@@ -36,12 +36,6 @@ static my_bool delete_dynamic_record(MARIA_HA *info,MARIA_RECORD_POS filepos,
 static my_bool _ma_cmp_buffer(File file, const uchar *buff, my_off_t filepos,
                               uint length);
 
-/* Play it safe; We have a small stack when using threads */
-#undef my_alloca
-#undef my_afree
-#define my_alloca(A) my_malloc((A),MYF(0))
-#define my_afree(A) my_free((A))
-
 	/* Interface function from MARIA_HA */
 
 #ifdef HAVE_MMAP
@@ -256,7 +250,8 @@ my_bool _ma_write_blob_record(MARIA_HA *info, const uchar *record)
 	  MARIA_DYN_DELETE_BLOCK_HEADER+1);
   reclength= (info->s->base.pack_reclength +
 	      _ma_calc_total_blob_length(info,record)+ extra);
-  if (!(rec_buff=(uchar*) my_alloca(reclength)))
+  if (!(rec_buff=(uchar*) my_safe_alloca(reclength,
+                                         MARIA_MAX_RECORD_ON_STACK)))
   {
     my_errno= HA_ERR_OUT_OF_MEM; /* purecov: inspected */
     return(1);
@@ -270,7 +265,7 @@ my_bool _ma_write_blob_record(MARIA_HA *info, const uchar *record)
   error= write_dynamic_record(info,
                               rec_buff+ALIGN_SIZE(MARIA_MAX_DYN_BLOCK_HEADER),
                               reclength2);
-  my_afree(rec_buff);
+  my_safe_afree(rec_buff, reclength, MARIA_MAX_RECORD_ON_STACK);
   return(error != 0);
 }
 
@@ -294,7 +289,8 @@ my_bool _ma_update_blob_record(MARIA_HA *info, MARIA_RECORD_POS pos,
     return 1;
   }
 #endif
-  if (!(rec_buff=(uchar*) my_alloca(reclength)))
+  if (!(rec_buff=(uchar*) my_safe_alloca(reclength,
+                                         MARIA_MAX_RECORD_ON_STACK)))
   {
     my_errno= HA_ERR_OUT_OF_MEM; /* purecov: inspected */
     return(1);
@@ -304,7 +300,7 @@ my_bool _ma_update_blob_record(MARIA_HA *info, MARIA_RECORD_POS pos,
   error=update_dynamic_record(info,pos,
 			      rec_buff+ALIGN_SIZE(MARIA_MAX_DYN_BLOCK_HEADER),
 			      reclength);
-  my_afree(rec_buff);
+  my_safe_afree(rec_buff, reclength, MARIA_MAX_RECORD_ON_STACK);
   return(error != 0);
 }
 
@@ -1559,7 +1555,8 @@ my_bool _ma_cmp_dynamic_unique(MARIA_HA *info, MARIA_UNIQUEDEF *def,
   my_bool error;
   DBUG_ENTER("_ma_cmp_dynamic_unique");
 
-  if (!(old_record=my_alloca(info->s->base.reclength)))
+  if (!(old_record= my_safe_alloca(info->s->base.reclength,
+                                   MARIA_MAX_RECORD_ON_STACK)))
     DBUG_RETURN(1);
 
   /* Don't let the compare destroy blobs that may be in use */
@@ -1580,7 +1577,8 @@ my_bool _ma_cmp_dynamic_unique(MARIA_HA *info, MARIA_UNIQUEDEF *def,
     info->rec_buff=      old_rec_buff;
     info->rec_buff_size= old_rec_buff_size;
   }
-  my_afree(old_record);
+  my_safe_afree(old_record, info->s->base.reclength,
+                MARIA_MAX_RECORD_ON_STACK);
   DBUG_RETURN(error);
 }
 
@@ -1595,6 +1593,9 @@ my_bool _ma_cmp_dynamic_record(register MARIA_HA *info,
   uchar *buffer;
   MARIA_BLOCK_INFO block_info;
   my_bool error= 1;
+  size_t buffer_length;
+  LINT_INIT(buffer_length);
+
   DBUG_ENTER("_ma_cmp_dynamic_record");
 
   if (info->opt_flag & WRITE_CACHE_USED)
@@ -1612,8 +1613,10 @@ my_bool _ma_cmp_dynamic_record(register MARIA_HA *info,
   {						/* If check isn't disabled  */
     if (info->s->base.blobs)
     {
-      if (!(buffer=(uchar*) my_alloca(info->s->base.pack_reclength+
-				     _ma_calc_total_blob_length(info,record))))
+      buffer_length= (info->s->base.pack_reclength +
+                      _ma_calc_total_blob_length(info,record));
+      if (!(buffer=(uchar*) my_safe_alloca(buffer_length,
+                                           MARIA_MAX_RECORD_ON_STACK)))
 	DBUG_RETURN(1);
     }
     reclength= _ma_rec_pack(info,buffer,record);
@@ -1665,7 +1668,7 @@ my_bool _ma_cmp_dynamic_record(register MARIA_HA *info,
   error= 0;
 err:
   if (buffer != info->rec_buff)
-    my_afree(buffer);
+    my_safe_afree(buffer, buffer_length, MARIA_MAX_RECORD_ON_STACK);
   DBUG_PRINT("exit", ("result: %d", error));
   DBUG_RETURN(error);
 }
diff --git a/storage/maria/maria_def.h b/storage/maria/maria_def.h
index 9b57dd2847d..ab4ade30c44 100644
--- a/storage/maria/maria_def.h
+++ b/storage/maria/maria_def.h
@@ -42,6 +42,7 @@
 
 #define MAX_NONMAPPED_INSERTS 1000
 #define MARIA_MAX_TREE_LEVELS 32
+#define MARIA_MAX_RECORD_ON_STACK 16384
 
 /* maria_open() flag, specific for maria_pack */
 #define HA_OPEN_IGNORE_MOVED_STATE (1U << 30)
diff --git a/storage/maria/maria_pack.c b/storage/maria/maria_pack.c
index 66085a8981b..26d57ade59a 100644
--- a/storage/maria/maria_pack.c
+++ b/storage/maria/maria_pack.c
@@ -861,7 +861,7 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts)
 
   reclength=  mrg->file[0]->s->base.reclength;
   null_bytes= mrg->file[0]->s->base.null_bytes;
-  record=(uchar*) my_alloca(reclength);
+  record=(uchar*) my_safe_alloca(reclength, MARIA_MAX_RECORD_ON_STACK);
   end_count=huff_counts+mrg->file[0]->s->base.fields;
   record_count=0; glob_crc=0;
   max_blob_length=0;
@@ -1145,7 +1145,7 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts)
 
   mrg->records=record_count;
   mrg->max_blob_length=max_blob_length;
-  my_afree(record);
+  my_safe_afree(record, reclength, MARIA_MAX_RECORD_ON_STACK);
   DBUG_RETURN(error != HA_ERR_END_OF_FILE);
 }
 
@@ -2415,7 +2415,8 @@ static int compress_maria_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts)
   DBUG_ENTER("compress_maria_file");
 
   /* Allocate a buffer for the records (excluding blobs). */
-  if (!(record=(uchar*) my_alloca(isam_file->s->base.reclength)))
+  if (!(record=(uchar*) my_safe_alloca(isam_file->s->base.reclength,
+                                       MARIA_MAX_RECORD_ON_STACK)))
     return -1;
 
   end_count=huff_counts+isam_file->s->base.fields;
@@ -2778,7 +2779,8 @@ static int compress_maria_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts)
   if (verbose >= 2)
     printf("wrote %s records.\n", llstr((longlong) record_count, llbuf));
 
-  my_afree(record);
+  my_safe_afree(record, isam_file->s->base.reclength,
+                MARIA_MAX_RECORD_ON_STACK);
   mrg->ref_length=max_pack_length;
   mrg->min_pack_length=max_record_length ? min_record_length : 0;
   mrg->max_pack_length=max_record_length;