Bug#11757464:SERVER CRASH IN RECURSIVE CALL WHEN OOM

Analysis:
---------

When the server is out of memory, an error is raised
to indicate the same. Handling the error requires
more memory to be allocated which fails, hence the
error handling loops in a recursion and causes the
server to crash.

Fix:
---
a) Prevents pushing the 'out of memory' error condition
to the diagnostic area as it requires memory allocation.
GET DIAGNOSTICS, SHOW WARNINGS and SHOW ERRORS statements
will not show information about this error. However the
'out of memory' error is returned to the client.
b) It sets the ME_FATALERROR flag when 'out of memory' errors
are reported (for places where the flag is not already set).
This flag prevents activation of SP error handlers which also
require memory allocation and therefore are likely to fail.

12 files changed, 39 insertions, 20 deletions

diff --git a/mysys/mf_keycache.c b/mysys/mf_keycache.c
index 7dfbf22d4b7..5829ab819f7 100644
--- a/mysys/mf_keycache.c
+++ b/mysys/mf_keycache.c
@@ -444,7 +444,8 @@ int init_key_cache(KEY_CACHE *keycache, uint key_cache_block_size,
       if (blocks < 8)
       {
         my_errno= ENOMEM;
-        my_error(EE_OUTOFMEMORY, MYF(0), blocks * keycache->key_cache_block_size);
+        my_error(EE_OUTOFMEMORY, MYF(ME_FATALERROR),
+                 blocks * keycache->key_cache_block_size);
         goto err;
       }
       blocks= blocks / 4*3;
diff --git a/mysys/my_lockmem.c b/mysys/my_lockmem.c
index 8f06192d9f8..2e036936c70 100644
--- a/mysys/my_lockmem.c
+++ b/mysys/my_lockmem.c
@@ -43,7 +43,7 @@ uchar *my_malloc_lock(uint size,myf MyFlags)
   if (!(ptr=memalign(pagesize,size)))
   {
     if (MyFlags & (MY_FAE+MY_WME))
-      my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_WAITTANG),size);
+      my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_WAITTANG+ME_FATALERROR), size);
     DBUG_RETURN(0);
   }
   success = mlock((uchar*) ptr,size);
diff --git a/mysys/my_malloc.c b/mysys/my_malloc.c
index fc2dc98c3c5..93eb547888f 100644
--- a/mysys/my_malloc.c
+++ b/mysys/my_malloc.c
@@ -41,6 +41,11 @@ void *my_malloc(size_t size, myf my_flags)
                     free(point);
                     point= NULL;
                   });
+  DBUG_EXECUTE_IF("simulate_persistent_out_of_memory",
+                  {
+                    free(point);
+                    point= NULL;
+                  });
 
   if (point == NULL)
   {
@@ -48,7 +53,8 @@ void *my_malloc(size_t size, myf my_flags)
     if (my_flags & MY_FAE)
       error_handler_hook=fatal_error_handler_hook;
     if (my_flags & (MY_FAE+MY_WME))
-      my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_WAITTANG+ME_NOREFRESH),size);
+      my_error(EE_OUTOFMEMORY, MYF(ME_BELL + ME_WAITTANG +
+                                   ME_NOREFRESH + ME_FATALERROR),size);
     DBUG_EXECUTE_IF("simulate_out_of_memory",
                     DBUG_SET("-d,simulate_out_of_memory"););
     if (my_flags & MY_FAE)
@@ -90,7 +96,7 @@ void *my_realloc(void *oldpoint, size_t size, myf my_flags)
       DBUG_RETURN(oldpoint);
     my_errno=errno;
     if (my_flags & MY_FAE+MY_WME)
-      my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_WAITTANG),size);
+      my_error(EE_OUTOFMEMORY, MYF(ME_BELL + ME_WAITTANG + ME_FATALERROR),size);
   }
   else
   {
@@ -106,7 +112,7 @@ void *my_realloc(void *oldpoint, size_t size, myf my_flags)
       DBUG_RETURN(oldpoint);
     my_errno=errno;
     if (my_flags & (MY_FAE+MY_WME))
-      my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_WAITTANG), size);
+      my_error(EE_OUTOFMEMORY, MYF(ME_BELL + ME_WAITTANG + ME_FATALERROR), size);
   }
 #endif
   DBUG_PRINT("exit",("ptr: %p", point));
diff --git a/mysys/my_once.c b/mysys/my_once.c
index 7df9b0a1981..b9232db9b2e 100644
--- a/mysys/my_once.c
+++ b/mysys/my_once.c
@@ -59,7 +59,7 @@ void* my_once_alloc(size_t Size, myf MyFlags)
     {
       my_errno=errno;
       if (MyFlags & (MY_FAE+MY_WME))
-	my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_WAITTANG),get_size);
+	my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_WAITTANG+ME_FATALERROR), get_size);
       return((uchar*) 0);
     }
     DBUG_PRINT("test",("my_once_malloc %lu byte malloced", (ulong) get_size));
diff --git a/sql/handler.cc b/sql/handler.cc
index f4eb89912e9..6d022630508 100644
--- a/sql/handler.cc
+++ b/sql/handler.cc
@@ -356,7 +356,8 @@ handler *get_ha_partition(partition_info *part_info)
   }
   else
   {
-    my_error(ER_OUTOFMEMORY, MYF(0), static_cast<int>(sizeof(ha_partition)));
+    my_error(ER_OUTOFMEMORY, MYF(ME_FATALERROR), 
+             static_cast<int>(sizeof(ha_partition)));
   }
   DBUG_RETURN(((handler*) partition));
 }
diff --git a/sql/item_sum.cc b/sql/item_sum.cc
index 75aec7918fa..81d151ffba0 100644
--- a/sql/item_sum.cc
+++ b/sql/item_sum.cc
@@ -3211,8 +3211,12 @@ bool Item_func_group_concat::add()
   TREE_ELEMENT *el= 0;                          // Only for safety
   if (row_eligible && tree)
   {
+    DBUG_EXECUTE_IF("trigger_OOM_in_gconcat_add",
+                     DBUG_SET("+d,simulate_persistent_out_of_memory"););
     el= tree_insert(tree, table->record[0] + table->s->null_bytes, 0,
                     tree->custom_arg);
+    DBUG_EXECUTE_IF("trigger_OOM_in_gconcat_add",
+                    DBUG_SET("-d,simulate_persistent_out_of_memory"););
     /* check if there was enough memory to insert the row */
     if (!el)
       return 1;
diff --git a/sql/sql_binlog.cc b/sql/sql_binlog.cc
index 8e7ee4f01c7..e32bafc73e4 100644
--- a/sql/sql_binlog.cc
+++ b/sql/sql_binlog.cc
@@ -107,7 +107,7 @@ void mysql_client_binlog_statement(THD* thd)
         rli->relay_log.description_event_for_exec &&
         buf))
   {
-    my_error(ER_OUTOFMEMORY, MYF(0), 1);  /* needed 1 bytes */
+    my_error(ER_OUTOFMEMORY, MYF(ME_FATALERROR), 1);  /* needed 1 bytes */
     goto end;
   }
 
diff --git a/sql/sql_class.cc b/sql/sql_class.cc
index 6e47dcb5795..556e7d66447 100644
--- a/sql/sql_class.cc
+++ b/sql/sql_class.cc
@@ -1136,10 +1136,14 @@ MYSQL_ERROR* THD::raise_condition(uint sql_errno,
 
   query_cache_abort(&query_cache_tls);
 
-  /* When simulating OOM, skip writing to error log to avoid mtr errors */
-  DBUG_EXECUTE_IF("simulate_out_of_memory", DBUG_RETURN(NULL););
-
-  cond= warning_info->push_warning(this, sql_errno, sqlstate, level, msg);
+  /* 
+     Avoid pushing a condition for out of memory errors as this will require
+     memory allocation and therefore might fail.
+  */
+  if (sql_errno != EE_OUTOFMEMORY && sql_errno != ER_OUTOFMEMORY)
+  {
+    cond= warning_info->push_warning(this, sql_errno, sqlstate, level, msg);
+  }
   DBUG_RETURN(cond);
 }
 
@@ -1938,7 +1942,7 @@ CHANGED_TABLE_LIST* THD::changed_table_dup(const char *key, long key_length)
 				      key_length + 1);
   if (!new_table)
   {
-    my_error(EE_OUTOFMEMORY, MYF(ME_BELL),
+    my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_FATALERROR),
              ALIGN_SIZE(sizeof(TABLE_LIST)) + key_length + 1);
     killed= KILL_CONNECTION;
     return 0;
@@ -2493,7 +2497,7 @@ bool select_export::send_data(List<Item> &items)
       set_if_smaller(estimated_bytes, UINT_MAX32);
       if (cvt_str.realloc((uint32) estimated_bytes))
       {
-        my_error(ER_OUTOFMEMORY, MYF(0), (uint32) estimated_bytes);
+        my_error(ER_OUTOFMEMORY, MYF(ME_FATALERROR), (uint32) estimated_bytes);
         goto err;
       }
 
diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc
index 066262c3919..8154f4cc404 100644
--- a/sql/sql_partition.cc
+++ b/sql/sql_partition.cc
@@ -7044,7 +7044,8 @@ void set_key_field_ptr(KEY *key_info, const uchar *new_buf,
 
 void mem_alloc_error(size_t size)
 {
-  my_error(ER_OUTOFMEMORY, MYF(0), static_cast<int>(size));
+  my_error(ER_OUTOFMEMORY, MYF(ME_FATALERROR), 
+           static_cast<int>(size));
 }
 
 #ifdef WITH_PARTITION_STORAGE_ENGINE
diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc
index 2122a2badf9..f3deafc6035 100644
--- a/sql/sql_prepare.cc
+++ b/sql/sql_prepare.cc
@@ -1475,7 +1475,8 @@ static int mysql_test_select(Prepared_statement *stmt,
 
   if (!lex->result && !(lex->result= new (stmt->mem_root) select_send))
   {
-    my_error(ER_OUTOFMEMORY, MYF(0), static_cast<int>(sizeof(select_send)));
+    my_error(ER_OUTOFMEMORY, MYF(ME_FATALERROR), 
+             static_cast<int>(sizeof(select_send)));
     goto error;
   }
 
@@ -1842,7 +1843,7 @@ static bool mysql_test_multidelete(Prepared_statement *stmt,
   stmt->thd->lex->current_select= &stmt->thd->lex->select_lex;
   if (add_item_to_list(stmt->thd, new Item_null()))
   {
-    my_error(ER_OUTOFMEMORY, MYF(0), 0);
+    my_error(ER_OUTOFMEMORY, MYF(ME_FATALERROR), 0);
     goto error;
   }
 
@@ -3755,7 +3756,7 @@ bool Prepared_statement::execute(String *expanded_query, bool open_cursor)
       alloc_query(thd, (char*) expanded_query->ptr(),
                   expanded_query->length()))
   {
-    my_error(ER_OUTOFMEMORY, 0, expanded_query->length());
+    my_error(ER_OUTOFMEMORY, MYF(ME_FATALERROR), expanded_query->length());
     goto error;
   }
   /*
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 181fa6ece0a..7ffba371b53 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -14417,7 +14417,7 @@ static int remove_dup_with_compare(THD *thd, TABLE *table, Field **first_field,
     }
     if (copy_blobs(first_field))
     {
-      my_message(ER_OUTOFMEMORY, ER(ER_OUTOFMEMORY), MYF(0));
+      my_message(ER_OUTOFMEMORY, ER(ER_OUTOFMEMORY), MYF(ME_FATALERROR));
       error=0;
       goto err;
     }
diff --git a/storage/myisam/myisampack.c b/storage/myisam/myisampack.c
index f9f31fefcb6..97f95487d01 100644
--- a/storage/myisam/myisampack.c
+++ b/storage/myisam/myisampack.c
@@ -2151,7 +2151,8 @@ static my_off_t write_huff_tree(HUFF_TREE *huff_tree, uint trees)
   */
   if (!(packed_tree=(uint*) my_alloca(sizeof(uint)*length*2)))
   {
-    my_error(EE_OUTOFMEMORY,MYF(ME_BELL),sizeof(uint)*length*2);
+    my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_FATALERROR), 
+             sizeof(uint)*length*2);
     return 0;
   }