Bug#37956 memory leak and / or crash with geometry and prepared statements!

Bug#37671 crash on prepared statement + cursor + geometry + too many open files!
if mysql_execute_command() returns error then free materialized_cursor object.
is_rnd_inited is added to satisfy rnd_end() assertion
(handler may be uninitialized in some cases)


sql/sql_cursor.cc:
  if mysql_execute_command() returns error then free materialized_cursor object.
  is_rnd_inited is added to satisfy rnd_end() assertion
  (handler may be uninitialized in some cases)
sql/sql_select.cc:
  added result check
tests/mysql_client_test.c:
  test case

3 files changed, 72 insertions, 7 deletions

diff --git a/sql/sql_cursor.cc b/sql/sql_cursor.cc
index 16567765ba6..83c60814cf3 100644
--- a/sql/sql_cursor.cc
+++ b/sql/sql_cursor.cc
@@ -85,6 +85,7 @@ class Materialized_cursor: public Server_side_cursor
   List<Item> item_list;
   ulong fetch_limit;
   ulong fetch_count;
+  bool is_rnd_inited;
 public:
   Materialized_cursor(select_result *result, TABLE *table);
 
@@ -191,7 +192,11 @@ int mysql_open_cursor(THD *thd, uint flags, select_result *result,
       such command is SHOW VARIABLES or SHOW STATUS.
   */
   if (rc)
+  {
+    if (result_materialize->materialized_cursor)
+      delete result_materialize->materialized_cursor;
     goto err_open;
+  }
 
   if (sensitive_cursor->is_open())
   {
@@ -532,7 +537,8 @@ Materialized_cursor::Materialized_cursor(select_result *result_arg,
   :Server_side_cursor(&table_arg->mem_root, result_arg),
   table(table_arg),
   fetch_limit(0),
-  fetch_count(0)
+  fetch_count(0),
+  is_rnd_inited(0)
 {
   fake_unit.init_query();
   fake_unit.thd= table->in_use;
@@ -589,11 +595,12 @@ int Materialized_cursor::open(JOIN *join __attribute__((unused)))
   THD *thd= fake_unit.thd;
   int rc;
   Query_arena backup_arena;
-
   thd->set_n_backup_active_arena(this, &backup_arena);
   /* Create a list of fields and start sequential scan */
-  rc= (result->prepare(item_list, &fake_unit) ||
-       table->file->ha_rnd_init(TRUE));
+  rc= result->prepare(item_list, &fake_unit);
+  if (!rc && !(rc= table->file->ha_rnd_init(TRUE)))
+    is_rnd_inited= 1;
+
   thd->restore_active_arena(this, &backup_arena);
   if (rc == 0)
   {
@@ -673,7 +680,8 @@ void Materialized_cursor::close()
 {
   /* Free item_list items */
   free_items();
-  (void) table->file->ha_rnd_end();
+  if (is_rnd_inited)
+    (void) table->file->ha_rnd_end();
   /*
     We need to grab table->mem_root to prevent free_tmp_table from freeing:
     the cursor object was allocated in this memory.
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 48276170caf..d2c469f99da 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -1599,8 +1599,11 @@ JOIN::exec()
 		      (zero_result_cause?zero_result_cause:"No tables used"));
     else
     {
-      result->send_fields(*columns_list,
-                          Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF);
+      if (result->send_fields(*columns_list,
+                              Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF))
+      {
+        DBUG_VOID_RETURN;
+      }
       /*
         We have to test for 'conds' here as the WHERE may not be constant
         even if we don't have any tables for prepared statements or if
diff --git a/tests/mysql_client_test.c b/tests/mysql_client_test.c
index ee3a053f8bd..ea4d363bdac 100644
--- a/tests/mysql_client_test.c
+++ b/tests/mysql_client_test.c
@@ -15899,6 +15899,59 @@ static void test_bug28934()
 }
 
 
+/**
+  Bug#37956 memory leak and / or crash with geometry and prepared statements! 
+*/
+
+static void test_bug37956(void)
+{
+  const char *query="select point(?,?)";
+  MYSQL_STMT *stmt=NULL;
+  unsigned int val=0;
+  MYSQL_BIND bind_param[2];
+  unsigned char buff[2]= { 134, 211 };
+  DBUG_ENTER("test_bug37956");
+  myheader("test_bug37956");
+
+  stmt= mysql_simple_prepare(mysql, query);
+  check_stmt(stmt);
+
+  val=1;
+  mysql_stmt_attr_set(stmt, STMT_ATTR_UPDATE_MAX_LENGTH, (void *)&val);
+  val=CURSOR_TYPE_READ_ONLY;
+  mysql_stmt_attr_set(stmt, STMT_ATTR_CURSOR_TYPE, (void *)&val);
+  val=0;
+  mysql_stmt_attr_set(stmt, STMT_ATTR_PREFETCH_ROWS, (void *)&val);
+
+  memset(bind_param, 0, sizeof(bind_param));
+  bind_param[0].buffer_type=MYSQL_TYPE_TINY;
+  bind_param[0].buffer= (void *)buff;
+  bind_param[0].is_null=NULL;
+  bind_param[0].error=NULL;
+  bind_param[0].is_unsigned=1;
+  bind_param[1].buffer_type=MYSQL_TYPE_TINY;
+  bind_param[1].buffer= (void *)(buff+1);
+  bind_param[1].is_null=NULL;
+  bind_param[1].error=NULL;
+  bind_param[1].is_unsigned=1;
+
+  if (mysql_stmt_bind_param(stmt, bind_param))
+  {
+    mysql_stmt_close(stmt);
+    DIE_UNLESS(0);
+  }
+
+  if (mysql_stmt_execute(stmt))
+  {
+    mysql_stmt_close(stmt);
+    DBUG_VOID_RETURN;
+  }
+  /* Should never reach here: execution returns an error. */
+  mysql_stmt_close(stmt);
+  DIE_UNLESS(0);
+  DBUG_VOID_RETURN;
+}
+
 /*
   Bug#27592 (stack overrun when storing datetime value using prepared statements)
 */
@@ -16595,6 +16648,7 @@ static struct my_tests_st my_tests[]= {
   { "test_bug32265", test_bug32265 },
   { "test_bug38486", test_bug38486 },
   { "test_bug40365", test_bug40365 },
+  { "test_bug37956", test_bug37956 },
   { 0, 0 }
 };