MDEV-29664 Assertion `!n_mysql_tables_in_use' failed in innobase_close_connection

When ha_end_bulk_insert() fails F_UNLCK was done twice: in
select_insert::prepare_eof() and in select_create::abort_result_set().

Now we avoid making F_UNLCK in prepare_eof() if error is non-zero.

5 files changed, 31 insertions, 1 deletions

diff --git a/mysql-test/main/create_or_replace2.result b/mysql-test/main/create_replace_debug.result
index 248d5dbc3f5..3a3ef21bdec 100644
--- a/mysql-test/main/create_or_replace2.result
+++ b/mysql-test/main/create_replace_debug.result
@@ -91,3 +91,13 @@ ERROR HY000: Table 't2' was not locked with LOCK TABLES
 unlock tables;
 drop tables t1;
 set @@debug_dbug= @saved_debug_dbug;
+#
+# MDEV-29664 Assertion `!n_mysql_tables_in_use' failed in innobase_close_connection
+#
+create table t1 (x int);
+set @old_dbug= @@debug_dbug;
+set @@debug_dbug= '+d,ha_end_bulk_insert_fail';
+create or replace table t2 (y int) engine innodb select * from t1;
+ERROR HY000: Out of memory.
+set @@debug_dbug= @old_dbug;
+drop table t1;
diff --git a/mysql-test/main/create_or_replace2.test b/mysql-test/main/create_replace_debug.test
index bd82d13f1d1..86e68c47743 100644
--- a/mysql-test/main/create_or_replace2.test
+++ b/mysql-test/main/create_replace_debug.test
@@ -86,3 +86,14 @@ unlock tables;
 
 drop tables t1;
 set @@debug_dbug= @saved_debug_dbug;
+
+--echo #
+--echo # MDEV-29664 Assertion `!n_mysql_tables_in_use' failed in innobase_close_connection
+--echo #
+create table t1 (x int);
+set @old_dbug= @@debug_dbug;
+set @@debug_dbug= '+d,ha_end_bulk_insert_fail';
+--error ER_OUT_OF_RESOURCES
+create or replace table t2 (y int) engine innodb select * from t1;
+set @@debug_dbug= @old_dbug;
+drop table t1;
diff --git a/sql/handler.cc b/sql/handler.cc
index 79fba1539cf..238b4a5c53d 100644
--- a/sql/handler.cc
+++ b/sql/handler.cc
@@ -5032,6 +5032,11 @@ int handler::ha_end_bulk_insert()
   DBUG_ENTER("handler::ha_end_bulk_insert");
   DBUG_EXECUTE_IF("crash_end_bulk_insert",
                   { extra(HA_EXTRA_FLUSH) ; DBUG_SUICIDE();});
+  if (DBUG_IF("ha_end_bulk_insert_fail"))
+  {
+    my_error(ER_OUT_OF_RESOURCES, MYF(0));
+    DBUG_RETURN(HA_ERR_OUT_OF_MEM);
+  }
   estimation_rows_to_insert= 0;
   DBUG_RETURN(end_bulk_insert());
 }
diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc
index 456bff18e4b..275c0c09fe6 100644
--- a/sql/sql_insert.cc
+++ b/sql/sql_insert.cc
@@ -4292,7 +4292,7 @@ bool select_insert::prepare_eof()
   table->file->extra(HA_EXTRA_NO_IGNORE_DUP_KEY);
   table->file->extra(HA_EXTRA_WRITE_CANNOT_REPLACE);
 
-  if (atomic_replace)
+  if (atomic_replace && !error)
   {
     DBUG_ASSERT(table->s->tmp_table);
 
@@ -5509,7 +5509,10 @@ void select_create::abort_result_set()
 
     if (atomic_replace)
     {
+      ulonglong save_options_bits= thd->variables.option_bits;
+      thd->variables.option_bits|= OPTION_NOT_AUTOCOMMIT;
       (void) table->file->ha_external_lock(thd, F_UNLCK);
+      thd->variables.option_bits= save_options_bits;
       (void) thd->drop_temporary_table(table, NULL, true);
     }
     else
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index 7470d18d048..23daa3293df 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -16096,6 +16096,7 @@ ha_innobase::external_lock(
 
 	/* MySQL is releasing a table lock */
 
+	ut_ad(trx->n_mysql_tables_in_use);
 	trx->n_mysql_tables_in_use--;
 	m_mysql_has_locked = false;