MDEV-13818 CREATE INDEX leaks memory if running out of undo log space

row_merge_create_index_graph(): Relay the internal state
from dict_create_index_step(). Our caller should free the index
only if it was not copied, added to the cache, and freed.

row_merge_create_index(): Free the index template if it was
not added to the cache. This is a safer variant of the logic
that was introduced in 65070beffd2e9279145b48d1f27c517b6588e543 in 10.2.

prepare_inplace_alter_table_dict(): Add additional fault injection
to exercise a code path where we have already added an index
to the cache.

8 files changed, 58 insertions, 30 deletions

diff --git a/mysql-test/suite/innodb/r/alter_crash.result b/mysql-test/suite/innodb/r/alter_crash.result
index 8de02cc5fbd..4b49a788cd4 100644
--- a/mysql-test/suite/innodb/r/alter_crash.result
+++ b/mysql-test/suite/innodb/r/alter_crash.result
@@ -3,6 +3,9 @@
 #
 CREATE TABLE t1(c1 INT PRIMARY KEY, c2 CHAR(1), c3 INT UNSIGNED) ENGINE=InnoDB;
 SET @saved_debug_dbug = @@SESSION.debug_dbug;
+SET DEBUG_DBUG='+d,create_index_metadata_fail';
+ALTER TABLE t1 ADD INDEX (c2), ADD INDEX (c3);
+ERROR HY000: The table 't1' is full
 SET DEBUG_DBUG='+d,ib_create_table_fail_too_many_trx';
 ALTER TABLE t1 ADD INDEX (c2), ADD INDEX (c3);
 ERROR HY000: Too many active concurrent transactions
@@ -11,21 +14,21 @@ ALTER TABLE t1 ADD INDEX (c2), ADD INDEX (c3);
 SET DEBUG_DBUG='+d,dict_set_index_corrupted';
 CHECK TABLE t1;
 Table	Op	Msg_type	Msg_text
-test.t1	check	Warning	InnoDB: Index c2 is marked as corrupted
-test.t1	check	Warning	InnoDB: Index c3 is marked as corrupted
+test.t1	check	Warning	InnoDB: Index "c2" is marked as corrupted
+test.t1	check	Warning	InnoDB: Index "c3" is marked as corrupted
 test.t1	check	error	Corrupt
 CHECK TABLE t1;
 Table	Op	Msg_type	Msg_text
-test.t1	check	Warning	InnoDB: Index c2 is marked as corrupted
-test.t1	check	Warning	InnoDB: Index c3 is marked as corrupted
+test.t1	check	Warning	InnoDB: Index "c2" is marked as corrupted
+test.t1	check	Warning	InnoDB: Index "c3" is marked as corrupted
 test.t1	check	error	Corrupt
 ALTER TABLE t1 DROP INDEX c2;
 CHECK TABLE t1;
 Table	Op	Msg_type	Msg_text
-test.t1	check	Warning	InnoDB: Index c3 is marked as corrupted
+test.t1	check	Warning	InnoDB: Index "c3" is marked as corrupted
 test.t1	check	error	Corrupt
 ALTER TABLE t1 ADD INDEX (c2,c3);
-ERROR HY000: Index c3 is corrupted
+ERROR HY000: Index "c3" is corrupted
 ALTER TABLE t1 CHANGE c3 c3 INT NOT NULL;
 CHECK TABLE t1;
 Table	Op	Msg_type	Msg_text
diff --git a/mysql-test/suite/innodb/t/alter_crash.test b/mysql-test/suite/innodb/t/alter_crash.test
index 54cc51aecf4..b417b441723 100644
--- a/mysql-test/suite/innodb/t/alter_crash.test
+++ b/mysql-test/suite/innodb/t/alter_crash.test
@@ -7,10 +7,7 @@
 --source include/not_crashrep.inc
 
 --disable_query_log
-call mtr.add_suppression('InnoDB: cannot find a free slot for an undo log');
-call mtr.add_suppression('InnoDB: row_merge_rename_index_to_add failed with error 47');
-call mtr.add_suppression('InnoDB: Flagged corruption of `c[23]`');
-call mtr.add_suppression('InnoDB: Index `c[23]` .*is corrupted');
+call mtr.add_suppression('InnoDB: Flagged corruption of c[23]');
 --enable_query_log
 
 --echo #
@@ -19,6 +16,10 @@ call mtr.add_suppression('InnoDB: Index `c[23]` .*is corrupted');
 
 CREATE TABLE t1(c1 INT PRIMARY KEY, c2 CHAR(1), c3 INT UNSIGNED) ENGINE=InnoDB;
 SET @saved_debug_dbug = @@SESSION.debug_dbug;
+SET DEBUG_DBUG='+d,create_index_metadata_fail';
+--error ER_RECORD_FILE_FULL
+ALTER TABLE t1 ADD INDEX (c2), ADD INDEX (c3);
+
 SET DEBUG_DBUG='+d,ib_create_table_fail_too_many_trx';
 --error ER_TOO_MANY_CONCURRENT_TRXS
 ALTER TABLE t1 ADD INDEX (c2), ADD INDEX (c3);
diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc
index 2bcec2ce51f..b03b70e488e 100644
--- a/storage/innobase/handler/handler0alter.cc
+++ b/storage/innobase/handler/handler0alter.cc
@@ -3090,10 +3090,18 @@ prepare_inplace_alter_table_dict(
 	/* Create the indexes in SYS_INDEXES and load into dictionary. */
 
 	for (ulint a = 0; a < ctx->num_to_add_index; a++) {
-
+		DBUG_EXECUTE_IF(
+			"create_index_metadata_fail",
+			if (a + 1 == ctx->num_to_add_index) {
+				ctx->trx->error_state = DB_OUT_OF_FILE_SPACE;
+				ctx->add_index[a] = NULL;
+				goto index_created;
+			});
 		ctx->add_index[a] = row_merge_create_index(
 			ctx->trx, ctx->new_table, &index_defs[a]);
-
+#ifndef DBUG_OFF
+index_created:
+#endif
 		add_key_nums[a] = index_defs[a].key_number;
 
 		if (!ctx->add_index[a]) {
diff --git a/storage/innobase/include/ut0dbg.h b/storage/innobase/include/ut0dbg.h
index 3f5baef0a3c..b4c941bc163 100644
--- a/storage/innobase/include/ut0dbg.h
+++ b/storage/innobase/include/ut0dbg.h
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -84,7 +85,7 @@ ut_dbg_assertion_failed(
 /** Debug assertion. Does nothing unless UNIV_DEBUG is defined. */
 #define ut_ad(EXPR)	ut_a(EXPR)
 /** Debug statement. Does nothing unless UNIV_DEBUG is defined. */
-#define ut_d(EXPR)	do {EXPR;} while (0)
+#define ut_d(EXPR)	EXPR
 #else
 /** Debug assertion. Does nothing unless UNIV_DEBUG is defined. */
 #define ut_ad(EXPR)
diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc
index 356eec8e7ee..9fa2eaccfa8 100644
--- a/storage/innobase/row/row0merge.cc
+++ b/storage/innobase/row/row0merge.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 2005, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2014, 2018, MariaDB Corporation.
+Copyright (c) 2014, 2019, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -3702,7 +3702,7 @@ row_merge_create_index_graph(
 /*=========================*/
 	trx_t*		trx,		/*!< in: trx */
 	dict_table_t*	table,		/*!< in: table */
-	dict_index_t*	index)		/*!< in: index */
+	dict_index_t*&	index)		/*!< in,out: index */
 {
 	ind_node_t*	node;		/*!< Index creation node */
 	mem_heap_t*	heap;		/*!< Memory heap */
@@ -3726,6 +3726,8 @@ row_merge_create_index_graph(
 
 	err = trx->error_state;
 
+	index = node->index;
+
 	que_graph_free((que_t*) que_node_get_parent(thr));
 
 	return(err);
@@ -3767,20 +3769,21 @@ row_merge_create_index(
 			ifield->prefix_len);
 	}
 
+	ut_d(const dict_index_t* const index_template = index);
 	/* Add the index to SYS_INDEXES, using the index prototype. */
 	err = row_merge_create_index_graph(trx, table, index);
 
 	if (err == DB_SUCCESS) {
-
-		index = dict_table_get_index_on_name(table, index_def->name);
-
-		ut_a(index);
-
+		ut_ad(index != index_template);
 		/* Note the id of the transaction that created this
 		index, we use it to restrict readers from accessing
 		this index, to ensure read consistency. */
 		ut_ad(index->trx_id == trx->id);
 	} else {
+		ut_ad(!index || index == index_template);
+		if (index) {
+			dict_mem_index_free(index);
+		}
 		index = NULL;
 	}
 
diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc
index fa41c963464..b0cfb17fe28 100644
--- a/storage/xtradb/handler/handler0alter.cc
+++ b/storage/xtradb/handler/handler0alter.cc
@@ -3097,10 +3097,18 @@ prepare_inplace_alter_table_dict(
 	/* Create the indexes in SYS_INDEXES and load into dictionary. */
 
 	for (ulint a = 0; a < ctx->num_to_add_index; a++) {
-
+		DBUG_EXECUTE_IF(
+			"create_index_metadata_fail",
+			if (a + 1 == ctx->num_to_add_index) {
+				ctx->trx->error_state = DB_OUT_OF_FILE_SPACE;
+				ctx->add_index[a] = NULL;
+				goto index_created;
+			});
 		ctx->add_index[a] = row_merge_create_index(
 			ctx->trx, ctx->new_table, &index_defs[a]);
-
+#ifndef DBUG_OFF
+index_created:
+#endif
 		add_key_nums[a] = index_defs[a].key_number;
 
 		if (!ctx->add_index[a]) {
diff --git a/storage/xtradb/include/ut0dbg.h b/storage/xtradb/include/ut0dbg.h
index 3f5baef0a3c..b4c941bc163 100644
--- a/storage/xtradb/include/ut0dbg.h
+++ b/storage/xtradb/include/ut0dbg.h
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -84,7 +85,7 @@ ut_dbg_assertion_failed(
 /** Debug assertion. Does nothing unless UNIV_DEBUG is defined. */
 #define ut_ad(EXPR)	ut_a(EXPR)
 /** Debug statement. Does nothing unless UNIV_DEBUG is defined. */
-#define ut_d(EXPR)	do {EXPR;} while (0)
+#define ut_d(EXPR)	EXPR
 #else
 /** Debug assertion. Does nothing unless UNIV_DEBUG is defined. */
 #define ut_ad(EXPR)
diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc
index e4440640431..02d272f095a 100644
--- a/storage/xtradb/row/row0merge.cc
+++ b/storage/xtradb/row/row0merge.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 2005, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2014, 2018, MariaDB Corporation.
+Copyright (c) 2014, 2019, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -3705,7 +3705,7 @@ row_merge_create_index_graph(
 /*=========================*/
 	trx_t*		trx,		/*!< in: trx */
 	dict_table_t*	table,		/*!< in: table */
-	dict_index_t*	index)		/*!< in: index */
+	dict_index_t*&	index)		/*!< in,out: index */
 {
 	ind_node_t*	node;		/*!< Index creation node */
 	mem_heap_t*	heap;		/*!< Memory heap */
@@ -3729,6 +3729,8 @@ row_merge_create_index_graph(
 
 	err = trx->error_state;
 
+	index = node->index;
+
 	que_graph_free((que_t*) que_node_get_parent(thr));
 
 	return(err);
@@ -3770,20 +3772,21 @@ row_merge_create_index(
 			ifield->prefix_len);
 	}
 
+	ut_d(const dict_index_t* const index_template = index);
 	/* Add the index to SYS_INDEXES, using the index prototype. */
 	err = row_merge_create_index_graph(trx, table, index);
 
 	if (err == DB_SUCCESS) {
-
-		index = dict_table_get_index_on_name(table, index_def->name);
-
-		ut_a(index);
-
+		ut_ad(index != index_template);
 		/* Note the id of the transaction that created this
 		index, we use it to restrict readers from accessing
 		this index, to ensure read consistency. */
 		ut_ad(index->trx_id == trx->id);
 	} else {
+		ut_ad(!index || index == index_template);
+		if (index) {
+			dict_mem_index_free(index);
+		}
 		index = NULL;
 	}