MariaDB adjustments for Oracle Bug#23070734 fix

Split the test case so that a server restart is not needed.
Reduce the test cases and use a simpler mechanism for triggering
and waiting for purge.

fil_table_accessible(): Check if a table can be accessed without
enjoying MDL protection.

12 files changed, 210 insertions, 179 deletions

diff --git a/mysql-test/suite/innodb/r/innodb-truncate-debug.result b/mysql-test/suite/innodb/r/truncate_debug.result
index 3012462d373..47316fed2f7 100644
--- a/mysql-test/suite/innodb/r/innodb-truncate-debug.result
+++ b/mysql-test/suite/innodb/r/truncate_debug.result
@@ -20,14 +20,11 @@ insert into t3 values (10,20),(30,40),(50,50);
 insert into t3 select * from t3;
 insert into t3 select * from t3;
 SET session lock_wait_timeout = 1;
-show variables like 'lock_wait_timeout%';
-Variable_name	Value
-lock_wait_timeout	1
-connection 1
-SET DEBUG_SYNC= 'simulate_buffer_pool_scan SIGNAL opened WAIT_FOR finish_scan';
-truncate table t1;;
-connection default
-SET DEBUG_SYNC= 'now WAIT_FOR opened';
+connect  con1,localhost,root,,;
+SET DEBUG_SYNC= 'buffer_pool_scan SIGNAL started WAIT_FOR finish_scan';
+truncate table t1;
+connection default;
+SET DEBUG_SYNC= 'now WAIT_FOR started';
 Check Analyze table. Gives lock time out error.
 analyze table t1;
 Table	Op	Msg_type	Msg_text
@@ -47,7 +44,7 @@ alter table t1 add column f4 int;
 ERROR HY000: Lock wait timeout exceeded; try restarting transaction
 check if table can be created with the same name
 create table t1 (bd int) engine=innodb;
-ERROR HY000: Lock wait timeout exceeded; try restarting transaction
+Got one of the listed errors
 check if index can be created on table being truncated
 create index idx1 on t1(f1);
 ERROR HY000: Lock wait timeout exceeded; try restarting transaction
@@ -74,51 +71,16 @@ check if index can be created on the other table
 create index idx1 on t2(f3);
 Check if we can turn off persistent stats off entire instance
 SET GLOBAL innodb_stats_persistent=off;
-connection con2
-set global innodb_adaptive_hash_index=off;;
-connection default
+connect  con2,localhost,root,,;
+set global innodb_adaptive_hash_index=off;
+connection default;
 SET DEBUG_SYNC= 'now SIGNAL finish_scan';
-connection con1
-connection con2
-connection default
+SET DEBUG_SYNC= 'RESET';
+connection con1;
+disconnect con1;
+connection con2;
+disconnect con2;
+connection default;
 SET session lock_wait_timeout=default;
-set global innodb_adaptive_hash_index=default;
+set global innodb_adaptive_hash_index=on;
 drop table t1,t2,t3;
-SET @@global.innodb_stats_persistent=default;
-Test_2 :- Check if purge thread ignores the undo  
-log records of the table being truncated.
-create table t1 (f1 int ,f2 int,key(f2)) engine=innodb;
-insert into t1 values (10,45),(20,78),(30,88),(40,23),(50,78),(60,11),(70,56),(80,90);
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-select count(*) from t1;
-count(*)
-131072
-Stop the purge thread
-set global innodb_purge_stop_now = 1;
-delete from t1;
-connection con1
-SET DEBUG_SYNC= 'simulate_buffer_pool_scan SIGNAL opened WAIT_FOR finish_scan';
-truncate table t1;;
-Connection default
-SET DEBUG_SYNC= 'now WAIT_FOR opened';
-set global innodb_purge_run_now=1;
-SET DEBUG_SYNC= 'now SIGNAL finish_scan';
-connection con1
-connection default
-drop table t1;
-Pattern "InnoDB: Record with space id \d+ belongs to table which is being truncated therefore skipping this undo record." found
-# restart server
-# restart:
diff --git a/mysql-test/suite/innodb/r/truncate_purge_debug.result b/mysql-test/suite/innodb/r/truncate_purge_debug.result
new file mode 100644
index 00000000000..edece3019bc
--- /dev/null
+++ b/mysql-test/suite/innodb/r/truncate_purge_debug.result
@@ -0,0 +1,29 @@
+#
+# Bug #23070734	CONCURRENT TRUNCATE TABLES CAUSE STALLS
+#
+create table t1 (f1 int ,f2 int,key(f2)) engine=innodb;
+begin;
+insert into t1 values (10,45),(20,78),(30,88),(40,23),(50,78),(60,11),(70,56),(80,90);
+delete from t1;
+connect  con2,localhost,root,,;
+START TRANSACTION WITH CONSISTENT SNAPSHOT;
+connection default;
+SET @saved_frequency = @@GLOBAL.innodb_purge_rseg_truncate_frequency;
+SET GLOBAL innodb_purge_rseg_truncate_frequency = 1;
+commit;
+connect  con1,localhost,root,,;
+SET DEBUG_SYNC= 'buffer_pool_scan SIGNAL started WAIT_FOR finish_scan';
+truncate table t1;
+connection con2;
+SET DEBUG_SYNC= 'now WAIT_FOR started';
+COMMIT;
+disconnect con2;
+connection default;
+InnoDB		0 transactions not purged
+SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency;
+SET DEBUG_SYNC = 'now SIGNAL finish_scan';
+SET DEBUG_SYNC = 'RESET';
+connection con1;
+disconnect con1;
+connection default;
+drop table t1;
diff --git a/mysql-test/suite/innodb/t/innodb-truncate-debug.test b/mysql-test/suite/innodb/t/truncate_debug.test
index 9705190332c..fed5cf9392b 100644
--- a/mysql-test/suite/innodb/t/innodb-truncate-debug.test
+++ b/mysql-test/suite/innodb/t/truncate_debug.test
@@ -2,6 +2,8 @@
 --source include/have_debug.inc
 --source include/have_debug_sync.inc
 
+--source include/count_sessions.inc
+
 --echo #
 --echo # Bug #23070734	CONCURRENT TRUNCATE TABLES CAUSE STALLS
 --echo #
@@ -30,19 +32,13 @@ insert into t3 select * from t3;
 insert into t3 select * from t3;
 
 SET session lock_wait_timeout = 1;
-show variables like 'lock_wait_timeout%';
 
 connect (con1,localhost,root,,);
-connect (con2,localhost,root,,);
-
---echo connection 1
-connection con1;
-SET DEBUG_SYNC= 'simulate_buffer_pool_scan SIGNAL opened WAIT_FOR finish_scan';
---send truncate table t1;
+SET DEBUG_SYNC= 'buffer_pool_scan SIGNAL started WAIT_FOR finish_scan';
+send truncate table t1;
 
---echo connection default
 connection default;
-SET DEBUG_SYNC= 'now WAIT_FOR opened';
+SET DEBUG_SYNC= 'now WAIT_FOR started';
 
 --echo Check Analyze table. Gives lock time out error.
 analyze table t1;
@@ -64,7 +60,7 @@ insert into t1 values (10,89,99);
 alter table t1 add column f4 int;
 
 --echo check if table can be created with the same name
---error ER_LOCK_WAIT_TIMEOUT
+--error ER_TABLE_EXISTS_ERROR,ER_LOCK_WAIT_TIMEOUT
 create table t1 (bd int) engine=innodb;
 
 --echo check if index can be created on table being truncated
@@ -99,96 +95,28 @@ alter table t2 add column f4 int;
 --echo check if index can be created on the other table
 create index idx1 on t2(f3);
 
-
 --echo Check if we can turn off persistent stats off entire instance
 SET GLOBAL innodb_stats_persistent=off;
 
---echo connection con2
-connection con2;
---send set global innodb_adaptive_hash_index=off;
+connect (con2,localhost,root,,);
+send set global innodb_adaptive_hash_index=off;
 
---echo connection default
 connection default;
 SET DEBUG_SYNC= 'now SIGNAL finish_scan';
+SET DEBUG_SYNC= 'RESET';
 
---echo connection con1
 connection con1;
 reap;
+disconnect con1;
 
---echo connection con2
 connection con2;
 reap;
+disconnect con2;
 
---echo connection default
 connection default;
-disconnect con1;
-disconnect con2;
 
 SET session lock_wait_timeout=default;
-set global innodb_adaptive_hash_index=default;
+set global innodb_adaptive_hash_index=on;
 
 drop table t1,t2,t3;
-SET @@global.innodb_stats_persistent=default;
-
-
---echo Test_2 :- Check if purge thread ignores the undo  
---echo		 log records of the table being truncated.
-
-let $restart_parameters = restart: --innodb_purge_threads=1 --innodb_purge_batch_size=1 --innodb_stats_persistent=OFF
---source include/restart_mysqld.inc;
-
-create table t1 (f1 int ,f2 int,key(f2)) engine=innodb;
-insert into t1 values (10,45),(20,78),(30,88),(40,23),(50,78),(60,11),(70,56),(80,90);
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-insert into t1 select * from t1;
-select count(*) from t1;
-
---echo Stop the purge thread
-set global innodb_purge_stop_now = 1;
-delete from t1;
-
-connect (con1,localhost,root,,);
-
---echo connection con1
-connection con1;
-SET DEBUG_SYNC= 'simulate_buffer_pool_scan SIGNAL opened WAIT_FOR finish_scan';
---send truncate table t1;
-
---echo Connection default
-connection default;
-
-SET DEBUG_SYNC= 'now WAIT_FOR opened';
-set global innodb_purge_run_now=1;
---sleep 60
-SET DEBUG_SYNC= 'now SIGNAL finish_scan';
-
---echo connection con1
-connection con1;
-reap;
-
---echo connection default
-connection default;
-disconnect con1;
-
-drop table t1;
-
-let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err;
-let SEARCH_PATTERN = InnoDB: Record with space id \d+ belongs to table which is being truncated therefore skipping this undo record.;
---source include/search_pattern.inc
-
-#cleanup
---echo # restart server
-let $restart_parameters = restart:;
---source include/restart_mysqld.inc
+--source include/wait_until_count_sessions.inc
diff --git a/mysql-test/suite/innodb/t/truncate_purge_debug.opt b/mysql-test/suite/innodb/t/truncate_purge_debug.opt
new file mode 100644
index 00000000000..8bed7e46166
--- /dev/null
+++ b/mysql-test/suite/innodb/t/truncate_purge_debug.opt
@@ -0,0 +1,3 @@
+--innodb-purge-threads=1
+--innodb-purge-batch-size=1
+--innodb-stats-persistent=OFF
diff --git a/mysql-test/suite/innodb/t/truncate_purge_debug.test b/mysql-test/suite/innodb/t/truncate_purge_debug.test
new file mode 100644
index 00000000000..70ebd28024d
--- /dev/null
+++ b/mysql-test/suite/innodb/t/truncate_purge_debug.test
@@ -0,0 +1,66 @@
+--source include/have_innodb.inc
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
+--source include/count_sessions.inc
+
+--echo #
+--echo # Bug #23070734	CONCURRENT TRUNCATE TABLES CAUSE STALLS
+--echo #
+
+create table t1 (f1 int ,f2 int,key(f2)) engine=innodb;
+begin;
+insert into t1 values (10,45),(20,78),(30,88),(40,23),(50,78),(60,11),(70,56),(80,90);
+delete from t1;
+
+connect (con2,localhost,root,,);
+# Stop the purge thread
+START TRANSACTION WITH CONSISTENT SNAPSHOT;
+
+connection default;
+# Ensure that the history list length will actually be decremented by purge.
+SET @saved_frequency = @@GLOBAL.innodb_purge_rseg_truncate_frequency;
+SET GLOBAL innodb_purge_rseg_truncate_frequency = 1;
+commit;
+
+connect (con1,localhost,root,,);
+SET DEBUG_SYNC= 'buffer_pool_scan SIGNAL started WAIT_FOR finish_scan';
+send truncate table t1;
+
+connection con2;
+SET DEBUG_SYNC= 'now WAIT_FOR started';
+# Allow purge to proceed, by discarding our read view.
+COMMIT;
+disconnect con2;
+
+connection default;
+
+# Wait for everything to be purged.
+
+let $wait_counter= 300;
+while ($wait_counter)
+{
+    --replace_regex /.*History list length ([0-9]+).*/\1/
+    let $remaining= `SHOW ENGINE INNODB STATUS`;
+    if ($remaining == 'InnoDB		0')
+    {
+        let $wait_counter= 0;
+    }
+    if ($wait_counter)
+    {
+        real_sleep 0.1;
+        dec $wait_counter;
+    }
+}
+echo $remaining transactions not purged;
+SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency;
+
+SET DEBUG_SYNC = 'now SIGNAL finish_scan';
+SET DEBUG_SYNC = 'RESET';
+
+connection con1;
+reap;
+disconnect con1;
+
+connection default;
+drop table t1;
+--source include/wait_until_count_sessions.inc
diff --git a/storage/innobase/dict/dict0stats_bg.cc b/storage/innobase/dict/dict0stats_bg.cc
index 986b6a2613f..876d1bcb342 100644
--- a/storage/innobase/dict/dict0stats_bg.cc
+++ b/storage/innobase/dict/dict0stats_bg.cc
@@ -319,14 +319,9 @@ dict_stats_process_entry_from_recalc_pool()
 		return;
 	}
 
-	if (fil_space_is_being_truncated(table->space)) {
-		dict_table_close(table, TRUE, FALSE);
-		mutex_exit(&dict_sys->mutex);
-		return;
-	}
+	ut_ad(!dict_table_is_temporary(table));
 
-	/* Check whether table is corrupted */
-	if (table->corrupted) {
+	if (!fil_table_accessible(table)) {
 		dict_table_close(table, TRUE, FALSE);
 		mutex_exit(&dict_sys->mutex);
 		return;
diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc
index 7fd41a8fcde..47c1c480a3f 100644
--- a/storage/innobase/fil/fil0fil.cc
+++ b/storage/innobase/fil/fil0fil.cc
@@ -3055,6 +3055,32 @@ fil_close_tablespace(
 	return(err);
 }
 
+/** Determine whether a table can be accessed in operations that are
+not (necessarily) protected by meta-data locks.
+(Rollback would generally be protected, but rollback of
+FOREIGN KEY CASCADE/SET NULL is not protected by meta-data locks
+but only by InnoDB table locks, which may be broken by TRUNCATE TABLE.)
+@param[in]	table	persistent table
+checked @return whether the table is accessible */
+bool
+fil_table_accessible(const dict_table_t* table)
+{
+	if (UNIV_UNLIKELY(table->ibd_file_missing || table->corrupted)) {
+		return(false);
+	}
+
+	if (fil_space_t* space = fil_space_acquire(table->space)) {
+		bool accessible = !space->is_stopping();
+		fil_space_release(space);
+		ut_ad(accessible || dict_table_is_file_per_table(table));
+		return(accessible);
+	} else {
+		/* The tablespace may momentarily be missing during
+		TRUNCATE TABLE. */
+		return(false);
+	}
+}
+
 /** Deletes an IBD tablespace, either general or single-table.
 The tablespace must be cached in the memory cache. This will delete the
 datafile, fil_space_t & fil_node_t entries from the file_system_t cache.
@@ -3288,7 +3314,7 @@ fil_reinit_space_header(
 	/* Lock the search latch in shared mode to prevent user
 	from disabling AHI during the scan */
 	btr_search_s_lock_all();
-	DEBUG_SYNC_C("simulate_buffer_pool_scan");
+	DEBUG_SYNC_C("buffer_pool_scan");
 	buf_LRU_flush_or_remove_pages(id, BUF_REMOVE_ALL_NO_WRITE, 0);
 	btr_search_s_unlock_all();
 	row_mysql_lock_data_dictionary(trx);
diff --git a/storage/innobase/include/btr0sea.h b/storage/innobase/include/btr0sea.h
index 8887a3f81ad..fad0dac93c4 100644
--- a/storage/innobase/include/btr0sea.h
+++ b/storage/innobase/include/btr0sea.h
@@ -247,6 +247,8 @@ btr_get_search_table(const dict_index_t* index);
 # define btr_search_drop_page_hash_index(block)
 # define btr_search_s_lock(index)
 # define btr_search_s_unlock(index)
+# define btr_search_s_lock_all(index)
+# define btr_search_s_unlock_all(index)
 # define btr_search_x_lock(index)
 # define btr_search_x_unlock(index)
 # define btr_search_info_update(index, cursor)
diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h
index 9c6d4f920ff..081362fb6f4 100644
--- a/storage/innobase/include/fil0fil.h
+++ b/storage/innobase/include/fil0fil.h
@@ -883,6 +883,17 @@ fil_op_replay_rename(
 	const char*	new_name)
 	MY_ATTRIBUTE((warn_unused_result));
 
+/** Determine whether a table can be accessed in operations that are
+not (necessarily) protected by meta-data locks.
+(Rollback would generally be protected, but rollback of
+FOREIGN KEY CASCADE/SET NULL is not protected by meta-data locks
+but only by InnoDB table locks, which may be broken by TRUNCATE TABLE.)
+@param[in]	table	persistent table
+checked @return whether the table is accessible */
+bool
+fil_table_accessible(const dict_table_t* table)
+	MY_ATTRIBUTE((warn_unused_result, nonnull));
+
 /** Deletes an IBD tablespace, either general or single-table.
 The tablespace must be cached in the memory cache. This will delete the
 datafile, fil_space_t & fil_node_t entries from the file_system_t cache.
diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc
index 5202fb15af8..492d864ec96 100644
--- a/storage/innobase/row/row0purge.cc
+++ b/storage/innobase/row/row0purge.cc
@@ -853,17 +853,10 @@ try_again:
 		/* The table has been dropped: no need to do purge */
 		goto err_exit;
 	}
-	ut_ad(!dict_table_is_temporary(node->table));
 
-	if (fil_space_is_being_truncated(node->table->space)) {
+	ut_ad(!dict_table_is_temporary(node->table));
 
-#if UNIV_DEBUG
-		ib::info() << "Record with space id "
-			   << node->table->space
-			   << " belongs to table which is being truncated"
-			   << " therefore skipping this undo record.";
-#endif
-		ut_ad(dict_table_is_file_per_table(node->table));
+	if (!fil_table_accessible(node->table)) {
 		dict_table_close(node->table, FALSE, FALSE);
 		node->table = NULL;
 		goto err_exit;
@@ -887,16 +880,6 @@ try_again:
 		innobase_init_vc_templ(node->table);
 	}
 
-	if (node->table->ibd_file_missing) {
-		/* We skip purge of missing .ibd files */
-
-		dict_table_close(node->table, FALSE, FALSE);
-
-		node->table = NULL;
-
-		goto err_exit;
-	}
-
 	clust_index = dict_table_get_first_index(node->table);
 
 	if (clust_index == NULL
diff --git a/storage/innobase/row/row0uins.cc b/storage/innobase/row/row0uins.cc
index e0b9508caed..9288adb21a4 100644
--- a/storage/innobase/row/row0uins.cc
+++ b/storage/innobase/row/row0uins.cc
@@ -346,12 +346,20 @@ row_undo_ins_parse_undo_rec(
 
 	/* Skip the UNDO if we can't find the table or the .ibd file. */
 	if (UNIV_UNLIKELY(node->table == NULL)) {
-	} else if (UNIV_UNLIKELY(node->table->ibd_file_missing)) {
-close_table:
-		dict_table_close(node->table, dict_locked, FALSE);
-		node->table = NULL;
-	} else if (fil_space_is_being_truncated(node->table->space)) {
+		return;
+	}
 
+	if (UNIV_UNLIKELY(!fil_table_accessible(node->table))) {
+close_table:
+		/* Normally, tables should not disappear or become
+		unaccessible during ROLLBACK, because they should be
+		protected by InnoDB table locks. TRUNCATE TABLE
+		or table corruption could be valid exceptions.
+
+		FIXME: When running out of temporary tablespace, it
+		would probably be better to just drop all temporary
+		tables (and temporary undo log records) of the current
+		connection, instead of doing this rollback. */
 		dict_table_close(node->table, dict_locked, FALSE);
 		node->table = NULL;
 	} else {
@@ -362,6 +370,9 @@ close_table:
 				ptr, clust_index, &node->ref, node->heap);
 
 			if (!row_undo_search_clust_to_pcur(node)) {
+				/* An error probably occurred during
+				an insert into the clustered index,
+				after we wrote the undo log record. */
 				goto close_table;
 			}
 			if (node->table->n_v_cols) {
diff --git a/storage/innobase/row/row0umod.cc b/storage/innobase/row/row0umod.cc
index c583a438988..ba072a72aa1 100644
--- a/storage/innobase/row/row0umod.cc
+++ b/storage/innobase/row/row0umod.cc
@@ -1139,8 +1139,17 @@ row_undo_mod_parse_undo_rec(
 		return;
 	}
 
-	if (node->table->ibd_file_missing ||
-	    fil_space_is_being_truncated(node->table->space) ) {
+	if (UNIV_UNLIKELY(!fil_table_accessible(node->table))) {
+close_table:
+		/* Normally, tables should not disappear or become
+		unaccessible during ROLLBACK, because they should be
+		protected by InnoDB table locks. TRUNCATE TABLE
+		or table corruption could be valid exceptions.
+
+		FIXME: When running out of temporary tablespace, it
+		would probably be better to just drop all temporary
+		tables (and temporary undo log records) of the current
+		connection, instead of doing this rollback. */
 		dict_table_close(node->table, dict_locked, FALSE);
 		node->table = NULL;
 		return;
@@ -1160,15 +1169,21 @@ row_undo_mod_parse_undo_rec(
 	node->new_trx_id = trx_id;
 	node->cmpl_info = cmpl_info;
 
-	if (!row_undo_search_clust_to_pcur(node)) {
-
-		dict_table_close(node->table, dict_locked, FALSE);
-
-		node->table = NULL;
+	if (UNIV_UNLIKELY(!row_undo_search_clust_to_pcur(node))) {
+		/* This should never occur. As long as this
+		rolling-back transaction exists, the PRIMARY KEY value
+		pointed to by the undo log record must exist.
+		btr_cur_upd_lock_and_undo() only writes the undo log
+		record after successfully acquiring an exclusive lock
+		on the the clustered index record. That lock will not
+		be released before the transaction is committed or
+		fully rolled back. */
+		ut_ad(0);
+		goto close_table;
 	}
 
 	/* Extract indexed virtual columns from undo log */
-	if (node->table && node->table->n_v_cols) {
+	if (node->table->n_v_cols) {
 		row_upd_replace_vcol(node->row, node->table,
 				     node->update, false, node->undo_row,
 				     (node->cmpl_info & UPD_NODE_NO_ORD_CHANGE)