MDEV-24695 Encryption modifies a freed page

During recovery, InnoDB fails if it tries to apply a FREE_PAGE
and WRITE record to the page. InnoDB encryption thread accesses
the freed page and writes redo log for it.

This is similar to commit deadec4e689c9435e20ebb89fd8f84d3f0f90ff3 (MDEV-24569)
InnoDB is missing buf_page_free() while freeing the segment.
To avoid accessing of freed page in buffer pool, InnoDB should
mark the pages as FREED while freeing the segment. Also to
avoid reading of freed page, InnoDB should check the
allocation bitmap page.

fseg_free_step(): Mark the page in buffer pool as FREED

fseg_free_step_not_header(): Mark the page in buffer pool as FREED

buf_dump(): Ignore the freed pages while dumping the buffer pool content

fil_crypt_get_page_throttle_func(): Skip the rotation for FREED page
to avoid the assert failure during recovery

fil_crypt_rotate_page(): Skip the rotation for the freed page

Reviewed-by: Marko Mäkelä

7 files changed, 236 insertions, 5 deletions

diff --git a/mysql-test/suite/encryption/r/innodb_encrypt_freed.result b/mysql-test/suite/encryption/r/innodb_encrypt_freed.result
new file mode 100644
index 00000000000..cdcdd17720e
--- /dev/null
+++ b/mysql-test/suite/encryption/r/innodb_encrypt_freed.result
@@ -0,0 +1,96 @@
+SHOW VARIABLES LIKE 'innodb_encrypt%';
+Variable_name	Value
+innodb_encrypt_log	ON
+innodb_encrypt_tables	ON
+innodb_encrypt_temporary_tables	OFF
+innodb_encryption_rotate_key_age	1
+innodb_encryption_rotation_iops	100
+innodb_encryption_threads	1
+SET GLOBAL innodb_encrypt_tables = ON;
+CREATE TABLE t1(f1 BIGINT PRIMARY KEY, f2 int not null,
+f3 int not null, index(f1), index idx_1(f2),
+index(f2, f3)) ENGINE=InnoDB;
+# Wait max 10 min for key encryption threads to encrypt all spaces
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+NAME
+innodb_system
+mysql/innodb_index_stats
+mysql/innodb_table_stats
+mysql/transaction_registry
+test/t1
+CREATE TABLE t2 (f1 int not null)engine=innodb;
+# restart: --debug=d,ib_log_checkpoint_avoid
+connect con1,localhost,root,,,;
+begin;
+insert into t2 values(1);
+connection default;
+set global innodb_encrypt_tables = OFF;
+# Wait max 10 min for key encryption threads to decrypt all spaces
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+NAME
+innodb_system
+mysql/innodb_index_stats
+mysql/innodb_table_stats
+mysql/transaction_registry
+test/t1
+test/t2
+alter table t1 drop index idx_1;
+set global innodb_encrypt_tables = ON;
+# Wait max 10 min for key encryption threads to encrypt all spaces
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+NAME
+innodb_system
+mysql/innodb_index_stats
+mysql/innodb_table_stats
+mysql/transaction_registry
+test/t1
+test/t2
+disconnect con1;
+# restart: --debug=d,ib_log_checkpoint_avoid
+drop table t1, t2;
+CREATE TABLE t1(f1 BIGINT PRIMARY KEY, f2 int not null,
+f3 int not null, index(f1), index idx_1(f2),
+index(f2, f3)) ENGINE=InnoDB;
+# Wait max 10 min for key encryption threads to encrypt all spaces
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+NAME
+innodb_system
+mysql/innodb_index_stats
+mysql/innodb_table_stats
+mysql/transaction_registry
+test/t1
+CREATE TABLE t2 (f1 int not null)engine=innodb;
+# restart: --debug=d,ib_log_checkpoint_avoid
+connect con1,localhost,root,,,;
+begin;
+insert into t2 values(1);
+connection default;
+set global innodb_encrypt_tables = OFF;
+# Wait max 10 min for key encryption threads to decrypt all spaces
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+NAME
+innodb_system
+mysql/innodb_index_stats
+mysql/innodb_table_stats
+mysql/transaction_registry
+test/t1
+test/t2
+alter table t1 drop index idx_1;
+disconnect con1;
+# restart: --debug=d,ib_log_checkpoint_avoid
+connect con1,localhost,root,,,;
+begin;
+insert into t2 values(1);
+connection default;
+set global innodb_encrypt_tables = ON;
+# Wait max 10 min for key encryption threads to encrypt all spaces
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+NAME
+innodb_system
+mysql/innodb_index_stats
+mysql/innodb_table_stats
+mysql/transaction_registry
+test/t1
+test/t2
+disconnect con1;
+drop table t2, t1;
diff --git a/mysql-test/suite/encryption/t/innodb_encrypt_freed.opt b/mysql-test/suite/encryption/t/innodb_encrypt_freed.opt
new file mode 100644
index 00000000000..221e38367f5
--- /dev/null
+++ b/mysql-test/suite/encryption/t/innodb_encrypt_freed.opt
@@ -0,0 +1,4 @@
+--innodb-encrypt-tables
+--innodb-encrypt-log
+--innodb-encryption-threads=1
+--innodb-tablespaces-encryption
diff --git a/mysql-test/suite/encryption/t/innodb_encrypt_freed.test b/mysql-test/suite/encryption/t/innodb_encrypt_freed.test
new file mode 100644
index 00000000000..785b4e9e498
--- /dev/null
+++ b/mysql-test/suite/encryption/t/innodb_encrypt_freed.test
@@ -0,0 +1,117 @@
+--source include/have_innodb.inc
+--source include/have_example_key_management_plugin.inc
+--source include/have_debug.inc
+--source include/not_embedded.inc
+
+SHOW VARIABLES LIKE 'innodb_encrypt%';
+
+SET GLOBAL innodb_encrypt_tables = ON;
+
+CREATE TABLE t1(f1 BIGINT PRIMARY KEY, f2 int not null,
+		f3 int not null, index(f1), index idx_1(f2),
+		index(f2, f3)) ENGINE=InnoDB;
+--let $tables_count= `select count(*) + 1 from information_schema.tables where engine = 'InnoDB'`
+
+--echo # Wait max 10 min for key encryption threads to encrypt all spaces
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+--source include/wait_condition.inc
+--sorted_result
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+
+CREATE TABLE t2 (f1 int not null)engine=innodb;
+let $restart_parameters="--debug=d,ib_log_checkpoint_avoid";
+--source include/restart_mysqld.inc
+
+# Stop the purge
+connect(con1,localhost,root,,,);
+begin;
+insert into t2 values(1);
+
+connection default;
+
+--let $tables_count= `select count(*) + 1 from information_schema.tables where engine = 'InnoDB'`
+set global innodb_encrypt_tables = OFF;
+
+--echo # Wait max 10 min for key encryption threads to decrypt all spaces
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+--source include/wait_condition.inc
+--sorted_result
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+
+# Free the index `idx_1`
+alter table t1 drop index idx_1;
+
+set global innodb_encrypt_tables = ON;
+
+--echo # Wait max 10 min for key encryption threads to encrypt all spaces
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+--source include/wait_condition.inc
+--sorted_result
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+
+disconnect con1;
+let $shutdown_timeout=0;
+--source include/restart_mysqld.inc
+
+drop table t1, t2;
+
+#
+#
+CREATE TABLE t1(f1 BIGINT PRIMARY KEY, f2 int not null,
+		f3 int not null, index(f1), index idx_1(f2),
+		index(f2, f3)) ENGINE=InnoDB;
+--let $tables_count= `select count(*) + 1 from information_schema.tables where engine = 'InnoDB'`
+
+--echo # Wait max 10 min for key encryption threads to encrypt all spaces
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+--source include/wait_condition.inc
+--sorted_result
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+
+CREATE TABLE t2 (f1 int not null)engine=innodb;
+--source include/restart_mysqld.inc
+
+# Stop the purge
+connect(con1,localhost,root,,,);
+begin;
+insert into t2 values(1);
+
+connection default;
+
+--let $tables_count= `select count(*) + 1 from information_schema.tables where engine = 'InnoDB'`
+set global innodb_encrypt_tables = OFF;
+
+--echo # Wait max 10 min for key encryption threads to decrypt all spaces
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+--source include/wait_condition.inc
+--sorted_result
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+
+# Free the index `idx_1`
+alter table t1 drop index idx_1;
+
+disconnect con1;
+--source include/restart_mysqld.inc
+
+# Stop the purge
+connect(con1,localhost,root,,,);
+begin;
+insert into t2 values(1);
+
+connection default;
+set global innodb_encrypt_tables = ON;
+
+--echo # Wait max 10 min for key encryption threads to encrypt all spaces
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+--source include/wait_condition.inc
+--sorted_result
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+
+disconnect con1;
+drop table t2, t1;
diff --git a/mysql-test/suite/encryption/t/innodb_encryption.test b/mysql-test/suite/encryption/t/innodb_encryption.test
index a1abfb51462..1c8d200458a 100644
--- a/mysql-test/suite/encryption/t/innodb_encryption.test
+++ b/mysql-test/suite/encryption/t/innodb_encryption.test
@@ -14,7 +14,7 @@ SHOW VARIABLES LIKE 'innodb_encrypt%';
 
 SET GLOBAL innodb_encrypt_tables = ON;
 
---let $tables_count= `select count(*) + 1 from information_schema.tables where engine = 'InnoDB'`
+--let $tables_count= `select count(*) + @@global.innodb_undo_tablespaces + 1 from information_schema.tables where engine = 'InnoDB'`
 
 --echo # Wait max 10 min for key encryption threads to encrypt all spaces
 --let $wait_timeout= 600
diff --git a/storage/innobase/buf/buf0dump.cc b/storage/innobase/buf/buf0dump.cc
index c62c0112e92..c6ddcb4fa48 100644
--- a/storage/innobase/buf/buf0dump.cc
+++ b/storage/innobase/buf/buf0dump.cc
@@ -332,6 +332,10 @@ buf_dump(
 			continue;
 		}
 
+		if (bpage->status == buf_page_t::FREED) {
+			continue;
+		}
+
 		dump[j++] = id;
 	}
 
diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc
index 26c73e7d0eb..2a8fa875e90 100644
--- a/storage/innobase/fil/fil0crypt.cc
+++ b/storage/innobase/fil/fil0crypt.cc
@@ -1771,6 +1771,11 @@ fil_crypt_get_page_throttle_func(
 		return NULL;
 	}
 
+	if (fseg_page_is_free(space, state->offset)) {
+		/* page is already freed */
+		return NULL;
+	}
+
 	state->crypt_stat.pages_read_from_disk++;
 
 	const ulonglong start = my_interval_timer();
@@ -1874,6 +1879,9 @@ fil_crypt_rotate_page(
 			some dummy pages will be allocated, with 0 in
 			the FIL_PAGE_TYPE. Those pages should be
 			skipped from key rotation forever. */
+		} else if (block->page.status == buf_page_t::FREED) {
+			/* Do not modify freed pages to avoid an assertion
+			failure on recovery.*/
 		} else if (fil_crypt_needs_rotation(
 				crypt_data,
 				kv,
diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc
index 1c88d98c3b8..8b821755227 100644
--- a/storage/innobase/fsp/fsp0fsp.cc
+++ b/storage/innobase/fsp/fsp0fsp.cc
@@ -2705,10 +2705,11 @@ fseg_free_step(
 		DBUG_RETURN(true);
 	}
 
-	fseg_free_page_low(
-		inode, iblock, space,
-		fseg_get_nth_frag_page_no(inode, n),
-		mtr);
+	page_no_t page_no = fseg_get_nth_frag_page_no(inode, n);
+
+	fseg_free_page_low(inode, iblock, space, page_no, mtr);
+
+	buf_page_free(space, page_no, mtr, __FILE__, __LINE__);
 
 	n = fseg_find_last_used_frag_page_slot(inode);
 
@@ -2770,6 +2771,7 @@ fseg_free_step_not_header(
 	}
 
 	fseg_free_page_low(inode, iblock, space, page_no, mtr);
+	buf_page_free(space, page_no, mtr, __FILE__, __LINE__);
 	return false;
 }