MDEV-8750: Server crashes in page_cur_is_after_last on altering table using a wrong encryption key

Analysis: Server tried to continue reading tablespace using a cursor after
we had resolved that pages in the tablespace can't be decrypted.

Fixed by addind check is tablespace still encrypted.

6 files changed, 109 insertions, 0 deletions

diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result
new file mode 100644
index 00000000000..dc1ae51576b
--- /dev/null
+++ b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result
@@ -0,0 +1,15 @@
+call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
+call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match.");
+call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
+call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
+call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");
+CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTION_KEY_ID=4;
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+select * from t1;
+alter table t1 discard tablespace;
+alter table t1 engine=InnoDB;
+ERROR 42S02: Table 'test.t1' doesn't exist in engine
+show warnings;
+Level	Code	Message
+Warning	155	Table test/t1 is encrypted but encryption service or used key_id is not available.  Can't continue reading table.
+Error	1932	Table 'test.t1' doesn't exist in engine
diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test
new file mode 100644
index 00000000000..e151312ed41
--- /dev/null
+++ b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test
@@ -0,0 +1,50 @@
+--source include/have_innodb.inc
+
+#
+# MDEV-8750: Server crashes in page_cur_is_after_last on altering table using a wrong encryption key
+#
+call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
+call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match.");
+call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
+call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
+call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");
+
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--shutdown_server
+--source include/wait_until_disconnected.inc
+
+--write_file $MYSQLTEST_VARDIR/keys1.txt
+1;770A8A65DA156D24EE2A093277530142
+4;770A8A65DA156D24EE2A093277530143
+EOF
+
+--exec echo "restart:--innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTION_KEY_ID=4;
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--shutdown_server
+--source include/wait_until_disconnected.inc
+
+--write_file $MYSQLTEST_VARDIR/keys2.txt
+1;770A8A65DA156D24EE2A093277530142
+4;770A8A65DA156D24EE2A093277530144
+EOF
+
+--exec echo "restart:--innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys2.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+--error 0,ER_NO_SUCH_TABLE_IN_ENGINE
+select * from t1;
+--error 0,ER_NO_SUCH_TABLE_IN_ENGINE
+alter table t1 discard tablespace;
+--error ER_NO_SUCH_TABLE_IN_ENGINE
+alter table t1 engine=InnoDB;
+show warnings;
+
+--remove_file $MYSQLTEST_VARDIR/keys1.txt
+--remove_file $MYSQLTEST_VARDIR/keys2.txt
diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc
index 316cbc75a8b..3ed0067f395 100644
--- a/storage/innobase/handler/handler0alter.cc
+++ b/storage/innobase/handler/handler0alter.cc
@@ -4179,6 +4179,10 @@ oom:
 			 : ha_alter_info->key_info_buffer[
 				 prebuilt->trx->error_key_num].name);
 		break;
+	case DB_ENCRYPTED_DECRYPT_FAILED:
+		my_error(ER_NO_SUCH_TABLE_IN_ENGINE, MYF(0),
+			table_share->db.str, table_share->table_name.str);
+		break;
 	default:
 		my_error_innodb(error,
 				table_share->table_name.str,
diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc
index c1d3e08beaa..ab2ea05b2f2 100644
--- a/storage/innobase/row/row0merge.cc
+++ b/storage/innobase/row/row0merge.cc
@@ -1431,6 +1431,13 @@ row_merge_read_clustered_index(
 		row_ext_t*	ext;
 		page_cur_t*	cur	= btr_pcur_get_page_cur(&pcur);
 
+		/* Do not continue if table pages are still encrypted */
+		if (old_table->is_encrypted || new_table->is_encrypted) {
+			err = DB_ENCRYPTED_DECRYPT_FAILED;
+			trx->error_key_num = 0;
+			goto func_exit;
+		}
+
 		page_cur_move_to_next(cur);
 
 		if (page_cur_is_after_last(cur)) {
@@ -3754,6 +3761,17 @@ row_merge_build_indexes(
 
 	pct_cost = COST_READ_CLUSTERED_INDEX * 100 / (total_static_cost + total_dynamic_cost);
 
+	/* Do not continue if we can't encrypt table pages */
+	if (old_table->is_encrypted || new_table->is_encrypted) {
+		error = DB_ENCRYPTED_DECRYPT_FAILED;
+		ib_push_warning(trx->mysql_thd, DB_ENCRYPTED_DECRYPT_FAILED,
+			"Table %s is encrypted but encryption service or"
+			" used key_id is not available. "
+			" Can't continue reading table.",
+			old_table->is_encrypted ? old_table->name : new_table->name);
+		goto func_exit;
+	}
+
 	/* Read clustered index of the table and create files for
 	secondary index entries for merge sort */
 
diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc
index 168921ae20b..5a190e4ca9a 100644
--- a/storage/xtradb/handler/handler0alter.cc
+++ b/storage/xtradb/handler/handler0alter.cc
@@ -4168,6 +4168,10 @@ oom:
 			 : ha_alter_info->key_info_buffer[
 				 prebuilt->trx->error_key_num].name);
 		break;
+	case DB_ENCRYPTED_DECRYPT_FAILED:
+		my_error(ER_NO_SUCH_TABLE_IN_ENGINE, MYF(0),
+			table_share->db.str, table_share->table_name.str);
+		break;
 	default:
 		my_error_innodb(error,
 				table_share->table_name.str,
diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc
index 2df9c2bea82..20fc1768eda 100644
--- a/storage/xtradb/row/row0merge.cc
+++ b/storage/xtradb/row/row0merge.cc
@@ -1433,6 +1433,13 @@ row_merge_read_clustered_index(
 		row_ext_t*	ext;
 		page_cur_t*	cur	= btr_pcur_get_page_cur(&pcur);
 
+		/* Do not continue if table pages are still encrypted */
+		if (old_table->is_encrypted || new_table->is_encrypted) {
+			err = DB_ENCRYPTED_DECRYPT_FAILED;
+			trx->error_key_num = 0;
+			goto func_exit;
+		}
+
 		page_cur_move_to_next(cur);
 
 		if (page_cur_is_after_last(cur)) {
@@ -3762,6 +3769,17 @@ row_merge_build_indexes(
 
 	pct_cost = COST_READ_CLUSTERED_INDEX * 100 / (total_static_cost + total_dynamic_cost);
 
+	/* Do not continue if we can't encrypt table pages */
+	if (old_table->is_encrypted || new_table->is_encrypted) {
+		error = DB_ENCRYPTED_DECRYPT_FAILED;
+		ib_push_warning(trx->mysql_thd, DB_ENCRYPTED_DECRYPT_FAILED,
+			"Table %s is encrypted but encryption service or"
+			" used key_id is not available. "
+			" Can't continue reading table.",
+			old_table->is_encrypted ? old_table->name : new_table->name);
+		goto func_exit;
+	}
+
 	/* Read clustered index of the table and create files for
 	secondary index entries for merge sort */