MDEV-17230: encryption_key_id from alter is ignored by encryption threads

Background: Used encryption key_id is stored to encryption metadata
i.e. crypt_data that is stored on page 0 of the tablespace of the
table. crypt_data is created only if implicit encryption/not encryption
is requested i.e. ENCRYPTED=[YES|NO] table option is used
fil_create_new_single_table_tablespace on fil0fil.cc.

Later if encryption is enabled all tables that use default encryption
mode (i.e. no encryption table option is set) are encrypted with
default encryption key_id that is 1. See fil_crypt_start_encrypting_space on
fil0crypt.cc.

ha_innobase::check_table_options()
	If default encryption is used and encryption is disabled, you may
        not use nondefault encryption_key_id as it is not stored anywhere.

2 files changed, 61 insertions, 0 deletions

diff --git a/mysql-test/suite/encryption/r/innodb-encryption-alter.result b/mysql-test/suite/encryption/r/innodb-encryption-alter.result
index 9ff0f492034..5245d1da7d0 100644
--- a/mysql-test/suite/encryption/r/innodb-encryption-alter.result
+++ b/mysql-test/suite/encryption/r/innodb-encryption-alter.result
@@ -50,3 +50,40 @@ Warning	140	InnoDB: ENCRYPTION_KEY_ID 99 not available
 Error	1478	Table storage engine 'InnoDB' does not support the create option 'ENCRYPTION_KEY_ID'
 set innodb_default_encryption_key_id = 1;
 drop table t1,t2;
+SET GLOBAL innodb_encrypt_tables=OFF;
+CREATE TABLE t1 (a int not null primary key) engine=innodb;
+ALTER TABLE t1 ENCRYPTION_KEY_ID=4;
+ERROR HY000: Table storage engine 'InnoDB' does not support the create option 'ENCRYPTION_KEY_ID'
+SHOW WARNINGS;
+Level	Code	Message
+Warning	140	InnoDB: innodb_encrypt_tables=OFF only allows ENCRYPTION_KEY_ID=1
+Error	1478	Table storage engine 'InnoDB' does not support the create option 'ENCRYPTION_KEY_ID'
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) NOT NULL,
+  PRIMARY KEY (`a`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+DROP TABLE t1;
+CREATE TABLE t2 (a int not null primary key) engine=innodb;
+ALTER TABLE t2 ENCRYPTION_KEY_ID=4, ALGORITHM=COPY;
+ERROR HY000: Can't create table `test`.`#sql-temporary` (errno: 140 "Wrong create options")
+SHOW WARNINGS;
+Level	Code	Message
+Warning	140	InnoDB: innodb_encrypt_tables=OFF only allows ENCRYPTION_KEY_ID=1
+Error	1005	Can't create table `test`.`#sql-temporary` (errno: 140 "Wrong create options")
+Warning	1030	Got error 140 "Wrong create options" from storage engine InnoDB
+SHOW CREATE TABLE t2;
+Table	Create Table
+t2	CREATE TABLE `t2` (
+  `a` int(11) NOT NULL,
+  PRIMARY KEY (`a`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+DROP TABLE t2;
+CREATE TABLE t3 (a int not null primary key) engine=innodb ENCRYPTION_KEY_ID=4;
+ERROR HY000: Can't create table `test`.`t3` (errno: 140 "Wrong create options")
+SHOW WARNINGS;
+Level	Code	Message
+Warning	140	InnoDB: innodb_encrypt_tables=OFF only allows ENCRYPTION_KEY_ID=1
+Error	1005	Can't create table `test`.`t3` (errno: 140 "Wrong create options")
+Warning	1030	Got error 140 "Wrong create options" from storage engine InnoDB
diff --git a/mysql-test/suite/encryption/t/innodb-encryption-alter.test b/mysql-test/suite/encryption/t/innodb-encryption-alter.test
index 9420fb74a4c..9465226dd96 100644
--- a/mysql-test/suite/encryption/t/innodb-encryption-alter.test
+++ b/mysql-test/suite/encryption/t/innodb-encryption-alter.test
@@ -87,6 +87,30 @@ connection default;
 
 drop table t1,t2;
 
+#
+# MDEV-17230: encryption_key_id from alter is ignored by encryption threads
+#
+SET GLOBAL innodb_encrypt_tables=OFF;
+CREATE TABLE t1 (a int not null primary key) engine=innodb;
+--error ER_ILLEGAL_HA_CREATE_OPTION
+ALTER TABLE t1 ENCRYPTION_KEY_ID=4;
+SHOW WARNINGS;
+SHOW CREATE TABLE t1;
+DROP TABLE t1;
+
+CREATE TABLE t2 (a int not null primary key) engine=innodb;
+--replace_regex /#sql-[0-9a-f_]*`/#sql-temporary`/
+--error ER_CANT_CREATE_TABLE
+ALTER TABLE t2 ENCRYPTION_KEY_ID=4, ALGORITHM=COPY;
+--replace_regex /#sql-[0-9a-f_]*`/#sql-temporary`/
+SHOW WARNINGS;
+SHOW CREATE TABLE t2;
+DROP TABLE t2;
+
+--error ER_CANT_CREATE_TABLE
+CREATE TABLE t3 (a int not null primary key) engine=innodb ENCRYPTION_KEY_ID=4;
+SHOW WARNINGS;
+
 # reset system
 --disable_query_log
 EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig;