MDEV-11782: Redefine the innodb_encrypt_log format

Write only one encryption key to the checkpoint page.
Use 4 bytes of nonce. Encrypt more of each redo log block,
only skipping the 4-byte field LOG_BLOCK_HDR_NO which the
initialization vector is derived from.

Issue notes, not warning messages for rewriting the redo log files.

recv_recovery_from_checkpoint_finish(): Do not generate any redo log,
because we must avoid that before rewriting the redo log files, or
otherwise a crash during a redo log rewrite (removing or adding
encryption) may end up making the database unrecoverable.
Instead, do these tasks in innobase_start_or_create_for_mysql().

Issue a firm "Missing MLOG_CHECKPOINT" error message. Remove some
unreachable code and duplicated error messages for log corruption.

LOG_HEADER_FORMAT_ENCRYPTED: A flag for identifying an encrypted redo
log format.

log_group_t::is_encrypted(), log_t::is_encrypted(): Determine
if the redo log is in encrypted format.

recv_find_max_checkpoint(): Interpret LOG_HEADER_FORMAT_ENCRYPTED.

srv_prepare_to_delete_redo_log_files(): Display NOTE messages about
adding or removing encryption. Do not issue warnings for redo log
resizing any more.

innobase_start_or_create_for_mysql(): Rebuild the redo logs also when
the encryption changes.

innodb_log_checksums_func_update(): Always use the CRC-32C checksum
if innodb_encrypt_log. If needed, issue a warning
that innodb_encrypt_log implies innodb_log_checksums.

log_group_write_buf(): Compute the checksum on the encrypted
block contents, so that transmission errors or incomplete blocks can be
detected without decrypting.

Rewrite most of the redo log encryption code. Only remember one
encryption key at a time (but remember up to 5 when upgrading from the
MariaDB 10.1 format.)

13 files changed, 14 insertions, 51 deletions

diff --git a/mysql-test/suite/innodb/r/innodb-32k.result b/mysql-test/suite/innodb/r/innodb-32k.result
index 29374689a3b..140b9b87c2f 100644
--- a/mysql-test/suite/innodb/r/innodb-32k.result
+++ b/mysql-test/suite/innodb/r/innodb-32k.result
@@ -1,7 +1,3 @@
-call mtr.add_suppression("InnoDB: Warning: innodb_page_size has been changed from default value ");
-call mtr.add_suppression("InnoDB: Resizing redo log from ");
-call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files");
-call mtr.add_suppression("InnoDB: New log files created, LSN=");
 call mtr.add_suppression("Innodb: Cannot add field.*row size is");
 # Test 1) Show the page size from Information Schema
 SELECT variable_value FROM information_schema.global_status
diff --git a/mysql-test/suite/innodb/r/innodb-64k.result b/mysql-test/suite/innodb/r/innodb-64k.result
index 9271ad70fb4..f72ba8ef8b5 100644
--- a/mysql-test/suite/innodb/r/innodb-64k.result
+++ b/mysql-test/suite/innodb/r/innodb-64k.result
@@ -1,7 +1,3 @@
-call mtr.add_suppression("InnoDB: Warning: innodb_page_size has been changed from default value *");
-call mtr.add_suppression("InnoDB: Resizing redo log from *");
-call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files.");
-call mtr.add_suppression("InnoDB: New log files created, LSN=*");
 # Test 1) Show the page size from Information Schema
 SELECT variable_value FROM information_schema.global_status
 WHERE LOWER(variable_name) = 'innodb_page_size';
diff --git a/mysql-test/suite/innodb/r/innodb-bigblob.result b/mysql-test/suite/innodb/r/innodb-bigblob.result
index 20fe0ce5c43..dcde9804cdc 100644
--- a/mysql-test/suite/innodb/r/innodb-bigblob.result
+++ b/mysql-test/suite/innodb/r/innodb-bigblob.result
@@ -1,7 +1,3 @@
-call mtr.add_suppression("Resizing redo log from *");
-call mtr.add_suppression("Starting to delete and rewrite log files.");
-call mtr.add_suppression("New log files created, LSN=*");
-call mtr.add_suppression("Writer thread is waiting this semaphore");
 create table foo (id varchar(37) not null, content longblob) engine=INNODB;
 insert into foo (id, content) values('xyz', '');
 update foo set content=repeat('a', 43941888) where id='xyz';
diff --git a/mysql-test/suite/innodb/r/log_corruption.result b/mysql-test/suite/innodb/r/log_corruption.result
index ad06398e4e2..ccf5f73a3a3 100644
--- a/mysql-test/suite/innodb/r/log_corruption.result
+++ b/mysql-test/suite/innodb/r/log_corruption.result
@@ -41,7 +41,7 @@ WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
 FOUND /InnoDB: Invalid log block checksum. block: 2372 checkpoint no: 1 expected: 3362026715 found: 144444122/ in mysqld.1.err
-FOUND /InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
+FOUND /InnoDB: Missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
 # --innodb-force-recovery=6 (skip the entire redo log)
 SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
@@ -54,7 +54,6 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
-FOUND /InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
 # --innodb-force-recovery=6 (skip the entire redo log)
 SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
@@ -90,7 +89,7 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
-FOUND /InnoDB: Redo log crypto: getting mysqld crypto key from key version failed err = 4294967295/ in mysqld.1.err
+FOUND /InnoDB: Obtaining redo log encryption key version 1 failed/ in mysqld.1.err
 FOUND /InnoDB: Decrypting checkpoint failed/ in mysqld.1.err
 ib_buffer_pool
 ib_logfile0
diff --git a/mysql-test/suite/innodb/r/log_file.result b/mysql-test/suite/innodb/r/log_file.result
index 0003f7d3c3a..352e4b76cf1 100644
--- a/mysql-test/suite/innodb/r/log_file.result
+++ b/mysql-test/suite/innodb/r/log_file.result
@@ -340,7 +340,7 @@ WHERE engine='innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 1
 1
-FOUND /Resizing redo log from 1\*\d+ to 3\*\d+ pages, LSN=\d+/ in mysqld.1.err
+FOUND /Resizing redo log from 1\*\d+ to 3\*\d+ pages; LSN=\d+/ in mysqld.1.err
 # Cleanup
 bak_ib_logfile0
 bak_ib_logfile1
diff --git a/mysql-test/suite/innodb/t/innodb-32k.test b/mysql-test/suite/innodb/t/innodb-32k.test
index 53a2d3a7442..80a05c350d0 100644
--- a/mysql-test/suite/innodb/t/innodb-32k.test
+++ b/mysql-test/suite/innodb/t/innodb-32k.test
@@ -3,10 +3,6 @@
 --source include/have_innodb.inc
 --source include/have_innodb_32k.inc
 
-call mtr.add_suppression("InnoDB: Warning: innodb_page_size has been changed from default value ");
-call mtr.add_suppression("InnoDB: Resizing redo log from ");
-call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files");
-call mtr.add_suppression("InnoDB: New log files created, LSN=");
 call mtr.add_suppression("Innodb: Cannot add field.*row size is");
 
 let $MYSQLD_DATADIR= `select @@datadir`;
diff --git a/mysql-test/suite/innodb/t/innodb-64k.test b/mysql-test/suite/innodb/t/innodb-64k.test
index c611b6cb2e2..13351450cfc 100644
--- a/mysql-test/suite/innodb/t/innodb-64k.test
+++ b/mysql-test/suite/innodb/t/innodb-64k.test
@@ -3,11 +3,6 @@
 --source include/have_innodb.inc
 --source include/have_innodb_64k.inc
 
-call mtr.add_suppression("InnoDB: Warning: innodb_page_size has been changed from default value *");
-call mtr.add_suppression("InnoDB: Resizing redo log from *");
-call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files.");
-call mtr.add_suppression("InnoDB: New log files created, LSN=*");
-
 let $MYSQLD_DATADIR= `select @@datadir`;
 
 --echo # Test 1) Show the page size from Information Schema
diff --git a/mysql-test/suite/innodb/t/innodb-bigblob.test b/mysql-test/suite/innodb/t/innodb-bigblob.test
index d72e20487e4..799dfe42d48 100644
--- a/mysql-test/suite/innodb/t/innodb-bigblob.test
+++ b/mysql-test/suite/innodb/t/innodb-bigblob.test
@@ -6,11 +6,6 @@
 let $status_orig=`SELECT @@innodb_status_output`;
 --enable_query_log
 
-call mtr.add_suppression("Resizing redo log from *");
-call mtr.add_suppression("Starting to delete and rewrite log files.");
-call mtr.add_suppression("New log files created, LSN=*");
-call mtr.add_suppression("Writer thread is waiting this semaphore");
-
 create table foo (id varchar(37) not null, content longblob) engine=INNODB;
 insert into foo (id, content) values('xyz', '');
 update foo set content=repeat('a', 43941888) where id='xyz';
diff --git a/mysql-test/suite/innodb/t/log_corruption.test b/mysql-test/suite/innodb/t/log_corruption.test
index 0f59f64ebdf..d193b3e74a8 100644
--- a/mysql-test/suite/innodb/t/log_corruption.test
+++ b/mysql-test/suite/innodb/t/log_corruption.test
@@ -9,11 +9,11 @@ call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE faile
 call mtr.add_suppression("InnoDB: Unsupported redo log format");
 call mtr.add_suppression("InnoDB: No valid checkpoint found");
 call mtr.add_suppression("InnoDB: Invalid (log block|redo log header) checksum");
-call mtr.add_suppression("InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT");
+call mtr.add_suppression("InnoDB: Missing MLOG_CHECKPOINT");
 call mtr.add_suppression("InnoDB: MLOG_FILE_NAME incorrect");
 call mtr.add_suppression("InnoDB: ############### CORRUPT LOG RECORD FOUND");
-call mtr.add_suppression("InnoDB: Found corrupted log");
-call mtr.add_suppression("InnoDB: Redo log crypto: getting mysqld crypto key from key version failed");
+call mtr.add_suppression("InnoDB: Log scan aborted at LSN");
+call mtr.add_suppression("InnoDB: Obtaining redo log encryption key version 1 failed");
 call mtr.add_suppression("InnoDB: Decrypting checkpoint failed");
 --enable_query_log
 
@@ -206,7 +206,7 @@ eval $check_no_innodb;
 --source include/shutdown_mysqld.inc
 let SEARCH_PATTERN=InnoDB: Invalid log block checksum. block: 2372 checkpoint no: 1 expected: 3362026715 found: 144444122;
 --source include/search_pattern_in_file.inc
-let SEARCH_PATTERN=InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\.;
+let SEARCH_PATTERN=InnoDB: Missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\.;
 --source include/search_pattern_in_file.inc
 --echo # --innodb-force-recovery=6 (skip the entire redo log)
 --let $restart_parameters= $dirs --innodb-force-recovery=6
@@ -232,14 +232,12 @@ print OUT pack("H*x[5]", "C0DEBA5E0022000c0000000138");
 print OUT pack("H*x[475]H*", "12860cb7809781e80006626f677573", "089C0ADA");
 EOF
 --copy_file $bugdir/ib_logfile0 $bugdir/ib_logfile
-# Anything below innodb_force_recovery=6 must find a valid redo log.
+# Anything below innodb_force_recovery=6 must find an invalid redo log.
 # Missing tablespace files are tolerated already with innodb_force_recovery=1.
 --let $restart_parameters= $dirs --innodb-force-recovery=5
 --source include/start_mysqld.inc
 eval $check_no_innodb;
 --source include/shutdown_mysqld.inc
-let SEARCH_PATTERN=InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\.;
---source include/search_pattern_in_file.inc
 --echo # --innodb-force-recovery=6 (skip the entire redo log)
 --let $restart_parameters= $dirs --innodb-force-recovery=6
 --source include/start_mysqld.inc
@@ -340,7 +338,7 @@ EOF
 if (!$no_cleanup) {
 eval $check_no_innodb;
 --source include/shutdown_mysqld.inc
---let SEARCH_PATTERN= InnoDB: Redo log crypto: getting mysqld crypto key from key version failed err = 4294967295
+--let SEARCH_PATTERN= InnoDB: Obtaining redo log encryption key version 1 failed
 --source include/search_pattern_in_file.inc
 --let SEARCH_PATTERN= InnoDB: Decrypting checkpoint failed
 --source include/search_pattern_in_file.inc
diff --git a/mysql-test/suite/innodb/t/log_file.test b/mysql-test/suite/innodb/t/log_file.test
index 0bb8ee8845c..c50257a69be 100644
--- a/mysql-test/suite/innodb/t/log_file.test
+++ b/mysql-test/suite/innodb/t/log_file.test
@@ -219,7 +219,7 @@ eval $check_no_innodb;
 --source include/start_mysqld.inc
 eval $check_yes_innodb;
 --source include/shutdown_mysqld.inc
---let SEARCH_PATTERN=Resizing redo log from 1\*\d+ to 3\*\d+ pages, LSN=\d+
+--let SEARCH_PATTERN=Resizing redo log from 1\*\d+ to 3\*\d+ pages; LSN=\d+
 --source include/search_pattern_in_file.inc
 
 --let $restart_parameters=
diff --git a/mysql-test/suite/innodb/t/log_file_name.test b/mysql-test/suite/innodb/t/log_file_name.test
index 78500a6a310..e528abc80d5 100644
--- a/mysql-test/suite/innodb/t/log_file_name.test
+++ b/mysql-test/suite/innodb/t/log_file_name.test
@@ -160,7 +160,7 @@ call mtr.add_suppression("InnoDB: Cannot read first page in datafile: .*test.*ib
 call mtr.add_suppression("InnoDB: Datafile '.*test.*ibd' is corrupted");
 call mtr.add_suppression("InnoDB:  Cannot replay file rename. Remove either file and try again");
 call mtr.add_suppression("InnoDB: Cannot rename.*because the target file exists");
-call mtr.add_suppression("InnoDB: Found corrupted log");
+call mtr.add_suppression("InnoDB: Log scan aborted at LSN");
 # The following are for the --innodb-force-recovery=1 with broken u* tables:
 call mtr.add_suppression("InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd");
 call mtr.add_suppression("InnoDB: The error means the system cannot find the path specified");
diff --git a/mysql-test/suite/innodb/t/log_file_size.test b/mysql-test/suite/innodb/t/log_file_size.test
index 179472a45a1..d8edcda28e5 100644
--- a/mysql-test/suite/innodb/t/log_file_size.test
+++ b/mysql-test/suite/innodb/t/log_file_size.test
@@ -10,9 +10,6 @@ if (`SELECT @@innodb_log_file_size = 1048576`) {
 }
 
 --disable_query_log
-call mtr.add_suppression("InnoDB: Resizing redo log");
-call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files");
-call mtr.add_suppression("InnoDB: New log files created");
 call mtr.add_suppression("InnoDB: The log sequence numbers [0-9]+ and [0-9]+ in ibdata files do not match the log sequence number [0-9]+ in the ib_logfiles");
 call mtr.add_suppression("syntax error in innodb_log_group_home_dir");
 call mtr.add_suppression("Plugin 'InnoDB' init function returned error");
@@ -100,7 +97,7 @@ let SEARCH_PATTERN= InnoDB: innodb_read_only prevents crash recovery;
 SELECT * FROM t1;
 let SEARCH_PATTERN= InnoDB: Starting an apply batch of log records;
 --source include/search_pattern_in_file.inc
-let SEARCH_PATTERN= InnoDB: Resizing redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
+let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
 --source include/search_pattern_in_file.inc
 
 --let $restart_parameters= --debug=d,innodb_log_abort_5
@@ -109,7 +106,7 @@ let SEARCH_PATTERN= InnoDB: Resizing redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
 SELECT * FROM t1;
 let SEARCH_PATTERN= InnoDB: Starting an apply batch of log records;
 --source include/search_pattern_in_file.inc
-let SEARCH_PATTERN= InnoDB: Resizing redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
+let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
 --source include/search_pattern_in_file.inc
 
 --let $restart_parameters= --innodb-read-only
@@ -126,7 +123,7 @@ SELECT * FROM t1;
 
 let SEARCH_PATTERN= InnoDB: Starting an apply batch of log records;
 --source include/search_pattern_in_file.inc
-let SEARCH_PATTERN= InnoDB: Resizing redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
+let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
 --source include/search_pattern_in_file.inc
 
 --let $restart_parameters= --debug=d,innodb_log_abort_7
diff --git a/mysql-test/suite/innodb/t/log_file_size_checkpoint.test b/mysql-test/suite/innodb/t/log_file_size_checkpoint.test
index 26e0bdf5e2c..16b71bfd1f4 100644
--- a/mysql-test/suite/innodb/t/log_file_size_checkpoint.test
+++ b/mysql-test/suite/innodb/t/log_file_size_checkpoint.test
@@ -8,11 +8,6 @@ let $n=250;
 let $t=veryLongTableNameToCreateMLOG_FILE_NAMErecords;
 
 --disable_query_log
-call mtr.add_suppression("InnoDB: Resizing redo log");
-call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files");
-call mtr.add_suppression("InnoDB: New log files created");
-FLUSH TABLES;
-
 let $i=$n;
 while ($i)
 {