MDEV-18187 Aria engine: Redo phase failed with "error 192 when executing record redo_index_new_page" upon startup on a restored datadir

The issue is that when recovery is about to create a new data or index
page it check if the page already exits.
If the page does not exists (file is too short) or contains wrong checksum,
then the recovery code will recreate the page.
The bug was that the code that checked if the page existed didn't take
into account encrypted pages.

Fixed by adding a check if page could not be encrypted solved the issue.
I also added some code to silence decryption errors for new pages.

Test case and some inspiration for how to solve this come from
the pull request by alexandr.miloslavsky

7 files changed, 171 insertions, 10 deletions

diff --git a/mysql-test/suite/large_tests/r/maria_recover_encrypted.result b/mysql-test/suite/large_tests/r/maria_recover_encrypted.result
new file mode 100644
index 00000000000..a7293d45db6
--- /dev/null
+++ b/mysql-test/suite/large_tests/r/maria_recover_encrypted.result
@@ -0,0 +1,53 @@
+DROP TABLE IF EXISTS t1;
+DROP PROCEDURE IF EXISTS proc_insert_many;
+CREATE TABLE t1 (
+field1 INTEGER NOT NULL,
+field2 INTEGER NOT NULL,
+field3 INTEGER NOT NULL,
+KEY i_1   (field1),
+KEY i_2   (field2),
+KEY i_3   (field3),
+KEY i_12  (field1, field2),
+KEY i_13  (field1, field3),
+KEY i_21  (field2, field1),
+KEY i_23  (field2, field3),
+KEY i_31  (field3, field1),
+KEY i_32  (field3, field2),
+KEY i_123 (field1, field2, field3),
+KEY i_132 (field1, field3, field2),
+KEY i_213 (field2, field1, field3),
+KEY i_231 (field2, field3, field1),
+KEY i_312 (field3, field1, field2),
+KEY i_321 (field3, field2, field1)
+) ENGINE=Aria;
+CREATE PROCEDURE proc_insert_many()
+BEGIN
+DECLARE iRow INT DEFAULT 0;
+insertRows: LOOP
+IF (iRow = 70000) THEN
+LEAVE insertRows;
+END IF;
+INSERT INTO t1 VALUES (1000000+iRow,2000000+iRow,3000000+iRow);
+SET iRow = iRow + 1;
+END LOOP insertRows;
+END|
+LOCK TABLES t1 WRITE;
+CALL proc_insert_many();
+UNLOCK TABLES;
+SET debug_dbug="d,crash_shutdown";
+shutdown;
+ERROR HY000: Lost connection to MySQL server during query
+SELECT * FROM t1 ORDER BY 1 DESC LIMIT 10;
+field1	field2	field3
+1069999	2069999	3069999
+1069998	2069998	3069998
+1069997	2069997	3069997
+1069996	2069996	3069996
+1069995	2069995	3069995
+1069994	2069994	3069994
+1069993	2069993	3069993
+1069992	2069992	3069992
+1069991	2069991	3069991
+1069990	2069990	3069990
+DROP TABLE IF EXISTS t1;
+DROP PROCEDURE IF EXISTS proc_insert_many;
diff --git a/mysql-test/suite/large_tests/t/maria_recover_encrypted.test b/mysql-test/suite/large_tests/t/maria_recover_encrypted.test
new file mode 100644
index 00000000000..4c590e5e1f9
--- /dev/null
+++ b/mysql-test/suite/large_tests/t/maria_recover_encrypted.test
@@ -0,0 +1,90 @@
+# MDEV-18187: If server crashes before flushing index pages in an
+# encrypted Aria table, it could permanently fail to repair the table
+
+--source include/have_maria.inc
+--source include/default_charset.inc
+
+# Cleanup
+--disable_warnings
+DROP TABLE IF EXISTS t1;
+DROP PROCEDURE IF EXISTS proc_insert_many;
+--enable_warnings
+
+# --------
+
+# Configure encryption
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--shutdown_server
+--source include/wait_until_disconnected.inc
+
+--write_file $MYSQLTEST_VARDIR/key.txt
+1;76025E3ADC78D74819927DB02AAA4C35
+EOF
+
+--exec echo "restart:--aria-encrypt-tables=1 --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/key.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+# Create table with many indexes so that its index size grows quickly
+# and it can be grown to needed size without too many inserts
+CREATE TABLE t1 (
+  field1 INTEGER NOT NULL,
+  field2 INTEGER NOT NULL,
+  field3 INTEGER NOT NULL,
+  KEY i_1   (field1),
+  KEY i_2   (field2),
+  KEY i_3   (field3),
+  KEY i_12  (field1, field2),
+  KEY i_13  (field1, field3),
+  KEY i_21  (field2, field1),
+  KEY i_23  (field2, field3),
+  KEY i_31  (field3, field1),
+  KEY i_32  (field3, field2),
+  KEY i_123 (field1, field2, field3),
+  KEY i_132 (field1, field3, field2),
+  KEY i_213 (field2, field1, field3),
+  KEY i_231 (field2, field3, field1),
+  KEY i_312 (field3, field1, field2),
+  KEY i_321 (field3, field2, field1)
+) ENGINE=Aria;
+
+# Create procedures to insert many rows.
+DELIMITER |;
+CREATE PROCEDURE proc_insert_many()
+BEGIN
+	DECLARE iRow INT DEFAULT 0;
+	insertRows: LOOP
+		IF (iRow = 70000) THEN
+			LEAVE insertRows;
+		END IF;
+
+		INSERT INTO t1 VALUES (1000000+iRow,2000000+iRow,3000000+iRow);
+		SET iRow = iRow + 1;
+	END LOOP insertRows;
+END|
+DELIMITER ;|
+
+# Call the procedure to insert rows.
+# Use 'LOCK TABLES' to make things a lot faster.
+# Note that his code doesn't reproduce for some reason:
+#   INSERT INTO t1 SELECT 1000000+seq,2000000+seq,3000000+seq FROM seq_1_to_70000;
+LOCK TABLES t1 WRITE;
+CALL proc_insert_many();
+UNLOCK TABLES;
+
+# Crash and restart the server while it's still flushing index
+--exec echo "restart:--aria-encrypt-tables=1 --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/key.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+SET debug_dbug="d,crash_shutdown";
+--error 2013
+shutdown;
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+# Access the table to trigger repair; validate repaired data
+SELECT * FROM t1 ORDER BY 1 DESC LIMIT 10;
+
+# --------
+
+# Cleanup
+DROP TABLE IF EXISTS t1;
+DROP PROCEDURE IF EXISTS proc_insert_many;
diff --git a/storage/maria/ma_blockrec.c b/storage/maria/ma_blockrec.c
index 592aab6da41..667cc26758d 100644
--- a/storage/maria/ma_blockrec.c
+++ b/storage/maria/ma_blockrec.c
@@ -6379,16 +6379,19 @@ uint _ma_apply_redo_insert_row_head_or_tail(MARIA_HA *info, LSN lsn,
     pin_method=  PAGECACHE_PIN_LEFT_PINNED;
 
     share->pagecache->readwrite_flags&= ~MY_WME;
+    share->silence_encryption_errors= 1;
     buff= pagecache_read(share->pagecache, &info->dfile,
                          page, 0, 0,
                          PAGECACHE_PLAIN_PAGE, PAGECACHE_LOCK_WRITE,
                          &page_link.link);
     share->pagecache->readwrite_flags= share->pagecache->org_readwrite_flags;
+    share->silence_encryption_errors= 0;
     if (!buff)
     {
       /* Skip errors when reading outside of file and uninitialized pages */
       if (!new_page || (my_errno != HA_ERR_FILE_TOO_SHORT &&
-                        my_errno != HA_ERR_WRONG_CRC))
+                        my_errno != HA_ERR_WRONG_CRC &&
+                        my_errno != HA_ERR_DECRYPTION_FAILED))
       {
         DBUG_PRINT("error", ("Error %d when reading page", (int) my_errno));
         goto err;
@@ -6880,6 +6883,7 @@ uint _ma_apply_redo_insert_row_blobs(MARIA_HA *info,
         else
         {
           share->pagecache->readwrite_flags&= ~MY_WME;
+          share->silence_encryption_errors= 1;
           buff= pagecache_read(share->pagecache,
                                &info->dfile,
                                page, 0, 0,
@@ -6887,10 +6891,12 @@ uint _ma_apply_redo_insert_row_blobs(MARIA_HA *info,
                                PAGECACHE_LOCK_WRITE, &page_link.link);
           share->pagecache->readwrite_flags= share->pagecache->
             org_readwrite_flags;
+          share->silence_encryption_errors= 0;
           if (!buff)
           {
             if (my_errno != HA_ERR_FILE_TOO_SHORT &&
-                my_errno != HA_ERR_WRONG_CRC)
+                my_errno != HA_ERR_WRONG_CRC &&
+                my_errno != HA_ERR_DECRYPTION_FAILED)
             {
               /* If not read outside of file */
               pagecache_unlock_by_link(share->pagecache, page_link.link,
diff --git a/storage/maria/ma_check.c b/storage/maria/ma_check.c
index 3a65cd61d08..98f9c49b9fa 100644
--- a/storage/maria/ma_check.c
+++ b/storage/maria/ma_check.c
@@ -4931,7 +4931,8 @@ static int sort_get_next_record(MARIA_SORT_PARAM *sort_param)
         DBUG_RETURN(-1);
       }
       /* Retry only if wrong record, not if disk error */
-      if (flag != HA_ERR_WRONG_IN_RECORD && flag != HA_ERR_WRONG_CRC)
+      if (flag != HA_ERR_WRONG_IN_RECORD && flag != HA_ERR_WRONG_CRC &&
+          flag != HA_ERR_DECRYPTION_FAILED)
       {
         retry_if_quick(sort_param, flag);
         DBUG_RETURN(flag);
@@ -6741,7 +6742,8 @@ read_next_page:
                            PAGECACHE_READ_UNKNOWN_PAGE,
                            PAGECACHE_LOCK_LEFT_UNLOCKED, 0)))
       {
-        if (my_errno == HA_ERR_WRONG_CRC)
+        if (my_errno == HA_ERR_WRONG_CRC ||
+            my_errno == HA_ERR_DECRYPTION_FAILED)
         {
           /*
             Don't give errors for zero filled blocks. These can
diff --git a/storage/maria/ma_crypt.c b/storage/maria/ma_crypt.c
index 564edb4bbe8..33cf72b0b1e 100644
--- a/storage/maria/ma_crypt.c
+++ b/storage/maria/ma_crypt.c
@@ -345,7 +345,14 @@ static my_bool ma_crypt_index_post_read_hook(int res,
   const uint block_size= share->block_size;
   const uint page_used= _ma_get_page_used(share, args->page);
 
-  if (res == 0 && page_used <= block_size - CRC_SIZE)
+  if (res ||
+      page_used < share->keypage_header ||
+      page_used >= block_size - CRC_SIZE)
+  {
+    res= 1;
+    my_errno= HA_ERR_DECRYPTION_FAILED;
+  }
+  else
   {
     const uchar *src= args->page;
     uchar* dst= args->crypt_buf;
@@ -506,10 +513,11 @@ static int ma_decrypt(MARIA_SHARE *share, MARIA_CRYPT_DATA *crypt_data,
   if (! (rc == MY_AES_OK && dstlen == size))
   {
     my_errno= HA_ERR_DECRYPTION_FAILED;
-    my_printf_error(HA_ERR_DECRYPTION_FAILED,
-                    "failed to decrypt '%s'  rc: %d  dstlen: %u  size: %u\n",
-                    MYF(ME_FATAL|ME_ERROR_LOG),
-                    share->open_file_name.str, rc, dstlen, size);
+    if (!share->silence_encryption_errors)
+      my_printf_error(HA_ERR_DECRYPTION_FAILED,
+                      "failed to decrypt '%s'  rc: %d  dstlen: %u  size: %u\n",
+                      MYF(ME_FATAL|ME_ERROR_LOG),
+                      share->open_file_name.str, rc, dstlen, size);
     return 1;
   }
   return 0;
diff --git a/storage/maria/ma_key_recover.c b/storage/maria/ma_key_recover.c
index afa69fce444..2f28ec8d175 100644
--- a/storage/maria/ma_key_recover.c
+++ b/storage/maria/ma_key_recover.c
@@ -767,7 +767,8 @@ uint _ma_apply_redo_index_new_page(MARIA_HA *info, LSN lsn,
                                &page_link.link)))
     {
       if (my_errno != HA_ERR_FILE_TOO_SHORT &&
-          my_errno != HA_ERR_WRONG_CRC)
+          my_errno != HA_ERR_WRONG_CRC &&
+          my_errno != HA_ERR_DECRYPTION_FAILED)
       {
         result= 1;
         goto err;
diff --git a/storage/maria/maria_def.h b/storage/maria/maria_def.h
index 6c9649ede45..c51ae2b95f7 100644
--- a/storage/maria/maria_def.h
+++ b/storage/maria/maria_def.h
@@ -502,6 +502,7 @@ typedef struct st_maria_share
   my_bool key_del_used;                         /* != 0 if key_del is locked */
   my_bool deleting;                     /* we are going to delete this table */
   my_bool redo_error_given;             /* Used during recovery */
+  my_bool silence_encryption_errors;    /* Used during recovery */
   THR_LOCK lock;
   void (*lock_restore_status)(void *);
   /**