MDEV-24197: Add "innodb_force_recovery" for "mariabackup --prepare"

During the prepare phase of restoring backups, "mariabackup" does
not seem to allow (or recognize) the option "innodb_force_recovery"
for the embedded InnoDB server instance that it starts.

If page corruption observed during page recovery, the prepare step
fails. While this is indeed the correct behavior ideally, allowing
this option to be set in case of emergencies might be useful when
the current backup is the only copy available. Some error messages
during "--prepare" suggest to set "innodb_force_recovery" to 1:

  [ERROR] InnoDB: Set innodb_force_recovery=1 to ignore corruption.

For backwards compatibility, "mariabackup --innobackupex --apply-log"
should also have this option.

Signed-off-by: Srinidhi Kaushik <shrinidhi.kaushik@gmail.com>

6 files changed, 233 insertions, 9 deletions

diff --git a/extra/mariabackup/innobackupex.cc b/extra/mariabackup/innobackupex.cc
index f1627dd1bc5..fefd81dc1be 100644
--- a/extra/mariabackup/innobackupex.cc
+++ b/extra/mariabackup/innobackupex.cc
@@ -206,7 +206,8 @@ enum innobackupex_options
 	OPT_STREAM,
 	OPT_TABLES_FILE,
 	OPT_THROTTLE,
-	OPT_USE_MEMORY
+	OPT_USE_MEMORY,
+	OPT_INNODB_FORCE_RECOVERY,
 };
 
 ibx_mode_t ibx_mode = IBX_MODE_BACKUP;
@@ -624,6 +625,16 @@ static struct my_option ibx_long_options[] =
 	 0, GET_LL, REQUIRED_ARG, 100*1024*1024L, 1024*1024L, LONGLONG_MAX, 0,
 	 1024*1024L, 0},
 
+	{"innodb-force-recovery", OPT_INNODB_FORCE_RECOVERY,
+	 "This option starts up the embedded InnoDB instance in crash "
+	 "recovery mode to ignore page corruption; should be used "
+	 "with the \"--apply-log\" option, in emergencies only. The "
+	 "default value is 0. Refer to \"innodb_force_recovery\" server "
+	 "system variable documentation for more details.",
+	 (uchar*)&xtrabackup_innodb_force_recovery,
+	 (uchar*)&xtrabackup_innodb_force_recovery,
+	 0, GET_ULONG, OPT_ARG, 0, 0, SRV_FORCE_IGNORE_CORRUPT, 0, 0, 0},
+
 	{ 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
 };
 
@@ -669,6 +680,7 @@ innobackupex [--compress] [--compress-threads=NUMBER-OF-THREADS] [--compress-chu
 innobackupex --apply-log [--use-memory=B]\n\
              [--defaults-file=MY.CNF]\n\
              [--export] [--ibbackup=IBBACKUP-BINARY]\n\
+             [--innodb-force-recovery=1]\n\
              BACKUP-DIR\n\
 \n\
 innobackupex --copy-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME] BACKUP-DIR\n\
diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc
index ffb09c73edb..59551a7d044 100644
--- a/extra/mariabackup/xtrabackup.cc
+++ b/extra/mariabackup/xtrabackup.cc
@@ -271,6 +271,12 @@ static char *xtrabackup_debug_sync = NULL;
 
 my_bool xtrabackup_incremental_force_scan = FALSE;
 
+/*
+ * Ignore corrupt pages (disabled by default; used
+ * by "innobackupex" as a command line argument).
+ */
+ulong xtrabackup_innodb_force_recovery = 0;
+
 /* The flushed lsn which is read from data files */
 lsn_t	flushed_lsn= 0;
 
@@ -1046,7 +1052,8 @@ enum options_xtrabackup
   OPT_ROCKSDB_DATADIR,
   OPT_BACKUP_ROCKSDB,
   OPT_XTRA_CHECK_PRIVILEGES,
-  OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION
+  OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION,
+  OPT_INNODB_FORCE_RECOVERY
 };
 
 struct my_option xb_client_options[]= {
@@ -1677,6 +1684,13 @@ struct my_option xb_server_options[] =
    &opt_check_privileges, &opt_check_privileges,
    0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 },
 
+  {"innodb_force_recovery", OPT_INNODB_FORCE_RECOVERY,
+   "(for --prepare): Crash recovery mode (ignores "
+   "page corruption; for emergencies only).",
+   (G_PTR*)&srv_force_recovery,
+   (G_PTR*)&srv_force_recovery,
+   0, GET_ULONG, OPT_ARG, 0, 0, SRV_FORCE_IGNORE_CORRUPT, 0, 0, 0},
+
   { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
 };
 
@@ -1808,24 +1822,26 @@ static int prepare_export()
       " --defaults-extra-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=."
       " --innodb --innodb-fast-shutdown=0 --loose-partition"
       " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu"
-      " --console  --skip-log-error --skip-log-bin --bootstrap  < "
+      " --console  --skip-log-error --skip-log-bin --bootstrap %s < "
       BOOTSTRAP_FILENAME IF_WIN("\"",""),
-      mariabackup_exe, 
+      mariabackup_exe,
       orig_argv1, (my_defaults_group_suffix?my_defaults_group_suffix:""),
-      xtrabackup_use_memory);
+      xtrabackup_use_memory,
+      (srv_force_recovery ? "--innodb-force-recovery=1" : ""));
   }
   else
   {
-    sprintf(cmdline,
+    snprintf(cmdline, sizeof cmdline,
      IF_WIN("\"","") "\"%s\" --mysqld"
       " --defaults-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=."
       " --innodb --innodb-fast-shutdown=0 --loose-partition"
       " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu"
-      " --console  --log-error= --skip-log-bin --bootstrap  < "
+      " --console  --log-error= --skip-log-bin --bootstrap %s < "
       BOOTSTRAP_FILENAME IF_WIN("\"",""),
       mariabackup_exe,
       (my_defaults_group_suffix?my_defaults_group_suffix:""),
-      xtrabackup_use_memory);
+      xtrabackup_use_memory,
+      (srv_force_recovery ? "--innodb-force-recovery=1" : ""));
   }
 
   msg("Prepare export : executing %s\n", cmdline);
@@ -1985,6 +2001,13 @@ xb_get_one_option(int optid,
     ADD_PRINT_PARAM_OPT(innobase_buffer_pool_filename);
     break;
 
+  case OPT_INNODB_FORCE_RECOVERY:
+
+    if (srv_force_recovery) {
+        ADD_PRINT_PARAM_OPT(srv_force_recovery);
+    }
+    break;
+
   case OPT_XTRA_TARGET_DIR:
     strmake(xtrabackup_real_target_dir,argument, sizeof(xtrabackup_real_target_dir)-1);
     xtrabackup_target_dir= xtrabackup_real_target_dir;
@@ -2250,6 +2273,29 @@ innodb_init_param(void)
 		srv_undo_dir = (char*) ".";
 	}
 
+	compile_time_assert(SRV_FORCE_IGNORE_CORRUPT == 1);
+
+	/*
+	 * This option can be read both from the command line, and the
+	 * defaults file. The assignment should account for both cases,
+	 * and for "--innobackupex". Since the command line argument is
+	 * parsed after the defaults file, it takes precedence.
+	 */
+	if (xtrabackup_innodb_force_recovery) {
+		srv_force_recovery = xtrabackup_innodb_force_recovery;
+	}
+
+	if (srv_force_recovery >= SRV_FORCE_IGNORE_CORRUPT) {
+		if (!xtrabackup_prepare) {
+			msg("mariabackup: The option \"innodb_force_recovery\""
+			    " should only be used with \"%s\".",
+			    (innobackupex_mode ? "--apply-log" : "--prepare"));
+			goto error;
+		} else {
+			msg("innodb_force_recovery = %lu", srv_force_recovery);
+		}
+	}
+
 	return(FALSE);
 
 error:
diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h
index 15d53d00db4..e2955f58d6a 100644
--- a/extra/mariabackup/xtrabackup.h
+++ b/extra/mariabackup/xtrabackup.h
@@ -172,6 +172,8 @@ enum binlog_info_enum { BINLOG_INFO_OFF, BINLOG_INFO_ON,
 
 extern ulong opt_binlog_info;
 
+extern ulong xtrabackup_innodb_force_recovery;
+
 void xtrabackup_io_throttling(void);
 my_bool xb_write_delta_metadata(const char *filename,
 				const xb_delta_info_t *info);
diff --git a/mysql-test/suite/mariabackup/innodb_force_recovery.result b/mysql-test/suite/mariabackup/innodb_force_recovery.result
new file mode 100644
index 00000000000..94a265cfabe
--- /dev/null
+++ b/mysql-test/suite/mariabackup/innodb_force_recovery.result
@@ -0,0 +1,26 @@
+CREATE TABLE t(i INT) ENGINE INNODB;
+INSERT INTO t VALUES(1);
+# "innodb_force_recovery=1" should be allowed with "--prepare" only (mariabackup)
+FOUND 1 /should only be used with "--prepare"/ in backup.log
+# "innodb_force_recovery=1" should be allowed with "--apply-log" only (innobackupex)
+FOUND 1 /should only be used with "--apply-log"/ in backup.log
+# "innodb_force_recovery" should be limited to "SRV_FORCE_IGNORE_CORRUPT" (mariabackup)
+FOUND 1 /innodb_force_recovery = 1/ in backup.log
+# "innodb_force_recovery" should be limited to "SRV_FORCE_IGNORE_CORRUPT" (innobackupex)
+FOUND 1 /innodb_force_recovery = 1/ in backup.log
+# "innodb_force_recovery" should be read from "backup-my.cnf" (mariabackup)
+FOUND 1 /innodb_force_recovery = 1/ in backup.log
+# "innodb_force_recovery=1" should be read from "backup-my.cnf" (innobackupex)
+FOUND 1 /innodb_force_recovery = 1/ in backup.log
+# "innodb_force_recovery" from the command line should override "backup-my.cnf" (mariabackup)
+NOT FOUND /innodb_force_recovery = 1/ in backup.log
+# "innodb_force_recovery" from the command line should override "backup-my.cnf" (innobackupex)
+NOT FOUND /innodb_force_recovery = 1/ in backup.log
+# shutdown server
+# remove datadir
+# xtrabackup move back
+# restart server
+SELECT * FROM t;
+i
+1
+DROP TABLE t;
diff --git a/mysql-test/suite/mariabackup/innodb_force_recovery.test b/mysql-test/suite/mariabackup/innodb_force_recovery.test
new file mode 100644
index 00000000000..3a7b3c6106c
--- /dev/null
+++ b/mysql-test/suite/mariabackup/innodb_force_recovery.test
@@ -0,0 +1,138 @@
+# This test checks if "innodb_force_recovery" is only allowed with "--prepare"
+# (for mariabackup) and "--apply-log" (for innobackupex), and is limited to
+# "SRV_FORCE_IGNORE_CORRUPT" only.
+
+# Setup.
+--source include/have_innodb.inc
+
+--let targetdir=$MYSQLTEST_VARDIR/tmp/backup
+--let backuplog=$MYSQLTEST_VARDIR/tmp/backup.log
+
+CREATE TABLE t(i INT) ENGINE INNODB;
+INSERT INTO t VALUES(1);
+
+# Check for command line arguments.
+--echo # "innodb_force_recovery=1" should be allowed with "--prepare" only (mariabackup)
+--disable_result_log
+--error 1
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --innodb-force-recovery=1 --target-dir=$targetdir >$backuplog;
+--enable_result_log
+--let SEARCH_PATTERN=should only be used with "--prepare"
+--let SEARCH_FILE=$backuplog
+--source include/search_pattern_in_file.inc
+
+--echo # "innodb_force_recovery=1" should be allowed with "--apply-log" only (innobackupex)
+--disable_result_log
+--error 1
+exec $XTRABACKUP --innobackupex --defaults-file=$MYSQLTEST_VARDIR/my.cnf --no-timestamp --innodb-force-recovery=1 $targetdir >$backuplog;
+--enable_result_log
+--let SEARCH_PATTERN=should only be used with "--apply-log"
+--let SEARCH_FILE=$backuplog
+--source include/search_pattern_in_file.inc
+
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir;
+--enable_result_log
+--echo # "innodb_force_recovery" should be limited to "SRV_FORCE_IGNORE_CORRUPT" (mariabackup)
+--disable_result_log
+exec $XTRABACKUP --prepare --innodb-force-recovery=2 --target-dir=$targetdir >$backuplog;
+--enable_result_log
+--let SEARCH_PATTERN=innodb_force_recovery = 1
+--let SEARCH_FILE=$backuplog
+--source include/search_pattern_in_file.inc
+rmdir $targetdir;
+
+--disable_result_log
+exec $XTRABACKUP --innobackupex --defaults-file=$MYSQLTEST_VARDIR/my.cnf --no-timestamp $targetdir;
+--enable_result_log
+--echo # "innodb_force_recovery" should be limited to "SRV_FORCE_IGNORE_CORRUPT" (innobackupex)
+--disable_result_log
+exec $XTRABACKUP --innobackupex --apply-log --innodb-force-recovery=2 $targetdir >$backuplog;
+--enable_result_log
+--let SEARCH_PATTERN=innodb_force_recovery = 1
+--let SEARCH_FILE=$backuplog
+--source include/search_pattern_in_file.inc
+rmdir $targetdir;
+
+# Check for default file ("backup-my.cnf").
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir;
+--enable_result_log
+perl;
+my $cfg_path="$ENV{'targetdir'}/backup-my.cnf";
+open(my $fd, '>>', "$cfg_path");
+print $fd "innodb_force_recovery=1\n";
+close $fd;
+EOF
+--echo # "innodb_force_recovery" should be read from "backup-my.cnf" (mariabackup)
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$targetdir/backup-my.cnf --prepare --export --target-dir=$targetdir >$backuplog;
+--enable_result_log
+--let SEARCH_PATTERN=innodb_force_recovery = 1
+--let SEARCH_FILE=$backuplog
+--source include/search_pattern_in_file.inc
+rmdir $targetdir;
+
+--disable_result_log
+exec $XTRABACKUP --innobackupex --defaults-file=$MYSQLTEST_VARDIR/my.cnf --no-timestamp $targetdir;
+--enable_result_log
+perl;
+my $cfg_path="$ENV{'targetdir'}/backup-my.cnf";
+open(my $fd, '>>', "$cfg_path");
+print $fd "innodb_force_recovery=2\n";
+close $fd;
+EOF
+--echo # "innodb_force_recovery=1" should be read from "backup-my.cnf" (innobackupex)
+--disable_result_log
+exec $XTRABACKUP --innobackupex --defaults-file=$targetdir/backup-my.cnf --apply-log --export $targetdir >$backuplog;
+--enable_result_log
+--let SEARCH_PATTERN=innodb_force_recovery = 1
+--let SEARCH_FILE=$backuplog
+--source include/search_pattern_in_file.inc
+rmdir $targetdir;
+
+# Check for command line argument precedence.
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir;
+--enable_result_log
+perl;
+my $cfg_path="$ENV{'targetdir'}/backup-my.cnf";
+open(my $fd, '>>', "$cfg_path");
+print $fd "innodb_force_recovery=1\n";
+close $fd;
+EOF
+--echo # "innodb_force_recovery" from the command line should override "backup-my.cnf" (mariabackup)
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$targetdir/backup-my.cnf --prepare --innodb-force-recovery=0 --target-dir=$targetdir >$backuplog;
+--enable_result_log
+--let SEARCH_PATTERN=innodb_force_recovery = 1
+--let SEARCH_FILE=$backuplog
+--source include/search_pattern_in_file.inc
+rmdir $targetdir;
+
+--disable_result_log
+exec $XTRABACKUP --innobackupex --defaults-file=$MYSQLTEST_VARDIR/my.cnf --no-timestamp $targetdir;
+--enable_result_log
+perl;
+my $cfg_path="$ENV{'targetdir'}/backup-my.cnf";
+open(my $fd, '>>', "$cfg_path");
+print $fd "innodb_force_recovery=2\n";
+close $fd;
+EOF
+--echo # "innodb_force_recovery" from the command line should override "backup-my.cnf" (innobackupex)
+--disable_result_log
+exec $XTRABACKUP --innobackupex --defaults-file=$targetdir/backup-my.cnf --apply-log --innodb-force-recovery=0 --export $targetdir >$backuplog;
+--enable_result_log
+--let SEARCH_PATTERN=innodb_force_recovery = 1
+--let SEARCH_FILE=$backuplog
+--source include/search_pattern_in_file.inc
+
+--source include/restart_and_restore.inc
+
+# Check for restore.
+SELECT * FROM t;
+
+# Clean-up.
+DROP TABLE t;
+--rmdir $targetdir
+--remove_file $backuplog
diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc
index a7be1cdbf21..9548730b359 100644
--- a/storage/innobase/srv/srv0start.cc
+++ b/storage/innobase/srv/srv0start.cc
@@ -2322,7 +2322,7 @@ files_checked:
 			to the data files and truncate or delete the log.
 			Unless --export is specified, no further change to
 			InnoDB files is needed. */
-			ut_ad(!srv_force_recovery);
+			ut_ad(srv_force_recovery <= SRV_FORCE_IGNORE_CORRUPT);
 			ut_ad(srv_n_log_files_found <= 1);
 			ut_ad(recv_no_log_write);
 			buf_flush_sync_all_buf_pools();