Merge 10.3 into 10.4

13 files changed, 109 insertions, 17 deletions

diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc
index 80191418388..f52d288f42d 100644
--- a/extra/mariabackup/backup_copy.cc
+++ b/extra/mariabackup/backup_copy.cc
@@ -1062,6 +1062,7 @@ copy_file(ds_ctxt_t *datasink,
 	ds_file_t		*dstfile = NULL;
 	datafile_cur_t		 cursor;
 	xb_fil_cur_result_t	 res;
+	DBUG_ASSERT(datasink->datasink->remove);
 	const char	*dst_path =
 		(xtrabackup_copy_back || xtrabackup_move_back)?
 		dst_file_path : trim_dotslash(dst_file_path);
@@ -1087,6 +1088,7 @@ copy_file(ds_ctxt_t *datasink,
 		if (ds_write(dstfile, cursor.buf, cursor.buf_read)) {
 			goto error;
 		}
+		DBUG_EXECUTE_IF("copy_file_error", errno=ENOSPC;goto error;);
 	}
 
 	if (res == XB_FIL_CUR_ERROR) {
@@ -1108,6 +1110,7 @@ copy_file(ds_ctxt_t *datasink,
 error:
 	datafile_close(&cursor);
 	if (dstfile != NULL) {
+		datasink->datasink->remove(dstfile->path);
 		ds_close(dstfile);
 	}
 
@@ -1152,17 +1155,18 @@ move_file(ds_ctxt_t *datasink,
 
 	if (my_rename(src_file_path, dst_file_path_abs, MYF(0)) != 0) {
 		if (my_errno == EXDEV) {
-			bool ret;
-			ret = copy_file(datasink, src_file_path,
-					dst_file_path, thread_n);
+			/* Fallback to copy/unlink */
+			if(!copy_file(datasink, src_file_path,
+					dst_file_path, thread_n))
+					return false;
 			msg(thread_n,"Removing %s", src_file_path);
 			if (unlink(src_file_path) != 0) {
 				my_strerror(errbuf, sizeof(errbuf), errno);
-				msg("Error: unlink %s failed: %s",
+				msg("Warning: unlink %s failed: %s",
 					src_file_path,
 					errbuf);
 			}
-			return(ret);
+			return true;
 		}
 		my_strerror(errbuf, sizeof(errbuf), my_errno);
 		msg("Can not move file %s to %s: %s",
diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc
index 4c0ef77760c..162980acb21 100644
--- a/extra/mariabackup/backup_mysql.cc
+++ b/extra/mariabackup/backup_mysql.cc
@@ -44,6 +44,7 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA
 #include <mysql.h>
 #include <mysqld.h>
 #include <my_sys.h>
+#include <stdlib.h>
 #include <string.h>
 #include <limits>
 #include "common.h"
@@ -108,6 +109,13 @@ xb_mysql_connect()
 		return(NULL);
 	}
 
+#if !defined(DONT_USE_MYSQL_PWD)
+	if (!opt_password)
+	{
+		opt_password=getenv("MYSQL_PWD");
+	}
+#endif
+
 	if (!opt_secure_auth) {
 		mysql_options(connection, MYSQL_SECURE_AUTH,
 			      (char *) &opt_secure_auth);
diff --git a/extra/mariabackup/datasink.h b/extra/mariabackup/datasink.h
index 201bbfd3267..5c82556b9ba 100644
--- a/extra/mariabackup/datasink.h
+++ b/extra/mariabackup/datasink.h
@@ -50,9 +50,15 @@ struct datasink_struct {
 	ds_file_t *(*open)(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat);
 	int (*write)(ds_file_t *file, const unsigned char *buf, size_t len);
 	int (*close)(ds_file_t *file);
+	int (*remove)(const char *path);
 	void (*deinit)(ds_ctxt_t *ctxt);
 };
 
+
+static inline int dummy_remove(const char *) {
+	return 0;
+}
+
 /* Supported datasink types */
 typedef enum {
 	DS_TYPE_STDOUT,
diff --git a/extra/mariabackup/ds_archive.cc b/extra/mariabackup/ds_archive.cc
index c8fcfa1f5f5..3a5081119b3 100644
--- a/extra/mariabackup/ds_archive.cc
+++ b/extra/mariabackup/ds_archive.cc
@@ -57,6 +57,7 @@ datasink_t datasink_archive = {
 	&archive_open,
 	&archive_write,
 	&archive_close,
+	&dummy_remove,
 	&archive_deinit
 };
 
diff --git a/extra/mariabackup/ds_buffer.cc b/extra/mariabackup/ds_buffer.cc
index 720a329c238..6e6ec6fcfbc 100644
--- a/extra/mariabackup/ds_buffer.cc
+++ b/extra/mariabackup/ds_buffer.cc
@@ -54,6 +54,7 @@ datasink_t datasink_buffer = {
 	&buffer_open,
 	&buffer_write,
 	&buffer_close,
+	&dummy_remove,
 	&buffer_deinit
 };
 
diff --git a/extra/mariabackup/ds_compress.cc b/extra/mariabackup/ds_compress.cc
index 487718e2ac0..3c28e55e1fe 100644
--- a/extra/mariabackup/ds_compress.cc
+++ b/extra/mariabackup/ds_compress.cc
@@ -75,6 +75,7 @@ datasink_t datasink_compress = {
 	&compress_open,
 	&compress_write,
 	&compress_close,
+	&dummy_remove,
 	&compress_deinit
 };
 
diff --git a/extra/mariabackup/ds_local.cc b/extra/mariabackup/ds_local.cc
index fb2ea0a1629..41a00dd9c39 100644
--- a/extra/mariabackup/ds_local.cc
+++ b/extra/mariabackup/ds_local.cc
@@ -43,12 +43,18 @@ static int local_write(ds_file_t *file, const uchar *buf, size_t len);
 static int local_close(ds_file_t *file);
 static void local_deinit(ds_ctxt_t *ctxt);
 
+static int local_remove(const char *path)
+{
+	return unlink(path);
+}
+
 extern "C" {
 datasink_t datasink_local = {
 	&local_init,
 	&local_open,
 	&local_write,
 	&local_close,
+	&local_remove,
 	&local_deinit
 };
 }
diff --git a/extra/mariabackup/ds_stdout.cc b/extra/mariabackup/ds_stdout.cc
index 85dbb83865b..3b3091bd426 100644
--- a/extra/mariabackup/ds_stdout.cc
+++ b/extra/mariabackup/ds_stdout.cc
@@ -40,6 +40,7 @@ datasink_t datasink_stdout = {
 	&stdout_open,
 	&stdout_write,
 	&stdout_close,
+	&dummy_remove,
 	&stdout_deinit
 };
 
diff --git a/extra/mariabackup/ds_tmpfile.cc b/extra/mariabackup/ds_tmpfile.cc
index 22dff165aa0..d5316803926 100644
--- a/extra/mariabackup/ds_tmpfile.cc
+++ b/extra/mariabackup/ds_tmpfile.cc
@@ -51,6 +51,7 @@ datasink_t datasink_tmpfile = {
 	&tmpfile_open,
 	&tmpfile_write,
 	&tmpfile_close,
+	&dummy_remove,
 	&tmpfile_deinit
 };
 
diff --git a/extra/mariabackup/ds_xbstream.cc b/extra/mariabackup/ds_xbstream.cc
index 5a753b08474..daf1cc73038 100644
--- a/extra/mariabackup/ds_xbstream.cc
+++ b/extra/mariabackup/ds_xbstream.cc
@@ -50,6 +50,7 @@ datasink_t datasink_xbstream = {
 	&xbstream_open,
 	&xbstream_write,
 	&xbstream_close,
+	&dummy_remove,
 	&xbstream_deinit
 };
 
diff --git a/extra/mariabackup/innobackupex.cc b/extra/mariabackup/innobackupex.cc
index bc944c18659..1341d2c46de 100644
--- a/extra/mariabackup/innobackupex.cc
+++ b/extra/mariabackup/innobackupex.cc
@@ -208,7 +208,8 @@ enum innobackupex_options
 	OPT_STREAM,
 	OPT_TABLES_FILE,
 	OPT_THROTTLE,
-	OPT_USE_MEMORY
+	OPT_USE_MEMORY,
+	OPT_INNODB_FORCE_RECOVERY,
 };
 
 ibx_mode_t ibx_mode = IBX_MODE_BACKUP;
@@ -626,6 +627,16 @@ static struct my_option ibx_long_options[] =
 	 0, GET_LL, REQUIRED_ARG, 100*1024*1024L, 1024*1024L, LONGLONG_MAX, 0,
 	 1024*1024L, 0},
 
+	{"innodb-force-recovery", OPT_INNODB_FORCE_RECOVERY,
+	 "This option starts up the embedded InnoDB instance in crash "
+	 "recovery mode to ignore page corruption; should be used "
+	 "with the \"--apply-log\" option, in emergencies only. The "
+	 "default value is 0. Refer to \"innodb_force_recovery\" server "
+	 "system variable documentation for more details.",
+	 (uchar*)&xtrabackup_innodb_force_recovery,
+	 (uchar*)&xtrabackup_innodb_force_recovery,
+	 0, GET_ULONG, OPT_ARG, 0, 0, SRV_FORCE_IGNORE_CORRUPT, 0, 0, 0},
+
 	{ 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
 };
 
@@ -671,6 +682,7 @@ innobackupex [--compress] [--compress-threads=NUMBER-OF-THREADS] [--compress-chu
 innobackupex --apply-log [--use-memory=B]\n\
              [--defaults-file=MY.CNF]\n\
              [--export] [--ibbackup=IBBACKUP-BINARY]\n\
+             [--innodb-force-recovery=1]\n\
              BACKUP-DIR\n\
 \n\
 innobackupex --copy-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME] BACKUP-DIR\n\
diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc
index 9a2a544fd8f..de4462bc01f 100644
--- a/extra/mariabackup/xtrabackup.cc
+++ b/extra/mariabackup/xtrabackup.cc
@@ -4,7 +4,7 @@ MariaBackup: hot backup tool for InnoDB
 Originally Created 3/3/2009 Yasufumi Kinoshita
 Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
 Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
-(c) 2017, 2020, MariaDB Corporation.
+(c) 2017, 2021, MariaDB Corporation.
 Portions written by Marko Mäkelä.
 
 This program is free software; you can redistribute it and/or modify
@@ -274,6 +274,12 @@ static char *xtrabackup_debug_sync = NULL;
 
 my_bool xtrabackup_incremental_force_scan = FALSE;
 
+/*
+ * Ignore corrupt pages (disabled by default; used
+ * by "innobackupex" as a command line argument).
+ */
+ulong xtrabackup_innodb_force_recovery = 0;
+
 /* The flushed lsn which is read from data files */
 lsn_t	flushed_lsn= 0;
 
@@ -1050,7 +1056,8 @@ enum options_xtrabackup
   OPT_BACKUP_ROCKSDB,
   OPT_XTRA_CHECK_PRIVILEGES,
   OPT_XTRA_MYSQLD_ARGS,
-  OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION
+  OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION,
+  OPT_INNODB_FORCE_RECOVERY
 };
 
 struct my_option xb_client_options[]= {
@@ -1677,6 +1684,13 @@ struct my_option xb_server_options[] =
    &opt_check_privileges, &opt_check_privileges,
    0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 },
 
+  {"innodb_force_recovery", OPT_INNODB_FORCE_RECOVERY,
+   "(for --prepare): Crash recovery mode (ignores "
+   "page corruption; for emergencies only).",
+   (G_PTR*)&srv_force_recovery,
+   (G_PTR*)&srv_force_recovery,
+   0, GET_ULONG, OPT_ARG, 0, 0, SRV_FORCE_IGNORE_CORRUPT, 0, 0, 0},
+
     {"mysqld-args", OPT_XTRA_MYSQLD_ARGS,
      "All arguments that follow this argument are considered as server "
      "options, and if some of them are not supported by mariabackup, they "
@@ -1813,31 +1827,33 @@ static int prepare_export()
 
   // Process defaults-file , it can have some --lc-language stuff,
   // which is* unfortunately* still necessary to get mysqld up
-  if (strncmp(orig_argv1,"--defaults-file=",16) == 0)
+  if (strncmp(orig_argv1,"--defaults-file=", 16) == 0)
   {
     snprintf(cmdline, sizeof cmdline,
-     IF_WIN("\"","") "\"%s\" --mysqld \"%s\" "
+      IF_WIN("\"","") "\"%s\" --mysqld \"%s\""
       " --defaults-extra-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=."
       " --innodb --innodb-fast-shutdown=0 --loose-partition"
       " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu"
-      " --console  --skip-log-error --skip-log-bin --bootstrap  < "
+      " --console --skip-log-error --skip-log-bin --bootstrap %s< "
       BOOTSTRAP_FILENAME IF_WIN("\"",""),
-      mariabackup_exe, 
+      mariabackup_exe,
       orig_argv1, (my_defaults_group_suffix?my_defaults_group_suffix:""),
-      xtrabackup_use_memory);
+      xtrabackup_use_memory,
+      (srv_force_recovery ? "--innodb-force-recovery=1 " : ""));
   }
   else
   {
-    sprintf(cmdline,
-     IF_WIN("\"","") "\"%s\" --mysqld"
+    snprintf(cmdline, sizeof cmdline,
+      IF_WIN("\"","") "\"%s\" --mysqld"
       " --defaults-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=."
       " --innodb --innodb-fast-shutdown=0 --loose-partition"
       " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu"
-      " --console  --log-error= --skip-log-bin --bootstrap  < "
+      " --console --log-error= --skip-log-bin --bootstrap %s< "
       BOOTSTRAP_FILENAME IF_WIN("\"",""),
       mariabackup_exe,
       (my_defaults_group_suffix?my_defaults_group_suffix:""),
-      xtrabackup_use_memory);
+      xtrabackup_use_memory,
+      (srv_force_recovery ? "--innodb-force-recovery=1 " : ""));
   }
 
   msg("Prepare export : executing %s\n", cmdline);
@@ -1985,6 +2001,13 @@ xb_get_one_option(int optid,
     ADD_PRINT_PARAM_OPT(innobase_buffer_pool_filename);
     break;
 
+  case OPT_INNODB_FORCE_RECOVERY:
+
+    if (srv_force_recovery) {
+        ADD_PRINT_PARAM_OPT(srv_force_recovery);
+    }
+    break;
+
   case OPT_XTRA_TARGET_DIR:
     strmake(xtrabackup_real_target_dir,argument, sizeof(xtrabackup_real_target_dir)-1);
     xtrabackup_target_dir= xtrabackup_real_target_dir;
@@ -2234,6 +2257,29 @@ static bool innodb_init_param()
 		srv_undo_dir = (char*) ".";
 	}
 
+	compile_time_assert(SRV_FORCE_IGNORE_CORRUPT == 1);
+
+	/*
+	 * This option can be read both from the command line, and the
+	 * defaults file. The assignment should account for both cases,
+	 * and for "--innobackupex". Since the command line argument is
+	 * parsed after the defaults file, it takes precedence.
+	 */
+	if (xtrabackup_innodb_force_recovery) {
+		srv_force_recovery = xtrabackup_innodb_force_recovery;
+	}
+
+	if (srv_force_recovery >= SRV_FORCE_IGNORE_CORRUPT) {
+		if (!xtrabackup_prepare) {
+			msg("mariabackup: The option \"innodb_force_recovery\""
+			    " should only be used with \"%s\".",
+			    (innobackupex_mode ? "--apply-log" : "--prepare"));
+			goto error;
+		} else {
+			msg("innodb_force_recovery = %lu", srv_force_recovery);
+		}
+	}
+
 #ifdef _WIN32
 	srv_use_native_aio = TRUE;
 #endif
@@ -6616,6 +6662,8 @@ int main(int argc, char **argv)
   char **client_defaults;
   char **backup_defaults;
 
+	my_getopt_prefix_matching= 0;
+
 	if (get_exepath(mariabackup_exe,FN_REFLEN, argv[0]))
     strncpy(mariabackup_exe,argv[0], FN_REFLEN-1);
 
diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h
index 4db1738e64d..bf5313d35db 100644
--- a/extra/mariabackup/xtrabackup.h
+++ b/extra/mariabackup/xtrabackup.h
@@ -174,6 +174,8 @@ enum binlog_info_enum { BINLOG_INFO_OFF, BINLOG_INFO_ON,
 
 extern ulong opt_binlog_info;
 
+extern ulong xtrabackup_innodb_force_recovery;
+
 void xtrabackup_io_throttling(void);
 my_bool xb_write_delta_metadata(const char *filename,
 				const xb_delta_info_t *info);