diff options
Diffstat (limited to 'extra/mariabackup/xtrabackup.cc')
| -rw-r--r-- | extra/mariabackup/xtrabackup.cc | 887 |
1 files changed, 543 insertions, 344 deletions
diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index aa0ed07c205..009a4891d93 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -36,8 +36,8 @@ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., 59 Temple -Place, Suite 330, Boston, MA 02111-1307 USA +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA *******************************************************/ @@ -185,7 +185,7 @@ xb_stream_fmt_t xtrabackup_stream_fmt = XB_STREAM_FMT_NONE; ibool xtrabackup_stream = FALSE; const char *xtrabackup_compress_alg = NULL; -ibool xtrabackup_compress = FALSE; +uint xtrabackup_compress = FALSE; uint xtrabackup_compress_threads; ulonglong xtrabackup_compress_chunk_size = 0; @@ -200,6 +200,8 @@ static char* log_ignored_opt; extern my_bool opt_use_ssl; my_bool opt_ssl_verify_server_cert; +my_bool opt_extended_validation; +my_bool opt_encrypted_backup; /* === metadata of backup === */ #define XTRABACKUP_METADATA_FILENAME "xtrabackup_checkpoints" @@ -301,6 +303,7 @@ my_bool opt_decompress = FALSE; my_bool opt_remove_original; my_bool opt_lock_ddl_per_table = FALSE; +static my_bool opt_check_privileges; extern const char *innodb_checksum_algorithm_names[]; extern TYPELIB innodb_checksum_algorithm_typelib; @@ -474,9 +477,18 @@ DECLARE_THREAD(dbug_execute_in_new_connection)(void *arg) dbug_thread_param_t *par= (dbug_thread_param_t *)arg; int err = mysql_query(par->con, par->query); int err_no = mysql_errno(par->con); - DBUG_ASSERT(par->expect_err == err); - if (err && par->expect_errno) - DBUG_ASSERT(err_no == par->expect_errno); + if(par->expect_err != err) + { + msg("FATAL: dbug_execute_in_new_connection : mysql_query '%s' returns %d, instead of expected %d", + par->query, err, par->expect_err); + _exit(1); + } + if (err && par->expect_errno && par->expect_errno != err_no) + { + msg("FATAL: dbug_execute_in_new_connection: mysql_query '%s' returns mysql_errno %d, instead of expected %d", + par->query, err_no, par->expect_errno); + _exit(1); + } mysql_close(par->con); mysql_thread_end(); os_event_t done = par->done_event; @@ -522,16 +534,18 @@ static os_event_t dbug_start_query_thread( mysql_thread_id(par->con), wait_state); for (;;) { MYSQL_RES *result = xb_mysql_query(mysql_connection,q, true, true); - if (mysql_fetch_row(result)) { + bool exists = mysql_fetch_row(result) != NULL; + mysql_free_result(result); + if (exists) { goto end; } - msg_ts("Waiting for query '%s' on connection %lu to " + msg("Waiting for query '%s' on connection %lu to " " reach state '%s'", query, mysql_thread_id(par->con), wait_state); my_sleep(1000); } end: - msg_ts("query '%s' on connection %lu reached state '%s'", query, + msg("query '%s' on connection %lu reached state '%s'", query, mysql_thread_id(par->con), wait_state); return par->done_event; } @@ -577,7 +591,9 @@ std::string filename_to_spacename(const byte *filename, size_t len) char *db = strrchr(f, '/'); ut_a(db); *table = '/'; - return std::string(db+1); + std::string s(db+1); + free(f); + return s; } /** Report an operation to create, delete, or rename a file during backup. @@ -587,7 +603,7 @@ std::string filename_to_spacename(const byte *filename, size_t len) @param[in] len length of name, in bytes @param[in] new_name new file name (NULL if not rename) @param[in] new_len length of new_name, in bytes (0 if NULL) */ -void backup_file_op(ulint space_id, const byte* flags, +static void backup_file_op(ulint space_id, const byte* flags, const byte* name, ulint len, const byte* new_name, ulint new_len) { @@ -600,41 +616,123 @@ void backup_file_op(ulint space_id, const byte* flags, if (flags) { ddl_tracker.id_to_name[space_id] = filename_to_spacename(name, len); - msg("DDL tracking : create %zu \"%.*s\": %x\n", + msg("DDL tracking : create %zu \"%.*s\": %x", space_id, int(len), name, mach_read_from_4(flags)); } else if (new_name) { ddl_tracker.id_to_name[space_id] = filename_to_spacename(new_name, new_len); - msg("DDL tracking : rename %zu \"%.*s\",\"%.*s\"\n", + msg("DDL tracking : rename %zu \"%.*s\",\"%.*s\"", space_id, int(len), name, int(new_len), new_name); } else { ddl_tracker.drops.insert(space_id); - msg("DDL tracking : delete %zu \"%.*s\"\n", space_id, int(len), name); + msg("DDL tracking : delete %zu \"%.*s\"", space_id, int(len), name); } pthread_mutex_unlock(&backup_mutex); } +/* + This callback is called if DDL operation is detected, + at the end of backup + + Normally, DDL operations are blocked due to FTWRL, + but in rare cases of --no-lock, they are not. + + We will abort backup in this case. +*/ +static void backup_file_op_fail(ulint space_id, const byte* flags, + const byte* name, ulint len, + const byte* new_name, ulint new_len) +{ + ut_a(opt_no_lock); + bool fail; + if (flags) { + msg("DDL tracking : create %zu \"%.*s\": %x", + space_id, int(len), name, mach_read_from_4(flags)); + std::string spacename = filename_to_spacename(name, len); + fail = !check_if_skip_table(spacename.c_str()); + } + else if (new_name) { + msg("DDL tracking : rename %zu \"%.*s\",\"%.*s\"", + space_id, int(len), name, int(new_len), new_name); + std::string spacename = filename_to_spacename(name, len); + std::string new_spacename = filename_to_spacename(new_name, new_len); + fail = !check_if_skip_table(spacename.c_str()) || !check_if_skip_table(new_spacename.c_str()); + } + else { + std::string spacename = filename_to_spacename(name, len); + fail = !check_if_skip_table(spacename.c_str()); + msg("DDL tracking : delete %zu \"%.*s\"", space_id, int(len), name); + } + if (fail) { + die("DDL operation detected in the late phase of backup." + "Backup is inconsistent. Remove --no-lock option to fix."); + } +} + + /** Callback whenever MLOG_INDEX_LOAD happens. @param[in] space_id space id to check */ static void backup_optimized_ddl_op(ulint space_id) { - // TODO : handle incremental - if (xtrabackup_incremental) - return; - pthread_mutex_lock(&backup_mutex); ddl_tracker.optimized_ddl.insert(space_id); pthread_mutex_unlock(&backup_mutex); } +/* + Optimized DDL callback at the end of backup that + run with --no-lock. Usually aborts the backup. +*/ +static void backup_optimized_ddl_op_fail(ulint space_id) { + ut_a(opt_no_lock); + msg("DDL tracking : optimized DDL on space %zu", space_id); + if (ddl_tracker.tables_in_backup.find(space_id) != ddl_tracker.tables_in_backup.end()) { + msg("ERROR : Optimized DDL operation detected in the late phase of backup." + "Backup is inconsistent. Remove --no-lock option to fix."); + exit(EXIT_FAILURE); + } +} + + +/* + Retrieve default data directory, to be used with --copy-back. + + On Windows, default datadir is ..\data, relative to the + directory where mariabackup.exe is located(usually "bin") + + Elsewhere, the compiled-in constant MYSQL_DATADIR is used. +*/ +static char *get_default_datadir() { + static char ddir[] = MYSQL_DATADIR; +#ifdef _WIN32 + static char buf[MAX_PATH]; + DWORD size = (DWORD)sizeof(buf) - 1; + if (GetModuleFileName(NULL, buf, size) <= size) + { + char *p; + if ((p = strrchr(buf, '\\'))) + { + *p = 0; + if ((p = strrchr(buf, '\\'))) + { + strncpy(p + 1, "data", buf + MAX_PATH - p); + return buf; + } + } + } +#endif + return ddir; +} + + /* ======== Date copying thread context ======== */ typedef struct { datafiles_iter_t *it; uint num; uint *count; - pthread_mutex_t count_mutex; + pthread_mutex_t* count_mutex; os_thread_id_t id; } data_thread_ctxt_t; @@ -661,6 +759,8 @@ enum options_xtrabackup OPT_XTRA_DATABASES, OPT_XTRA_DATABASES_FILE, OPT_XTRA_PARALLEL, + OPT_XTRA_EXTENDED_VALIDATION, + OPT_XTRA_ENCRYPTED_BACKUP, OPT_XTRA_STREAM, OPT_XTRA_COMPRESS, OPT_XTRA_COMPRESS_THREADS, @@ -734,7 +834,8 @@ enum options_xtrabackup OPT_PROTOCOL, OPT_LOCK_DDL_PER_TABLE, OPT_ROCKSDB_DATADIR, - OPT_BACKUP_ROCKSDB + OPT_BACKUP_ROCKSDB, + OPT_XTRA_CHECK_PRIVILEGES }; struct my_option xb_client_options[] = @@ -1117,6 +1218,22 @@ struct my_option xb_server_options[] = (G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, + {"extended_validation", OPT_XTRA_EXTENDED_VALIDATION, + "Enable extended validation for Innodb data pages during backup phase. " + "Will slow down backup considerably, in case encryption is used. " + "May fail if tables are created during the backup.", + (G_PTR*)&opt_extended_validation, + (G_PTR*)&opt_extended_validation, + 0, GET_BOOL, NO_ARG, FALSE, 0, 0, 0, 0, 0}, + + {"encrypted_backup", OPT_XTRA_ENCRYPTED_BACKUP, + "In --backup, assume that nonzero key_version implies that the page" + " is encrypted. Use --backup --skip-encrypted-backup to allow" + " copying unencrypted that were originally created before MySQL 5.1.48.", + (G_PTR*)&opt_encrypted_backup, + (G_PTR*)&opt_encrypted_backup, + 0, GET_BOOL, NO_ARG, TRUE, 0, 0, 0, 0, 0}, + {"log", OPT_LOG, "Ignored option for MySQL option compatibility", (G_PTR*) &log_ignored_opt, (G_PTR*) &log_ignored_opt, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, @@ -1287,6 +1404,10 @@ struct my_option xb_server_options[] = &xb_backup_rocksdb, &xb_backup_rocksdb, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 }, + {"check-privileges", OPT_XTRA_CHECK_PRIVILEGES, "Check database user " + "privileges fro the backup user", + &opt_check_privileges, &opt_check_privileges, + 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 }, { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} }; @@ -1327,14 +1448,13 @@ debug_sync_point(const char *name) xtrabackup_target_dir); fp = fopen(pid_path, "w"); if (fp == NULL) { - msg("mariabackup: Error: cannot open %s\n", pid_path); - exit(EXIT_FAILURE); + die("Can't open open %s", pid_path); } fprintf(fp, "%u\n", (uint) pid); fclose(fp); msg("mariabackup: DEBUG: Suspending at debug sync point '%s'. " - "Resume with 'kill -SIGCONT %u'.\n", name, (uint) pid); + "Resume with 'kill -SIGCONT %u'.", name, (uint) pid); debug_sync_resumed= 0; kill(pid, SIGSTOP); @@ -1343,13 +1463,13 @@ debug_sync_point(const char *name) } /* On resume */ - msg("mariabackup: DEBUG: removing the pid file.\n"); + msg("mariabackup: DEBUG: removing the pid file."); my_delete(pid_path, MYF(MY_WME)); #endif } -static std::vector<std::string> tables_for_export; +static std::set<std::string> tables_for_export; static void append_export_table(const char *dbname, const char *tablename, bool is_remote) { @@ -1361,7 +1481,15 @@ static void append_export_table(const char *dbname, const char *tablename, bool char *p=strrchr(buf, '.'); if (p) *p=0; - tables_for_export.push_back(ut_get_name(0,buf)); + std::string name=ut_get_name(0, buf); + /* Strip partition name comment from table name, if any */ + if (ends_with(name.c_str(), "*/")) + { + size_t pos= name.rfind("/*"); + if (pos != std::string::npos) + name.resize(pos); + } + tables_for_export.insert(name); } } @@ -1376,9 +1504,10 @@ static int create_bootstrap_file() fputs("SET NAMES UTF8;\n",f); enumerate_ibd_files(append_export_table); - for (size_t i= 0; i < tables_for_export.size(); i++) + for (std::set<std::string>::iterator it = tables_for_export.begin(); + it != tables_for_export.end(); it++) { - const char *tab = tables_for_export[i].c_str(); + const char *tab = it->c_str(); fprintf(f, "BEGIN NOT ATOMIC " "DECLARE CONTINUE HANDLER FOR NOT FOUND,SQLEXCEPTION BEGIN END;" @@ -1408,7 +1537,7 @@ static int prepare_export() snprintf(cmdline, sizeof cmdline, IF_WIN("\"","") "\"%s\" --mysqld \"%s\" " " --defaults-extra-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=." - " --innodb --innodb-fast-shutdown=0" + " --innodb --innodb-fast-shutdown=0 --loose-partition" " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu" " --console --skip-log-error --bootstrap < " BOOTSTRAP_FILENAME IF_WIN("\"",""), mariabackup_exe, @@ -1420,7 +1549,7 @@ static int prepare_export() sprintf(cmdline, IF_WIN("\"","") "\"%s\" --mysqld" " --defaults-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=." - " --innodb --innodb-fast-shutdown=0" + " --innodb --innodb-fast-shutdown=0 --loose-partition" " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu" " --console --log-error= --bootstrap < " BOOTSTRAP_FILENAME IF_WIN("\"",""), mariabackup_exe, @@ -1454,7 +1583,7 @@ static const char *xb_server_default_groups[]= static void print_version(void) { - msg("%s based on MariaDB server %s %s (%s) \n", + msg("%s based on MariaDB server %s %s (%s)", my_progname, MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE); } @@ -1579,7 +1708,7 @@ xb_get_one_option(int optid, xtrabackup_stream_fmt = XB_STREAM_FMT_XBSTREAM; else { - msg("Invalid --stream argument: %s\n", argument); + msg("Invalid --stream argument: %s", argument); return 1; } xtrabackup_stream = TRUE; @@ -1589,7 +1718,7 @@ xb_get_one_option(int optid, xtrabackup_compress_alg = "quicklz"; else if (strcasecmp(argument, "quicklz")) { - msg("Invalid --compress argument: %s\n", argument); + msg("Invalid --compress argument: %s", argument); return 1; } xtrabackup_compress = TRUE; @@ -1654,8 +1783,10 @@ static bool innodb_init_param() /* === some variables from mysqld === */ memset((G_PTR) &mysql_tmpdir_list, 0, sizeof(mysql_tmpdir_list)); - if (init_tmpdir(&mysql_tmpdir_list, opt_mysql_tmpdir)) + if (init_tmpdir(&mysql_tmpdir_list, opt_mysql_tmpdir)) { + msg("init_tmpdir() failed"); return true; + } xtrabackup_tmpdir = my_tmpdir(&mysql_tmpdir_list); /* dummy for initialize all_charsets[] */ get_charset_name(0); @@ -1673,9 +1804,9 @@ static bool innodb_init_param() srv_page_size_shift = ulong(n_shift); srv_page_size = 1U << n_shift; msg("InnoDB: The universal page size of the " - "database is set to %lu.\n", srv_page_size); + "database is set to %lu.", srv_page_size); } else { - msg("InnoDB: Error: invalid value of " + msg("invalid value of " "innobase_page_size: %lld", innobase_page_size); goto error; } @@ -1688,12 +1819,12 @@ static bool innodb_init_param() if (sizeof(ulint) == 4) { if (xtrabackup_use_memory > UINT_MAX32) { msg("mariabackup: use-memory can't be over 4GB" - " on 32-bit systems\n"); + " on 32-bit systems"); } if (innobase_buffer_pool_size > UINT_MAX32) { msg("mariabackup: innobase_buffer_pool_size can't be " - "over 4GB on 32-bit systems\n"); + "over 4GB on 32-bit systems"); goto error; } @@ -1706,10 +1837,10 @@ static bool innodb_init_param() read from MySQL .cnf file */ if (xtrabackup_backup) { - msg("mariabackup: using the following InnoDB configuration:\n"); + msg("mariabackup: using the following InnoDB configuration:"); } else { msg("mariabackup: using the following InnoDB configuration " - "for recovery:\n"); + "for recovery:"); } /*--------------- Data files -------------------------*/ @@ -1718,7 +1849,7 @@ static bool innodb_init_param() srv_data_home = (xtrabackup_backup && innobase_data_home_dir ? innobase_data_home_dir : default_path); - msg("mariabackup: innodb_data_home_dir = %s\n", srv_data_home); + msg("innodb_data_home_dir = %s", srv_data_home); /* Set default InnoDB data file size to 10 MB and let it be auto-extending. Thus users can use InnoDB in >= 4.0 without having @@ -1727,7 +1858,7 @@ static bool innodb_init_param() if (!innobase_data_file_path) { innobase_data_file_path = (char*) "ibdata1:10M:autoextend"; } - msg("mariabackup: innodb_data_file_path = %s\n", + msg("innodb_data_file_path = %s", innobase_data_file_path); /* This is the first time univ_page_size is used. @@ -1757,13 +1888,12 @@ static bool innodb_init_param() if (xtrabackup_prepare && xtrabackup_incremental_dir) { srv_log_group_home_dir = xtrabackup_incremental_dir; } - msg("mariabackup: innodb_log_group_home_dir = %s\n", + msg("innodb_log_group_home_dir = %s", srv_log_group_home_dir); os_normalize_path(srv_log_group_home_dir); if (strchr(srv_log_group_home_dir, ';')) { - msg("syntax error in innodb_log_group_home_dir, "); goto error; } @@ -1811,8 +1941,7 @@ static bool innodb_init_param() #elif defined(LINUX_NATIVE_AIO) if (srv_use_native_aio) { - ut_print_timestamp(stderr); - msg(" InnoDB: Using Linux native AIO\n"); + msg("InnoDB: Using Linux native AIO"); } #else /* Currently native AIO is supported only on windows and linux @@ -1857,7 +1986,7 @@ static bool innodb_init() } if (err != DB_SUCCESS) { - msg("mariabackup: innodb_init() returned %d (%s).\n", + msg("mariabackup: innodb_init() returned %d (%s).", err, ut_strerr(err)); innodb_shutdown(); return(TRUE); @@ -1881,7 +2010,7 @@ xtrabackup_read_metadata(char *filename) fp = fopen(filename,"r"); if(!fp) { - msg("mariabackup: Error: cannot open %s\n", filename); + msg("Error: cannot open %s", filename); return(FALSE); } @@ -1960,8 +2089,7 @@ xtrabackup_stream_metadata(ds_ctxt_t *ds_ctxt) stream = ds_open(ds_ctxt, XTRABACKUP_METADATA_FILENAME, &mystat); if (stream == NULL) { - msg("mariabackup: Error: cannot open output stream " - "for %s\n", XTRABACKUP_METADATA_FILENAME); + msg("Error: cannot open output stream for %s", XTRABACKUP_METADATA_FILENAME); return(FALSE); } @@ -1993,7 +2121,7 @@ xtrabackup_write_metadata(const char *filepath) fp = fopen(filepath, "w"); if(!fp) { - msg("mariabackup: Error: cannot open %s\n", filepath); + msg("Error: cannot open %s", filepath); return(FALSE); } if (fwrite(buf, len, 1, fp) < 1) { @@ -2042,7 +2170,7 @@ xb_read_delta_metadata(const char *filepath, xb_delta_info_t *info) fclose(fp); if (page_size == ULINT_UNDEFINED) { - msg("mariabackup: page_size is required in %s\n", filepath); + msg("page_size is required in %s", filepath); r = FALSE; } else { info->page_size = page_size_t(zip_size ? zip_size : page_size, @@ -2052,7 +2180,7 @@ xb_read_delta_metadata(const char *filepath, xb_delta_info_t *info) if (info->space_id == ULINT_UNDEFINED) { msg("mariabackup: Warning: This backup was taken with XtraBackup 2.0.1 " "or earlier, some DDL operations between full and incremental " - "backups may be handled incorrectly\n"); + "backups may be handled incorrectly"); } return(r); @@ -2085,8 +2213,7 @@ xb_write_delta_metadata(const char *filename, const xb_delta_info_t *info) f = ds_open(ds_meta, filename, &mystat); if (f == NULL) { - msg("mariabackup: Error: cannot open output stream for %s\n", - filename); + msg("Error: Can't open output stream for %s",filename); return(FALSE); } @@ -2260,6 +2387,18 @@ check_if_skip_table( const char *ptr; char *eptr; + + dbname = NULL; + tbname = name; + while ((ptr = strchr(tbname, '/')) != NULL) { + dbname = tbname; + tbname = ptr + 1; + } + + if (strncmp(tbname, tmp_file_prefix, tmp_file_prefix_length) == 0) { + return TRUE; + } + if (regex_exclude_list.empty() && regex_include_list.empty() && tables_include_hash == NULL && @@ -2269,13 +2408,6 @@ check_if_skip_table( return(FALSE); } - dbname = NULL; - tbname = name; - while ((ptr = strchr(tbname, '/')) != NULL) { - dbname = tbname; - tbname = ptr + 1; - } - if (dbname == NULL) { return(FALSE); } @@ -2394,7 +2526,7 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n, const char *dest_name= if (fil_is_user_tablespace_id(node->space->id) && check_if_skip_table(node_name)) { - msg("[%02u] Skipping %s.\n", thread_n, node_name); + msg(thread_n, "Skipping %s.", node_name); return(FALSE); } @@ -2440,26 +2572,22 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n, const char *dest_name= if (write_filter->init != NULL && !write_filter->init(&write_filt_ctxt, dst_name, &cursor)) { - msg("[%02u] mariabackup: error: " - "failed to initialize page write filter.\n", thread_n); + msg (thread_n, "mariabackup: error: failed to initialize page write filter."); goto error; } dstfile = ds_open(ds_data, dst_name, &cursor.statinfo); if (dstfile == NULL) { - msg("[%02u] mariabackup: error: " - "cannot open the destination stream for %s\n", - thread_n, dst_name); + msg(thread_n,"mariabackup: error: can't open the destination stream for %s", dst_name); goto error; } action = xb_get_copy_action(); if (xtrabackup_stream) { - msg_ts("[%02u] %s %s\n", thread_n, action, node_path); + msg(thread_n, "%s %s", action, node_path); } else { - msg_ts("[%02u] %s %s to %s\n", thread_n, action, - node_path, dstfile->path); + msg(thread_n, "%s %s to %s", action, node_path, dstfile->path); } /* The main copy loop */ @@ -2483,7 +2611,7 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n, const char *dest_name= pthread_mutex_unlock(&backup_mutex); /* close */ - msg_ts("[%02u] ...done\n", thread_n); + msg(thread_n," ...done"); xb_fil_cur_close(&cursor); if (ds_close(dstfile)) { rc = TRUE; @@ -2501,8 +2629,7 @@ error: if (write_filter && write_filter->deinit) { write_filter->deinit(&write_filt_ctxt);; } - msg("[%02u] mariabackup: Error: " - "xtrabackup_copy_datafile() failed.\n", thread_n); + msg(thread_n, "mariabackup: xtrabackup_copy_datafile() failed."); return(TRUE); /*ERROR*/ skip: @@ -2513,11 +2640,7 @@ skip: if (write_filter && write_filter->deinit) { write_filter->deinit(&write_filt_ctxt); } - msg("[%02u] mariabackup: Warning: We assume the " - "table was dropped during xtrabackup execution " - "and ignore the file.\n", thread_n); - msg("[%02u] mariabackup: Warning: skipping tablespace %s.\n", - thread_n, node_name); + msg(thread_n,"Warning: We assume the table was dropped during xtrabackup execution and ignore the tablespace %s", node_name); return(FALSE); } @@ -2542,9 +2665,8 @@ static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn, bool last) && scanned_checkpoint - checkpoint >= 0x80000000UL) { /* Garbage from a log buffer flush which was made before the most recent database recovery */ - msg("mariabackup: checkpoint wrap: " - LSN_PF ",%zx,%zx\n", - scanned_lsn, scanned_checkpoint, checkpoint); + msg(0,"checkpoint wrap: " LSN_PF ",%zx,%zx", + scanned_lsn, scanned_checkpoint, checkpoint); break; } @@ -2564,8 +2686,7 @@ static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn, bool last) } else if (data_len >= log_sys.trailer_offset() || data_len <= LOG_BLOCK_HDR_SIZE) { /* We got a garbage block (abrupt end of the log). */ - msg("mariabackup: garbage block: " LSN_PF ",%zu\n", - scanned_lsn, data_len); + msg(0,"garbage block: " LSN_PF ",%zu",scanned_lsn, data_len); break; } else { /* We got a partial block (abrupt end of the log). */ @@ -2576,7 +2697,7 @@ static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn, bool last) if (more_data && recv_parse_log_recs(0, STORE_NO, false)) { - msg("mariabackup: copying the log failed \n"); + msg("Error: copying the log failed"); return(0); } @@ -2595,8 +2716,7 @@ static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn, bool last) } if (ds_write(dst_log_file, log_sys.buf, write_size)) { - msg("mariabackup: Error: " - "write to logfile failed\n"); + msg("Error: write to logfile failed"); return(0); } } @@ -2632,7 +2752,7 @@ static bool xtrabackup_copy_logfile(bool last = false) || lsn != start_lsn) { break; } - msg("Retrying read of log at LSN=" LSN_PF "\n", lsn); + msg("Retrying read of log at LSN=" LSN_PF, lsn); my_sleep(1000); } @@ -2642,15 +2762,13 @@ static bool xtrabackup_copy_logfile(bool last = false) log_mutex_exit(); if (!start_lsn) { - msg("mariabackup: Error: xtrabackup_copy_logfile()" - " failed.\n"); - exit(EXIT_FAILURE); + die("xtrabackup_copy_logfile() failed."); } } while (start_lsn == end_lsn); ut_ad(start_lsn == log_sys.log.scanned_lsn); - msg_ts(">> log scanned up to (" LSN_PF ")\n", start_lsn); + msg(">> log scanned up to (" LSN_PF ")", start_lsn); /* update global variable*/ pthread_mutex_lock(&backup_mutex); @@ -2759,7 +2877,7 @@ static void dbug_mariabackup_event(const char *event,const char *key) } char *sql = getenv(envvar); if (sql) { - msg("dbug_mariabackup_event : executing '%s'\n", sql); + msg("dbug_mariabackup_event : executing '%s'", sql); xb_mysql_query(mysql_connection, sql, false, true); } @@ -2790,24 +2908,19 @@ DECLARE_THREAD(data_copy_thread_func)( debug_sync_point("data_copy_thread_func"); while ((node = datafiles_iter_next(ctxt->it)) != NULL) { - DBUG_MARIABACKUP_EVENT("before_copy", node->space->name); - - /* copy the datafile */ if(xtrabackup_copy_datafile(node, num)) { - msg("[%02u] mariabackup: Error: " - "failed to copy datafile.\n", num); - exit(EXIT_FAILURE); + die("failed to copy datafile."); } DBUG_MARIABACKUP_EVENT("after_copy", node->space->name); } - pthread_mutex_lock(&ctxt->count_mutex); + pthread_mutex_lock(ctxt->count_mutex); (*ctxt->count)--; - pthread_mutex_unlock(&ctxt->count_mutex); + pthread_mutex_unlock(ctxt->count_mutex); my_thread_end(); os_thread_exit(); @@ -2974,12 +3087,19 @@ xb_load_single_table_tablespace( Datafile *file = xb_new_datafile(name, is_remote); if (file->open_read_only(true) != DB_SUCCESS) { - msg("Can't open datafile %s\n", name); - ut_free(name); - exit(EXIT_FAILURE); + die("Can't open datafile %s", name); } - err = file->validate_first_page(&flush_lsn); + for (int i = 0; i < 10; i++) { + err = file->validate_first_page(&flush_lsn); + if (err != DB_CORRUPTION) { + break; + } + + my_sleep(1000); + } + + bool is_empty_file = file->exists() && file->is_empty_file(); if (err == DB_SUCCESS && file->space_id() != SRV_TMP_SPACE_ID) { os_offset_t node_size = os_file_get_size(file->handle()); @@ -2995,11 +3115,8 @@ xb_load_single_table_tablespace( ut_a(space != NULL); - if (!fil_node_create(file->filepath(), ulint(n_pages), space, - false, false)) { - ut_error; - } - + space->add(file->filepath(), OS_FILE_CLOSED, ulint(n_pages), + false, false); /* by opening the tablespace we forcing node and space objects in the cache to be populated with fields from space header */ space->open(); @@ -3014,10 +3131,8 @@ xb_load_single_table_tablespace( delete file; - if (err != DB_SUCCESS && err != DB_CORRUPTION && xtrabackup_backup) { - /* allow corrupted first page for xtrabackup, it could be just - zero-filled page, which we restore from redo log later */ - exit(EXIT_FAILURE); + if (err != DB_SUCCESS && xtrabackup_backup && !is_empty_file) { + die("Failed to not validate first page of the file %s, error %d",name, (int)err); } } @@ -3163,35 +3278,24 @@ the first slot rollback segments of TRX_SYS_PAGE_NO. @retval DB_SUCCESS if srv_undo_space_id assigned successfully. */ static dberr_t xb_assign_undo_space_start() { - ulint dirnamelen; - char name[1000]; + pfs_os_file_t file; byte* buf; byte* page; bool ret; dberr_t error = DB_SUCCESS; ulint space, page_no __attribute__((unused)); + int n_retries = 5; if (srv_undo_tablespaces == 0) { return error; } - os_normalize_path(srv_data_home); - dirnamelen = strlen(srv_data_home); - memcpy(name, srv_data_home, dirnamelen); - - if (dirnamelen && name[dirnamelen - 1] != OS_PATH_SEPARATOR) { - name[dirnamelen++] = OS_PATH_SEPARATOR; - } - - snprintf(name + dirnamelen, (sizeof name) - dirnamelen, - "%s", "ibdata1"); - - file = os_file_create(0, name, OS_FILE_OPEN, - OS_FILE_NORMAL, OS_DATA_FILE, true, &ret); + file = os_file_create(0, srv_sys_space.first_datafile()->filepath(), + OS_FILE_OPEN, OS_FILE_NORMAL, OS_DATA_FILE, true, &ret); if (!ret) { - msg("mariabackup: Error in opening %s\n", name); + msg("Error opening %s", srv_sys_space.first_datafile()->filepath()); return DB_ERROR; } @@ -3202,14 +3306,21 @@ retry: if (!os_file_read(IORequestRead, file, page, TRX_SYS_PAGE_NO << srv_page_size_shift, srv_page_size)) { - msg("mariabackup: Reading TRX_SYS page failed.\n"); + msg("Reading TRX_SYS page failed.\n"); error = DB_ERROR; goto func_exit; } /* TRX_SYS page can't be compressed or encrypted. */ if (buf_page_is_corrupted(false, page, univ_page_size)) { - goto retry; + if (n_retries--) { + os_thread_sleep(1000); + goto retry; + } else { + msg("mariabackup: TRX_SYS page corrupted.\n"); + error = DB_ERROR; + goto func_exit; + } } /* 0th slot always points to system tablespace. @@ -3253,16 +3364,22 @@ xb_load_tablespaces() /* create_new_db must not be true. */ if (err != DB_SUCCESS || create_new_db) { - msg("mariabackup: could not find data files at the " - "specified datadir\n"); + msg("Could not find data files at the specified datadir"); return(DB_ERROR); } - err = srv_sys_space.open_or_create(false, false, &sum_of_new_sizes, - &flush_lsn); + for (int i= 0; i < 10; i++) { + err = srv_sys_space.open_or_create(false, false, &sum_of_new_sizes, + &flush_lsn); + if (err == DB_PAGE_CORRUPTED || err == DB_CORRUPTION) { + my_sleep(1000); + } + else + break; + } if (err != DB_SUCCESS) { - msg("mariabackup: Could not open data files.\n"); + msg("Could not open data files.\n"); return(err); } @@ -3284,7 +3401,7 @@ xb_load_tablespaces() srv_undo_tablespaces_init(), because fil_is_user_tablespace_id() * relies on srv_undo_tablespaces_open to be properly initialized */ - msg("mariabackup: Generating a list of tablespaces\n"); + msg("mariabackup: Generating a list of tablespaces"); err = enumerate_ibd_files(xb_load_single_table_tablespace); if (err != DB_SUCCESS) { @@ -3387,13 +3504,11 @@ xb_validate_name( /* perform only basic validation. validate length and path symbols */ if (len > NAME_LEN) { - msg("mariabackup: name `%s` is too long.\n", name); - exit(EXIT_FAILURE); + die("name `%s` is too long.", name); } p = strpbrk(name, "/\\~"); if (p && (uint) (p - name) < NAME_LEN) { - msg("mariabackup: name `%s` is not valid.\n", name); - exit(EXIT_FAILURE); + die("name `%s` is not valid.", name); } } @@ -3471,8 +3586,7 @@ xb_register_table( const char* name) /*!< in: name of table */ { if (strchr(name, '.') == NULL) { - msg("mariabackup: `%s` is not fully qualified name.\n", name); - exit(EXIT_FAILURE); + die("`%s` is not fully qualified name.", name); } xb_register_include_filter_entry(name); @@ -3493,7 +3607,7 @@ xb_add_regex_to_list( if (ret != 0) { regerror(ret, &compiled_regex, errbuf, sizeof(errbuf)); - msg("mariabackup: error: %s regcomp(%s): %s\n", + msg("mariabackup: error: %s regcomp(%s): %s", error_context, regex, errbuf); exit(EXIT_FAILURE); } @@ -3562,17 +3676,15 @@ xb_load_list_file( /* read and store the filenames */ fp = fopen(filename, "r"); if (!fp) { - msg("mariabackup: cannot open %s\n", + die("Can't open %s", filename); - exit(EXIT_FAILURE); } while (fgets(name_buf, sizeof(name_buf), fp) != NULL) { char* p = strchr(name_buf, '\n'); if (p) { *p = '\0'; } else { - msg("mariabackup: `%s...` name is too long", name_buf); - exit(EXIT_FAILURE); + die("`%s...` name is too long", name_buf); } ins(name_buf); @@ -3680,22 +3792,17 @@ xb_filters_free() } /*********************************************************************//** -Creates or opens the log files and closes them. -@return DB_SUCCESS or error code */ +Create log file metadata. */ static -ulint +void open_or_create_log_file( /*====================*/ fil_space_t* space, - ibool* log_file_created, /*!< out: TRUE if new log file - created */ ulint i) /*!< in: log file number in group */ { char name[10000]; ulint dirnamelen; - *log_file_created = FALSE; - os_normalize_path(srv_log_group_home_dir); dirnamelen = strlen(srv_log_group_home_dir); @@ -3707,14 +3814,13 @@ open_or_create_log_file( name[dirnamelen++] = OS_PATH_SEPARATOR; } - sprintf(name + dirnamelen, "%s%lu", "ib_logfile", (ulong) i); + sprintf(name + dirnamelen, "%s%zu", "ib_logfile", i); ut_a(fil_validate()); - ut_a(fil_node_create(name, ulint(srv_log_file_size >> srv_page_size_shift), - space, false, false)); - - return(DB_SUCCESS); + space->add(name, OS_FILE_CLOSED, + ulint(srv_log_file_size >> srv_page_size_shift), + false, false); } /*********************************************************************** @@ -3818,10 +3924,10 @@ static bool xtrabackup_backup_low() metadata_to_lsn = mach_read_from_8( log_sys.checkpoint_buf + LOG_CHECKPOINT_LSN); msg("mariabackup: The latest check point" - " (for incremental): '" LSN_PF "'\n", + " (for incremental): '" LSN_PF "'", metadata_to_lsn); } else { - msg("mariabackup: Error: recv_find_max_checkpoint() failed.\n"); + msg("Error: recv_find_max_checkpoint() failed."); } log_mutex_exit(); } @@ -3851,7 +3957,7 @@ static bool xtrabackup_backup_low() metadata_last_lsn = log_copy_scanned_lsn; if (!xtrabackup_stream_metadata(ds_meta)) { - msg("mariabackup: Error: failed to stream metadata.\n"); + msg("Error: failed to stream metadata."); return false; } if (xtrabackup_extra_lsndir) { @@ -3860,15 +3966,15 @@ static bool xtrabackup_backup_low() sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, XTRABACKUP_METADATA_FILENAME); if (!xtrabackup_write_metadata(filename)) { - msg("mariabackup: Error: failed to write metadata " - "to '%s'.\n", filename); + msg("Error: failed to write metadata " + "to '%s'.", filename); return false; } sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, XTRABACKUP_INFO); if (!write_xtrabackup_info(mysql_connection, filename, false)) { - msg("mariabackup: Error: failed to write info " - "to '%s'.\n", filename); + msg("Error: failed to write info " + "to '%s'.", filename); return false; } } @@ -3891,19 +3997,19 @@ xtrabackup_backup_func() pthread_cond_init(&scanned_lsn_cond, NULL); #ifdef USE_POSIX_FADVISE - msg("mariabackup: uses posix_fadvise().\n"); + msg("uses posix_fadvise()."); #endif /* cd to datadir */ if (my_setwd(mysql_real_data_home,MYF(MY_WME))) { - msg("mariabackup: cannot my_setwd %s\n", mysql_real_data_home); + msg("my_setwd() failed , %s", mysql_real_data_home); return(false); } - msg("mariabackup: cd to %s\n", mysql_real_data_home); + msg("cd to %s", mysql_real_data_home); encryption_plugin_backup_init(mysql_connection); - msg("mariabackup: open files limit requested %u, set to %u\n", + msg("open files limit requested %u, set to %u", (uint) xb_open_files_limit, xb_set_max_open_files(xb_open_files_limit)); @@ -3918,12 +4024,6 @@ xtrabackup_backup_func() log_file_op = backup_file_op; metadata_to_lsn = 0; - if (xb_close_files) - msg("mariabackup: warning: close-files specified. Use it " - "at your own risk. If there are DDL operations like table DROP TABLE " - "or RENAME TABLE during the backup, inconsistent backup will be " - "produced.\n"); - /* initialize components */ if(innodb_init_param()) { fail: @@ -3974,13 +4074,6 @@ fail: xb_filters_init(); - { - ibool log_file_created; - ibool log_created = FALSE; - ibool log_opened = FALSE; - ulint err; - ulint i; - xb_fil_io_init(); srv_n_file_io_threads = srv_n_read_io_threads; @@ -3993,43 +4086,15 @@ fail: "innodb_redo_log", SRV_LOG_SPACE_FIRST_ID, 0, FIL_TYPE_LOG, NULL); - for (i = 0; i < srv_n_log_files; i++) { - err = open_or_create_log_file(space, &log_file_created, i); - if (err != DB_SUCCESS) { - goto fail; - } - - if (log_file_created) { - log_created = TRUE; - } else { - log_opened = TRUE; - } - if ((log_opened && log_created)) { - msg( - "mariabackup: Error: all log files must be created at the same time.\n" - "mariabackup: All log files must be created also in database creation.\n" - "mariabackup: If you want bigger or smaller log files, shut down the\n" - "mariabackup: database and make sure there were no errors in shutdown.\n" - "mariabackup: Then delete the existing log files. Edit the .cnf file\n" - "mariabackup: and start the database again.\n"); - - goto fail; - } - } - - /* log_file_created must not be TRUE, if online */ - if (log_file_created) { - msg("mariabackup: Something wrong with source files...\n"); - goto fail; - } - + for (ulint i = 0; i < srv_n_log_files; i++) { + open_or_create_log_file(space, i); } /* create extra LSN dir if it does not exist. */ if (xtrabackup_extra_lsndir &&!my_stat(xtrabackup_extra_lsndir,&stat_info,MYF(0)) && (my_mkdir(xtrabackup_extra_lsndir,0777,MYF(0)) < 0)) { - msg("mariabackup: Error: cannot mkdir %d: %s\n", + msg("Error: cannot mkdir %d: %s\n", my_errno, xtrabackup_extra_lsndir); goto fail; } @@ -4037,7 +4102,7 @@ fail: /* create target dir if not exist */ if (!xtrabackup_stream_str && !my_stat(xtrabackup_target_dir,&stat_info,MYF(0)) && (my_mkdir(xtrabackup_target_dir,0777,MYF(0)) < 0)){ - msg("mariabackup: Error: cannot mkdir %d: %s\n", + msg("Error: cannot mkdir %d: %s\n", my_errno, xtrabackup_target_dir); goto fail; } @@ -4048,7 +4113,6 @@ fail: /* start back ground thread to copy newer log */ os_thread_id_t log_copying_thread_id; - datafiles_iter_t *it; /* get current checkpoint_lsn */ /* Look for the latest checkpoint from any of the log groups */ @@ -4065,8 +4129,8 @@ log_fail: if (log_sys.log.format == 0) { old_format: - msg("mariabackup: Error: cannot process redo log" - " before MariaDB 10.2.2\n"); + msg("Error: cannot process redo log" + " before MariaDB 10.2.2"); log_mutex_exit(); goto log_fail; } @@ -4105,8 +4169,8 @@ reread_log_header: memset(&stat_info, 0, sizeof(MY_STAT)); dst_log_file = ds_open(ds_redo, "ib_logfile0", &stat_info); if (dst_log_file == NULL) { - msg("mariabackup: error: failed to open the target stream for " - "'ib_logfile0'.\n"); + msg("§rror: failed to open the target stream for " + "'ib_logfile0'."); goto fail; } @@ -4124,7 +4188,7 @@ reread_log_header: /* Write the log header. */ if (ds_write(dst_log_file, log_hdr, sizeof log_hdr)) { log_write_fail: - msg("mariabackup: error: write to logfile failed\n"); + msg("error: write to logfile failed"); goto fail; } /* Adjust the checkpoint page. */ @@ -4159,8 +4223,8 @@ reread_log_header: /* Populate fil_system with tablespaces to copy */ err = xb_load_tablespaces(); if (err != DB_SUCCESS) { - msg("mariabackup: error: xb_load_tablespaces() failed with" - " error %s.\n", ut_strerr(err)); + msg("merror: xb_load_tablespaces() failed with" + " error %s.", ut_strerr(err)); fail_before_log_copying_thread_start: log_copying_running = false; goto fail; @@ -4183,25 +4247,12 @@ fail_before_log_copying_thread_start: } debug_sync_point("xtrabackup_suspend_at_start"); - if (xtrabackup_incremental) { - if (!xtrabackup_incremental_force_scan) { - changed_page_bitmap = xb_page_bitmap_init(); - } - if (!changed_page_bitmap) { - msg("mariabackup: using the full scan for incremental " - "backup\n"); - } else if (incremental_lsn != checkpoint_lsn_start) { - /* Do not print that bitmaps are used when dummy bitmap - is build for an empty LSN range. */ - msg("mariabackup: using the changed page bitmap\n"); - } - } ut_a(xtrabackup_parallel > 0); if (xtrabackup_parallel > 1) { msg("mariabackup: Starting %u threads for parallel data " - "files transfer\n", xtrabackup_parallel); + "files transfer", xtrabackup_parallel); } if (opt_lock_ddl_per_table) { @@ -4210,12 +4261,12 @@ fail_before_log_copying_thread_start: DBUG_EXECUTE_IF("check_mdl_lock_works", dbug_alter_thread_done = dbug_start_query_thread("ALTER TABLE test.t ADD COLUMN mdl_lock_column int", - "Waiting for table metadata lock", 1, ER_QUERY_INTERRUPTED);); + "Waiting for table metadata lock", 0, 0);); } - it = datafiles_iter_new(); + datafiles_iter_t *it = datafiles_iter_new(); if (it == NULL) { - msg("mariabackup: Error: datafiles_iter_new() failed.\n"); + msg("mariabackup: Error: datafiles_iter_new() failed."); goto fail; } @@ -4229,7 +4280,7 @@ fail_before_log_copying_thread_start: data_threads[i].it = it; data_threads[i].num = i+1; data_threads[i].count = &count; - data_threads[i].count_mutex = count_mutex; + data_threads[i].count_mutex = &count_mutex; os_thread_create(data_copy_thread_func, data_threads + i, &data_threads[i].id); } @@ -4276,16 +4327,16 @@ fail_before_log_copying_thread_start: } xtrabackup_destroy_datasinks(); - msg("mariabackup: Redo log (from LSN " LSN_PF " to " LSN_PF - ") was copied.\n", checkpoint_lsn_start, log_copy_scanned_lsn); + msg("Redo log (from LSN " LSN_PF " to " LSN_PF + ") was copied.", checkpoint_lsn_start, log_copy_scanned_lsn); xb_filters_free(); xb_data_files_close(); /* Make sure that the latest checkpoint was included */ if (metadata_to_lsn > log_copy_scanned_lsn) { - msg("mariabackup: error: failed to copy enough redo log (" - "LSN=" LSN_PF "; checkpoint LSN=" LSN_PF ").\n", + msg("Error: failed to copy enough redo log (" + "LSN=" LSN_PF "; checkpoint LSN=" LSN_PF ").", log_copy_scanned_lsn, metadata_to_lsn); goto fail; } @@ -4322,6 +4373,14 @@ void backup_fix_ddl(void) std::set<std::string> dropped_tables; std::map<std::string, std::string> renamed_tables; + /* Disable further DDL on backed up tables (only needed for --no-lock).*/ + pthread_mutex_lock(&backup_mutex); + log_file_op = backup_file_op_fail; + log_optimized_ddl_op = backup_optimized_ddl_op_fail; + pthread_mutex_unlock(&backup_mutex); + + DBUG_MARIABACKUP_EVENT("backup_fix_ddl",0); + for (space_id_to_name_t::iterator iter = ddl_tracker.tables_in_backup.begin(); iter != ddl_tracker.tables_in_backup.end(); iter++) { @@ -4418,8 +4477,9 @@ void backup_fix_ddl(void) } fil_space_free(n->space->id, false); } + datafiles_iter_free(it); - + DBUG_EXECUTE_IF("check_mdl_lock_works", DBUG_ASSERT(new_tables.size() == 0);); for (std::set<std::string>::iterator iter = new_tables.begin(); iter != new_tables.end(); iter++) { const char *space_name = iter->c_str(); @@ -4434,7 +4494,7 @@ void backup_fix_ddl(void) const char *dbname = buf; char *p = strchr(buf, '/'); if (p == 0) { - msg("Unexpected tablespace %s filename %s\n", space_name, name.c_str()); + msg("Unexpected tablespace %s filename %s", space_name, name.c_str()); ut_a(0); } ut_a(p); @@ -4453,18 +4513,10 @@ void backup_fix_ddl(void) continue; std::string dest_name(node->space->name); dest_name.append(".new"); -#if 0 - bool do_full_copy = ddl_tracker.optimized_ddl.find(n->space->id) != ddl_tracker.optimized_ddl.end(); - if (do_full_copy) { - msg( - "Performing a full copy of the tablespace %s, because optimized (without redo logging) DDL operation" - "ran during backup. You can use set innodb_log_optimize_ddl=OFF to improve backup performance" - "in the future.\n", - n->space->name); - } -#endif xtrabackup_copy_datafile(node, 0, dest_name.c_str()/*, do_full_copy ? ULONGLONG_MAX:UNIV_PAGE_SIZE */); } + + datafiles_iter_free(it); } /* ================= prepare ================= */ @@ -4515,7 +4567,7 @@ xb_space_create_file( *file = os_file_create_simple_no_error_handling( 0, path, OS_FILE_CREATE, OS_FILE_READ_WRITE, false, &ret); if (!ret) { - msg("mariabackup: cannot create file %s\n", path); + msg("Can't create file %s", path); return ret; } @@ -4523,7 +4575,7 @@ xb_space_create_file( FIL_IBD_FILE_INITIAL_SIZE << srv_page_size_shift); if (!ret) { - msg("mariabackup: cannot set size for file %s\n", path); + msg("mariabackup: cannot set size for file %s", path); os_file_close(*file); os_file_delete(0, path); return ret; @@ -4567,7 +4619,7 @@ xb_space_create_file( free(buf); if (!ret) { - msg("mariabackup: could not write the first page to %s\n", + msg("mariabackup: could not write the first page to %s", path); os_file_close(*file); os_file_delete(0, path); @@ -4638,7 +4690,7 @@ xb_delta_open_matching_space( /* Create the database directory if it doesn't exist yet */ if (!os_file_create_directory(dest_dir, FALSE)) { - msg("mariabackup: error: cannot create dir %s\n", dest_dir); + msg("mariabackup: error: cannot create dir %s", dest_dir); return file; } @@ -4685,12 +4737,12 @@ exit: snprintf(tmpname, FN_REFLEN, "%s/xtrabackup_tmp_#" ULINTPF, dbname, fil_space->id); - msg("mariabackup: Renaming %s to %s.ibd\n", + msg("mariabackup: Renaming %s to %s.ibd", fil_space->name, tmpname); if (fil_space->rename(tmpname, NULL, false) != DB_SUCCESS) { - msg("mariabackup: Cannot rename %s to %s\n", + msg("mariabackup: Cannot rename %s to %s", fil_space->name, tmpname); goto exit; } @@ -4699,9 +4751,8 @@ exit: if (info.space_id == ULINT_UNDEFINED) { - msg("mariabackup: Error: Cannot handle DDL operation on tablespace " + die("Can't handle DDL operation on tablespace " "%s\n", dest_space_name); - exit(EXIT_FAILURE); } mutex_enter(&fil_system.mutex); fil_space = fil_space_get_by_id(info.space_id); @@ -4711,12 +4762,12 @@ exit: strncpy(tmpname, dest_space_name, FN_REFLEN); - msg("mariabackup: Renaming %s to %s\n", + msg("mariabackup: Renaming %s to %s", fil_space->name, dest_space_name); if (fil_space->rename(tmpname, NULL, false) != DB_SUCCESS) { - msg("mariabackup: Cannot rename %s to %s\n", + msg("mariabackup: Cannot rename %s to %s", fil_space->name, dest_space_name); goto exit; } @@ -4744,8 +4795,7 @@ exit: *success = xb_space_create_file(real_name, info.space_id, flags, &file); } else { - msg("mariabackup: Cannot create tablespace %s\n", - dest_space_name); + msg("Can't create tablespace %s\n", dest_space_name); } goto exit; @@ -4815,12 +4865,12 @@ xtrabackup_apply_delta( page_size = info.page_size.physical(); page_size_shift = get_bit_shift(page_size); - msg("mariabackup: page size for %s is %zu bytes\n", + msg("page size for %s is %zu bytes", src_path, page_size); if (page_size_shift < 10 || page_size_shift > UNIV_PAGE_SIZE_SHIFT_MAX) { - msg("mariabackup: error: invalid value of page_size " - "(%zu bytes) read from %s\n", page_size, meta_path); + msg("error: invalid value of page_size " + "(%zu bytes) read from %s", page_size, meta_path); goto error; } @@ -4829,7 +4879,7 @@ xtrabackup_apply_delta( OS_FILE_OPEN, OS_FILE_READ_WRITE, false, &success); if (!success) { os_file_get_last_error(TRUE); - msg("mariabackup: error: cannot open %s\n", src_path); + msg("error: can't open %s", src_path); goto error; } @@ -4839,7 +4889,7 @@ xtrabackup_apply_delta( dbname, space_name, info, dst_path, sizeof(dst_path), &success); if (!success) { - msg("mariabackup: error: cannot open %s\n", dst_path); + msg("error: can't open %s", dst_path); goto error; } @@ -4852,7 +4902,7 @@ xtrabackup_apply_delta( (ut_align(incremental_buffer_base, page_size)); - msg("Applying %s to %s...\n", src_path, dst_path); + msg("Applying %s to %s...", src_path, dst_path); while (!last_buffer) { ulint cluster_header; @@ -4875,8 +4925,8 @@ xtrabackup_apply_delta( last_buffer = TRUE; break; default: - msg("mariabackup: error: %s seems not " - ".delta file.\n", src_path); + msg("error: %s seems not " + ".delta file.", src_path); goto error; } @@ -4974,7 +5024,7 @@ error: os_file_close(src_file); if (dst_file != OS_FILE_CLOSED) os_file_close(dst_file); - msg("mariabackup: Error: xtrabackup_apply_delta(): " + msg("Error: xtrabackup_apply_delta(): " "failed to apply %s to %s.\n", src_path, dst_path); return FALSE; } @@ -4992,8 +5042,7 @@ std::string change_extension(std::string filename, std::string new_ext) { static void rename_file(const char *from,const char *to) { msg("Renaming %s to %s\n", from, to); if (my_rename(from, to, MY_WME)) { - msg("Cannot rename %s to %s errno %d", from, to, errno); - exit(EXIT_FAILURE); + die("Can't rename %s to %s errno %d", from, to, errno); } } @@ -5014,7 +5063,7 @@ typedef ibool (*handle_datadir_entry_func_t)( /** Rename, and replace destination file, if exists */ static void rename_force(const char *from, const char *to) { if (access(to, R_OK) == 0) { - msg("Removing %s\n", to); + msg("Removing %s", to); if (my_delete(to, MYF(MY_WME))) { msg("Can't remove %s, errno %d", to, errno); exit(EXIT_FAILURE); @@ -5122,6 +5171,7 @@ xb_process_datadir( path, NULL, fileinfo.name, NULL)) { + os_file_closedir(dbdir); return(FALSE); } } @@ -5133,16 +5183,14 @@ next_file_item_1: os_file_closedir(dbdir); } else { - msg("mariabackup: Cannot open dir %s\n", - path); + msg("Can't open dir %s", path); } /* single table tablespaces */ dir = os_file_opendir(path, FALSE); if (dir == NULL) { - msg("mariabackup: Cannot open dir %s\n", - path); + msg("Can't open dir %s", path); } ret = fil_file_readdir_next_file(&err, path, dir, @@ -5183,6 +5231,7 @@ next_file_item_1: dbinfo.name, fileinfo.name, NULL)) { + os_file_closedir(dbdir); return(FALSE); } } @@ -5237,7 +5286,7 @@ store_binlog_info(const char* filename, const char* name, ulonglong pos) FILE *fp = fopen(filename, "w"); if (!fp) { - msg("mariabackup: failed to open '%s'\n", filename); + msg("mariabackup: failed to open '%s'", filename); return(false); } @@ -5258,7 +5307,7 @@ static std::string read_file_as_string(const std::string file) { char content[FN_REFLEN]; FILE *f = fopen(file.c_str(), "r"); if (!f) { - msg("Can not open %s\n", file.c_str()); + msg("Can not open %s", file.c_str()); } size_t len = fread(content, 1, FN_REFLEN, f); fclose(f); @@ -5270,8 +5319,7 @@ static void delete_file(const std::string& file, bool if_exists = false) { if (if_exists && !file_exists(file)) return; if (my_delete(file.c_str(), MYF(MY_WME))) { - msg("Can't remove %s, errno %d", file.c_str(), errno); - exit(EXIT_FAILURE); + die("Can't remove %s, errno %d", file.c_str(), errno); } } @@ -5355,14 +5403,21 @@ xtrabackup_prepare_func(char** argv) if (my_setwd(xtrabackup_real_target_dir,MYF(MY_WME))) { - msg("mariabackup: cannot my_setwd %s\n", - xtrabackup_real_target_dir); + msg("can't my_setwd %s", xtrabackup_real_target_dir); return(false); } - msg("mariabackup: cd to %s\n", xtrabackup_real_target_dir); + msg("cd to %s", xtrabackup_real_target_dir); fil_path_to_mysql_datadir = "."; + /* Fix DDL for prepare. Process .del,.ren, and .new files. + The order in which files are processed, is important + (see MDEV-18185, MDEV-18201) + */ + xb_process_datadir(xtrabackup_incremental_dir ? xtrabackup_incremental_dir : ".", + ".del", prepare_handle_del_files); + xb_process_datadir(xtrabackup_incremental_dir? xtrabackup_incremental_dir:".", + ".ren", prepare_handle_ren_files); if (xtrabackup_incremental_dir) { xb_process_datadir(xtrabackup_incremental_dir, ".new.meta", prepare_handle_new_files); xb_process_datadir(xtrabackup_incremental_dir, ".new.delta", prepare_handle_new_files); @@ -5370,11 +5425,6 @@ xtrabackup_prepare_func(char** argv) else { xb_process_datadir(".", ".new", prepare_handle_new_files); } - xb_process_datadir(xtrabackup_incremental_dir? xtrabackup_incremental_dir:".", - ".ren", prepare_handle_ren_files); - xb_process_datadir(xtrabackup_incremental_dir ? xtrabackup_incremental_dir : ".", - ".del", prepare_handle_del_files); - int argc; for (argc = 0; argv[argc]; argc++) {} encryption_plugin_prepare_init(argc, argv); @@ -5392,32 +5442,31 @@ xtrabackup_prepare_func(char** argv) XTRABACKUP_METADATA_FILENAME); if (!xtrabackup_read_metadata(metadata_path)) { - msg("mariabackup: Error: failed to read metadata from '%s'\n", + msg("Error: failed to read metadata from '%s'\n", metadata_path); return(false); } if (!strcmp(metadata_type, "full-backuped")) { if (xtrabackup_incremental) { - msg("mariabackup: error: applying incremental backup " - "needs a prepared target.\n"); + msg("error: applying incremental backup " + "needs a prepared target."); return(false); } - msg("mariabackup: This target seems to be not prepared yet.\n"); + msg("This target seems to be not prepared yet."); } else if (!strcmp(metadata_type, "log-applied")) { - msg("mariabackup: This target seems to be already prepared.\n"); + msg("This target seems to be already prepared."); } else { - msg("mariabackup: This target does not have correct metadata.\n"); + msg("This target does not have correct metadata."); return(false); } bool ok = !xtrabackup_incremental || metadata_to_lsn == incremental_lsn; if (!ok) { - msg("mariabackup: error: This incremental backup seems " - "not to be proper for the target.\n" - "mariabackup: Check 'to_lsn' of the target and " - "'from_lsn' of the incremental.\n"); + msg("error: This incremental backup seems " + "not to be proper for the target. Check 'to_lsn' of the target and " + "'from_lsn' of the incremental."); return(false); } @@ -5493,9 +5542,10 @@ xtrabackup_prepare_func(char** argv) srv_n_write_io_threads = 4; } - msg("mariabackup: Starting InnoDB instance for recovery.\n" - "mariabackup: Using %lld bytes for buffer pool " - "(set by --use-memory parameter)\n", xtrabackup_use_memory); + msg("Starting InnoDB instance for recovery."); + + msg("mariabackup: Using %lld bytes for buffer pool " + "(set by --use-memory parameter)", xtrabackup_use_memory); srv_max_buf_pool_modified_pct = (double)max_buf_pool_modified_pct; @@ -5508,7 +5558,7 @@ xtrabackup_prepare_func(char** argv) } if (ok) { - msg("Last binlog file %s, position %lld\n", + msg("Last binlog file %s, position %lld", trx_sys.recovered_binlog_filename, longlong(trx_sys.recovered_binlog_offset)); @@ -5532,7 +5582,7 @@ xtrabackup_prepare_func(char** argv) && srv_start_lsn < target_lsn) { msg("mariabackup: error: " "The log was only applied up to LSN " LSN_PF - ", instead of " LSN_PF "\n", + ", instead of " LSN_PF, srv_start_lsn, target_lsn); ok = false; } @@ -5560,13 +5610,13 @@ xtrabackup_prepare_func(char** argv) if (!xtrabackup_write_metadata(filename)) { msg("mariabackup: Error: failed to write metadata " - "to '%s'\n", filename); + "to '%s'", filename); ok = false; } else if (xtrabackup_extra_lsndir) { sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, XTRABACKUP_METADATA_FILENAME); if (!xtrabackup_write_metadata(filename)) { msg("mariabackup: Error: failed to write " - "metadata to '%s'\n", filename); + "metadata to '%s'", filename); ok = false; } } @@ -5601,6 +5651,153 @@ append_defaults_group(const char *group, const char *default_groups[], ut_a(appended); } +static const char* +normalize_privilege_target_name(const char* name) +{ + if (strcmp(name, "*") == 0) { + return "\\*"; + } + else { + /* should have no regex special characters. */ + ut_ad(strpbrk(name, ".()[]*+?") == 0); + } + return name; +} + +/******************************************************************//** +Check if specific privilege is granted. +Uses regexp magic to check if requested privilege is granted for given +database.table or database.* or *.* +or if user has 'ALL PRIVILEGES' granted. +@return true if requested privilege is granted, false otherwise. */ +static bool +has_privilege(const std::list<std::string> &granted, + const char* required, + const char* db_name, + const char* table_name) +{ + char buffer[1000]; + regex_t priv_re; + regmatch_t tables_regmatch[1]; + bool result = false; + + db_name = normalize_privilege_target_name(db_name); + table_name = normalize_privilege_target_name(table_name); + + int written = snprintf(buffer, sizeof(buffer), + "GRANT .*(%s)|(ALL PRIVILEGES).* ON (\\*|`%s`)\\.(\\*|`%s`)", + required, db_name, table_name); + if (written < 0 || written == sizeof(buffer) + || regcomp(&priv_re, buffer, REG_EXTENDED)) { + die("regcomp() failed for '%s'", buffer); + } + + typedef std::list<std::string>::const_iterator string_iter; + for (string_iter i = granted.begin(), e = granted.end(); i != e; ++i) { + int res = regexec(&priv_re, i->c_str(), + 1, tables_regmatch, 0); + + if (res != REG_NOMATCH) { + result = true; + break; + } + } + + xb_regfree(&priv_re); + return result; +} + +enum { + PRIVILEGE_OK = 0, + PRIVILEGE_WARNING = 1, + PRIVILEGE_ERROR = 2, +}; + +/******************************************************************//** +Check if specific privilege is granted. +Prints error message if required privilege is missing. +@return PRIVILEGE_OK if requested privilege is granted, error otherwise. */ +static +int check_privilege( + const std::list<std::string> &granted_priv, /* in: list of + granted privileges*/ + const char* required, /* in: required privilege name */ + const char* target_database, /* in: required privilege target + database name */ + const char* target_table, /* in: required privilege target + table name */ + int error = PRIVILEGE_ERROR) /* in: return value if privilege + is not granted */ +{ + if (!has_privilege(granted_priv, + required, target_database, target_table)) { + msg("%s: missing required privilege %s on %s.%s", + (error == PRIVILEGE_ERROR ? "Error" : "Warning"), + required, target_database, target_table); + return error; + } + return PRIVILEGE_OK; +} + + +/** +Check DB user privileges according to the intended actions. + +Fetches DB user privileges, determines intended actions based on +command-line arguments and prints missing privileges. +@return whether all the necessary privileges are granted */ +static bool check_all_privileges() +{ + if (!mysql_connection) { + /* Not connected, no queries is going to be executed. */ + return true; + } + + /* Fetch effective privileges. */ + std::list<std::string> granted_privileges; + MYSQL_RES* result = xb_mysql_query(mysql_connection, "SHOW GRANTS", + true); + while (MYSQL_ROW row = mysql_fetch_row(result)) { + granted_privileges.push_back(*row); + } + mysql_free_result(result); + + int check_result = PRIVILEGE_OK; + + /* FLUSH TABLES WITH READ LOCK */ + if (!opt_no_lock) + { + check_result |= check_privilege( + granted_privileges, + "RELOAD", "*", "*"); + check_result |= check_privilege( + granted_privileges, + "PROCESS", "*", "*"); + } + + /* KILL ... */ + if ((!opt_no_lock && (opt_kill_long_queries_timeout || opt_lock_ddl_per_table)) + /* START SLAVE SQL_THREAD */ + /* STOP SLAVE SQL_THREAD */ + || opt_safe_slave_backup) { + check_result |= check_privilege( + granted_privileges, + "SUPER", "*", "*", + PRIVILEGE_WARNING); + } + + /* SHOW MASTER STATUS */ + /* SHOW SLAVE STATUS */ + if (opt_galera_info || opt_slave_info + || (opt_no_lock && opt_safe_slave_backup)) { + check_result |= check_privilege(granted_privileges, + "REPLICATION CLIENT", "*", "*", + PRIVILEGE_WARNING); + } + + return !(check_result & PRIVILEGE_ERROR); +} + bool xb_init() { @@ -5614,7 +5811,7 @@ xb_init() && !opt_safe_slave_backup) { msg("Error: --slave-info is used with --no-lock but " "without --safe-slave-backup. The binlog position " - "cannot be consistent with the backup data.\n"); + "cannot be consistent with the backup data."); return(false); } @@ -5665,7 +5862,9 @@ xb_init() if (!get_mysql_vars(mysql_connection)) { return(false); } - + if (opt_check_privileges && !check_all_privileges()) { + return(false); + } history_start_time = time(NULL); } @@ -5834,21 +6033,13 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) char *optend = strcend((argv)[i], '='); if (optend - argv[i] == 15 && - !strncmp(argv[i], "--defaults-file", optend - argv[i])) { - - msg("mariabackup: Error: --defaults-file " - "must be specified first on the command " - "line\n"); - exit(EXIT_FAILURE); + !strncmp(argv[i], "--defaults-file", optend - argv[i])) { + die("--defaults-file must be specified first on the command line"); } - if (optend - argv[i] == 21 && - !strncmp(argv[i], "--defaults-extra-file", - optend - argv[i])) { - - msg("mariabackup: Error: --defaults-extra-file " - "must be specified first on the command " - "line\n"); - exit(EXIT_FAILURE); + if (optend - argv[i] == 21 && + !strncmp(argv[i], "--defaults-extra-file", + optend - argv[i])) { + die("--defaults-extra-file must be specified first on the command line"); } } @@ -5895,7 +6086,7 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) if (!server_option) { msg("mariabackup: Error:" - " unknown argument: '%s'\n", opt); + " unknown argument: '%s'", opt); exit(EXIT_FAILURE); } } @@ -5951,7 +6142,7 @@ int main(int argc, char **argv) if (mysql_server_init(-1, NULL, NULL)) { - exit(EXIT_FAILURE); + die("mysql_server_init() failed"); } system_charset_info = &my_charset_utf8_general_ci; @@ -5995,7 +6186,7 @@ int main(int argc, char **argv) mysql_mutex_destroy(&LOCK_error_log); if (status == EXIT_SUCCESS) { - msg_ts("completed OK!\n"); + msg("completed OK!"); } return status; @@ -6013,7 +6204,7 @@ static int main_low(char** argv) && !strcmp(mysql_data_home, "./")) { if (!xtrabackup_print_param) usage(); - msg("\nmariabackup: Error: Please set parameter 'datadir'\n"); + msg("mariabackup: Error: Please set parameter 'datadir'"); return(EXIT_FAILURE); } @@ -6084,7 +6275,7 @@ static int main_low(char** argv) if (error) { msg("mariabackup: value '%s' may be wrong format for " - "incremental option.\n", xtrabackup_incremental); + "incremental option.", xtrabackup_incremental); return(EXIT_FAILURE); } } else if (xtrabackup_backup && xtrabackup_incremental_basedir) { @@ -6094,7 +6285,7 @@ static int main_low(char** argv) if (!xtrabackup_read_metadata(filename)) { msg("mariabackup: error: failed to read metadata from " - "%s\n", filename); + "%s", filename); return(EXIT_FAILURE); } @@ -6107,7 +6298,7 @@ static int main_low(char** argv) if (!xtrabackup_read_metadata(filename)) { msg("mariabackup: error: failed to read metadata from " - "%s\n", filename); + "%s", filename); return(EXIT_FAILURE); } @@ -6125,7 +6316,7 @@ static int main_low(char** argv) } if (xtrabackup_stream && !xtrabackup_backup) { - msg("Warning: --stream parameter is ignored, it only works together with --backup.\n"); + msg("Warning: --stream parameter is ignored, it only works together with --backup."); } if (!xb_init()) { @@ -6140,13 +6331,13 @@ static int main_low(char** argv) print_version(); if (xtrabackup_incremental) { - msg("incremental backup from " LSN_PF " is enabled.\n", + msg("incremental backup from " LSN_PF " is enabled.", incremental_lsn); } if (xtrabackup_export && innobase_file_per_table == FALSE) { msg("mariabackup: auto-enabling --innodb-file-per-table due to " - "the --export option\n"); + "the --export option"); innobase_file_per_table = TRUE; } @@ -6184,8 +6375,7 @@ static int main_low(char** argv) if (xtrabackup_copy_back || xtrabackup_move_back) { if (!check_if_param_set("datadir")) { - msg("Error: datadir must be specified.\n"); - return(EXIT_FAILURE); + mysql_data_home = get_default_datadir(); } if (!copy_back()) return(EXIT_FAILURE); @@ -6213,3 +6403,12 @@ static int get_exepath(char *buf, size_t size, const char *argv0) return my_realpath(buf, argv0, 0); } + + +#if defined (__SANITIZE_ADDRESS__) && defined (__linux__) +/* Avoid LeakSanitizer's false positives. */ +const char* __asan_default_options() +{ + return "detect_leaks=0"; +} +#endif |