diff options
Diffstat (limited to 'extra/mariabackup')
| -rw-r--r-- | extra/mariabackup/CMakeLists.txt | 4 | ||||
| -rw-r--r-- | extra/mariabackup/backup_copy.cc | 212 | ||||
| -rw-r--r-- | extra/mariabackup/backup_mysql.cc | 57 | ||||
| -rw-r--r-- | extra/mariabackup/backup_wsrep.h (renamed from extra/mariabackup/wsrep.h) | 4 | ||||
| -rw-r--r-- | extra/mariabackup/changed_page_bitmap.cc | 63 | ||||
| -rw-r--r-- | extra/mariabackup/encryption_plugin.cc | 21 | ||||
| -rw-r--r-- | extra/mariabackup/fil_cur.cc | 183 | ||||
| -rw-r--r-- | extra/mariabackup/fil_cur.h | 21 | ||||
| -rw-r--r-- | extra/mariabackup/innobackupex.cc | 21 | ||||
| -rw-r--r-- | extra/mariabackup/read_filt.cc | 15 | ||||
| -rw-r--r-- | extra/mariabackup/read_filt.h | 8 | ||||
| -rw-r--r-- | extra/mariabackup/write_filt.cc | 20 | ||||
| -rw-r--r-- | extra/mariabackup/xb0xb.h | 36 | ||||
| -rw-r--r-- | extra/mariabackup/xtrabackup.cc | 2636 | ||||
| -rw-r--r-- | extra/mariabackup/xtrabackup.h | 50 |
15 files changed, 1296 insertions, 2055 deletions
diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt index ac15460660c..f7202a7f3b7 100644 --- a/extra/mariabackup/CMakeLists.txt +++ b/extra/mariabackup/CMakeLists.txt @@ -30,7 +30,7 @@ ENDIF() INCLUDE_DIRECTORIES( ${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/storage/xtradb/include + ${CMAKE_SOURCE_DIR}/storage/innodb/include ${CMAKE_SOURCE_DIR}/sql ${CMAKE_CURRENT_SOURCE_DIR}/quicklz ${CMAKE_CURRENT_SOURCE_DIR} @@ -74,9 +74,9 @@ MYSQL_ADD_EXECUTABLE(mariabackup backup_mysql.cc backup_copy.cc encryption_plugin.cc - ${PROJECT_SOURCE_DIR}/libmysql/libmysql.c ${PROJECT_SOURCE_DIR}/sql/net_serv.cc ${NT_SERVICE_SOURCE} + ${PROJECT_SOURCE_DIR}/libmysqld/libmysql.c COMPONENT backup ) diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index 156e0b20e7c..d9e2db0cb66 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -89,7 +89,7 @@ struct datadir_iter_t { ulint filepath_len; char *filepath_rel; ulint filepath_rel_len; - os_ib_mutex_t mutex; + pthread_mutex_t mutex; os_file_dir_t dir; os_file_dir_t dbdir; os_file_stat_t dbinfo; @@ -107,7 +107,7 @@ struct datadir_thread_ctxt_t { datadir_iter_t *it; uint n_thread; uint *count; - os_ib_mutex_t count_mutex; + pthread_mutex_t count_mutex; os_thread_id_t id; bool ret; }; @@ -134,12 +134,12 @@ datadir_node_fill(datadir_node_t *node, datadir_iter_t *it) { if (node->filepath_len < it->filepath_len) { free(node->filepath); - node->filepath = (char*)(ut_malloc(it->filepath_len)); + node->filepath = (char*)(malloc(it->filepath_len)); node->filepath_len = it->filepath_len; } if (node->filepath_rel_len < it->filepath_rel_len) { free(node->filepath_rel); - node->filepath_rel = (char*)(ut_malloc(it->filepath_rel_len)); + node->filepath_rel = (char*)(malloc(it->filepath_rel_len)); node->filepath_rel_len = it->filepath_rel_len; } @@ -153,8 +153,8 @@ static void datadir_node_free(datadir_node_t *node) { - ut_free(node->filepath); - ut_free(node->filepath_rel); + free(node->filepath); + free(node->filepath_rel); memset(node, 0, sizeof(datadir_node_t)); } @@ -178,10 +178,10 @@ datadir_iter_new(const char *path, bool skip_first_level = true) { datadir_iter_t *it; - it = static_cast<datadir_iter_t *>(ut_malloc(sizeof(datadir_iter_t))); + it = static_cast<datadir_iter_t *>(malloc(sizeof(datadir_iter_t))); memset(it, 0, sizeof(datadir_iter_t)); - it->mutex = os_mutex_create(); + pthread_mutex_init(&it->mutex, NULL); it->datadir_path = strdup(path); it->dir = os_file_opendir(it->datadir_path, TRUE); @@ -194,20 +194,20 @@ datadir_iter_new(const char *path, bool skip_first_level = true) it->err = DB_SUCCESS; it->dbpath_len = FN_REFLEN; - it->dbpath = static_cast<char*>(ut_malloc(it->dbpath_len)); + it->dbpath = static_cast<char*>(malloc(it->dbpath_len)); it->filepath_len = FN_REFLEN; - it->filepath = static_cast<char*>(ut_malloc(it->filepath_len)); + it->filepath = static_cast<char*>(malloc(it->filepath_len)); it->filepath_rel_len = FN_REFLEN; - it->filepath_rel = static_cast<char*>(ut_malloc(it->filepath_rel_len)); + it->filepath_rel = static_cast<char*>(malloc(it->filepath_rel_len)); it->skip_first_level = skip_first_level; return(it); error: - ut_free(it); + free(it); return(NULL); } @@ -246,19 +246,15 @@ datadir_iter_next_database(datadir_iter_t *it) + strlen (it->dbinfo.name) + 2; if (len > it->dbpath_len) { it->dbpath_len = len; + free(it->dbpath); - if (it->dbpath) { - - ut_free(it->dbpath); - } - - it->dbpath = static_cast<char*> - (ut_malloc(it->dbpath_len)); + it->dbpath = static_cast<char*>( + malloc(it->dbpath_len)); } ut_snprintf(it->dbpath, it->dbpath_len, "%s/%s", it->datadir_path, it->dbinfo.name); - srv_normalize_path_for_win(it->dbpath); + os_normalize_path(it->dbpath); if (it->dbinfo.type == OS_FILE_TYPE_FILE) { it->is_file = true; @@ -306,8 +302,8 @@ make_path_n(int n, char **path, ulint *path_len, ...) va_end(vl); if (len_needed < *path_len) { - ut_free(*path); - *path = static_cast<char*>(ut_malloc(len_needed)); + free(*path); + *path = static_cast<char*>(malloc(len_needed)); } va_start(vl, path_len); @@ -378,7 +374,7 @@ datadir_iter_next(datadir_iter_t *it, datadir_node_t *node) { bool ret = true; - os_mutex_enter(it->mutex); + pthread_mutex_lock(&it->mutex); if (datadir_iter_next_file(it)) { @@ -413,7 +409,7 @@ datadir_iter_next(datadir_iter_t *it, datadir_node_t *node) ret = false; done: - os_mutex_exit(it->mutex); + pthread_mutex_unlock(&it->mutex); return(ret); } @@ -427,7 +423,7 @@ static void datadir_iter_free(datadir_iter_t *it) { - os_mutex_free(it->mutex); + pthread_mutex_destroy(&it->mutex); if (it->dbdir) { @@ -439,11 +435,11 @@ datadir_iter_free(datadir_iter_t *it) os_file_closedir(it->dir); } - ut_free(it->dbpath); - ut_free(it->filepath); - ut_free(it->filepath_rel); + free(it->dbpath); + free(it->filepath); + free(it->filepath_rel); free(it->datadir_path); - ut_free(it); + free(it); } @@ -466,17 +462,17 @@ static void datafile_close(datafile_cur_t *cursor) { - if (cursor->file != 0) { + if (cursor->file != OS_FILE_CLOSED) { os_file_close(cursor->file); } - ut_free(cursor->buf); + free(cursor->buf); } static bool datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n) { - ulint success; + bool success; memset(cursor, 0, sizeof(datafile_cur_t)); @@ -490,11 +486,9 @@ datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n) xb_get_relative_path(cursor->abs_path, FALSE), sizeof(cursor->rel_path)); - cursor->file = os_file_create_simple_no_error_handling(0, - cursor->abs_path, - OS_FILE_OPEN, - OS_FILE_READ_ONLY, - &success, 0); + cursor->file = os_file_create_simple_no_error_handling( + 0, cursor->abs_path, + OS_FILE_OPEN, OS_FILE_READ_ALLOW_DELETE, true, &success); if (!success) { /* The following call prints an error message */ os_file_get_last_error(TRUE); @@ -518,7 +512,7 @@ datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n) posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL); cursor->buf_size = 10 * 1024 * 1024; - cursor->buf = static_cast<byte *>(ut_malloc((ulint)cursor->buf_size)); + cursor->buf = static_cast<byte *>(malloc((ulint)cursor->buf_size)); return(true); } @@ -528,7 +522,6 @@ static xb_fil_cur_result_t datafile_read(datafile_cur_t *cursor) { - ulint success; ulint to_read; xtrabackup_io_throttling(); @@ -540,14 +533,14 @@ datafile_read(datafile_cur_t *cursor) return(XB_FIL_CUR_EOF); } - success = os_file_read(cursor->file, cursor->buf, cursor->buf_offset, - to_read); - if (!success) { + if (!os_file_read(IORequestRead, + cursor->file, cursor->buf, cursor->buf_offset, + to_read)) { return(XB_FIL_CUR_ERROR); } posix_fadvise(cursor->file, cursor->buf_offset, to_read, - POSIX_FADV_DONTNEED); + POSIX_FADV_DONTNEED); cursor->buf_read = to_read; cursor->buf_offset += to_read; @@ -917,13 +910,13 @@ run_data_threads(datadir_iter_t *it, os_thread_func_t func, uint n) { datadir_thread_ctxt_t *data_threads; uint i, count; - os_ib_mutex_t count_mutex; + pthread_mutex_t count_mutex; bool ret; data_threads = (datadir_thread_ctxt_t*) - (ut_malloc(sizeof(datadir_thread_ctxt_t) * n)); + malloc(sizeof(datadir_thread_ctxt_t) * n); - count_mutex = os_mutex_create(); + pthread_mutex_init(&count_mutex, NULL); count = n; for (i = 0; i < n; i++) { @@ -937,15 +930,15 @@ run_data_threads(datadir_iter_t *it, os_thread_func_t func, uint n) /* Wait for threads to exit */ while (1) { os_thread_sleep(100000); - os_mutex_enter(count_mutex); + pthread_mutex_lock(&count_mutex); if (count == 0) { - os_mutex_exit(count_mutex); + pthread_mutex_unlock(&count_mutex); break; } - os_mutex_exit(count_mutex); + pthread_mutex_unlock(&count_mutex); } - os_mutex_free(count_mutex); + pthread_mutex_destroy(&count_mutex); ret = true; for (i = 0; i < n; i++) { @@ -955,7 +948,7 @@ run_data_threads(datadir_iter_t *it, os_thread_func_t func, uint n) } } - ut_free(data_threads); + free(data_threads); return(ret); } @@ -974,7 +967,6 @@ copy_file(ds_ctxt_t *datasink, ds_file_t *dstfile = NULL; datafile_cur_t cursor; xb_fil_cur_result_t res; - const char *action; if (!datafile_open(src_file_path, &cursor, thread_n)) { goto error_close; @@ -991,9 +983,8 @@ copy_file(ds_ctxt_t *datasink, goto error; } - action = xb_get_copy_action(); msg_ts("[%02u] %s %s to %s\n", - thread_n, action, src_file_path, dstfile->path); + thread_n, xb_get_copy_action(), src_file_path, dstfile->path); /* The main copy loop */ while ((res = datafile_read(&cursor)) == XB_FIL_CUR_SUCCESS) { @@ -1111,7 +1102,7 @@ read_link_file(const char *ibd_filepath, const char *link_filepath) while (lastch > 4 && filepath[lastch] <= 0x20) { filepath[lastch--] = 0x00; } - srv_normalize_path_for_win(filepath); + os_normalize_path(filepath); } tablespace_locations[ibd_filepath] = filepath; @@ -1629,14 +1620,9 @@ apply_log_finish() return(true); } -extern void -os_io_init_simple(void); - bool copy_back() { - char *innobase_data_file_path_copy; - ulint i; bool ret; datadir_iter_t *it = NULL; datadir_node_t node; @@ -1679,24 +1665,16 @@ copy_back() if (!innobase_data_file_path) { innobase_data_file_path = (char*) "ibdata1:10M:autoextend"; } - innobase_data_file_path_copy = strdup(innobase_data_file_path); - if (!(ret = srv_parse_data_file_paths_and_sizes( - innobase_data_file_path_copy))) { + srv_sys_space.set_path("."); + + if (!srv_sys_space.parse_params(innobase_data_file_path, true)) { msg("syntax error in innodb_data_file_path\n"); return(false); } srv_max_n_threads = 1000; - //os_sync_mutex = NULL; - ut_mem_init(); - /* temporally dummy value to avoid crash */ - srv_page_size_shift = 14; - srv_page_size = (1 << srv_page_size_shift); - os_sync_init(); - sync_init(); - os_io_init_simple(); - mem_init(srv_mem_pool_size); + sync_check_init(); ut_crc32_init(); /* copy undo tablespaces */ @@ -1707,9 +1685,9 @@ copy_back() ds_data = ds_create(dst_dir, DS_TYPE_LOCAL); - for (i = 1; i <= srv_undo_tablespaces; i++) { + for (ulong i = 1; i <= srv_undo_tablespaces; i++) { char filename[20]; - sprintf(filename, "undo%03u", (uint)i); + sprintf(filename, "undo%03lu", i); if (!(ret = copy_or_move_file(filename, filename, dst_dir, 1))) { goto cleanup; @@ -1720,29 +1698,30 @@ copy_back() ds_data = NULL; } - /* copy redo logs */ - dst_dir = (srv_log_group_home_dir && *srv_log_group_home_dir) - ? srv_log_group_home_dir : mysql_data_home; - - ds_data = ds_create(dst_dir, DS_TYPE_LOCAL); + ? srv_log_group_home_dir : mysql_data_home; - for (i = 0; i < (ulong)innobase_log_files_in_group; i++) { - char filename[20]; - sprintf(filename, "ib_logfile%lu", i); - - if (!file_exists(filename)) { - continue; - } + /* --backup generates a single ib_logfile0, which we must copy + if it exists. */ - if (!(ret = copy_or_move_file(filename, filename, - dst_dir, 1))) { - goto cleanup; + ds_data = ds_create(dst_dir, DS_TYPE_LOCAL); + if (!file_exists("ib_logfile0")) { + /* After completed --prepare, redo log files are redundant. + We must delete any redo logs at the destination, so that + the database will not jump to a different log sequence number + (LSN). */ + + for (uint i = 0; i <= SRV_N_LOG_FILES_MAX + 1; i++) { + char filename[FN_REFLEN]; + snprintf(filename, sizeof filename, "%s/ib_logfile%u", + dst_dir, i); + unlink(filename); } + } else if (!(ret = copy_or_move_file("ib_logfile0", "ib_logfile0", + dst_dir, 1))) { + goto cleanup; } - ds_destroy(ds_data); - ds_data = NULL; /* copy innodb system tablespace(s) */ @@ -1751,17 +1730,19 @@ copy_back() ds_data = ds_create(dst_dir, DS_TYPE_LOCAL); - for (i = 0; i < srv_n_data_files; i++) { - const char *filename = base_name(srv_data_file_names[i]); + for (Tablespace::const_iterator iter(srv_sys_space.begin()), + end(srv_sys_space.end()); + iter != end; + ++iter) { + const char *filename = base_name(iter->name()); - if (!(ret = copy_or_move_file(filename, srv_data_file_names[i], + if (!(ret = copy_or_move_file(filename, iter->name(), dst_dir, 1))) { goto cleanup; } } ds_destroy(ds_data); - ds_data = NULL; /* copy the rest of tablespaces */ ds_data = ds_create(mysql_data_home, DS_TYPE_LOCAL); @@ -1771,7 +1752,7 @@ copy_back() datadir_node_init(&node); while (datadir_iter_next(it, &node)) { - const char *ext_list[] = {"backup-my.cnf", "xtrabackup_logfile", + const char *ext_list[] = {"backup-my.cnf", "xtrabackup_binary", "xtrabackup_binlog_info", "xtrabackup_checkpoints", ".qp", ".pmap", ".tmp", NULL}; @@ -1817,21 +1798,18 @@ copy_back() continue; } - /* skip redo logs */ - if (sscanf(filename, "ib_logfile%d%c", &i_tmp, &c_tmp) == 1) { + /* skip the redo log (it was already copied) */ + if (!strcmp(filename, "ib_logfile0")) { continue; } /* skip innodb data files */ is_ibdata_file = false; - for (i = 0; i < srv_n_data_files; i++) { - const char *ibfile; - - ibfile = base_name(srv_data_file_names[i]); - - if (strcmp(ibfile, filename) == 0) { + for (Tablespace::const_iterator iter(srv_sys_space.begin()), + end(srv_sys_space.end()); iter != end; ++iter) { + if (strcmp(iter->name(), filename) == 0) { is_ibdata_file = true; - continue; + break; } } if (is_ibdata_file) { @@ -1873,20 +1851,13 @@ cleanup: datadir_node_free(&node); - free(innobase_data_file_path_copy); - if (ds_data != NULL) { ds_destroy(ds_data); } ds_data = NULL; - //os_sync_free(); - mem_close(); - //os_sync_mutex = NULL; - ut_free_all_mem(); - sync_close(); - sync_initialized = FALSE; + sync_check_close(); return(ret); } @@ -1965,9 +1936,9 @@ cleanup: datadir_node_free(&node); - os_mutex_enter(ctxt->count_mutex); + pthread_mutex_lock(&ctxt->count_mutex); --(*ctxt->count); - os_mutex_exit(ctxt->count_mutex); + pthread_mutex_unlock(&ctxt->count_mutex); ctxt->ret = ret; @@ -1982,10 +1953,7 @@ decrypt_decompress() datadir_iter_t *it = NULL; srv_max_n_threads = 1000; - //os_sync_mutex = NULL; - ut_mem_init(); - os_sync_init(); - sync_init(); + sync_check_init(); /* cd to backup directory */ if (my_setwd(xtrabackup_target_dir, MYF(MY_WME))) @@ -2014,11 +1982,7 @@ decrypt_decompress() ds_data = NULL; - sync_close(); - sync_initialized = FALSE; - //os_sync_free(); - //os_sync_mutex = NULL; - ut_free_all_mem(); + sync_check_close(); return(ret); } diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index 2353c7692cb..c6ec54c3a6c 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -54,6 +54,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "mysqld.h" #include "encryption_plugin.h" #include <sstream> +#include <sql_error.h> char *tool_name; @@ -494,7 +495,7 @@ get_mysql_vars(MYSQL *connection) && innodb_log_files_in_group_var) { char *endptr; - innobase_log_files_in_group = strtol( + srv_n_log_files = strtol( innodb_log_files_in_group_var, &endptr, 10); ut_ad(*endptr == 0); } @@ -503,7 +504,7 @@ get_mysql_vars(MYSQL *connection) && innodb_log_file_size_var) { char *endptr; - innobase_log_file_size = strtoll( + srv_log_file_size = strtoll( innodb_log_file_size_var, &endptr, 10); ut_ad(*endptr == 0); } @@ -850,9 +851,9 @@ static void start_query_killer() { - kill_query_thread_stop = os_event_create(); - kill_query_thread_started = os_event_create(); - kill_query_thread_stopped = os_event_create(); + kill_query_thread_stop = os_event_create(0); + kill_query_thread_started = os_event_create(0); + kill_query_thread_stopped = os_event_create(0); os_thread_create(kill_query_thread, NULL, &kill_query_thread_id); @@ -1368,17 +1369,27 @@ cleanup: return(result); } -static string escape_and_quote(MYSQL *mysql,const char *str) +struct escape_and_quote { - if (!str) - return "NULL"; - size_t len = strlen(str); - char* escaped = (char *)alloca(2 * len + 3); - escaped[0] = '\''; - size_t new_len = mysql_real_escape_string(mysql, escaped+1, str, len); - escaped[new_len + 1] = '\''; - escaped[new_len + 2] = 0; - return string(escaped); + escape_and_quote(MYSQL *mysql, const char *str) + : mysql(mysql), str(str) {} + MYSQL * const mysql; + const char * const str; +}; + +static +std::ostream& +operator<<(std::ostream& s, const escape_and_quote& eq) +{ + if (!eq.str) + return s << "NULL"; + s << '\''; + size_t len = strlen(eq.str); + char* escaped = (char *)alloca(2 * len + 1); + len = mysql_real_escape_string(eq.mysql, escaped, eq.str, len); + s << std::string(escaped, len); + s << '\''; + return s; } /*********************************************************************//** @@ -1395,12 +1406,9 @@ write_xtrabackup_info(MYSQL *connection) char buf_start_time[100]; char buf_end_time[100]; tm tm; - ostringstream oss; + std::ostringstream oss; const char *xb_stream_name[] = {"file", "tar", "xbstream"}; - - ut_ad(xtrabackup_stream_fmt < 3); - uuid = read_mysql_one_value(connection, "SELECT UUID()"); server_version = read_mysql_one_value(connection, "SELECT VERSION()"); localtime_r(&history_start_time, &tm); @@ -1528,28 +1536,21 @@ bool write_backup_config_file() "# The MySQL server\n" "[mysqld]\n" "innodb_checksum_algorithm=%s\n" - "innodb_log_checksum_algorithm=%s\n" "innodb_data_file_path=%s\n" "innodb_log_files_in_group=%lu\n" - "innodb_log_file_size=%lld\n" + "innodb_log_file_size=%llu\n" "innodb_page_size=%lu\n" - "innodb_log_block_size=%lu\n" "innodb_undo_directory=%s\n" "innodb_undo_tablespaces=%lu\n" "%s%s\n" - "%s%s\n" "%s\n", innodb_checksum_algorithm_names[srv_checksum_algorithm], - innodb_checksum_algorithm_names[srv_log_checksum_algorithm], innobase_data_file_path, srv_n_log_files, - innobase_log_file_size, + srv_log_file_size, srv_page_size, - srv_log_block_size, srv_undo_dir, srv_undo_tablespaces, - innobase_doublewrite_file ? "innodb_doublewrite_file=" : "", - innobase_doublewrite_file ? innobase_doublewrite_file : "", innobase_buffer_pool_filename ? "innodb_buffer_pool_filename=" : "", innobase_buffer_pool_filename ? diff --git a/extra/mariabackup/wsrep.h b/extra/mariabackup/backup_wsrep.h index 7638d1f2b54..6537b304e12 100644 --- a/extra/mariabackup/wsrep.h +++ b/extra/mariabackup/backup_wsrep.h @@ -19,8 +19,8 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA *******************************************************/ -#ifndef WSREP_H -#define WSREP_H +#ifndef BACKUP_WSREP_H +#define BACKUP_WSREP_H /*********************************************************************** Store Galera checkpoint info in the 'xtrabackup_galera_info' file, if that diff --git a/extra/mariabackup/changed_page_bitmap.cc b/extra/mariabackup/changed_page_bitmap.cc index 86a873ef69c..ce769375a16 100644 --- a/extra/mariabackup/changed_page_bitmap.cc +++ b/extra/mariabackup/changed_page_bitmap.cc @@ -193,9 +193,9 @@ log_online_read_bitmap_page( ut_a(bitmap_file->offset <= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE); ut_a(bitmap_file->offset % MODIFIED_PAGE_BLOCK_SIZE == 0); - - success = os_file_read(bitmap_file->file, page, bitmap_file->offset, - MODIFIED_PAGE_BLOCK_SIZE); + success = os_file_read(IORequestRead, + bitmap_file->file, page, bitmap_file->offset, + MODIFIED_PAGE_BLOCK_SIZE); if (UNIV_UNLIKELY(!success)) { @@ -355,8 +355,7 @@ log_online_setup_bitmap_file_range( bitmap_files->files = static_cast<log_online_bitmap_file_range_t::files_t *> - (ut_malloc(bitmap_files->count - * sizeof(bitmap_files->files[0]))); + (malloc(bitmap_files->count * sizeof(bitmap_files->files[0]))); memset(bitmap_files->files, 0, bitmap_files->count * sizeof(bitmap_files->files[0])); @@ -426,9 +425,9 @@ log_online_setup_bitmap_file_range( /****************************************************************//** Open a bitmap file for reading. -@return TRUE if opened successfully */ +@return whether opened successfully */ static -ibool +bool log_online_open_bitmap_file_read_only( /*==================================*/ const char* name, /*!<in: bitmap file @@ -438,23 +437,21 @@ log_online_open_bitmap_file_read_only( log_online_bitmap_file_t* bitmap_file) /*!<out: opened bitmap file */ { - ibool success = FALSE; + bool success = false; xb_ad(name[0] != '\0'); ut_snprintf(bitmap_file->name, FN_REFLEN, "%s%s", srv_data_home, name); - bitmap_file->file - = os_file_create_simple_no_error_handling(0, bitmap_file->name, - OS_FILE_OPEN, - OS_FILE_READ_ONLY, - &success,0); + bitmap_file->file = os_file_create_simple_no_error_handling( + 0, bitmap_file->name, + OS_FILE_OPEN, OS_FILE_READ_ONLY, true, &success); if (UNIV_UNLIKELY(!success)) { /* Here and below assume that bitmap file names do not contain apostrophes, thus no need for ut_print_filename(). */ msg("InnoDB: Warning: error opening the changed page " "bitmap \'%s\'\n", bitmap_file->name); - return FALSE; + return success; } bitmap_file->size = os_file_get_size(bitmap_file->file); @@ -465,7 +462,7 @@ log_online_open_bitmap_file_read_only( posix_fadvise(bitmap_file->file, 0, 0, POSIX_FADV_NOREUSE); #endif - return TRUE; + return success; } /****************************************************************//** @@ -909,6 +906,37 @@ xb_page_bitmap_setup_next_page( return TRUE; } +/** Find the node with the smallest key that greater than equal to search key. +@param[in] tree red-black tree +@param[in] key search key +@return node with the smallest greater-than-or-equal key +@retval NULL if none was found */ +static +const ib_rbt_node_t* +rbt_lower_bound(const ib_rbt_t* tree, const void* key) +{ + ut_ad(!tree->cmp_arg); + const ib_rbt_node_t* ge = NULL; + + for (const ib_rbt_node_t *node = tree->root->left; + node != tree->nil; ) { + int result = tree->compare(node->value, key); + + if (result < 0) { + node = node->right; + } else { + ge = node; + if (result == 0) { + break; + } + + node = node->left; + } + } + + return(ge); +} + /****************************************************************//** Set up a new bitmap range iterator over a given space id changed pages in a given bitmap. @@ -922,8 +950,7 @@ xb_page_bitmap_range_init( { byte search_page[MODIFIED_PAGE_BLOCK_SIZE]; xb_page_bitmap_range *result - = static_cast<xb_page_bitmap_range *> - (ut_malloc(sizeof(*result))); + = static_cast<xb_page_bitmap_range *>(malloc(sizeof(*result))); memset(result, 0, sizeof(*result)); result->bitmap = bitmap; @@ -1014,5 +1041,5 @@ xb_page_bitmap_range_deinit( /*========================*/ xb_page_bitmap_range* bitmap_range) /*! in/out: bitmap range */ { - ut_free(bitmap_range); + free(bitmap_range); } diff --git a/extra/mariabackup/encryption_plugin.cc b/extra/mariabackup/encryption_plugin.cc index 8f7741b057a..76512c185e2 100644 --- a/extra/mariabackup/encryption_plugin.cc +++ b/extra/mariabackup/encryption_plugin.cc @@ -8,6 +8,7 @@ #include <vector> #include <common.h> #include <backup_mysql.h> +#include <log0crypt.h> extern struct st_maria_plugin *mysql_optional_plugins[]; @@ -18,14 +19,14 @@ extern char *xb_plugin_load; extern char *xb_plugin_dir; const int PLUGIN_MAX_ARGS = 1024; -vector<string> backup_plugins_args; +std::vector<std::string> backup_plugins_args; const char *QUERY_PLUGIN = "SELECT plugin_name, plugin_library, @@plugin_dir" " FROM information_schema.plugins WHERE plugin_type='ENCRYPTION'" " AND plugin_status='ACTIVE'"; -string encryption_plugin_config; +std::string encryption_plugin_config; static void add_to_plugin_load_list(const char *plugin_def) { @@ -38,7 +39,7 @@ void encryption_plugin_backup_init(MYSQL *mysql) { MYSQL_RES *result; MYSQL_ROW row; - ostringstream oss; + std::ostringstream oss; char *argv[PLUGIN_MAX_ARGS]; int argc; @@ -59,17 +60,17 @@ void encryption_plugin_backup_init(MYSQL *mysql) if (*p == '\\') *p = '/'; #endif - string plugin_load(name); + std::string plugin_load(name); if (library) - plugin_load += string("=") + library; + plugin_load += std::string("=") + library; - oss << "plugin_load=" << plugin_load << endl; + oss << "plugin_load=" << plugin_load << std::endl; /* Required to load the plugin later.*/ add_to_plugin_load_list(plugin_load.c_str()); strncpy(opt_plugin_dir, dir, FN_REFLEN); - oss << "plugin_dir=" << '"' << dir << '"' << endl; + oss << "plugin_dir=" << '"' << dir << '"' << std::endl; /* Read plugin variables. */ @@ -80,12 +81,12 @@ void encryption_plugin_backup_init(MYSQL *mysql) result = xb_mysql_query(mysql, query, true, true); while ((row = mysql_fetch_row(result))) { - string arg("--"); + std::string arg("--"); arg += row[0]; arg += "="; arg += row[1]; backup_plugins_args.push_back(arg); - oss << row[0] << "=" << row[1] << endl; + oss << row[0] << "=" << row[1] << std::endl; } mysql_free_result(result); @@ -94,7 +95,7 @@ void encryption_plugin_backup_init(MYSQL *mysql) result = xb_mysql_query(mysql, "select @@innodb_encrypt_log", true, true); row = mysql_fetch_row(result); srv_encrypt_log = (row != 0 && row[0][0] == '1'); - oss << "innodb_encrypt_log=" << row[0] << endl; + oss << "innodb_encrypt_log=" << row[0] << std::endl; mysql_free_result(result); diff --git a/extra/mariabackup/fil_cur.cc b/extra/mariabackup/fil_cur.cc index 820d8e10c29..9df5f8e8fa3 100644 --- a/extra/mariabackup/fil_cur.cc +++ b/extra/mariabackup/fil_cur.cc @@ -24,8 +24,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include <my_base.h> -#include <univ.i> #include <fil0fil.h> +#include <fsp0fsp.h> #include <srv0start.h> #include <trx0sys.h> @@ -63,7 +63,7 @@ xb_get_relative_path( prev = NULL; cur = path; - while ((next = strchr(cur, SRV_PATH_SEPARATOR)) != NULL) { + while ((next = strchr(cur, OS_PATH_SEPARATOR)) != NULL) { prev = cur; cur = next + 1; @@ -96,7 +96,7 @@ xb_fil_node_close_file( ut_a(node->n_pending_flushes == 0); ut_a(!node->being_extended); - if (!node->open) { + if (!node->is_open()) { mutex_exit(&fil_system->mutex); @@ -106,19 +106,19 @@ xb_fil_node_close_file( ret = os_file_close(node->handle); ut_a(ret); - node->open = FALSE; + node->handle = OS_FILE_CLOSED; ut_a(fil_system->n_open > 0); fil_system->n_open--; fil_n_file_opened--; - if (node->space->purpose == FIL_TABLESPACE && + if (node->space->purpose == FIL_TYPE_TABLESPACE && fil_is_user_tablespace_id(node->space->id)) { ut_a(UT_LIST_GET_LEN(fil_system->LRU) > 0); /* The node is in the LRU list, remove it */ - UT_LIST_REMOVE(LRU, fil_system->LRU, node); + UT_LIST_REMOVE(fil_system->LRU, node); } mutex_exit(&fil_system->mutex); @@ -137,10 +137,7 @@ xb_fil_cur_open( fil_node_t* node, /*!< in: source tablespace node */ uint thread_n) /*!< thread number for diagnostics */ { - ulint page_size; - ulint page_size_shift; - ulint zip_size; - ibool success; + bool success; /* Initialize these first so xb_fil_cur_close() handles them correctly in case of error */ @@ -148,27 +145,26 @@ xb_fil_cur_open( cursor->node = NULL; cursor->space_id = node->space->id; - cursor->is_system = !fil_is_user_tablespace_id(node->space->id); strncpy(cursor->abs_path, node->name, sizeof(cursor->abs_path)); /* Get the relative path for the destination tablespace name, i.e. the one that can be appended to the backup root directory. Non-system - tablespaces may have absolute paths for remote tablespaces in MySQL - 5.6+. We want to make "local" copies for the backup. */ + tablespaces may have absolute paths for DATA DIRECTORY. + We want to make "local" copies for the backup. */ strncpy(cursor->rel_path, - xb_get_relative_path(cursor->abs_path, cursor->is_system), + xb_get_relative_path(cursor->abs_path, cursor->is_system()), sizeof(cursor->rel_path)); /* In the backup mode we should already have a tablespace handle created - by fil_load_single_table_tablespace() unless it is a system + by fil_ibd_load() unless it is a system tablespace. Otherwise we open the file here. */ - if (cursor->is_system || !srv_backup_mode || srv_close_files) { - node->handle = - os_file_create_simple_no_error_handling(0, node->name, - OS_FILE_OPEN, - OS_FILE_READ_ONLY, - &success,0); + if (cursor->is_system() || srv_operation == SRV_OPERATION_RESTORE + || xb_close_files) { + node->handle = os_file_create_simple_no_error_handling( + 0, node->name, + OS_FILE_OPEN, + OS_FILE_READ_ALLOW_DELETE, true, &success); if (!success) { /* The following call prints an error message */ os_file_get_last_error(TRUE); @@ -181,22 +177,20 @@ xb_fil_cur_open( } mutex_enter(&fil_system->mutex); - node->open = TRUE; - fil_system->n_open++; fil_n_file_opened++; - if (node->space->purpose == FIL_TABLESPACE && + if (node->space->purpose == FIL_TYPE_TABLESPACE && fil_is_user_tablespace_id(node->space->id)) { /* Put the node to the LRU list */ - UT_LIST_ADD_FIRST(LRU, fil_system->LRU, node); + UT_LIST_ADD_FIRST(fil_system->LRU, node); } mutex_exit(&fil_system->mutex); } - ut_ad(node->open); + ut_ad(node->is_open()); cursor->node = node; cursor->file = node->handle; @@ -210,8 +204,8 @@ xb_fil_cur_open( return(XB_FIL_CUR_ERROR); } - if (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT - || srv_unix_file_flush_method == SRV_UNIX_O_DIRECT_NO_FSYNC) { + if (srv_file_flush_method == SRV_O_DIRECT + || srv_file_flush_method == SRV_O_DIRECT_NO_FSYNC) { os_file_set_nocache(cursor->file, node->name, "OPEN"); } @@ -219,32 +213,29 @@ xb_fil_cur_open( posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL); /* Determine the page size */ - zip_size = xb_get_zip_size(cursor->file); - if (zip_size == ULINT_UNDEFINED) { + ulint flags = xb_get_space_flags(cursor->file); + if (flags == ULINT_UNDEFINED) { xb_fil_cur_close(cursor); return(XB_FIL_CUR_SKIP); - } else if (zip_size) { - page_size = zip_size; - page_size_shift = get_bit_shift(page_size); - msg("[%02u] %s is compressed with page size = " - "%lu bytes\n", thread_n, node->name, page_size); - if (page_size_shift < 10 || page_size_shift > 14) { + } + + if (!fsp_flags_is_valid(flags)) { + ulint cflags = fsp_flags_convert_from_101(flags); + if (cflags == ULINT_UNDEFINED) { msg("[%02u] xtrabackup: Error: Invalid " - "page size: %lu.\n", thread_n, page_size); - ut_error; + "tablespace flags: %x.\n", thread_n, uint(flags)); + return(XB_FIL_CUR_SKIP); } - } else { - page_size = UNIV_PAGE_SIZE; - page_size_shift = UNIV_PAGE_SIZE_SHIFT; + flags = cflags; } + + const page_size_t page_size(flags); cursor->page_size = page_size; - cursor->page_size_shift = page_size_shift; - cursor->zip_size = zip_size; /* Allocate read buffer */ - cursor->buf_size = XB_FIL_CUR_PAGES * page_size; + cursor->buf_size = XB_FIL_CUR_PAGES * page_size.physical(); cursor->orig_buf = static_cast<byte *> - (ut_malloc(cursor->buf_size + UNIV_PAGE_SIZE)); + (malloc(cursor->buf_size + UNIV_PAGE_SIZE)); cursor->buf = static_cast<byte *> (ut_align(cursor->orig_buf, UNIV_PAGE_SIZE)); @@ -254,7 +245,8 @@ xb_fil_cur_open( cursor->buf_page_no = 0; cursor->thread_n = thread_n; - cursor->space_size = (ulint)(cursor->statinfo.st_size / page_size); + cursor->space_size = (ulint)(cursor->statinfo.st_size + / page_size.physical()); cursor->read_filter = read_filter; cursor->read_filter->init(&cursor->read_filter_ctxt, cursor, @@ -282,6 +274,8 @@ xb_fil_cur_read( xb_fil_cur_result_t ret; ib_int64_t offset; ib_int64_t to_read; + const ulint page_size = cursor->page_size.physical(); + xb_ad(!cursor->is_system() || page_size == UNIV_PAGE_SIZE); cursor->read_filter->get_next_batch(&cursor->read_filter_ctxt, &offset, &to_read); @@ -296,10 +290,10 @@ xb_fil_cur_read( xb_a(to_read > 0 && to_read <= 0xFFFFFFFFLL); - if (to_read % cursor->page_size != 0 && - offset + to_read == cursor->statinfo.st_size) { + if ((to_read & ~(page_size - 1)) + && offset + to_read == cursor->statinfo.st_size) { - if (to_read < (ib_int64_t) cursor->page_size) { + if (to_read < (ib_int64_t) page_size) { msg("[%02u] xtrabackup: Warning: junk at the end of " "%s:\n", cursor->thread_n, cursor->abs_path); msg("[%02u] xtrabackup: Warning: offset = %llu, " @@ -312,12 +306,12 @@ xb_fil_cur_read( } to_read = (ib_int64_t) (((ulint) to_read) & - ~(cursor->page_size - 1)); + ~(page_size - 1)); } - xb_a(to_read % cursor->page_size == 0); + xb_a((to_read & (page_size - 1)) == 0); - npages = (ulint) (to_read >> cursor->page_size_shift); + npages = (ulint) (to_read / cursor->page_size.physical()); retry_count = 10; ret = XB_FIL_CUR_SUCCESS; @@ -328,59 +322,59 @@ read_retry: cursor->buf_read = 0; cursor->buf_npages = 0; cursor->buf_offset = offset; - cursor->buf_page_no = (ulint)(offset >> cursor->page_size_shift); + cursor->buf_page_no = (ulint)(offset / cursor->page_size.physical()); - success = os_file_read(cursor->file, cursor->buf, offset, - (ulint)to_read); - if (!success) { + FilSpace space(cursor->space_id); + + if (!space()) { return(XB_FIL_CUR_ERROR); } - fil_system_enter(); - fil_space_t *space = fil_space_get_by_id(cursor->space_id); - fil_system_exit(); + success = os_file_read(IORequestRead, + cursor->file, cursor->buf, offset, + (ulint) to_read); + if (!success) { + return(XB_FIL_CUR_ERROR); + } /* check pages for corruption and re-read if necessary. i.e. in case of partially written pages */ for (page = cursor->buf, i = 0; i < npages; - page += cursor->page_size, i++) { - ib_int64_t page_no = cursor->buf_page_no + i; - - bool checksum_ok = fil_space_verify_crypt_checksum(page, cursor->zip_size,space, (ulint)page_no); - - if (!checksum_ok && - buf_page_is_corrupted(true, page, cursor->zip_size,space)) { - - if (cursor->is_system && - page_no >= (ib_int64_t)FSP_EXTENT_SIZE && - page_no < (ib_int64_t) FSP_EXTENT_SIZE * 3) { - /* skip doublewrite buffer pages */ - xb_a(cursor->page_size == UNIV_PAGE_SIZE); + page += page_size, i++) { + ulint page_no = cursor->buf_page_no + i; + + if (cursor->space_id == TRX_SYS_SPACE && + page_no >= FSP_EXTENT_SIZE && + page_no < FSP_EXTENT_SIZE * 3) { + /* We ignore the doublewrite buffer pages */ + } else if (!fil_space_verify_crypt_checksum( + page, cursor->page_size, space->id, page_no) + && buf_page_is_corrupted(true, page, + cursor->page_size, + space)) { + retry_count--; + if (retry_count == 0) { msg("[%02u] xtrabackup: " - "Page %lu is a doublewrite buffer page, " - "skipping.\n", cursor->thread_n, page_no); - } else { - retry_count--; - if (retry_count == 0) { - msg("[%02u] xtrabackup: " - "Error: failed to read page after " - "10 retries. File %s seems to be " - "corrupted.\n", cursor->thread_n, - cursor->abs_path); - ret = XB_FIL_CUR_ERROR; - break; - } + "Error: failed to read page after " + "10 retries. File %s seems to be " + "corrupted.\n", cursor->thread_n, + cursor->abs_path); + ret = XB_FIL_CUR_ERROR; + break; + } + + if (retry_count == 9) { msg("[%02u] xtrabackup: " "Database page corruption detected at page " - "%lu, retrying...\n", cursor->thread_n, - page_no); + ULINTPF ", retrying...\n", + cursor->thread_n, page_no); + } - os_thread_sleep(100000); + os_thread_sleep(100000); - goto read_retry; - } + goto read_retry; } - cursor->buf_read += cursor->page_size; + cursor->buf_read += page_size; cursor->buf_npages++; } @@ -399,11 +393,10 @@ xb_fil_cur_close( { cursor->read_filter->deinit(&cursor->read_filter_ctxt); - if (cursor->orig_buf != NULL) { - ut_free(cursor->orig_buf); - } + free(cursor->orig_buf); + if (cursor->node != NULL) { xb_fil_node_close_file(cursor->node); - cursor->file = XB_FILE_UNDEFINED; + cursor->file = OS_FILE_CLOSED; } } diff --git a/extra/mariabackup/fil_cur.h b/extra/mariabackup/fil_cur.h index f3601823a5a..e3f356a346c 100644 --- a/extra/mariabackup/fil_cur.h +++ b/extra/mariabackup/fil_cur.h @@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include <my_dir.h> #include "read_filt.h" +#include "srv0start.h" struct xb_fil_cur_t { pfs_os_file_t file; /*!< source file handle */ @@ -36,14 +37,7 @@ struct xb_fil_cur_t { char abs_path[FN_REFLEN]; /*!< absolute file path */ MY_STAT statinfo; /*!< information about the file */ - ulint zip_size; /*!< compressed page size in bytes or 0 - for uncompressed pages */ - ulint page_size; /*!< = zip_size for compressed pages or - UNIV_PAGE_SIZE for uncompressed ones */ - ulint page_size_shift;/*!< bit shift corresponding to - page_size */ - my_bool is_system; /*!< TRUE for system tablespace, FALSE - otherwise */ + page_size_t page_size; /*!< page size */ xb_read_filt_t* read_filter; /*!< read filter */ xb_read_filt_ctxt_t read_filter_ctxt; /*!< read filter context */ @@ -61,6 +55,17 @@ struct xb_fil_cur_t { uint thread_n; /*!< thread number for diagnostics */ ulint space_id; /*!< ID of tablespace */ ulint space_size; /*!< space size in pages */ + + /** TODO: remove this default constructor */ + xb_fil_cur_t() : page_size(0), read_filter_ctxt() {} + + /** @return whether this is not a file-per-table tablespace */ + bool is_system() const + { + ut_ad(space_id != SRV_TMP_SPACE_ID); + return(space_id == TRX_SYS_SPACE + || srv_is_undo_tablespace(space_id)); + } }; typedef enum { diff --git a/extra/mariabackup/innobackupex.cc b/extra/mariabackup/innobackupex.cc index 7e97b258489..4fb85208eff 100644 --- a/extra/mariabackup/innobackupex.cc +++ b/extra/mariabackup/innobackupex.cc @@ -45,7 +45,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include <mysql.h> #include <my_dir.h> #include <ut0mem.h> -#include <os0sync.h> #include <os0file.h> #include <srv0start.h> #include <algorithm> @@ -70,7 +69,6 @@ using std::max; my_bool opt_ibx_version = FALSE; my_bool opt_ibx_help = FALSE; my_bool opt_ibx_apply_log = FALSE; -my_bool opt_ibx_redo_only = FALSE; my_bool opt_ibx_incremental = FALSE; my_bool opt_ibx_notimestamp = FALSE; @@ -226,21 +224,11 @@ static struct my_option ibx_long_options[] = GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"apply-log", OPT_APPLY_LOG, "Prepare a backup in BACKUP-DIR by " - "applying the transaction log file named \"xtrabackup_logfile\" " - "located in the same directory. Also, create new transaction logs. " + "applying the redo log 'ib_logfile0' and creating new redo log. " "The InnoDB configuration is read from the file \"backup-my.cnf\".", (uchar*) &opt_ibx_apply_log, (uchar*) &opt_ibx_apply_log, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"redo-only", OPT_REDO_ONLY, "This option should be used when " - "preparing the base full backup and when merging all incrementals " - "except the last one. This forces xtrabackup to skip the \"rollback\" " - "phase and do a \"redo\" only. This is necessary if the backup will " - "have incremental changes applied to it later. See the xtrabackup " - "documentation for details.", - (uchar *) &opt_ibx_redo_only, (uchar *) &opt_ibx_redo_only, 0, - GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"copy-back", OPT_COPY_BACK, "Copy all the files in a previously made " "backup from the backup directory to their original locations.", (uchar *) &opt_ibx_copy_back, (uchar *) &opt_ibx_copy_back, 0, @@ -682,7 +670,7 @@ innobackupex [--compress] [--compress-threads=NUMBER-OF-THREADS] [--compress-chu \n\ innobackupex --apply-log [--use-memory=B]\n\ [--defaults-file=MY.CNF]\n\ - [--export] [--redo-only] [--ibbackup=IBBACKUP-BINARY]\n\ + [--export] [--ibbackup=IBBACKUP-BINARY]\n\ BACKUP-DIR\n\ \n\ innobackupex --copy-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME] BACKUP-DIR\n\ @@ -710,7 +698,7 @@ process.\n\ \n\ The --apply-log command prepares a backup for starting a MySQL\n\ server on the backup. This command recovers InnoDB data files as specified\n\ -in BACKUP-DIR/backup-my.cnf using BACKUP-DIR/xtrabackup_logfile,\n\ +in BACKUP-DIR/backup-my.cnf using BACKUP-DIR/ib_logfile0,\n\ and creates new InnoDB log files as specified in BACKUP-DIR/backup-my.cnf.\n\ The BACKUP-DIR should be the path to a backup directory created by\n\ xtrabackup. This command runs xtrabackup as a child process, but it does not \n\ @@ -980,9 +968,6 @@ ibx_init() switch (ibx_mode) { case IBX_MODE_APPLY_LOG: xtrabackup_prepare = TRUE; - if (opt_ibx_redo_only) { - xtrabackup_apply_log_only = TRUE; - } xtrabackup_target_dir = ibx_position_arg; run = "apply-log"; break; diff --git a/extra/mariabackup/read_filt.cc b/extra/mariabackup/read_filt.cc index 05e6b7c86c7..a48591abf29 100644 --- a/extra/mariabackup/read_filt.cc +++ b/extra/mariabackup/read_filt.cc @@ -127,10 +127,11 @@ rf_bitmap_get_next_batch( of pages */ { ulint start_page_id; + const ulint page_size = ctxt->page_size.physical(); - start_page_id = (ulint)(ctxt->offset / ctxt->page_size); + start_page_id = (ulint)(ctxt->offset / page_size); - xb_a (ctxt->offset % ctxt->page_size == 0); + xb_a (ctxt->offset % page_size == 0); if (start_page_id == ctxt->filter_batch_end) { @@ -146,7 +147,7 @@ rf_bitmap_get_next_batch( return; } - ctxt->offset = next_page_id * ctxt->page_size; + ctxt->offset = next_page_id * page_size; /* Find the end of the current changed page block by searching for the next cleared bitmap bit */ @@ -162,7 +163,7 @@ rf_bitmap_get_next_batch( remaining pages. */ *read_batch_len = ctxt->data_file_size - ctxt->offset; } else { - *read_batch_len = ctxt->filter_batch_end * ctxt->page_size + *read_batch_len = ctxt->filter_batch_end * page_size - ctxt->offset; } @@ -175,9 +176,9 @@ rf_bitmap_get_next_batch( } ctxt->offset += *read_batch_len; - xb_a (ctxt->offset % ctxt->page_size == 0); - xb_a (*read_batch_start % ctxt->page_size == 0); - xb_a (*read_batch_len % ctxt->page_size == 0); + xb_a (ctxt->offset % page_size == 0); + xb_a (*read_batch_start % page_size == 0); + xb_a (*read_batch_len % page_size == 0); } /****************************************************************//** diff --git a/extra/mariabackup/read_filt.h b/extra/mariabackup/read_filt.h index d16f4e1093d..cebc714eed8 100644 --- a/extra/mariabackup/read_filt.h +++ b/extra/mariabackup/read_filt.h @@ -27,6 +27,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include "changed_page_bitmap.h" +typedef ulint space_id_t; + struct xb_fil_cur_t; /* The read filter context */ @@ -34,15 +36,17 @@ struct xb_read_filt_ctxt_t { ib_int64_t offset; /*!< current file offset */ ib_int64_t data_file_size; /*!< data file size */ size_t buffer_capacity;/*!< read buffer capacity */ - ib_int64_t space_id; /*!< space id */ + space_id_t space_id; /*!< space id */ /* The following fields used only in bitmap filter */ /* Move these to union if any other filters are added in future */ xb_page_bitmap_range *bitmap_range; /*!< changed page bitmap range iterator for space_id */ - size_t page_size; /*!< page size */ + page_size_t page_size; /*!< page size */ ulint filter_batch_end;/*!< the ending page id of the current changed page block in the bitmap */ + /** TODO: remove this default constructor */ + xb_read_filt_ctxt_t() : page_size(0) {} }; /* The read filter */ diff --git a/extra/mariabackup/write_filt.cc b/extra/mariabackup/write_filt.cc index cf7753bf380..a0633818405 100644 --- a/extra/mariabackup/write_filt.cc +++ b/extra/mariabackup/write_filt.cc @@ -67,7 +67,6 @@ wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, xb_fil_cur_t *cursor) { char meta_name[FN_REFLEN]; - xb_delta_info_t info; ulint buf_size; xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt); @@ -75,8 +74,9 @@ wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, ctxt->cursor = cursor; /* allocate buffer for incremental backup (4096 pages) */ - buf_size = (cursor->page_size / 4 + 1) * cursor->page_size; - cp->delta_buf_base = static_cast<byte *>(ut_malloc(buf_size)); + buf_size = (cursor->page_size.physical() / 4 + 1) + * cursor->page_size.physical(); + cp->delta_buf_base = static_cast<byte *>(malloc(buf_size)); memset(cp->delta_buf_base, 0, buf_size); cp->delta_buf = static_cast<byte *> (ut_align(cp->delta_buf_base, UNIV_PAGE_SIZE_MAX)); @@ -84,9 +84,7 @@ wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, /* write delta meta info */ snprintf(meta_name, sizeof(meta_name), "%s%s", dst_name, XB_DELTA_INFO_SUFFIX); - info.page_size = cursor->page_size; - info.zip_size = cursor->zip_size; - info.space_id = cursor->space_id; + const xb_delta_info_t info(cursor->page_size, cursor->space_id); if (!xb_write_delta_metadata(meta_name, &info)) { msg("[%02u] xtrabackup: Error: " "failed to write meta info for %s\n", @@ -113,8 +111,9 @@ wf_incremental_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) { ulint i; xb_fil_cur_t *cursor = ctxt->cursor; - ulint page_size = cursor->page_size; byte *page; + const ulint page_size + = cursor->page_size.physical(); xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt); for (i = 0, page = cursor->buf; i < cursor->buf_npages; @@ -159,7 +158,8 @@ static my_bool wf_incremental_finalize(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) { xb_fil_cur_t *cursor = ctxt->cursor; - ulint page_size = cursor->page_size; + const ulint page_size + = cursor->page_size.physical(); xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt); if (cp->npages != page_size / 4) { @@ -184,9 +184,7 @@ wf_incremental_deinit(xb_write_filt_ctxt_t *ctxt) { xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt); - if (cp->delta_buf_base != NULL) { - ut_free(cp->delta_buf_base); - } + free(cp->delta_buf_base); } /************************************************************************ diff --git a/extra/mariabackup/xb0xb.h b/extra/mariabackup/xb0xb.h index a8b17f59579..59938a014c6 100644 --- a/extra/mariabackup/xb0xb.h +++ b/extra/mariabackup/xb0xb.h @@ -21,43 +21,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #ifndef xb0xb_h #define xb0xb_h - -extern void os_io_init_simple(void); -extern pfs_os_file_t files[1000]; extern const char *innodb_checksum_algorithm_names[]; extern TYPELIB innodb_checksum_algorithm_typelib; -extern dberr_t open_or_create_data_files( - bool* create_new_db, -#ifdef UNIV_LOG_ARCHIVE - lsn_t* min_arch_log_no, - lsn_t* max_arch_log_no, -#endif - lsn_t* flushed_lsn, - ulint* sum_of_new_sizes) - ; -int -fil_file_readdir_next_file( -/*=======================*/ -dberr_t* err, /*!< out: this is set to DB_ERROR if an error - was encountered, otherwise not changed */ - const char* dirname,/*!< in: directory name or path */ - os_file_dir_t dir, /*!< in: directory stream */ - os_file_stat_t* info) /*!< in/out: buffer where the - info is returned */; -fil_space_t* -fil_space_get_by_name(const char *); -ibool -recv_check_cp_is_consistent(const byte* buf); -void -innodb_log_checksum_func_update( -/*============================*/ -ulint algorithm) /*!< in: algorithm */; -dberr_t -srv_undo_tablespaces_init( -/*======================*/ -ibool create_new_db, -ibool backup_mode, -const ulint n_conf_tablespaces, -ulint* n_opened); #endif diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index baad5b12118..6e9c172efc1 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -4,6 +4,8 @@ XtraBackup: hot backup tool for InnoDB Originally Created 3/3/2009 Yasufumi Kinoshita Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. +(c) 2017, MariaDB Corporation. +Portions written by Marko Mäkelä. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -41,6 +43,8 @@ Place, Suite 330, Boston, MA 02111-1307 USA //#define XTRABACKUP_TARGET_IS_PLUGIN +#include <my_config.h> +#include <unireg.h> #include <mysql_version.h> #include <my_base.h> #include <my_getopt.h> @@ -61,6 +65,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include <dict0priv.h> #include <lock0lock.h> #include <log0recv.h> +#include <log0crypt.h> #include <row0mysql.h> #include <row0quiesce.h> #include <srv0start.h> @@ -85,7 +90,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "xbstream.h" #include "changed_page_bitmap.h" #include "read_filt.h" -#include "wsrep.h" +#include "backup_wsrep.h" #include "innobackupex.h" #include "backup_mysql.h" #include "backup_copy.h" @@ -95,48 +100,39 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include <sql_plugin.h> #include <srv0srv.h> #include <crc_glue.h> - -/* TODO: replace with appropriate macros used in InnoDB 5.6 */ -#define PAGE_ZIP_MIN_SIZE_SHIFT 10 -#define DICT_TF_ZSSIZE_SHIFT 1 -#define DICT_TF_FORMAT_ZIP 1 -#define DICT_TF_FORMAT_SHIFT 5 +#include <log.h> int sys_var_init(); -my_bool innodb_inited= 0; - /* === xtrabackup specific options === */ char xtrabackup_real_target_dir[FN_REFLEN] = "./xtrabackup_backupfiles/"; char *xtrabackup_target_dir= xtrabackup_real_target_dir; -my_bool xtrabackup_version = FALSE; -my_bool xtrabackup_backup = FALSE; -my_bool xtrabackup_prepare = FALSE; -my_bool xtrabackup_copy_back = FALSE; -my_bool xtrabackup_move_back = FALSE; -my_bool xtrabackup_decrypt_decompress = FALSE; -my_bool xtrabackup_print_param = FALSE; - -my_bool xtrabackup_export = FALSE; -my_bool xtrabackup_apply_log_only = FALSE; - -longlong xtrabackup_use_memory = 100*1024*1024L; -my_bool xtrabackup_create_ib_logfile = FALSE; - -long xtrabackup_throttle = 0; /* 0:unlimited */ -lint io_ticket; -os_event_t wait_throttle = NULL; -os_event_t log_copying_stop = NULL; - -char *xtrabackup_incremental = NULL; +static my_bool xtrabackup_version; +my_bool xtrabackup_backup; +my_bool xtrabackup_prepare; +my_bool xtrabackup_copy_back; +my_bool xtrabackup_move_back; +my_bool xtrabackup_decrypt_decompress; +my_bool xtrabackup_print_param; + +my_bool xtrabackup_export; + +longlong xtrabackup_use_memory; + +long xtrabackup_throttle; /* 0:unlimited */ +static lint io_ticket; +static os_event_t wait_throttle; +static os_event_t log_copying_stop; + +char *xtrabackup_incremental; lsn_t incremental_lsn; lsn_t incremental_to_lsn; lsn_t incremental_last_lsn; -xb_page_bitmap *changed_page_bitmap = NULL; +xb_page_bitmap *changed_page_bitmap; -char *xtrabackup_incremental_basedir = NULL; /* for --backup */ -char *xtrabackup_extra_lsndir = NULL; /* for --backup with --extra-lsndir */ -char *xtrabackup_incremental_dir = NULL; /* for --prepare */ +char *xtrabackup_incremental_basedir; /* for --backup */ +char *xtrabackup_extra_lsndir; /* for --backup with --extra-lsndir */ +char *xtrabackup_incremental_dir; /* for --prepare */ char xtrabackup_real_incremental_basedir[FN_REFLEN]; char xtrabackup_real_extra_lsndir[FN_REFLEN]; @@ -144,9 +140,9 @@ char xtrabackup_real_incremental_dir[FN_REFLEN]; char *xtrabackup_tmpdir; -char *xtrabackup_tables = NULL; -char *xtrabackup_tables_file = NULL; -char *xtrabackup_tables_exclude = NULL; +char *xtrabackup_tables; +char *xtrabackup_tables_file; +char *xtrabackup_tables_exclude; typedef std::list<regex_t> regex_list_t; static regex_list_t regex_include_list; @@ -170,17 +166,12 @@ struct xb_filter_entry_struct{ }; typedef struct xb_filter_entry_struct xb_filter_entry_t; -static ulint thread_nr[SRV_MAX_N_IO_THREADS + 6]; -static os_thread_id_t thread_ids[SRV_MAX_N_IO_THREADS + 6]; - lsn_t checkpoint_lsn_start; lsn_t checkpoint_no_start; -lsn_t log_copy_scanned_lsn; -ibool log_copying = TRUE; -ibool log_copying_running = FALSE; -ibool io_watching_thread_running = FALSE; - -ibool xtrabackup_logfile_is_renamed = FALSE; +static lsn_t log_copy_scanned_lsn; +static bool log_copying; +static bool log_copying_running; +static bool io_watching_thread_running; int xtrabackup_parallel; @@ -199,19 +190,16 @@ ulint xtrabackup_log_copy_interval = 1000; static ulong max_buf_pool_modified_pct; /* Ignored option (--log) for MySQL option compatibility */ -char* log_ignored_opt = NULL; +static char* log_ignored_opt; /* === metadata of backup === */ #define XTRABACKUP_METADATA_FILENAME "xtrabackup_checkpoints" -char metadata_type[30] = ""; /*[full-backuped|log-applied| - full-prepared|incremental]*/ -lsn_t metadata_from_lsn = 0; -lsn_t metadata_to_lsn = 0; -lsn_t metadata_last_lsn = 0; - -#define XB_LOG_FILENAME "xtrabackup_logfile" +char metadata_type[30] = ""; /*[full-backuped|log-applied|incremental]*/ +lsn_t metadata_from_lsn; +lsn_t metadata_to_lsn; +lsn_t metadata_last_lsn; -ds_file_t *dst_log_file = NULL; +static ds_file_t* dst_log_file; static char mysql_data_home_buff[2]; @@ -227,24 +215,17 @@ ulong innobase_large_page_size = 0; /* The default values for the following, type long or longlong, start-up parameters are declared in mysqld.cc: */ -long innobase_additional_mem_pool_size = 1*1024*1024L; long innobase_buffer_pool_awe_mem_mb = 0; long innobase_file_io_threads = 4; long innobase_read_io_threads = 4; long innobase_write_io_threads = 4; -long innobase_force_recovery = 0; long innobase_log_buffer_size = 1024*1024L; -long innobase_log_files_in_group = 2; long innobase_open_files = 300L; longlong innobase_page_size = (1LL << 14); /* 16KB */ -static ulong innobase_log_block_size = 512; -my_bool innobase_fast_checksum = FALSE; -char* innobase_doublewrite_file = NULL; char* innobase_buffer_pool_filename = NULL; longlong innobase_buffer_pool_size = 8*1024*1024L; -longlong innobase_log_file_size = 48*1024*1024L; /* The default values for the following char* start-up parameters are determined in innobase_init below: */ @@ -256,20 +237,12 @@ char* innobase_data_file_path = NULL; affects Windows: */ char* innobase_unix_file_flush_method = NULL; -/* Below we have boolean-valued start-up parameters, and their default -values */ - -ulong innobase_fast_shutdown = 1; -my_bool innobase_use_doublewrite = TRUE; -my_bool innobase_use_checksums = TRUE; -my_bool innobase_use_large_pages = FALSE; -my_bool innobase_file_per_table = FALSE; -my_bool innobase_locks_unsafe_for_binlog = FALSE; -my_bool innobase_rollback_on_timeout = FALSE; -my_bool innobase_create_status_file = FALSE; -my_bool innobase_adaptive_hash_index = TRUE; - -static char *internal_innobase_data_file_path = NULL; +my_bool innobase_use_doublewrite; +my_bool innobase_use_large_pages; +my_bool innobase_file_per_table; +my_bool innobase_locks_unsafe_for_binlog; +my_bool innobase_rollback_on_timeout; +my_bool innobase_create_status_file; /* The following counter is used to convey information to InnoDB about server activity: in selects it is not sensible to call @@ -290,7 +263,7 @@ lsn_t flushed_lsn= 0; ulong xb_open_files_limit= 0; char *xb_plugin_dir; char *xb_plugin_load; -my_bool xb_close_files= FALSE; +my_bool xb_close_files; /* Datasinks */ ds_ctxt_t *ds_data = NULL; @@ -299,10 +272,6 @@ ds_ctxt_t *ds_redo = NULL; static bool innobackupex_mode = false; -static long innobase_log_files_in_group_save; -static char *srv_log_group_home_dir_save; -static longlong innobase_log_file_size_save; - /* String buffer used by --print-param to accumulate server options as they are parsed from the defaults file */ static std::ostringstream print_param_str; @@ -313,6 +282,7 @@ std::set<std::string> param_set; static ulonglong global_max_value; extern "C" sig_handler handle_fatal_signal(int sig); +extern LOGGER logger; my_bool opt_galera_info = FALSE; my_bool opt_slave_info = FALSE; @@ -361,9 +331,6 @@ const char *opt_history = NULL; #if defined(HAVE_OPENSSL) my_bool opt_ssl_verify_server_cert = FALSE; -#if !defined(HAVE_YASSL) -char *opt_server_public_key = NULL; -#endif #endif /* Whether xtrabackup_binlog_info should be created on recovery */ @@ -383,14 +350,23 @@ xtrabackup_add_datasink(ds_ctxt_t *ds) } /* ======== Datafiles iterator ======== */ +struct datafiles_iter_t { + fil_system_t *system; + fil_space_t *space; + fil_node_t *node; + ibool started; + pthread_mutex_t mutex; +}; + +/* ======== Datafiles iterator ======== */ +static datafiles_iter_t * datafiles_iter_new(fil_system_t *f_system) { datafiles_iter_t *it; - it = static_cast<datafiles_iter_t *> - (ut_malloc(sizeof(datafiles_iter_t))); - it->mutex = os_mutex_create(); + it = static_cast<datafiles_iter_t *>(malloc(sizeof(datafiles_iter_t))); + pthread_mutex_init(&it->mutex, NULL); it->system = f_system; it->space = NULL; @@ -400,12 +376,13 @@ datafiles_iter_new(fil_system_t *f_system) return it; } +static fil_node_t * datafiles_iter_next(datafiles_iter_t *it) { fil_node_t *new_node; - os_mutex_enter(it->mutex); + pthread_mutex_lock(&it->mutex); if (it->node == NULL) { if (it->started) @@ -422,7 +399,7 @@ datafiles_iter_next(datafiles_iter_t *it) UT_LIST_GET_NEXT(space_list, it->space); while (it->space != NULL && - (it->space->purpose != FIL_TABLESPACE || + (it->space->purpose != FIL_TYPE_TABLESPACE || UT_LIST_GET_LEN(it->space->chain) == 0)) it->space = UT_LIST_GET_NEXT(space_list, it->space); if (it->space == NULL) @@ -432,16 +409,17 @@ datafiles_iter_next(datafiles_iter_t *it) end: new_node = it->node; - os_mutex_exit(it->mutex); + pthread_mutex_unlock(&it->mutex); return new_node; } +static void datafiles_iter_free(datafiles_iter_t *it) { - os_mutex_free(it->mutex); - ut_free(it); + pthread_mutex_destroy(&it->mutex); + free(it); } /* ======== Date copying thread context ======== */ @@ -450,7 +428,7 @@ typedef struct { datafiles_iter_t *it; uint num; uint *count; - os_ib_mutex_t count_mutex; + pthread_mutex_t count_mutex; os_thread_id_t id; } data_thread_ctxt_t; @@ -463,7 +441,6 @@ enum options_xtrabackup OPT_XTRA_BACKUP, OPT_XTRA_PREPARE, OPT_XTRA_EXPORT, - OPT_XTRA_APPLY_LOG_ONLY, OPT_XTRA_PRINT_PARAM, OPT_XTRA_USE_MEMORY, OPT_XTRA_THROTTLE, @@ -476,7 +453,6 @@ enum options_xtrabackup OPT_XTRA_TABLES_FILE, OPT_XTRA_DATABASES, OPT_XTRA_DATABASES_FILE, - OPT_XTRA_CREATE_IB_LOGFILE, OPT_XTRA_PARALLEL, OPT_XTRA_STREAM, OPT_XTRA_COMPRESS, @@ -484,12 +460,10 @@ enum options_xtrabackup OPT_XTRA_COMPRESS_CHUNK_SIZE, OPT_LOG, OPT_INNODB, - OPT_INNODB_CHECKSUMS, OPT_INNODB_DATA_FILE_PATH, OPT_INNODB_DATA_HOME_DIR, OPT_INNODB_ADAPTIVE_HASH_INDEX, OPT_INNODB_DOUBLEWRITE, - OPT_INNODB_FAST_SHUTDOWN, OPT_INNODB_FILE_PER_TABLE, OPT_INNODB_FLUSH_LOG_AT_TRX_COMMIT, OPT_INNODB_FLUSH_METHOD, @@ -499,7 +473,6 @@ enum options_xtrabackup OPT_INNODB_MAX_PURGE_LAG, OPT_INNODB_ROLLBACK_ON_TIMEOUT, OPT_INNODB_STATUS_FILE, - OPT_INNODB_ADDITIONAL_MEM_POOL_SIZE, OPT_INNODB_AUTOEXTEND_INCREMENT, OPT_INNODB_BUFFER_POOL_SIZE, OPT_INNODB_COMMIT_CONCURRENCY, @@ -510,25 +483,17 @@ enum options_xtrabackup OPT_INNODB_WRITE_IO_THREADS, OPT_INNODB_USE_NATIVE_AIO, OPT_INNODB_PAGE_SIZE, - OPT_INNODB_LOG_BLOCK_SIZE, - OPT_INNODB_FAST_CHECKSUM, - OPT_INNODB_EXTRA_UNDOSLOTS, - OPT_INNODB_DOUBLEWRITE_FILE, OPT_INNODB_BUFFER_POOL_FILENAME, - OPT_INNODB_FORCE_RECOVERY, OPT_INNODB_LOCK_WAIT_TIMEOUT, OPT_INNODB_LOG_BUFFER_SIZE, OPT_INNODB_LOG_FILE_SIZE, OPT_INNODB_LOG_FILES_IN_GROUP, OPT_INNODB_OPEN_FILES, - OPT_INNODB_SYNC_SPIN_LOOPS, - OPT_INNODB_THREAD_CONCURRENCY, - OPT_INNODB_THREAD_SLEEP_DELAY, OPT_XTRA_DEBUG_SYNC, OPT_INNODB_CHECKSUM_ALGORITHM, OPT_INNODB_UNDO_DIRECTORY, OPT_INNODB_UNDO_TABLESPACES, - OPT_INNODB_LOG_CHECKSUM_ALGORITHM, + OPT_INNODB_LOG_CHECKSUMS, OPT_XTRA_INCREMENTAL_FORCE_SCAN, OPT_DEFAULTS_GROUP, OPT_OPEN_FILES_LIMIT, @@ -563,10 +528,6 @@ enum options_xtrabackup OPT_BINLOG_INFO, OPT_XB_SECURE_AUTH, - OPT_SSL_SSL, - OPT_SSL_VERIFY_SERVER_CERT, - OPT_SERVER_PUBLIC_KEY, - OPT_XTRA_TABLES_EXCLUDE, OPT_XTRA_DATABASES_EXCLUDE, }; @@ -587,10 +548,6 @@ struct my_option xb_client_options[] = {"export", OPT_XTRA_EXPORT, "create files to import to another database when prepare.", (G_PTR*) &xtrabackup_export, (G_PTR*) &xtrabackup_export, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"apply-log-only", OPT_XTRA_APPLY_LOG_ONLY, - "stop recovery process not to progress LSN after applying log when prepare.", - (G_PTR*) &xtrabackup_apply_log_only, (G_PTR*) &xtrabackup_apply_log_only, - 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"print-param", OPT_XTRA_PRINT_PARAM, "print parameter of mysqld needed for copyback.", (G_PTR*) &xtrabackup_print_param, (G_PTR*) &xtrabackup_print_param, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, @@ -628,7 +585,7 @@ struct my_option xb_client_options[] = {"databases", OPT_XTRA_DATABASES, "filtering by list of databases.", (G_PTR*) &xtrabackup_databases, (G_PTR*) &xtrabackup_databases, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"databases_file", OPT_XTRA_TABLES_FILE, + {"databases_file", OPT_XTRA_DATABASES_FILE, "filtering by list of databases in the file.", (G_PTR*) &xtrabackup_databases_file, (G_PTR*) &xtrabackup_databases_file, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -642,9 +599,6 @@ struct my_option xb_client_options[] = "Note that this option has a higher priority than --databases.", (G_PTR*) &xtrabackup_databases_exclude, (G_PTR*) &xtrabackup_databases_exclude, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"create-ib-logfile", OPT_XTRA_CREATE_IB_LOGFILE, "** not work for now** creates ib_logfile* also after '--prepare'. ### If you want create ib_logfile*, only re-execute this command in same options. ###", - (G_PTR*) &xtrabackup_create_ib_logfile, (G_PTR*) &xtrabackup_create_ib_logfile, - 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"stream", OPT_XTRA_STREAM, "Stream all backup files to the standard output " "in the specified format." @@ -922,6 +876,11 @@ struct my_option xb_client_options[] = uint xb_client_options_count = array_elements(xb_client_options); +#ifndef DBUG_OFF +/** Parameters to DBUG */ +static const char *dbug_option; +#endif + struct my_option xb_server_options[] = { {"datadir", 'h', "Path to the database root.", (G_PTR*) &mysql_data_home, @@ -952,40 +911,34 @@ struct my_option xb_server_options[] = {"innodb", OPT_INNODB, "Ignored option for MySQL option compatibility", (G_PTR*) &innobase_ignored_opt, (G_PTR*) &innobase_ignored_opt, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, - +#ifdef BTR_CUR_HASH_ADAPT {"innodb_adaptive_hash_index", OPT_INNODB_ADAPTIVE_HASH_INDEX, "Enable InnoDB adaptive hash index (enabled by default). " "Disable with --skip-innodb-adaptive-hash-index.", - (G_PTR*) &innobase_adaptive_hash_index, - (G_PTR*) &innobase_adaptive_hash_index, + &btr_search_enabled, + &btr_search_enabled, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, - {"innodb_additional_mem_pool_size", OPT_INNODB_ADDITIONAL_MEM_POOL_SIZE, - "Size of a memory pool InnoDB uses to store data dictionary information and other internal data structures.", - (G_PTR*) &innobase_additional_mem_pool_size, - (G_PTR*) &innobase_additional_mem_pool_size, 0, GET_LONG, REQUIRED_ARG, - 1*1024*1024L, 512*1024L, LONG_MAX, 0, 1024, 0}, +#endif /* BTR_CUR_HASH_ADAPT */ {"innodb_autoextend_increment", OPT_INNODB_AUTOEXTEND_INCREMENT, "Data file autoextend increment in megabytes", - (G_PTR*) &srv_auto_extend_increment, - (G_PTR*) &srv_auto_extend_increment, + (G_PTR*) &sys_tablespace_auto_extend_increment, + (G_PTR*) &sys_tablespace_auto_extend_increment, 0, GET_ULONG, REQUIRED_ARG, 8L, 1L, 1000L, 0, 1L, 0}, {"innodb_buffer_pool_size", OPT_INNODB_BUFFER_POOL_SIZE, "The size of the memory buffer InnoDB uses to cache data and indexes of its tables.", (G_PTR*) &innobase_buffer_pool_size, (G_PTR*) &innobase_buffer_pool_size, 0, GET_LL, REQUIRED_ARG, 8*1024*1024L, 1024*1024L, LONGLONG_MAX, 0, 1024*1024L, 0}, - {"innodb_checksums", OPT_INNODB_CHECKSUMS, "Enable InnoDB checksums validation (enabled by default). \ -Disable with --skip-innodb-checksums.", (G_PTR*) &innobase_use_checksums, - (G_PTR*) &innobase_use_checksums, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, {"innodb_data_file_path", OPT_INNODB_DATA_FILE_PATH, "Path to individual files and their sizes.", &innobase_data_file_path, &innobase_data_file_path, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"innodb_data_home_dir", OPT_INNODB_DATA_HOME_DIR, "The common part for InnoDB table spaces.", &innobase_data_home_dir, &innobase_data_home_dir, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"innodb_doublewrite", OPT_INNODB_DOUBLEWRITE, "Enable InnoDB doublewrite buffer (enabled by default). \ -Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, - (G_PTR*) &innobase_use_doublewrite, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, + {"innodb_doublewrite", OPT_INNODB_DOUBLEWRITE, + "Enable InnoDB doublewrite buffer during --prepare.", + (G_PTR*) &innobase_use_doublewrite, + (G_PTR*) &innobase_use_doublewrite, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"innodb_io_capacity", OPT_INNODB_IO_CAPACITY, "Number of IOPs the server can do. Tunes the background IO rate", (G_PTR*) &srv_io_capacity, (G_PTR*) &srv_io_capacity, @@ -1013,26 +966,19 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, (G_PTR*) &innobase_unix_file_flush_method, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, -/* ####### Should we use this option? ####### */ - {"innodb_force_recovery", OPT_INNODB_FORCE_RECOVERY, - "Helps to save your data in case the disk image of the database becomes corrupt.", - (G_PTR*) &innobase_force_recovery, (G_PTR*) &innobase_force_recovery, 0, - GET_LONG, REQUIRED_ARG, 0, 0, 6, 0, 1, 0}, - {"innodb_log_buffer_size", OPT_INNODB_LOG_BUFFER_SIZE, "The size of the buffer which InnoDB uses to write log to the log files on disk.", (G_PTR*) &innobase_log_buffer_size, (G_PTR*) &innobase_log_buffer_size, 0, GET_LONG, REQUIRED_ARG, 1024*1024L, 256*1024L, LONG_MAX, 0, 1024, 0}, {"innodb_log_file_size", OPT_INNODB_LOG_FILE_SIZE, - "Size of each log file in a log group.", - (G_PTR*) &innobase_log_file_size, (G_PTR*) &innobase_log_file_size, 0, - GET_LL, REQUIRED_ARG, 48*1024*1024L, 1*1024*1024L, LONGLONG_MAX, 0, - 1024*1024L, 0}, + "Ignored for mysqld option compatibility", + (G_PTR*) &srv_log_file_size, (G_PTR*) &srv_log_file_size, 0, + GET_ULL, REQUIRED_ARG, 48 << 20, 1 << 20, 512ULL << 30, 0, + UNIV_PAGE_SIZE_MAX, 0}, {"innodb_log_files_in_group", OPT_INNODB_LOG_FILES_IN_GROUP, - "Number of log files in the log group. InnoDB writes to the files in a " - "circular fashion. Value 3 is recommended here.", - &innobase_log_files_in_group, &innobase_log_files_in_group, - 0, GET_LONG, REQUIRED_ARG, 2, 2, 100, 0, 1, 0}, + "Ignored for mysqld option compatibility", + &srv_n_log_files, &srv_n_log_files, + 0, GET_LONG, REQUIRED_ARG, 1, 1, 100, 0, 1, 0}, {"innodb_log_group_home_dir", OPT_INNODB_LOG_GROUP_HOME_DIR, "Path to InnoDB log files.", &srv_log_group_home_dir, &srv_log_group_home_dir, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -1054,25 +1000,17 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, /* Use GET_LL to support numeric suffixes in 5.6 */ GET_LL, REQUIRED_ARG, (1LL << 14), (1LL << 12), (1LL << UNIV_PAGE_SIZE_SHIFT_MAX), 0, 1L, 0}, - {"innodb_log_block_size", OPT_INNODB_LOG_BLOCK_SIZE, - "The log block size of the transaction log file. " - "Changing for created log file is not supported. Use on your own risk!", - (G_PTR*) &innobase_log_block_size, (G_PTR*) &innobase_log_block_size, 0, - GET_ULONG, REQUIRED_ARG, 512, 512, 1 << UNIV_PAGE_SIZE_SHIFT_MAX, 0, 1L, 0}, - {"innodb_fast_checksum", OPT_INNODB_FAST_CHECKSUM, - "Change the algorithm of checksum for the whole of datapage to 4-bytes word based.", - (G_PTR*) &innobase_fast_checksum, - (G_PTR*) &innobase_fast_checksum, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"innodb_doublewrite_file", OPT_INNODB_DOUBLEWRITE_FILE, - "Path to special datafile for doublewrite buffer. (default is "": not used)", - (G_PTR*) &innobase_doublewrite_file, (G_PTR*) &innobase_doublewrite_file, - 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"innodb_buffer_pool_filename", OPT_INNODB_BUFFER_POOL_FILENAME, - "Filename to/from which to dump/load the InnoDB buffer pool", + "Ignored for mysqld option compatibility", (G_PTR*) &innobase_buffer_pool_filename, (G_PTR*) &innobase_buffer_pool_filename, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, +#ifndef DBUG_OFF /* unfortunately "debug" collides with existing options */ + {"dbug", '#', "Built in DBUG debugger.", + &dbug_option, &dbug_option, 0, GET_STR, OPT_ARG, + 0, 0, 0, 0, 0, 0}, +#endif #ifndef __WIN__ {"debug-sync", OPT_XTRA_DEBUG_SYNC, "Debug sync point. This is only used by the xtrabackup test suite", @@ -1086,11 +1024,7 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, "INNODB, STRICT_INNODB, NONE, STRICT_NONE]", &srv_checksum_algorithm, &srv_checksum_algorithm, &innodb_checksum_algorithm_typelib, GET_ENUM, REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_INNODB, 0, 0, 0, 0, 0}, - {"innodb_log_checksum_algorithm", OPT_INNODB_LOG_CHECKSUM_ALGORITHM, - "The algorithm InnoDB uses for log checksumming. [CRC32, STRICT_CRC32, " - "INNODB, STRICT_INNODB, NONE, STRICT_NONE]", &srv_log_checksum_algorithm, - &srv_log_checksum_algorithm, &innodb_checksum_algorithm_typelib, GET_ENUM, - REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_INNODB, 0, 0, 0, 0, 0}, + {"innodb_undo_directory", OPT_INNODB_UNDO_DIRECTORY, "Directory where undo tablespace files live, this path can be absolute.", &srv_undo_dir, &srv_undo_dir, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, @@ -1116,7 +1050,12 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, { "innodb-encrypt-log", OPT_INNODB_ENCRYPT_LOG, "encrypton plugin to load", &srv_encrypt_log, &srv_encrypt_log, 0, GET_BOOL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, - + + {"innodb-log-checksums", OPT_INNODB_LOG_CHECKSUMS, + "Whether to require checksums for InnoDB redo log blocks", + &innodb_log_checksums, &innodb_log_checksums, + 0, GET_BOOL, REQUIRED_ARG, 1, 0, 0, 0, 0, 0 }, + {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file " "descriptors to reserve with setrlimit().", (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG, @@ -1270,13 +1209,7 @@ xb_get_one_option(int optid, break; case OPT_INNODB_LOG_FILES_IN_GROUP: - - ADD_PRINT_PARAM_OPT(innobase_log_files_in_group); - break; - case OPT_INNODB_LOG_FILE_SIZE: - - ADD_PRINT_PARAM_OPT(innobase_log_file_size); break; case OPT_INNODB_FLUSH_METHOD: @@ -1289,21 +1222,6 @@ xb_get_one_option(int optid, ADD_PRINT_PARAM_OPT(innobase_page_size); break; - case OPT_INNODB_FAST_CHECKSUM: - - ADD_PRINT_PARAM_OPT(!!innobase_fast_checksum); - break; - - case OPT_INNODB_LOG_BLOCK_SIZE: - - ADD_PRINT_PARAM_OPT(innobase_log_block_size); - break; - - case OPT_INNODB_DOUBLEWRITE_FILE: - - ADD_PRINT_PARAM_OPT(innobase_doublewrite_file); - break; - case OPT_INNODB_UNDO_DIRECTORY: ADD_PRINT_PARAM_OPT(srv_undo_dir); @@ -1321,13 +1239,6 @@ xb_get_one_option(int optid, ADD_PRINT_PARAM_OPT(innodb_checksum_algorithm_names[srv_checksum_algorithm]); break; - case OPT_INNODB_LOG_CHECKSUM_ALGORITHM: - - ut_a(srv_log_checksum_algorithm <= SRV_CHECKSUM_ALGORITHM_STRICT_NONE); - - ADD_PRINT_PARAM_OPT(innodb_checksum_algorithm_names[srv_log_checksum_algorithm]); - break; - case OPT_INNODB_BUFFER_POOL_FILENAME: ADD_PRINT_PARAM_OPT(innobase_buffer_pool_filename); @@ -1400,40 +1311,9 @@ xb_get_one_option(int optid, return 0; } -/*********************************************************************** -Initializes log_block_size */ -static -ibool -xb_init_log_block_size(void) -{ - srv_log_block_size = 0; - if (innobase_log_block_size != 512) { - uint n_shift = (uint)get_bit_shift(innobase_log_block_size);; - - if (n_shift > 0) { - srv_log_block_size = (ulint)(1LL << n_shift); - msg("InnoDB: The log block size is set to %lu.\n", - srv_log_block_size); - } - } else { - srv_log_block_size = 512; - } - if (!srv_log_block_size) { - msg("InnoDB: Error: %lu is not valid value for " - "innodb_log_block_size.\n", innobase_log_block_size); - return FALSE; - } - - return TRUE; -} - static my_bool innodb_init_param(void) { - /* innobase_init */ - static char current_dir[3]; /* Set if using current lib */ - my_bool ret; - char *default_path; srv_is_being_started = TRUE; /* === some variables from mysqld === */ memset((G_PTR) &mysql_tmpdir_list, 0, sizeof(mysql_tmpdir_list)); @@ -1465,10 +1345,6 @@ innodb_init_param(void) srv_page_size = (1 << srv_page_size_shift); } - if (!xb_init_log_block_size()) { - goto error; - } - /* Check that values don't overflow on 32-bit systems. */ if (sizeof(ulint) == 4) { if (xtrabackup_use_memory > UINT_MAX32) { @@ -1482,30 +1358,10 @@ innodb_init_param(void) goto error; } - - if (innobase_log_file_size > UINT_MAX32) { - msg("xtrabackup: innobase_log_file_size can't be " - "over 4GB on 32-bit systemsi\n"); - - goto error; - } } - os_innodb_umask = (ulint)0664; - - /* First calculate the default path for innodb_data_home_dir etc., - in case the user has not given any value. - - Note that when using the embedded server, the datadirectory is not - necessarily the current directory of this program. */ - - /* It's better to use current lib, to keep paths short */ - current_dir[0] = FN_CURLIB; - current_dir[1] = FN_LIBCHAR; - current_dir[2] = 0; - default_path = current_dir; - - ut_a(default_path); + static char default_path[2] = { FN_CURLIB, 0 }; + fil_path_to_mysql_datadir = default_path; /* Set InnoDB initialization parameters according to the values read from MySQL .cnf file */ @@ -1535,37 +1391,20 @@ innodb_init_param(void) msg("xtrabackup: innodb_data_file_path = %s\n", innobase_data_file_path); - /* Since InnoDB edits the argument in the next call, we make another - copy of it: */ + /* This is the first time univ_page_size is used. + It was initialized to 16k pages before srv_page_size was set */ + univ_page_size.copy_from( + page_size_t(srv_page_size, srv_page_size, false)); - internal_innobase_data_file_path = strdup(innobase_data_file_path); + srv_sys_space.set_space_id(TRX_SYS_SPACE); + srv_sys_space.set_name("innodb_system"); + srv_sys_space.set_path(srv_data_home); + srv_sys_space.set_flags(FSP_FLAGS_PAGE_SSIZE()); - ret = (my_bool) srv_parse_data_file_paths_and_sizes( - internal_innobase_data_file_path); - if (ret == FALSE) { - msg("xtrabackup: syntax error in innodb_data_file_path\n"); -mem_free_and_error: - free(internal_innobase_data_file_path); - internal_innobase_data_file_path = NULL; + if (!srv_sys_space.parse_params(innobase_data_file_path, true)) { goto error; } - if (xtrabackup_prepare) { - /* "--prepare" needs filenames only */ - ulint i; - - for (i=0; i < srv_n_data_files; i++) { - char *p; - - p = srv_data_file_names[i]; - while ((p = strchr(p, SRV_PATH_SEPARATOR)) != NULL) - { - p++; - srv_data_file_names[i] = p; - } - } - } - /* -------------- Log files ---------------------------*/ /* The default dir for log files is the datadir of MySQL */ @@ -1579,60 +1418,38 @@ mem_free_and_error: msg("xtrabackup: innodb_log_group_home_dir = %s\n", srv_log_group_home_dir); - srv_normalize_path_for_win(srv_log_group_home_dir); + os_normalize_path(srv_log_group_home_dir); if (strchr(srv_log_group_home_dir, ';')) { msg("syntax error in innodb_log_group_home_dir, "); - - goto mem_free_and_error; + goto error; } srv_adaptive_flushing = FALSE; - srv_use_sys_malloc = TRUE; srv_file_format = 1; /* Barracuda */ srv_max_file_format_at_startup = UNIV_FORMAT_MIN; /* on */ /* --------------------------------------------------*/ srv_file_flush_method_str = innobase_unix_file_flush_method; - srv_n_log_files = (ulint) innobase_log_files_in_group; - srv_log_file_size = (ulint) innobase_log_file_size; - msg("xtrabackup: innodb_log_files_in_group = %ld\n", - srv_n_log_files); - msg("xtrabackup: innodb_log_file_size = %lld\n", - (long long int) srv_log_file_size); - srv_log_buffer_size = (ulint) innobase_log_buffer_size; /* We set srv_pool_size here in units of 1 kB. InnoDB internally changes the value so that it becomes the number of database pages. */ - //srv_buf_pool_size = (ulint) innobase_buffer_pool_size; srv_buf_pool_size = (ulint) xtrabackup_use_memory; - - srv_mem_pool_size = (ulint) innobase_additional_mem_pool_size; + srv_buf_pool_chunk_unit = srv_buf_pool_size; + srv_buf_pool_instances = 1; srv_n_file_io_threads = (ulint) innobase_file_io_threads; srv_n_read_io_threads = (ulint) innobase_read_io_threads; srv_n_write_io_threads = (ulint) innobase_write_io_threads; - srv_force_recovery = (ulint) innobase_force_recovery; - srv_use_doublewrite_buf = (ibool) innobase_use_doublewrite; - if (!innobase_use_checksums) { - - srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_NONE; - } - - btr_search_enabled = (char) innobase_adaptive_hash_index; - btr_search_index_num = 1; - os_use_large_pages = (ibool) innobase_use_large_pages; os_large_page_size = (ulint) innobase_large_page_size; - static char default_dir[3] = "./"; - srv_arch_dir = default_dir; row_rollback_on_timeout = (ibool) innobase_rollback_on_timeout; srv_file_per_table = (my_bool) innobase_file_per_table; @@ -1650,18 +1467,8 @@ mem_free_and_error: /* We cannot treat characterset here for now!! */ data_mysql_default_charset_coll = (ulint)default_charset_info->number; - ut_a(DATA_MYSQL_LATIN1_SWEDISH_CHARSET_COLL == - my_charset_latin1.number); ut_a(DATA_MYSQL_BINARY_CHARSET_COLL == my_charset_bin.number); - /* Store the latin1_swedish_ci character ordering table to InnoDB. For - non-latin1_swedish_ci charsets we use the MySQL comparison functions, - and consequently we do not need to know the ordering internally in - InnoDB. */ - - ut_a(0 == strcmp(my_charset_latin1.name, "latin1_swedish_ci")); - srv_latin1_ordering = my_charset_latin1.sort_order; - //innobase_commit_concurrency_init_default(); /* Since we in this module access directly the fields of a trx @@ -1675,31 +1482,8 @@ mem_free_and_error: innobase_start_or_create_for_mysql(). As we don't call it in xtrabackup, we have to duplicate checks from that function here. */ -#ifdef __WIN__ - switch (os_get_os_version()) { - case OS_WIN95: - case OS_WIN31: - case OS_WINNT: - /* On Win 95, 98, ME, Win32 subsystem for Windows 3.1, - and NT use simulated aio. In NT Windows provides async i/o, - but when run in conjunction with InnoDB Hot Backup, it seemed - to corrupt the data files. */ - - srv_use_native_aio = FALSE; - break; - - case OS_WIN2000: - case OS_WINXP: - /* On 2000 and XP, async IO is available. */ - srv_use_native_aio = TRUE; - break; - - default: - /* Vista and later have both async IO and condition variables */ - srv_use_native_aio = TRUE; - srv_use_native_conditions = TRUE; - break; - } +#ifdef _WIN32 + srv_use_native_aio = TRUE; #elif defined(LINUX_NATIVE_AIO) @@ -1726,7 +1510,9 @@ mem_free_and_error: srv_undo_dir = my_strdup(".", MYF(MY_FAE)); } - innodb_log_checksum_func_update(srv_log_checksum_algorithm); + log_checksum_algorithm_ptr = innodb_log_checksums || srv_encrypt_log + ? log_block_calc_checksum_crc32 + : log_block_calc_checksum_none; return(FALSE); @@ -1735,57 +1521,17 @@ error: return(TRUE); } -static my_bool -innodb_init(void) +static bool innodb_init() { - int err; - srv_is_being_started = TRUE; - err = innobase_start_or_create_for_mysql(); - + dberr_t err = innobase_start_or_create_for_mysql(); if (err != DB_SUCCESS) { - free(internal_innobase_data_file_path); - internal_innobase_data_file_path = NULL; - goto error; + msg("xtrabackup: innodb_init() returned %d (%s).\n", + err, ut_strerr(err)); + innodb_shutdown(); + return(TRUE); } - /* They may not be needed for now */ -// (void) hash_init(&innobase_open_tables,system_charset_info, 32, 0, 0, -// (hash_get_key) innobase_get_key, 0, 0); -// pthread_mutex_init(&innobase_share_mutex, MY_MUTEX_INIT_FAST); -// pthread_mutex_init(&prepare_commit_mutex, MY_MUTEX_INIT_FAST); -// pthread_mutex_init(&commit_threads_m, MY_MUTEX_INIT_FAST); -// pthread_mutex_init(&commit_cond_m, MY_MUTEX_INIT_FAST); -// pthread_cond_init(&commit_cond, NULL); - - innodb_inited= 1; - return(FALSE); - -error: - msg("xtrabackup: innodb_init(): Error occured.\n"); - return(TRUE); -} - -static void -innodb_end() -{ - srv_fast_shutdown = (ulint) innobase_fast_shutdown; - innodb_inited = 0; - - msg("xtrabackup: starting shutdown with innodb_fast_shutdown = %lu\n", - srv_fast_shutdown); - - innodb_shutdown(); - free(internal_innobase_data_file_path); - internal_innobase_data_file_path = NULL; - - /* They may not be needed for now */ -// hash_free(&innobase_open_tables); -// pthread_mutex_destroy(&innobase_share_mutex); -// pthread_mutex_destroy(&prepare_commit_mutex); -// pthread_mutex_destroy(&commit_threads_m); -// pthread_mutex_destroy(&commit_cond_m); -// pthread_cond_destroy(&commit_cond); } /* ================= common ================= */ @@ -1940,8 +1686,7 @@ xb_read_delta_metadata(const char *filepath, xb_delta_info_t *info) my_bool r = TRUE; /* set defaults */ - info->page_size = ULINT_UNDEFINED; - info->zip_size = ULINT_UNDEFINED; + ulint page_size = ULINT_UNDEFINED, zip_size = 0; info->space_id = ULINT_UNDEFINED; fp = fopen(filepath, "r"); @@ -1953,9 +1698,9 @@ xb_read_delta_metadata(const char *filepath, xb_delta_info_t *info) while (!feof(fp)) { if (fscanf(fp, "%50s = %50s\n", key, value) == 2) { if (strcmp(key, "page_size") == 0) { - info->page_size = strtoul(value, NULL, 10); + page_size = strtoul(value, NULL, 10); } else if (strcmp(key, "zip_size") == 0) { - info->zip_size = strtoul(value, NULL, 10); + zip_size = strtoul(value, NULL, 10); } else if (strcmp(key, "space_id") == 0) { info->space_id = strtoul(value, NULL, 10); } @@ -1964,10 +1709,14 @@ xb_read_delta_metadata(const char *filepath, xb_delta_info_t *info) fclose(fp); - if (info->page_size == ULINT_UNDEFINED) { + if (page_size == ULINT_UNDEFINED) { msg("xtrabackup: page_size is required in %s\n", filepath); r = FALSE; + } else { + info->page_size = page_size_t(zip_size ? zip_size : page_size, + page_size, zip_size != 0); } + if (info->space_id == ULINT_UNDEFINED) { msg("xtrabackup: Warning: This backup was taken with XtraBackup 2.0.1 " "or earlier, some DDL operations between full and incremental " @@ -1993,7 +1742,10 @@ xb_write_delta_metadata(const char *filename, const xb_delta_info_t *info) "page_size = %lu\n" "zip_size = %lu\n" "space_id = %lu\n", - info->page_size, info->zip_size, info->space_id); + info->page_size.logical(), + info->page_size.is_compressed() + ? info->page_size.physical() : 0, + info->space_id); len = strlen(buf); mystat.st_size = len; @@ -2150,7 +1902,7 @@ check_if_skip_database_by_path( return(FALSE); } - const char* db_name = strrchr(path, SRV_PATH_SEPARATOR); + const char* db_name = strrchr(path, OS_PATH_SEPARATOR); if (db_name == NULL) { db_name = path; } else { @@ -2258,33 +2010,26 @@ check_if_skip_table( return(FALSE); } -/*********************************************************************** -Reads the space flags from a given data file and returns the compressed -page size, or 0 if the space is not compressed. */ -ulint -xb_get_zip_size(pfs_os_file_t file) +/** @return the tablespace flags from a given data file +@retval ULINT_UNDEFINED if the file is not readable */ +ulint xb_get_space_flags(pfs_os_file_t file) { byte *buf; byte *page; - ulint zip_size = ULINT_UNDEFINED; - ibool success; - ulint space; + ulint flags; - buf = static_cast<byte *>(ut_malloc(2 * UNIV_PAGE_SIZE)); + buf = static_cast<byte *>(malloc(2 * UNIV_PAGE_SIZE)); page = static_cast<byte *>(ut_align(buf, UNIV_PAGE_SIZE)); - success = os_file_read(file, page, 0, UNIV_PAGE_SIZE); - if (!success) { - goto end; + if (os_file_read(IORequestRead, file, page, 0, UNIV_PAGE_SIZE)) { + flags = fsp_header_get_flags(page); + } else { + flags = ULINT_UNDEFINED; } - space = mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - zip_size = (space == 0 ) ? 0 : - dict_tf_get_zip_size(fsp_header_get_flags(page)); -end: - ut_free(buf); + free(buf); - return(zip_size); + return(flags); } const char* @@ -2323,7 +2068,6 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n) xb_write_filt_ctxt_t write_filt_ctxt; const char *action; xb_read_filt_t *read_filter; - ibool is_system; my_bool rc = FALSE; /* Get the name and the path for the tablespace. node->name always @@ -2338,9 +2082,8 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n) const char* const node_name = node->space->name; const char* const node_path = node->name; - is_system = !fil_is_user_tablespace_id(node->space->id); - - if (!is_system && check_if_skip_table(node_name)) { + if (fil_is_user_tablespace_id(node->space->id) + && check_if_skip_table(node_name)) { msg("[%02u] Skipping %s.\n", thread_n, node_name); return(FALSE); } @@ -2449,50 +2192,29 @@ skip: return(FALSE); } -extern ibool log_block_checksum_is_ok_or_old_format(const byte* block); - -/*******************************************************//** -Scans log from a buffer and writes new log data to the outpud datasinc. -@return true if success */ +/** Copy redo log blocks to the data sink. +@param[in] is_last whether this is the last log segment to copy +@param[in] start_lsn buffer start LSN +@param[in] end_lsn buffer end LSN +@return last copied LSN +@retval 0 on failure */ static -bool -xtrabackup_scan_log_recs( -/*===============*/ - log_group_t* group, /*!< in: log group */ - bool is_last, /*!< in: whether it is last segment - to copy */ - lsn_t start_lsn, /*!< in: buffer start lsn */ - lsn_t* contiguous_lsn, /*!< in/out: it is known that all log - groups contain contiguous log data up - to this lsn */ - lsn_t* group_scanned_lsn,/*!< out: scanning succeeded up to - this lsn */ - bool* finished) /*!< out: false if is not able to scan - any more in this log group */ +lsn_t +xtrabackup_copy_log(bool is_last, lsn_t start_lsn, lsn_t end_lsn) { - lsn_t scanned_lsn; - ulint data_len; - ulint write_size; - const byte* log_block; - - ulint scanned_checkpoint_no = 0; + lsn_t scanned_lsn = start_lsn; - *finished = false; - scanned_lsn = start_lsn; - log_block = log_sys->buf; + const ulint blocks_in_group = log_block_convert_lsn_to_no( + log_sys->log.capacity()) - 1; + const byte* log_block = log_sys->buf; - while (log_block < log_sys->buf + RECV_SCAN_SIZE && !*finished) { + for (ulint scanned_checkpoint = 0; + scanned_lsn < end_lsn; + log_block += OS_FILE_LOG_BLOCK_SIZE) { ulint no = log_block_get_hdr_no(log_block); ulint scanned_no = log_block_convert_lsn_to_no(scanned_lsn); - ibool checksum_is_ok = - log_block_checksum_is_ok_or_old_format(log_block); - - if (no != scanned_no && checksum_is_ok) { - ulint blocks_in_group; - - blocks_in_group = log_block_convert_lsn_to_no( - log_group_get_capacity(group)) - 1; + if (no != scanned_no) { if ((no < scanned_no && ((scanned_no - no) % blocks_in_group) == 0) || no == 0 || @@ -2501,15 +2223,15 @@ xtrabackup_scan_log_recs( blocks_in_group == 0) { /* old log block, do nothing */ - *finished = true; break; } msg("xtrabackup: error:" " log block numbers mismatch:\n" - "xtrabackup: error: expected log block no. %lu," - " but got no. %lu from the log file.\n", - (ulong) scanned_no, (ulong) no); + "xtrabackup: error: expected log block no. " + ULINTPF ", but got no. " ULINTPF + " from the log file.\n", + scanned_no, no); if ((no - scanned_no) % blocks_in_group == 0) { msg("xtrabackup: error:" @@ -2520,155 +2242,115 @@ xtrabackup_scan_log_recs( " log files being too small.\n"); } - return(false); - } else if (!checksum_is_ok) { - /* Garbage or an incompletely written log block */ - - msg("xtrabackup: warning: Log block checksum mismatch" - " (block no %lu at lsn " LSN_PF "): \n" - "expected %lu, calculated checksum %lu\n", - (ulong) no, - scanned_lsn, - (ulong) log_block_get_checksum(log_block), - (ulong) log_block_calc_checksum(log_block)); - msg("xtrabackup: warning: this is possible when the " - "log block has not been fully written by the " - "server, will retry later.\n"); - *finished = true; - break; - } - - if (log_block_get_flush_bit(log_block)) { - /* This block was a start of a log flush operation: - we know that the previous flush operation must have - been completed for all log groups before this block - can have been flushed to any of the groups. Therefore, - we know that log data is contiguous up to scanned_lsn - in all non-corrupt log groups. */ - - if (scanned_lsn > *contiguous_lsn) { - - *contiguous_lsn = scanned_lsn; - } + return(0); } - data_len = log_block_get_data_len(log_block); - - if ( - (scanned_checkpoint_no > 0) - && (log_block_get_checkpoint_no(log_block) - < scanned_checkpoint_no) - && (scanned_checkpoint_no - - log_block_get_checkpoint_no(log_block) - > 0x80000000UL)) { + ulint checkpoint = log_block_get_checkpoint_no(log_block); + if (scanned_checkpoint > checkpoint + && scanned_checkpoint - checkpoint >= 0x80000000UL) { /* Garbage from a log buffer flush which was made before the most recent database recovery */ - - *finished = true; break; } - scanned_lsn = scanned_lsn + data_len; - scanned_checkpoint_no = log_block_get_checkpoint_no(log_block); - - if (data_len < OS_FILE_LOG_BLOCK_SIZE) { - /* Log data for this group ends here */ + scanned_checkpoint = checkpoint; + ulint data_len = log_block_get_data_len(log_block); + scanned_lsn += data_len; - *finished = true; - } else { - log_block += OS_FILE_LOG_BLOCK_SIZE; + if (data_len != OS_FILE_LOG_BLOCK_SIZE) { + break; } } - *group_scanned_lsn = scanned_lsn; - - /* ===== write log to 'xtrabackup_logfile' ====== */ - if (!*finished) { - write_size = RECV_SCAN_SIZE; - } else { - write_size = (ulint)(ut_uint64_align_up(scanned_lsn, - OS_FILE_LOG_BLOCK_SIZE) - start_lsn); - if (!is_last && scanned_lsn % OS_FILE_LOG_BLOCK_SIZE) { - write_size -= OS_FILE_LOG_BLOCK_SIZE; - } + if (!is_last && scanned_lsn & (OS_FILE_LOG_BLOCK_SIZE - 1)) { + /* Omit the partial last block. */ + scanned_lsn &= ~(OS_FILE_LOG_BLOCK_SIZE - 1); } - if (write_size == 0) { - return(true); - } + log_sys->log.scanned_lsn = scanned_lsn; - if (srv_encrypt_log) { - log_encrypt_before_write(scanned_checkpoint_no, - log_sys->buf, write_size); - } + if (ulint write_size = ulint(ut_uint64_align_up(scanned_lsn, + OS_FILE_LOG_BLOCK_SIZE) + - start_lsn)) { + if (srv_encrypt_log) { + log_crypt(log_sys->buf, write_size); + } - if (ds_write(dst_log_file, log_sys->buf, write_size)) { - msg("xtrabackup: Error: " - "write to logfile failed\n"); - return(false); + if (ds_write(dst_log_file, log_sys->buf, write_size)) { + msg("xtrabackup: Error: " + "write to logfile failed\n"); + return(0); + } } - return(true); + return(scanned_lsn); } -static my_bool -xtrabackup_copy_logfile(lsn_t from_lsn, my_bool is_last) +/** Copy redo log until the current end of the log is reached +@param is_last whether this is the last run at the end of backup +@return whether the operation failed */ +static bool +xtrabackup_copy_logfile(bool is_last) { - /* definition from recv_recovery_from_checkpoint_start() */ - lsn_t contiguous_lsn; - ut_a(dst_log_file != NULL); + ut_ad(recv_sys != NULL); - /* read from checkpoint_lsn_start to current */ - contiguous_lsn = ut_uint64_align_down(from_lsn, OS_FILE_LOG_BLOCK_SIZE); - - /* TODO: We must check the contiguous_lsn still exists in log file.. */ - - bool finished; lsn_t start_lsn; lsn_t end_lsn; - /* reference recv_group_scan_log_recs() */ - - start_lsn = contiguous_lsn; + start_lsn = ut_uint64_align_down(log_copy_scanned_lsn, + OS_FILE_LOG_BLOCK_SIZE); - do { + /* When copying the last part of the log, retry a few times to + ensure that all log up to the last checkpoint will be read. */ + for (unsigned retries = is_last ? 3 : 0;;) { end_lsn = start_lsn + RECV_SCAN_SIZE; xtrabackup_io_throttling(); log_mutex_enter(); - log_group_read_log_seg(LOG_RECOVER, log_sys->buf, - &log_sys->log, start_lsn, end_lsn); + lsn_t lsn = log_group_read_log_seg(log_sys->buf, &log_sys->log, + start_lsn, end_lsn); - bool success = xtrabackup_scan_log_recs( - &log_sys->log, is_last, - start_lsn, &contiguous_lsn, - &log_sys->log.scanned_lsn, - &finished); + start_lsn = xtrabackup_copy_log(is_last, start_lsn, lsn); log_mutex_exit(); - if (!success) { + if (!start_lsn) { ds_close(dst_log_file); + dst_log_file = NULL; msg("xtrabackup: Error: xtrabackup_copy_logfile()" " failed.\n"); - return(TRUE); + return(true); } - start_lsn = end_lsn; - } while (!finished); + if (start_lsn == end_lsn) { + /* All RECV_SCAN_SIZE bytes that were read + were valid redo log. Proceed to read more. */ + } else if (start_lsn & (OS_FILE_LOG_BLOCK_SIZE - 1)) { + /* The end of the log was encountered. */ + ut_ad(is_last); + break; + } else if (retries--) { + /* Retry copying the last log segment. */ + ut_ad(is_last); + os_thread_sleep(xtrabackup_log_copy_interval * 1000); + } else { + break; + } + } - msg_ts(">> log scanned up to (" LSN_PF ")\n", - log_sys->log.scanned_lsn); + ut_ad(start_lsn == log_sys->log.scanned_lsn); + + msg_ts(">> log scanned up to (" LSN_PF ")\n", start_lsn); /* update global variable*/ - log_copy_scanned_lsn = log_sys->log.scanned_lsn; + log_copy_scanned_lsn = start_lsn; debug_sync_point("xtrabackup_copy_logfile_pause"); - return(FALSE); + return(false); } static @@ -2677,8 +2359,7 @@ void* #else ulint #endif -log_copying_thread( - void* arg __attribute__((unused))) +log_copying_thread(void*) { /* Initialize mysys thread-specific memory so we can @@ -2686,31 +2367,17 @@ log_copying_thread( */ my_thread_init(); - ut_a(dst_log_file != NULL); - - log_copying_running = TRUE; - - while(log_copying) { + while (log_copying) { os_event_reset(log_copying_stop); os_event_wait_time_low(log_copying_stop, xtrabackup_log_copy_interval * 1000ULL, 0); - if (log_copying) { - if(xtrabackup_copy_logfile(log_copy_scanned_lsn, - FALSE)) { - - exit(EXIT_FAILURE); - } + if (log_copying && xtrabackup_copy_logfile(false)) { + break; } } - /* last copying */ - if(xtrabackup_copy_logfile(log_copy_scanned_lsn, TRUE)) { - - exit(EXIT_FAILURE); - } - - log_copying_running = FALSE; + log_copying_running = false; my_thread_end(); os_thread_exit(NULL); @@ -2724,15 +2391,11 @@ void* #else ulint #endif -io_watching_thread( - void* arg) +io_watching_thread(void*) { - (void)arg; /* currently, for --backup only */ ut_a(xtrabackup_backup); - io_watching_thread_running = TRUE; - while (log_copying) { os_thread_sleep(1000000); /*1 sec*/ io_ticket = xtrabackup_throttle; @@ -2743,47 +2406,11 @@ io_watching_thread( xtrabackup_throttle = 0; os_event_set(wait_throttle); - io_watching_thread_running = FALSE; - - os_thread_exit(NULL); - - return(0); -} - -/************************************************************************ -I/o-handler thread function. */ -static - -#ifndef __WIN__ -void* -#else -ulint -#endif -io_handler_thread( -/*==============*/ - void* arg) -{ - ulint segment; - - - segment = *((ulint*)arg); - - while (srv_shutdown_state != SRV_SHUTDOWN_EXIT_THREADS) { - fil_aio_wait(segment); - } - - /* We count the number of threads in os_thread_exit(). A created - thread should always use that to exit and not use return() to exit. - The thread actually never comes here because it is exited in an - os_event_wait(). */ + io_watching_thread_running = false; os_thread_exit(NULL); -#ifndef __WIN__ - return(NULL); /* Not reached */ -#else return(0); -#endif } /************************************************************************** @@ -2816,9 +2443,9 @@ data_copy_thread_func( } } - os_mutex_enter(ctxt->count_mutex); + pthread_mutex_lock(&ctxt->count_mutex); (*ctxt->count)--; - os_mutex_exit(ctxt->count_mutex); + pthread_mutex_unlock(&ctxt->count_mutex); my_thread_end(); os_thread_exit(NULL); @@ -2831,7 +2458,7 @@ Initialize the appropriate datasink(s). Both local backups and streaming in the Otherwise (i.e. when streaming in the 'tar' format) we need 2 separate datasinks for the data stream (and don't allow parallel data copying) and for metainfo -files (including xtrabackup_logfile). The second datasink writes to temporary +files (including ib_logfile0). The second datasink writes to temporary files first, and then streams them in a serialized way when closed. */ static void xtrabackup_init_datasinks(void) @@ -2917,41 +2544,261 @@ static void xtrabackup_destroy_datasinks(void) ds_redo = NULL; } -#define SRV_N_PENDING_IOS_PER_THREAD OS_AIO_N_PENDING_IOS_PER_THREAD #define SRV_MAX_N_PENDING_SYNC_IOS 100 -/************************************************************************ -@return TRUE if table should be opened. */ +/** Initialize the tablespace cache subsystem. */ static -ibool -xb_check_if_open_tablespace( - const char* db, - const char* table) +void +xb_fil_io_init() { - char buf[FN_REFLEN]; - - snprintf(buf, sizeof(buf), "%s/%s", db, table); + fil_init(srv_file_per_table ? 50000 : 5000, LONG_MAX); + fsp_init(); +} - return !check_if_skip_table(buf); +static +Datafile* +xb_new_datafile(const char *name, bool is_remote) +{ + if (is_remote) { + RemoteDatafile *remote_file = new RemoteDatafile(); + remote_file->set_name(name); + return(remote_file); + } else { + Datafile *file = new Datafile(); + file->set_name(name); + file->make_filepath(".", name, IBD); + return(file); + } } -/************************************************************************ -Initializes the I/O and tablespace cache subsystems. */ static void -xb_fil_io_init(void) -/*================*/ +xb_load_single_table_tablespace( + const char *dirname, + const char *filname, + bool is_remote) { - srv_n_file_io_threads = srv_n_read_io_threads; + /* Ignore .isl files on XtraBackup recovery. All tablespaces must be + local. */ + if (is_remote && srv_operation == SRV_OPERATION_RESTORE) { + return; + } + if (check_if_skip_table(filname)) { + return; + } - os_aio_init(8 * SRV_N_PENDING_IOS_PER_THREAD, - srv_n_read_io_threads, - srv_n_write_io_threads, - SRV_MAX_N_PENDING_SYNC_IOS); + /* The name ends in .ibd or .isl; + try opening the file */ + char* name; + size_t dirlen = dirname == NULL ? 0 : strlen(dirname); + size_t namelen = strlen(filname); + ulint pathlen = dirname == NULL ? namelen + 1: dirlen + namelen + 2; + lsn_t flush_lsn; + dberr_t err; + fil_space_t *space; - fil_init(srv_file_per_table ? 50000 : 5000, LONG_MAX); + name = static_cast<char*>(ut_malloc_nokey(pathlen)); - fsp_init(); + if (dirname != NULL) { + ut_snprintf(name, pathlen, "%s/%s", dirname, filname); + name[pathlen - 5] = 0; + } else { + ut_snprintf(name, pathlen, "%s", filname); + name[pathlen - 5] = 0; + } + + Datafile *file = xb_new_datafile(name, is_remote); + + if (file->open_read_only(true) != DB_SUCCESS) { + ut_free(name); + exit(EXIT_FAILURE); + } + + err = file->validate_first_page(&flush_lsn); + + if (err == DB_SUCCESS && file->space_id() != SRV_TMP_SPACE_ID) { + os_offset_t node_size = os_file_get_size(file->handle()); + os_offset_t n_pages; + + ut_a(node_size != (os_offset_t) -1); + + n_pages = node_size / page_size_t(file->flags()).physical(); + + space = fil_space_create( + name, file->space_id(), file->flags(), + FIL_TYPE_TABLESPACE, NULL/* TODO: crypt_data */); + + ut_a(space != NULL); + + if (!fil_node_create(file->filepath(), n_pages, space, + false, false)) { + ut_error; + } + + /* by opening the tablespace we forcing node and space objects + in the cache to be populated with fields from space header */ + fil_space_open(space->name); + + if (srv_operation == SRV_OPERATION_RESTORE || xb_close_files) { + fil_space_close(space->name); + } + } + + ut_free(name); + + if (fil_space_crypt_t* crypt_info = file->get_crypt_info()) { + fil_space_destroy_crypt_data(&crypt_info); + } + + delete file; + + if (err != DB_SUCCESS && err != DB_CORRUPTION && xtrabackup_backup) { + /* allow corrupted first page for xtrabackup, it could be just + zero-filled page, which we restore from redo log later */ + exit(EXIT_FAILURE); + } +} + +/** Scan the database directories under the MySQL datadir, looking for +.ibd files and determining the space id in each of them. +@return DB_SUCCESS or error number */ +static +dberr_t +xb_load_single_table_tablespaces() +{ + int ret; + char* dbpath = NULL; + ulint dbpath_len = 100; + os_file_dir_t dir; + os_file_dir_t dbdir; + os_file_stat_t dbinfo; + os_file_stat_t fileinfo; + dberr_t err = DB_SUCCESS; + + /* The datadir of MySQL is always the default directory of mysqld */ + + dir = os_file_opendir(fil_path_to_mysql_datadir, true); + + if (dir == NULL) { + + return(DB_ERROR); + } + + dbpath = static_cast<char*>(ut_malloc_nokey(dbpath_len)); + + /* Scan all directories under the datadir. They are the database + directories of MySQL. */ + + ret = fil_file_readdir_next_file(&err, fil_path_to_mysql_datadir, dir, + &dbinfo); + while (ret == 0) { + size_t len = strlen(dbinfo.name); + + /* General tablespaces are always at the first level of the + data home dir */ + if (dbinfo.type == OS_FILE_TYPE_FILE && len > 4) { + bool is_isl = !strcmp(dbinfo.name + len - 4, ".isl"); + bool is_ibd = !is_isl + && !strcmp(dbinfo.name + len - 4, ".ibd"); + + if (is_isl || is_ibd) { + xb_load_single_table_tablespace( + NULL, dbinfo.name, is_isl); + } + } + + if (dbinfo.type == OS_FILE_TYPE_FILE + || dbinfo.type == OS_FILE_TYPE_UNKNOWN) { + + goto next_datadir_item; + } + + /* We found a symlink or a directory; try opening it to see + if a symlink is a directory */ + + len = strlen(fil_path_to_mysql_datadir) + + strlen (dbinfo.name) + 2; + if (len > dbpath_len) { + dbpath_len = len; + + if (dbpath) { + ut_free(dbpath); + } + + dbpath = static_cast<char*>(ut_malloc_nokey(dbpath_len)); + } + ut_snprintf(dbpath, dbpath_len, + "%s/%s", fil_path_to_mysql_datadir, dbinfo.name); + os_normalize_path(dbpath); + + if (check_if_skip_database_by_path(dbpath)) { + fprintf(stderr, "Skipping db: %s\n", dbpath); + goto next_datadir_item; + } + + /* We want wrong directory permissions to be a fatal error for + XtraBackup. */ + dbdir = os_file_opendir(dbpath, true); + + if (dbdir != NULL) { + + /* We found a database directory; loop through it, + looking for possible .ibd files in it */ + + for (ret = fil_file_readdir_next_file(&err, dbpath, + dbdir, + &fileinfo); + ret == 0; + ret = fil_file_readdir_next_file(&err, dbpath, + dbdir, + &fileinfo)) { + if (fileinfo.type == OS_FILE_TYPE_DIR) { + continue; + } + + size_t len = strlen(fileinfo.name); + + /* We found a symlink or a file */ + if (len > 4 + && !strcmp(fileinfo.name + len - 4, + ".ibd")) { + xb_load_single_table_tablespace( + dbinfo.name, fileinfo.name, + false); + } + } + + if (0 != os_file_closedir(dbdir)) { + fprintf(stderr, "InnoDB: Warning: could not" + " close database directory %s\n", + dbpath); + + err = DB_ERROR; + } + + } else { + + err = DB_ERROR; + break; + + } + +next_datadir_item: + ret = fil_file_readdir_next_file(&err, + fil_path_to_mysql_datadir, + dir, &dbinfo); + } + + ut_free(dbpath); + + if (0 != os_file_closedir(dir)) { + fprintf(stderr, + "InnoDB: Error: could not close MySQL datadir\n"); + + return(DB_ERROR); + } + + return(err); } /**************************************************************************** @@ -2961,25 +2808,26 @@ static dberr_t xb_load_tablespaces() { - ulint i; bool create_new_db; dberr_t err; ulint sum_of_new_sizes; - lsn_t min_arch_logno, max_arch_logno; + lsn_t flush_lsn; - for (i = 0; i < srv_n_file_io_threads; i++) { - thread_nr[i] = i; + ut_ad(srv_operation == SRV_OPERATION_BACKUP + || srv_operation == SRV_OPERATION_RESTORE); - os_thread_create(io_handler_thread, thread_nr + i, - thread_ids + i); - } + err = srv_sys_space.check_file_spec(&create_new_db, 0); - os_thread_sleep(200000); /*0.2 sec*/ + /* create_new_db must not be true. */ + if (err != DB_SUCCESS || create_new_db) { + msg("xtrabackup: could not find data files at the " + "specified datadir\n"); + return(DB_ERROR); + } + + err = srv_sys_space.open_or_create(false, false, &sum_of_new_sizes, + &flush_lsn); - err = open_or_create_data_files(&create_new_db, - &min_arch_logno, &max_arch_logno, - &flushed_lsn, - &sum_of_new_sizes); if (err != DB_SUCCESS) { msg("xtrabackup: Could not open or create data files.\n" "xtrabackup: If you tried to add new data files, and it " @@ -2997,30 +2845,21 @@ xb_load_tablespaces() return(err); } - /* create_new_db must not be TRUE.. */ - if (create_new_db) { - msg("xtrabackup: could not find data files at the " - "specified datadir\n"); - return(DB_ERROR); - } - /* Add separate undo tablespaces to fil_system */ - err = srv_undo_tablespaces_init(FALSE, - TRUE, - srv_undo_tablespaces, - &srv_undo_tablespaces_open); + err = srv_undo_tablespaces_init(false); + if (err != DB_SUCCESS) { return(err); } - /* It is important to call fil_load_single_table_tablespace() after + /* It is important to call xb_load_single_table_tablespaces() after srv_undo_tablespaces_init(), because fil_is_user_tablespace_id() * relies on srv_undo_tablespaces_open to be properly initialized */ msg("xtrabackup: Generating a list of tablespaces\n"); - err = fil_load_single_table_tablespaces(xb_check_if_open_tablespace); + err = xb_load_single_table_tablespaces(); if (err != DB_SUCCESS) { return(err); } @@ -3049,42 +2888,11 @@ static void xb_data_files_close() { - ulint i; - - /* Shutdown the aio threads. This has been copied from - innobase_shutdown_for_mysql(). */ - - srv_shutdown_state = SRV_SHUTDOWN_EXIT_THREADS; - - for (i = 0; i < 1000; i++) { - os_aio_wake_all_threads_at_shutdown(); - - if (os_thread_count == 0) { - break; - } - os_thread_sleep(10000); - } - - if (i == 1000) { - msg("xtrabackup: Warning: %lu threads created by InnoDB" - " had not exited at shutdown!\n", - (ulong) os_thread_count); - } - - os_aio_free(); - + ut_ad(!os_thread_count); fil_close_all_files(); - - /* Free the double write data structures. */ if (buf_dblwr) { buf_dblwr_free(); } - - /* Reset srv_file_io_threads to its default value to avoid confusing - warning on --prepare in innobase_start_or_create_for_mysql()*/ - srv_n_file_io_threads = 4; - - srv_shutdown_state = SRV_SHUTDOWN_NONE; } /*********************************************************************** @@ -3103,7 +2911,7 @@ xb_new_filter_entry( ut_a(namelen <= NAME_LEN * 2 + 1); entry = static_cast<xb_filter_entry_t *> - (ut_malloc(sizeof(xb_filter_entry_t) + namelen + 1)); + (malloc(sizeof(xb_filter_entry_t) + namelen + 1)); memset(entry, '\0', sizeof(xb_filter_entry_t) + namelen + 1); entry->name = ((char*)entry) + sizeof(xb_filter_entry_t); strcpy(entry->name, name); @@ -3402,7 +3210,7 @@ xb_filter_hash_free(hash_table_t* hash) HASH_DELETE(xb_filter_entry_t, name_hash, hash, ut_fold_string(prev_table->name), prev_table); - ut_free(prev_table); + free(prev_table); } } @@ -3451,77 +3259,33 @@ static ulint open_or_create_log_file( /*====================*/ - ibool create_new_db, /*!< in: TRUE if we should create a - new database */ + fil_space_t* space, ibool* log_file_created, /*!< out: TRUE if new log file created */ - ibool log_file_has_been_opened,/*!< in: TRUE if a log file has been - opened before: then it is an error - to try to create another log file */ ulint i) /*!< in: log file number in group */ { - ibool ret; - os_offset_t size; char name[10000]; ulint dirnamelen; - UT_NOT_USED(create_new_db); - UT_NOT_USED(log_file_has_been_opened); - *log_file_created = FALSE; - srv_normalize_path_for_win(srv_log_group_home_dir); + os_normalize_path(srv_log_group_home_dir); dirnamelen = strlen(srv_log_group_home_dir); ut_a(dirnamelen < (sizeof name) - 10 - sizeof "ib_logfile"); memcpy(name, srv_log_group_home_dir, dirnamelen); /* Add a path separator if needed. */ - if (dirnamelen && name[dirnamelen - 1] != SRV_PATH_SEPARATOR) { - name[dirnamelen++] = SRV_PATH_SEPARATOR; + if (dirnamelen && name[dirnamelen - 1] != OS_PATH_SEPARATOR) { + name[dirnamelen++] = OS_PATH_SEPARATOR; } sprintf(name + dirnamelen, "%s%lu", "ib_logfile", (ulong) i); - files[i] = os_file_create(innodb_file_log_key, name, - OS_FILE_OPEN, OS_FILE_NORMAL, - OS_LOG_FILE, &ret,0); - if (ret == FALSE) { - fprintf(stderr, "InnoDB: Error in opening %s\n", name); - - return(DB_ERROR); - } - - size = os_file_get_size(files[i]); - - if (size != srv_log_file_size * UNIV_PAGE_SIZE) { - - fprintf(stderr, - "InnoDB: Error: log file %s is" - " of different size " UINT64PF " bytes\n" - "InnoDB: than specified in the .cnf" - " file " UINT64PF " bytes!\n", - name, size, srv_log_file_size * UNIV_PAGE_SIZE); - - return(DB_ERROR); - } - - ret = os_file_close(files[i]); - ut_a(ret); - - if (i == 0) { - /* Create in memory the file space object - which is for this log group */ - - fil_space_create(name, - SRV_LOG_SPACE_FIRST_ID, 0, FIL_TYPE_LOG, 0, 0); - log_init(srv_n_log_files, srv_log_file_size * UNIV_PAGE_SIZE); - } - ut_a(fil_validate()); - ut_a(fil_node_create(name, (ulint)srv_log_file_size, - SRV_LOG_SPACE_FIRST_ID, FALSE)); + ut_a(fil_node_create(name, srv_log_file_size >> srv_page_size_shift, + space, false, false)); return(DB_SUCCESS); } @@ -3534,20 +3298,8 @@ void xb_normalize_init_values(void) /*==========================*/ { - ulint i; - - for (i = 0; i < srv_n_data_files; i++) { - srv_data_file_sizes[i] = srv_data_file_sizes[i] - * ((1024 * 1024) / UNIV_PAGE_SIZE); - } - - srv_last_file_size_max = srv_last_file_size_max - * ((1024 * 1024) / UNIV_PAGE_SIZE); - - srv_log_file_size = srv_log_file_size / UNIV_PAGE_SIZE; - - srv_log_buffer_size = srv_log_buffer_size / UNIV_PAGE_SIZE; - + srv_sys_space.normalize(); + srv_log_buffer_size /= UNIV_PAGE_SIZE; srv_lock_table_size = 5 * (srv_buf_pool_size / UNIV_PAGE_SIZE); } @@ -3608,8 +3360,11 @@ end: #endif } -void -xtrabackup_backup_func(void) +/** Implement --backup +@return whether the operation succeeded */ +static +bool +xtrabackup_backup_func() { MY_STAT stat_info; lsn_t latest_cp; @@ -3627,7 +3382,7 @@ xtrabackup_backup_func(void) if (my_setwd(mysql_real_data_home,MYF(MY_WME))) { msg("xtrabackup: cannot my_setwd %s\n", mysql_real_data_home); - exit(EXIT_FAILURE); + return(false); } msg("xtrabackup: cd to %s\n", mysql_real_data_home); @@ -3642,49 +3397,50 @@ xtrabackup_backup_func(void) srv_n_purge_threads = 1; srv_read_only_mode = TRUE; - srv_backup_mode = TRUE; - srv_close_files = (bool)xb_close_files; + srv_operation = SRV_OPERATION_BACKUP; - if (srv_close_files) + if (xb_close_files) msg("xtrabackup: warning: close-files specified. Use it " "at your own risk. If there are DDL operations like table DROP TABLE " "or RENAME TABLE during the backup, inconsistent backup will be " "produced.\n"); /* initialize components */ - if(innodb_init_param()) - exit(EXIT_FAILURE); + if(innodb_init_param()) { +fail: + innodb_shutdown(); + return(false); + } xb_normalize_init_values(); if (srv_file_flush_method_str == NULL) { - /* These are the default options */ - srv_unix_file_flush_method = SRV_UNIX_FSYNC; + /* These are the default options */ + srv_file_flush_method = SRV_FSYNC; } else if (0 == ut_strcmp(srv_file_flush_method_str, "fsync")) { - srv_unix_file_flush_method = SRV_UNIX_FSYNC; + srv_file_flush_method = SRV_FSYNC; } else if (0 == ut_strcmp(srv_file_flush_method_str, "O_DSYNC")) { - srv_unix_file_flush_method = SRV_UNIX_O_DSYNC; + srv_file_flush_method = SRV_O_DSYNC; } else if (0 == ut_strcmp(srv_file_flush_method_str, "O_DIRECT")) { - srv_unix_file_flush_method = SRV_UNIX_O_DIRECT; + srv_file_flush_method = SRV_O_DIRECT; msg("xtrabackup: using O_DIRECT\n"); } else if (0 == ut_strcmp(srv_file_flush_method_str, "littlesync")) { - srv_unix_file_flush_method = SRV_UNIX_LITTLESYNC; - + srv_file_flush_method = SRV_LITTLESYNC; } else if (0 == ut_strcmp(srv_file_flush_method_str, "nosync")) { - srv_unix_file_flush_method = SRV_UNIX_NOSYNC; + srv_file_flush_method = SRV_NOSYNC; } else if (0 == ut_strcmp(srv_file_flush_method_str, "ALL_O_DIRECT")) { - srv_unix_file_flush_method = SRV_UNIX_ALL_O_DIRECT; + srv_file_flush_method = SRV_ALL_O_DIRECT_FSYNC; msg("xtrabackup: using ALL_O_DIRECT\n"); } else if (0 == ut_strcmp(srv_file_flush_method_str, "O_DIRECT_NO_FSYNC")) { - srv_unix_file_flush_method = SRV_UNIX_O_DIRECT_NO_FSYNC; + srv_file_flush_method = SRV_O_DIRECT_NO_FSYNC; msg("xtrabackup: using O_DIRECT_NO_FSYNC\n"); } else { - msg("xtrabackup: Unrecognized value %s for " + msg("xtrabackup: Unrecognized value %s for " "innodb_flush_method\n", srv_file_flush_method_str); - exit(EXIT_FAILURE); + goto fail; } /* We can only use synchronous unbuffered IO on Windows for now */ @@ -3694,8 +3450,8 @@ xtrabackup_backup_func(void) } #ifdef _WIN32 - srv_win_file_flush_method = SRV_WIN_IO_UNBUFFERED; - srv_use_native_aio = FALSE; + srv_file_flush_method = SRV_ALL_O_DIRECT_FSYNC; + srv_use_native_aio = TRUE; #endif if (srv_buf_pool_size >= 1000 * 1024 * 1024) { @@ -3715,12 +3471,19 @@ xtrabackup_backup_func(void) computers */ } - srv_general_init(); + sync_check_init(); + ut_d(sync_check_enable()); + /* Reset the system variables in the recovery module. */ + recv_sys_var_init(); + trx_pool_init(); + row_mysql_init(); + ut_crc32_init(); crc_init(); + recv_sys_init(); #ifdef WITH_INNODB_DISALLOW_WRITES - srv_allow_writes_event = os_event_create(); + srv_allow_writes_event = os_event_create(0); os_event_set(srv_allow_writes_event); #endif @@ -3734,18 +3497,23 @@ xtrabackup_backup_func(void) ulint i; xb_fil_io_init(); + srv_n_file_io_threads = srv_n_read_io_threads; + + os_aio_init(srv_n_read_io_threads, srv_n_write_io_threads, + SRV_MAX_N_PENDING_SYNC_IOS); log_sys_init(); + log_init(srv_n_log_files); + fil_space_t* space = fil_space_create( + "innodb_redo_log", SRV_LOG_SPACE_FIRST_ID, 0, + FIL_TYPE_LOG, NULL); lock_sys_create(srv_lock_table_size); for (i = 0; i < srv_n_log_files; i++) { - err = open_or_create_log_file(FALSE, &log_file_created, - log_opened, i); + err = open_or_create_log_file(space, &log_file_created, i); if (err != DB_SUCCESS) { - - //return((int) err); - exit(EXIT_FAILURE); + goto fail; } if (log_file_created) { @@ -3762,15 +3530,14 @@ xtrabackup_backup_func(void) "xtrabackup: Then delete the existing log files. Edit the .cnf file\n" "xtrabackup: and start the database again.\n"); - //return(DB_ERROR); - exit(EXIT_FAILURE); + goto fail; } } /* log_file_created must not be TRUE, if online */ if (log_file_created) { msg("xtrabackup: Something wrong with source files...\n"); - exit(EXIT_FAILURE); + goto fail; } } @@ -3781,7 +3548,7 @@ xtrabackup_backup_func(void) && (my_mkdir(xtrabackup_extra_lsndir,0777,MYF(0)) < 0)) { msg("xtrabackup: Error: cannot mkdir %d: %s\n", my_errno, xtrabackup_extra_lsndir); - exit(EXIT_FAILURE); + goto fail; } /* create target dir if not exist */ @@ -3789,12 +3556,10 @@ xtrabackup_backup_func(void) && (my_mkdir(xtrabackup_target_dir,0777,MYF(0)) < 0)){ msg("xtrabackup: Error: cannot mkdir %d: %s\n", my_errno, xtrabackup_target_dir); - exit(EXIT_FAILURE); + goto fail; } { - fil_system_t* f_system = fil_system; - /* definition from recv_recovery_from_checkpoint_start() */ ulint max_cp_field; @@ -3810,14 +3575,17 @@ xtrabackup_backup_func(void) dberr_t err = recv_find_max_checkpoint(&max_cp_field); if (err != DB_SUCCESS) { - exit(EXIT_FAILURE); +log_fail: + log_mutex_exit(); + goto fail; } if (log_sys->log.format == 0) { old_format: msg("xtrabackup: Error: cannot process redo log" " before MariaDB 10.2.2\n"); - exit(EXIT_FAILURE); + log_mutex_exit(); + goto log_fail; } ut_ad(!((log_sys->log.format ^ LOG_HEADER_FORMAT_CURRENT) @@ -3832,7 +3600,7 @@ reread_log_header: err = recv_find_max_checkpoint(&max_cp_field); if (err != DB_SUCCESS) { - exit(EXIT_FAILURE); + goto log_fail; } if (log_sys->log.format == 0) { @@ -3842,7 +3610,7 @@ reread_log_header: ut_ad(!((log_sys->log.format ^ LOG_HEADER_FORMAT_CURRENT) & ~LOG_HEADER_FORMAT_ENCRYPTED)); - if(checkpoint_no_start != mach_read_from_8(buf + LOG_CHECKPOINT_NO)) { + if (checkpoint_no_start != mach_read_from_8(buf + LOG_CHECKPOINT_NO)) { checkpoint_lsn_start = mach_read_from_8(buf + LOG_CHECKPOINT_LSN); checkpoint_no_start = mach_read_from_8(buf + LOG_CHECKPOINT_NO); @@ -3854,34 +3622,50 @@ reread_log_header: xtrabackup_init_datasinks(); if (!select_history()) { - exit(EXIT_FAILURE); + goto fail; } /* open the log file */ memset(&stat_info, 0, sizeof(MY_STAT)); - dst_log_file = ds_open(ds_redo, XB_LOG_FILENAME, &stat_info); + dst_log_file = ds_open(ds_redo, "ib_logfile0", &stat_info); if (dst_log_file == NULL) { msg("xtrabackup: error: failed to open the target stream for " - "'%s'.\n", XB_LOG_FILENAME); - ut_free(log_hdr_buf_); - exit(EXIT_FAILURE); + "'ib_logfile0'.\n"); + goto fail; } /* label it */ - strcpy((char*) log_hdr_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, - "xtrabkup "); - ut_sprintf_timestamp( - (char*) log_hdr_buf + (LOG_FILE_WAS_CREATED_BY_HOT_BACKUP - + (sizeof "xtrabkup ") - 1)); - - if (ds_write(dst_log_file, log_hdr_buf, LOG_FILE_HDR_SIZE)) { + byte MY_ALIGNED(OS_FILE_LOG_BLOCK_SIZE) log_hdr[OS_FILE_LOG_BLOCK_SIZE]; + memset(log_hdr, 0, sizeof log_hdr); + mach_write_to_4(LOG_HEADER_FORMAT + log_hdr, log_sys->log.format); + mach_write_to_8(LOG_HEADER_START_LSN + log_hdr, checkpoint_lsn_start); + strcpy(reinterpret_cast<char*>(LOG_HEADER_CREATOR + log_hdr), + "Backup " MYSQL_SERVER_VERSION); + log_block_set_checksum(log_hdr, + log_block_calc_checksum_crc32(log_hdr)); + + /* Write the log header. */ + if (ds_write(dst_log_file, log_hdr, sizeof log_hdr)) { + log_write_fail: msg("xtrabackup: error: write to logfile failed\n"); - ut_free(log_hdr_buf_); - exit(EXIT_FAILURE); + goto fail; + } + /* Adjust the checkpoint page. */ + memcpy(log_hdr, buf, OS_FILE_LOG_BLOCK_SIZE); + mach_write_to_8(log_hdr + LOG_CHECKPOINT_OFFSET, + (checkpoint_lsn_start & (OS_FILE_LOG_BLOCK_SIZE - 1)) + | LOG_FILE_HDR_SIZE); + log_block_set_checksum(log_hdr, + log_block_calc_checksum_crc32(log_hdr)); + /* Write checkpoint page 1 and two empty log pages before the + payload. */ + if (ds_write(dst_log_file, log_hdr, OS_FILE_LOG_BLOCK_SIZE) + || !memset(log_hdr, 0, sizeof log_hdr) + || ds_write(dst_log_file, log_hdr, sizeof log_hdr) + || ds_write(dst_log_file, log_hdr, sizeof log_hdr)) { + goto log_write_fail; } - ut_free(log_hdr_buf_); - /* start flag */ log_copying = TRUE; @@ -3890,18 +3674,20 @@ reread_log_header: os_thread_id_t io_watching_thread_id; io_ticket = xtrabackup_throttle; - wait_throttle = os_event_create(); + wait_throttle = os_event_create(0); + io_watching_thread_running = true; os_thread_create(io_watching_thread, NULL, &io_watching_thread_id); } /* copy log file by current position */ - if(xtrabackup_copy_logfile(checkpoint_lsn_start, FALSE)) - exit(EXIT_FAILURE); + log_copy_scanned_lsn = checkpoint_lsn_start; + if (xtrabackup_copy_logfile(false)) + goto fail; - - log_copying_stop = os_event_create(); + log_copying_stop = os_event_create(0); + log_copying_running = true; os_thread_create(log_copying_thread, NULL, &log_copying_thread_id); /* Populate fil_system with tablespaces to copy */ @@ -3909,12 +3695,12 @@ reread_log_header: if (err != DB_SUCCESS) { msg("xtrabackup: error: xb_load_tablespaces() failed with" "error code %u\n", err); - exit(EXIT_FAILURE); + goto fail; } /* FLUSH CHANGED_PAGE_BITMAPS call */ if (!flush_changed_page_bitmaps()) { - exit(EXIT_FAILURE); + goto fail; } debug_sync_point("xtrabackup_suspend_at_start"); @@ -3939,17 +3725,17 @@ reread_log_header: "files transfer\n", xtrabackup_parallel); } - it = datafiles_iter_new(f_system); + it = datafiles_iter_new(fil_system); if (it == NULL) { msg("xtrabackup: Error: datafiles_iter_new() failed.\n"); - exit(EXIT_FAILURE); + goto fail; } /* Create data copying threads */ data_threads = (data_thread_ctxt_t *) - ut_malloc(sizeof(data_thread_ctxt_t) * xtrabackup_parallel); + malloc(sizeof(data_thread_ctxt_t) * xtrabackup_parallel); count = xtrabackup_parallel; - count_mutex = os_mutex_create(); + pthread_mutex_init(&count_mutex, NULL); for (i = 0; i < (uint) xtrabackup_parallel; i++) { data_threads[i].it = it; @@ -3963,16 +3749,16 @@ reread_log_header: /* Wait for threads to exit */ while (1) { os_thread_sleep(1000000); - os_mutex_enter(count_mutex); - if (count == 0) { - os_mutex_exit(count_mutex); + pthread_mutex_lock(&count_mutex); + bool stop = count == 0; + pthread_mutex_unlock(&count_mutex); + if (stop) { break; } - os_mutex_exit(count_mutex); } - os_mutex_free(count_mutex); - ut_free(data_threads); + pthread_mutex_destroy(&count_mutex); + free(data_threads); datafiles_iter_free(it); if (changed_page_bitmap) { @@ -3981,7 +3767,7 @@ reread_log_header: } if (!backup_start()) { - exit(EXIT_FAILURE); + goto fail; } /* read the latest checkpoint lsn */ @@ -4013,11 +3799,19 @@ reread_log_header: } msg("\n"); - os_event_free(log_copying_stop); + os_event_destroy(log_copying_stop); + + if (!dst_log_file || xtrabackup_copy_logfile(true)) { + goto fail; + } + if (ds_close(dst_log_file)) { - exit(EXIT_FAILURE); + dst_log_file = NULL; + goto fail; } + dst_log_file = NULL; + if(!xtrabackup_incremental) { strcpy(metadata_type, "full-backuped"); metadata_from_lsn = 0; @@ -4030,7 +3824,7 @@ reread_log_header: if (!xtrabackup_stream_metadata(ds_meta)) { msg("xtrabackup: Error: failed to stream metadata.\n"); - exit(EXIT_FAILURE); + goto fail; } if (xtrabackup_extra_lsndir) { char filename[FN_REFLEN]; @@ -4040,13 +3834,13 @@ reread_log_header: if (!xtrabackup_write_metadata(filename)) { msg("xtrabackup: Error: failed to write metadata " "to '%s'.\n", filename); - exit(EXIT_FAILURE); + goto fail; } } if (!backup_finish()) { - exit(EXIT_FAILURE); + goto fail; } xtrabackup_destroy_datasinks(); @@ -4056,319 +3850,29 @@ reread_log_header: while (io_watching_thread_running) { os_thread_sleep(1000000); } - os_event_free(wait_throttle); - wait_throttle = NULL; + os_event_destroy(wait_throttle); } - msg("xtrabackup: Transaction log of lsn (" LSN_PF ") to (" LSN_PF + msg("xtrabackup: Redo log (from LSN " LSN_PF " to " LSN_PF ") was copied.\n", checkpoint_lsn_start, log_copy_scanned_lsn); xb_filters_free(); xb_data_files_close(); - /* Make sure that the latest checkpoint made it to xtrabackup_logfile */ + /* Make sure that the latest checkpoint was included */ if (latest_cp > log_copy_scanned_lsn) { msg("xtrabackup: error: last checkpoint LSN (" LSN_PF ") is larger than last copied LSN (" LSN_PF ").\n", latest_cp, log_copy_scanned_lsn); - exit(EXIT_FAILURE); + goto fail; } + + innodb_shutdown(); + return(true); } /* ================= prepare ================= */ -static my_bool -xtrabackup_init_temp_log(void) -{ - pfs_os_file_t src_file; - char src_path[FN_REFLEN]; - char dst_path[FN_REFLEN]; - ibool success; - - ulint field; - byte* log_buf= (byte *)malloc(UNIV_PAGE_SIZE_MAX * 128); /* 2 MB */ - - ib_int64_t file_size; - - lsn_t max_no; - lsn_t max_lsn; - lsn_t checkpoint_no; - - ulint fold; - - bool checkpoint_found; - - max_no = 0; - - if (!log_buf) { - goto error; - } - - if (!xb_init_log_block_size()) { - goto error; - } - - if(!xtrabackup_incremental_dir) { - sprintf(dst_path, "%s/ib_logfile0", xtrabackup_target_dir); - sprintf(src_path, "%s/%s", xtrabackup_target_dir, - XB_LOG_FILENAME); - } else { - sprintf(dst_path, "%s/ib_logfile0", xtrabackup_incremental_dir); - sprintf(src_path, "%s/%s", xtrabackup_incremental_dir, - XB_LOG_FILENAME); - } - - srv_normalize_path_for_win(dst_path); - srv_normalize_path_for_win(src_path); -retry: - src_file = os_file_create_simple_no_error_handling(0, src_path, - OS_FILE_OPEN, - OS_FILE_READ_WRITE, - &success,0); - if (!success) { - /* The following call prints an error message */ - os_file_get_last_error(TRUE); - - msg("xtrabackup: Warning: cannot open %s. will try to find.\n", - src_path); - - /* check if ib_logfile0 may be xtrabackup_logfile */ - src_file = os_file_create_simple_no_error_handling(0, dst_path, - OS_FILE_OPEN, - OS_FILE_READ_WRITE, - &success,0); - if (!success) { - os_file_get_last_error(TRUE); - msg(" xtrabackup: Fatal error: cannot find %s.\n", - src_path); - - goto error; - } - - success = os_file_read(src_file, log_buf, 0, - LOG_FILE_HDR_SIZE); - if (!success) { - goto error; - } - - if ( ut_memcmp(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, - (byte*)"xtrabkup", (sizeof "xtrabkup") - 1) == 0) { - msg(" xtrabackup: 'ib_logfile0' seems to be " - "'xtrabackup_logfile'. will retry.\n"); - - os_file_close(src_file); - src_file = XB_FILE_UNDEFINED; - - /* rename and try again */ - success = os_file_rename(0, dst_path, src_path); - if (!success) { - goto error; - } - - goto retry; - } - - msg(" xtrabackup: Fatal error: cannot find %s.\n", - src_path); - - os_file_close(src_file); - src_file = XB_FILE_UNDEFINED; - - goto error; - } - - file_size = os_file_get_size(src_file); - - - /* TODO: We should skip the following modifies, if it is not the first time. */ - - /* read log file header */ - success = os_file_read(src_file, log_buf, 0, LOG_FILE_HDR_SIZE); - if (!success) { - goto error; - } - - if ( ut_memcmp(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, - (byte*)"xtrabkup", (sizeof "xtrabkup") - 1) != 0 ) { - msg("xtrabackup: notice: xtrabackup_logfile was already used " - "to '--prepare'.\n"); - goto skip_modify; - } else { - /* clear it later */ - //memset(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, - // ' ', 4); - } - - checkpoint_found = false; - - /* read last checkpoint lsn */ - for (field = LOG_CHECKPOINT_1; field <= LOG_CHECKPOINT_2; - field += LOG_CHECKPOINT_2 - LOG_CHECKPOINT_1) { - if (!recv_check_cp_is_consistent(const_cast<const byte *> - (log_buf + field))) - goto not_consistent; - - checkpoint_no = mach_read_from_8(log_buf + field + - LOG_CHECKPOINT_NO); - - if (checkpoint_no >= max_no) { - - max_no = checkpoint_no; - max_lsn = mach_read_from_8(log_buf + field + - LOG_CHECKPOINT_LSN); - checkpoint_found = true; - } -not_consistent: - ; - } - - if (!checkpoint_found) { - msg("xtrabackup: No valid checkpoint found.\n"); - goto error; - } - - - /* It seems to be needed to overwrite the both checkpoint area. */ - mach_write_to_8(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_LSN, - max_lsn); - mach_write_to_4(log_buf + LOG_CHECKPOINT_1 - + LOG_CHECKPOINT_OFFSET_LOW32, - LOG_FILE_HDR_SIZE + - (ulint)(max_lsn - - ut_uint64_align_down(max_lsn, - OS_FILE_LOG_BLOCK_SIZE))); - mach_write_to_4(log_buf + LOG_CHECKPOINT_1 - + LOG_CHECKPOINT_OFFSET_HIGH32, 0); - fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_1, LOG_CHECKPOINT_CHECKSUM_1); - mach_write_to_4(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_CHECKSUM_1, fold); - - fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_LSN, - LOG_CHECKPOINT_CHECKSUM_2 - LOG_CHECKPOINT_LSN); - mach_write_to_4(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_CHECKSUM_2, fold); - - mach_write_to_8(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_LSN, - max_lsn); - mach_write_to_4(log_buf + LOG_CHECKPOINT_2 - + LOG_CHECKPOINT_OFFSET_LOW32, - LOG_FILE_HDR_SIZE + - (ulint)(max_lsn - - ut_uint64_align_down(max_lsn, - OS_FILE_LOG_BLOCK_SIZE))); - mach_write_to_4(log_buf + LOG_CHECKPOINT_2 - + LOG_CHECKPOINT_OFFSET_HIGH32, 0); - fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_2, LOG_CHECKPOINT_CHECKSUM_1); - mach_write_to_4(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_CHECKSUM_1, fold); - - fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_LSN, - LOG_CHECKPOINT_CHECKSUM_2 - LOG_CHECKPOINT_LSN); - mach_write_to_4(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_CHECKSUM_2, fold); - - - success = os_file_write(src_path, src_file, log_buf, 0, - LOG_FILE_HDR_SIZE); - if (!success) { - goto error; - } - - /* expand file size (9/8) and align to UNIV_PAGE_SIZE_MAX */ - - if (file_size % UNIV_PAGE_SIZE_MAX) { - memset(log_buf, 0, UNIV_PAGE_SIZE_MAX); - success = os_file_write(src_path, src_file, log_buf, - file_size, - UNIV_PAGE_SIZE_MAX - - (ulint) (file_size - % UNIV_PAGE_SIZE_MAX)); - if (!success) { - goto error; - } - - file_size = os_file_get_size(src_file); - } - - /* TODO: We should judge whether the file is already expanded or not... */ - { - ulint expand; - - memset(log_buf, 0, UNIV_PAGE_SIZE_MAX * 128); - expand = (ulint) (file_size / UNIV_PAGE_SIZE_MAX / 8); - - for (; expand > 128; expand -= 128) { - success = os_file_write(src_path, src_file, log_buf, - file_size, - UNIV_PAGE_SIZE_MAX * 128); - if (!success) { - goto error; - } - file_size += UNIV_PAGE_SIZE_MAX * 128; - } - - if (expand) { - success = os_file_write(src_path, src_file, log_buf, - file_size, - expand * UNIV_PAGE_SIZE_MAX); - if (!success) { - goto error; - } - file_size += UNIV_PAGE_SIZE_MAX * expand; - } - } - - /* make larger than 2MB */ - if (file_size < 2*1024*1024L) { - memset(log_buf, 0, UNIV_PAGE_SIZE_MAX); - while (file_size < 2*1024*1024L) { - success = os_file_write(src_path, src_file, log_buf, - file_size, - UNIV_PAGE_SIZE_MAX); - if (!success) { - goto error; - } - file_size += UNIV_PAGE_SIZE_MAX; - } - file_size = os_file_get_size(src_file); - } - - msg("xtrabackup: xtrabackup_logfile detected: size=" INT64PF ", " - "start_lsn=(" LSN_PF ")\n", file_size, max_lsn); - - os_file_close(src_file); - src_file = XB_FILE_UNDEFINED; - - /* fake InnoDB */ - innobase_log_files_in_group_save = innobase_log_files_in_group; - srv_log_group_home_dir_save = srv_log_group_home_dir; - innobase_log_file_size_save = innobase_log_file_size; - - srv_log_group_home_dir = NULL; - innobase_log_file_size = file_size; - innobase_log_files_in_group = 1; - - srv_thread_concurrency = 0; - - /* rename 'xtrabackup_logfile' to 'ib_logfile0' */ - success = os_file_rename(0, src_path, dst_path); - if (!success) { - goto error; - } - xtrabackup_logfile_is_renamed = TRUE; - free(log_buf); - return(FALSE); - -skip_modify: - free(log_buf); - os_file_close(src_file); - src_file = XB_FILE_UNDEFINED; - return(FALSE); - -error: - free(log_buf); - if (src_file != XB_FILE_UNDEFINED) - os_file_close(src_file); - msg("xtrabackup: Error: xtrabackup_init_temp_log() failed.\n"); - return(TRUE); /*ERROR*/ -} - /*********************************************************************** Generates path to the meta file path from a given path to an incremental .delta by replacing trailing ".delta" with ".meta", or returns error if 'delta_path' @@ -4398,31 +3902,30 @@ file. Code adopted from fil_create_new_single_table_tablespace with the main difference that only disk file is created without updating the InnoDB in-memory dictionary data structures. -@return TRUE on success, FALSE on error. */ +@return true on success, false on error. */ static -ibool +bool xb_space_create_file( /*==================*/ const char* path, /*!<in: path to tablespace */ ulint space_id, /*!<in: space id */ - ulint flags __attribute__((unused)),/*!<in: tablespace - flags */ + ulint flags, /*!<in: tablespace flags */ pfs_os_file_t* file) /*!<out: file handle */ { - ibool ret; + bool ret; byte* buf; byte* page; - *file = os_file_create_simple_no_error_handling(0, path, OS_FILE_CREATE, - OS_FILE_READ_WRITE, - &ret,0); + *file = os_file_create_simple_no_error_handling( + 0, path, OS_FILE_CREATE, OS_FILE_READ_WRITE, false, &ret); if (!ret) { msg("xtrabackup: cannot create file %s\n", path); return ret; } ret = os_file_set_size(path, *file, - FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE); + FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE, + false); if (!ret) { msg("xtrabackup: cannot set size for file %s\n", path); os_file_close(*file); @@ -4430,7 +3933,7 @@ xb_space_create_file( return ret; } - buf = static_cast<byte *>(ut_malloc(3 * UNIV_PAGE_SIZE)); + buf = static_cast<byte *>(malloc(3 * UNIV_PAGE_SIZE)); /* Align the memory for file i/o if we might have O_DIRECT set */ page = static_cast<byte *>(ut_align(buf, UNIV_PAGE_SIZE)); @@ -4439,19 +3942,19 @@ xb_space_create_file( fsp_header_init_fields(page, space_id, flags); mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id); - if (!fsp_flags_is_compressed(flags)) { - buf_flush_init_for_writing(page, NULL, 0); + const page_size_t page_size(flags); - ret = os_file_write(path, *file, page, 0, UNIV_PAGE_SIZE); - } - else { - page_zip_des_t page_zip; - ulint zip_size; + if (!page_size.is_compressed()) { + buf_flush_init_for_writing(NULL, page, NULL, 0, false); - zip_size = fsp_flags_get_zip_size(flags); + ret = os_file_write(IORequestWrite, path, *file, page, 0, + UNIV_PAGE_SIZE); + } else { + page_zip_des_t page_zip; + ulint zip_size = page_size.physical(); page_zip_set_size(&page_zip, zip_size); page_zip.data = page + UNIV_PAGE_SIZE; - fprintf(stderr, "zip_size = %lu\n", zip_size); + fprintf(stderr, "zip_size = " ULINTPF "\n", zip_size); #ifdef UNIV_DEBUG page_zip.m_start = @@ -4459,13 +3962,13 @@ xb_space_create_file( page_zip.m_end = page_zip.m_nonempty = page_zip.n_blobs = 0; - buf_flush_init_for_writing(page, &page_zip, 0); + buf_flush_init_for_writing(NULL, page, &page_zip, 0, false); - ret = os_file_write(path, *file, page_zip.data, 0, - zip_size); + ret = os_file_write(IORequestWrite, path, *file, + page_zip.data, 0, zip_size); } - ut_free(buf); + free(buf); if (!ret) { msg("xtrabackup: could not write the first page to %s\n", @@ -4490,35 +3993,32 @@ pfs_os_file_t xb_delta_open_matching_space( const char* dbname, /* in: path to destination database dir */ const char* name, /* in: name of delta file (without .delta) */ - ulint space_id, /* in: space id of delta file */ - ulint zip_size, /* in: zip_size of tablespace */ + const xb_delta_info_t& info, char* real_name, /* out: full path of destination file */ size_t real_name_len, /* out: buffer size for real_name */ - ibool* success) /* out: indicates error. TRUE = success */ + bool* success) /* out: indicates error. true = success */ { char dest_dir[FN_REFLEN]; char dest_space_name[FN_REFLEN]; - ibool ok; fil_space_t* fil_space; pfs_os_file_t file; - ulint tablespace_flags; xb_filter_entry_t* table; ut_a(dbname != NULL || - !fil_is_user_tablespace_id(space_id) || - space_id == ULINT_UNDEFINED); + !fil_is_user_tablespace_id(info.space_id) || + info.space_id == ULINT_UNDEFINED); - *success = FALSE; + *success = false; if (dbname) { snprintf(dest_dir, FN_REFLEN, "%s/%s", xtrabackup_target_dir, dbname); - srv_normalize_path_for_win(dest_dir); + os_normalize_path(dest_dir); snprintf(dest_space_name, FN_REFLEN, "%s/%s", dbname, name); } else { snprintf(dest_dir, FN_REFLEN, "%s", xtrabackup_target_dir); - srv_normalize_path_for_win(dest_dir); + os_normalize_path(dest_dir); snprintf(dest_space_name, FN_REFLEN, "%s", name); } @@ -4526,7 +4026,7 @@ xb_delta_open_matching_space( snprintf(real_name, real_name_len, "%s/%s", xtrabackup_target_dir, dest_space_name); - srv_normalize_path_for_win(real_name); + os_normalize_path(real_name); /* Truncate ".ibd" */ dest_space_name[strlen(dest_space_name) - 4] = '\0'; @@ -4536,13 +4036,26 @@ xb_delta_open_matching_space( return file; } - if (!fil_is_user_tablespace_id(space_id)) { - goto found; + log_mutex_enter(); + if (!fil_is_user_tablespace_id(info.space_id)) { +found: + /* open the file and return its handle */ + + file = os_file_create_simple_no_error_handling( + 0, real_name, + OS_FILE_OPEN, OS_FILE_READ_WRITE, false, success); + + if (!*success) { + msg("xtrabackup: Cannot open file %s\n", real_name); + } +exit: + log_mutex_exit(); + return file; } /* remember space name for further reference */ table = static_cast<xb_filter_entry_t *> - (ut_malloc(sizeof(xb_filter_entry_t) + + (malloc(sizeof(xb_filter_entry_t) + strlen(dest_space_name) + 1)); table->name = ((char*)table) + sizeof(xb_filter_entry_t); @@ -4555,7 +4068,8 @@ xb_delta_open_matching_space( mutex_exit(&fil_system->mutex); if (fil_space != NULL) { - if (fil_space->id == space_id || space_id == ULINT_UNDEFINED) { + if (fil_space->id == info.space_id + || info.space_id == ULINT_UNDEFINED) { /* we found matching space */ goto found; } else { @@ -4568,8 +4082,10 @@ xb_delta_open_matching_space( msg("xtrabackup: Renaming %s to %s.ibd\n", fil_space->name, tmpname); - if (!fil_rename_tablespace(NULL, fil_space->id, - tmpname, NULL)) + if (!fil_rename_tablespace( + fil_space->id, + fil_space->chain.start->name, + tmpname, NULL)) { msg("xtrabackup: Cannot rename %s to %s\n", fil_space->name, tmpname); @@ -4578,14 +4094,14 @@ xb_delta_open_matching_space( } } - if (space_id == ULINT_UNDEFINED) + if (info.space_id == ULINT_UNDEFINED) { msg("xtrabackup: Error: Cannot handle DDL operation on tablespace " "%s\n", dest_space_name); exit(EXIT_FAILURE); } mutex_enter(&fil_system->mutex); - fil_space = fil_space_get_by_id(space_id); + fil_space = fil_space_get_by_id(info.space_id); mutex_exit(&fil_system->mutex); if (fil_space != NULL) { char tmpname[FN_REFLEN]; @@ -4595,7 +4111,9 @@ xb_delta_open_matching_space( msg("xtrabackup: Renaming %s to %s\n", fil_space->name, dest_space_name); - if (!fil_rename_tablespace(NULL, fil_space->id, tmpname, + if (!fil_rename_tablespace(fil_space->id, + fil_space->chain.start->name, + tmpname, NULL)) { msg("xtrabackup: Cannot rename %s to %s\n", @@ -4607,49 +4125,30 @@ xb_delta_open_matching_space( } /* No matching space found. create the new one. */ - - if (!fil_space_create(dest_space_name, space_id, 0, - FIL_TABLESPACE, 0, false)) { + const ulint flags = info.page_size.is_compressed() + ? get_bit_shift(info.page_size.physical() + >> (UNIV_ZIP_SIZE_SHIFT_MIN - 1)) + << FSP_FLAGS_POS_ZIP_SSIZE + | FSP_FLAGS_MASK_POST_ANTELOPE + | FSP_FLAGS_MASK_ATOMIC_BLOBS + | (info.page_size.logical() == UNIV_PAGE_SIZE_ORIG + ? 0 + : get_bit_shift(info.page_size.logical() + >> (UNIV_ZIP_SIZE_SHIFT_MIN - 1)) + << FSP_FLAGS_POS_PAGE_SSIZE) + : FSP_FLAGS_PAGE_SSIZE(); + ut_ad(page_size_t(flags).equals_to(info.page_size)); + + if (fil_space_create(dest_space_name, info.space_id, flags, + FIL_TYPE_TABLESPACE, 0)) { + *success = xb_space_create_file(real_name, info.space_id, + flags, &file); + } else { msg("xtrabackup: Cannot create tablespace %s\n", - dest_space_name); - goto exit; + dest_space_name); } - /* Calculate correct tablespace flags for compressed tablespaces. */ - if (!zip_size || zip_size == ULINT_UNDEFINED) { - tablespace_flags = 0; - } - else { - tablespace_flags - = (get_bit_shift(zip_size >> PAGE_ZIP_MIN_SIZE_SHIFT - << 1) - << DICT_TF_ZSSIZE_SHIFT) - | DICT_TF_COMPACT - | (DICT_TF_FORMAT_ZIP << DICT_TF_FORMAT_SHIFT); - ut_a(dict_tf_get_zip_size(tablespace_flags) - == zip_size); - } - *success = xb_space_create_file(real_name, space_id, tablespace_flags, - &file); goto exit; - -found: - /* open the file and return it's handle */ - - file = os_file_create_simple_no_error_handling(0, real_name, - OS_FILE_OPEN, - OS_FILE_READ_WRITE, - &ok,0); - - if (ok) { - *success = TRUE; - } else { - msg("xtrabackup: Cannot open file %s\n", real_name); - } - -exit: - - return file; } /************************************************************************ @@ -4670,13 +4169,13 @@ xtrabackup_apply_delta( char dst_path[FN_REFLEN]; char meta_path[FN_REFLEN]; char space_name[FN_REFLEN]; - ibool success; + bool success; ibool last_buffer = FALSE; ulint page_in_buffer; ulint incremental_buffers = 0; - xb_delta_info_t info; + xb_delta_info_t info(univ_page_size, SRV_TMP_SPACE_ID); ulint page_size; ulint page_size_shift; byte* incremental_buffer_base = NULL; @@ -4706,15 +4205,15 @@ xtrabackup_apply_delta( goto error; } - srv_normalize_path_for_win(dst_path); - srv_normalize_path_for_win(src_path); - srv_normalize_path_for_win(meta_path); + os_normalize_path(dst_path); + os_normalize_path(src_path); + os_normalize_path(meta_path); if (!xb_read_delta_metadata(meta_path, &info)) { goto error; } - page_size = info.page_size; + page_size = info.page_size.physical(); page_size_shift = get_bit_shift(page_size); msg("xtrabackup: page size for %s is %lu bytes\n", src_path, page_size); @@ -4725,10 +4224,9 @@ xtrabackup_apply_delta( goto error; } - src_file = os_file_create_simple_no_error_handling(0, src_path, - OS_FILE_OPEN, - OS_FILE_READ_WRITE, - &success,0); + src_file = os_file_create_simple_no_error_handling( + 0, src_path, + OS_FILE_OPEN, OS_FILE_READ_WRITE, false, &success); if (!success) { os_file_get_last_error(TRUE); msg("xtrabackup: error: cannot open %s\n", src_path); @@ -4740,7 +4238,7 @@ xtrabackup_apply_delta( os_file_set_nocache(src_file, src_path, "OPEN"); dst_file = xb_delta_open_matching_space( - dbname, space_name, info.space_id, info.zip_size, + dbname, space_name, info, dst_path, sizeof(dst_path), &success); if (!success) { msg("xtrabackup: error: cannot open %s\n", dst_path); @@ -4753,8 +4251,7 @@ xtrabackup_apply_delta( /* allocate buffer for incremental backup (4096 pages) */ incremental_buffer_base = static_cast<byte *> - (ut_malloc((page_size / 4 + 1) * - page_size)); + (malloc((page_size / 4 + 1) * page_size)); incremental_buffer = static_cast<byte *> (ut_align(incremental_buffer_base, page_size)); @@ -4768,8 +4265,8 @@ xtrabackup_apply_delta( /* first block of block cluster */ offset = ((incremental_buffers * (page_size / 4)) << page_size_shift); - success = os_file_read(src_file, incremental_buffer, - offset, page_size); + success = os_file_read(IORequestRead, src_file, + incremental_buffer, offset, page_size); if (!success) { goto error; } @@ -4787,6 +4284,9 @@ xtrabackup_apply_delta( goto error; } + /* FIXME: If the .delta modifies FSP_SIZE on page 0, + extend the file to that size. */ + for (page_in_buffer = 1; page_in_buffer < page_size / 4; page_in_buffer++) { if (mach_read_from_4(incremental_buffer + page_in_buffer * 4) @@ -4797,8 +4297,9 @@ xtrabackup_apply_delta( ut_a(last_buffer || page_in_buffer == page_size / 4); /* read whole of the cluster */ - success = os_file_read(src_file, incremental_buffer, - offset, page_in_buffer * page_size); + success = os_file_read(IORequestRead, src_file, + incremental_buffer, + offset, page_in_buffer * page_size); if (!success) { goto error; } @@ -4815,7 +4316,8 @@ xtrabackup_apply_delta( if (offset_on_page == 0xFFFFFFFFUL) break; - success = os_file_write(dst_path, dst_file, + success = os_file_write(IORequestWrite, + dst_path, dst_file, incremental_buffer + page_in_buffer * page_size, (offset_on_page << @@ -4829,20 +4331,18 @@ xtrabackup_apply_delta( incremental_buffers++; } - if (incremental_buffer_base) - ut_free(incremental_buffer_base); - if (src_file != XB_FILE_UNDEFINED) + free(incremental_buffer_base); + if (src_file != OS_FILE_CLOSED) os_file_close(src_file); - if (dst_file != XB_FILE_UNDEFINED) + if (dst_file != OS_FILE_CLOSED) os_file_close(dst_file); return TRUE; error: - if (incremental_buffer_base) - ut_free(incremental_buffer_base); - if (src_file != XB_FILE_UNDEFINED) + free(incremental_buffer_base); + if (src_file != OS_FILE_CLOSED) os_file_close(src_file); - if (dst_file != XB_FILE_UNDEFINED) + if (dst_file != OS_FILE_CLOSED) os_file_close(dst_file); msg("xtrabackup: Error: xtrabackup_apply_delta(): " "failed to apply %s to %s.\n", src_path, dst_path); @@ -4903,9 +4403,7 @@ xb_process_datadir( const char* path, /*!<in: datadir path */ const char* suffix, /*!<in: suffix to match against */ - handle_datadir_entry_func_t func, /*!<in: callback */ - void* data) /*!<in: additional argument for - callback */ + handle_datadir_entry_func_t func) /*!<in: callback */ { ulint ret; char dbpath[FN_REFLEN]; @@ -4940,7 +4438,7 @@ xb_process_datadir( suffix)) { if (!func( path, NULL, - fileinfo.name, data)) + fileinfo.name, NULL)) { return(FALSE); } @@ -4976,7 +4474,7 @@ next_file_item_1: sprintf(dbpath, "%s/%s", path, dbinfo.name); - srv_normalize_path_for_win(dbpath); + os_normalize_path(dbpath); dbdir = os_file_opendir(dbpath, FALSE); @@ -5001,7 +4499,7 @@ next_file_item_1: if (!func( path, dbinfo.name, - fileinfo.name, data)) + fileinfo.name, NULL)) { return(FALSE); } @@ -5033,82 +4531,9 @@ ibool xtrabackup_apply_deltas() { return xb_process_datadir(xtrabackup_incremental_dir, ".delta", - xtrabackup_apply_delta, NULL); -} - -static my_bool -xtrabackup_close_temp_log(my_bool clear_flag) -{ - pfs_os_file_t src_file; - char src_path[FN_REFLEN]; - char dst_path[FN_REFLEN]; - ibool success; - byte log_buf[UNIV_PAGE_SIZE_MAX]; - - if (!xtrabackup_logfile_is_renamed) - return(FALSE); - - /* rename 'ib_logfile0' to 'xtrabackup_logfile' */ - if(!xtrabackup_incremental_dir) { - sprintf(dst_path, "%s/ib_logfile0", xtrabackup_target_dir); - sprintf(src_path, "%s/%s", xtrabackup_target_dir, - XB_LOG_FILENAME); - } else { - sprintf(dst_path, "%s/ib_logfile0", xtrabackup_incremental_dir); - sprintf(src_path, "%s/%s", xtrabackup_incremental_dir, - XB_LOG_FILENAME); - } - - srv_normalize_path_for_win(dst_path); - srv_normalize_path_for_win(src_path); - - success = os_file_rename(0, dst_path, src_path); - if (!success) { - goto error; - } - xtrabackup_logfile_is_renamed = FALSE; - - if (!clear_flag) - return(FALSE); - - /* clear LOG_FILE_WAS_CREATED_BY_HOT_BACKUP field */ - src_file = os_file_create_simple_no_error_handling(0, src_path, - OS_FILE_OPEN, - OS_FILE_READ_WRITE, - &success,0); - if (!success) { - goto error; - } - - success = os_file_read(src_file, log_buf, 0, LOG_FILE_HDR_SIZE); - if (!success) { - goto error; - } - - memset(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, ' ', 4); - - success = os_file_write(src_path, src_file, log_buf, 0, - LOG_FILE_HDR_SIZE); - if (!success) { - goto error; - } - - os_file_close(src_file); - src_file = XB_FILE_UNDEFINED; - - innobase_log_files_in_group = innobase_log_files_in_group_save; - srv_log_group_home_dir = srv_log_group_home_dir_save; - innobase_log_file_size = innobase_log_file_size_save; - - return(FALSE); -error: - if (src_file != XB_FILE_UNDEFINED) - os_file_close(src_file); - msg("xtrabackup: Error: xtrabackup_close_temp_log() failed.\n"); - return(TRUE); /*ERROR*/ + xtrabackup_apply_delta); } - /*********************************************************************//** Write the meta data (index user fields) config file. @return true in case of success otherwise false. */ @@ -5359,14 +4784,15 @@ xb_export_cfg_write_header( } /* The table name includes the NUL byte. */ - ut_a(table->name != 0); - len = (ib_uint32_t)strlen(table->name) + 1; + const char* table_name = table->name.m_name; + ut_a(table_name != 0); + len = (ib_uint32_t)strlen(table_name) + 1; /* Write the table name. */ mach_write_to_4(value, len); if (fwrite(&value, 1, sizeof(value), file) != sizeof(value) - || fwrite(table->name, 1, len, file) != len) { + || fwrite(table_name, 1, len, file) != len) { msg("xtrabackup: Error: writing table name."); @@ -5456,48 +4882,38 @@ static void innodb_free_param() { - srv_free_paths_and_sizes(); - free(internal_innobase_data_file_path); - internal_innobase_data_file_path = NULL; + srv_sys_space.shutdown(); free_tmpdir(&mysql_tmpdir_list); } -/************************************************************************** -Store the current binary log coordinates in a specified file. -@return 'false' on error. */ +/** Store the current binary log coordinates in a specified file. +@param[in] filename file name +@param[in] name binary log file name +@param[in] pos binary log file position +@return whether the operation succeeded */ static bool -store_binlog_info( -/*==============*/ - const char *filename) /*!< in: output file name */ +store_binlog_info(const char* filename, const char* name, ulonglong pos) { - FILE *fp; - - if (trx_sys_mysql_bin_log_name[0] == '\0') { - return(true); - } - - fp = fopen(filename, "w"); + FILE *fp = fopen(filename, "w"); if (!fp) { msg("xtrabackup: failed to open '%s'\n", filename); return(false); } - fprintf(fp, "%s\t" UINT64PF "\n", - trx_sys_mysql_bin_log_name, trx_sys_mysql_bin_log_pos); + fprintf(fp, "%s\t%llu\n", name, pos); fclose(fp); return(true); } -static void -xtrabackup_prepare_func(int argc, char ** argv) +/** Implement --prepare +@return whether the operation succeeded */ +static bool +xtrabackup_prepare_func(char** argv) { - ulint err; datafiles_iter_t *it; - fil_node_t *node; - fil_space_t *space; char metadata_path[FN_REFLEN]; /* cd to target-dir */ @@ -5506,10 +4922,11 @@ xtrabackup_prepare_func(int argc, char ** argv) { msg("xtrabackup: cannot my_setwd %s\n", xtrabackup_real_target_dir); - exit(EXIT_FAILURE); + return(false); } msg("xtrabackup: cd to %s\n", xtrabackup_real_target_dir); + int argc; for (argc = 0; argv[argc]; argc++) {} encryption_plugin_prepare_init(argc, argv); xtrabackup_target_dir= mysql_data_home_buff; @@ -5525,116 +4942,100 @@ xtrabackup_prepare_func(int argc, char ** argv) if (!xtrabackup_read_metadata(metadata_path)) { msg("xtrabackup: Error: failed to read metadata from '%s'\n", metadata_path); - exit(EXIT_FAILURE); + return(false); } if (!strcmp(metadata_type, "full-backuped")) { + if (xtrabackup_incremental) { + msg("xtrabackup: error: applying incremental backup " + "needs a prepared target.\n"); + return(false); + } msg("xtrabackup: This target seems to be not prepared yet.\n"); } else if (!strcmp(metadata_type, "log-applied")) { - msg("xtrabackup: This target seems to be already " - "prepared with --apply-log-only.\n"); - goto skip_check; - } else if (!strcmp(metadata_type, "full-prepared")) { msg("xtrabackup: This target seems to be already prepared.\n"); } else { - msg("xtrabackup: This target seems not to have correct " - "metadata...\n"); - exit(EXIT_FAILURE); + msg("xtrabackup: This target does not have correct metadata.\n"); + return(false); } - if (xtrabackup_incremental) { - msg("xtrabackup: error: applying incremental backup " - "needs target prepared with --apply-log-only.\n"); - exit(EXIT_FAILURE); - } -skip_check: - if (xtrabackup_incremental - && metadata_to_lsn != incremental_lsn) { + bool ok = !xtrabackup_incremental + || metadata_to_lsn == incremental_lsn; + if (!ok) { msg("xtrabackup: error: This incremental backup seems " "not to be proper for the target.\n" "xtrabackup: Check 'to_lsn' of the target and " "'from_lsn' of the incremental.\n"); - exit(EXIT_FAILURE); + return(false); } - /* Create logfiles for recovery from 'xtrabackup_logfile', before start InnoDB */ srv_max_n_threads = 1000; + srv_undo_logs = 1; srv_n_purge_threads = 1; - ut_mem_init(); - /* temporally dummy value to avoid crash */ - srv_page_size_shift = 14; - srv_page_size = (1 << srv_page_size_shift); - os_sync_init(); - sync_init(); - os_io_init_simple(); - mem_init(srv_mem_pool_size); - ut_crc32_init(); - -#ifdef WITH_INNODB_DISALLOW_WRITES - srv_allow_writes_event = os_event_create(); - os_event_set(srv_allow_writes_event); -#endif xb_filters_init(); - if (xtrabackup_init_temp_log()) - goto error_cleanup; + srv_log_group_home_dir = NULL; + srv_thread_concurrency = 1; - if(innodb_init_param()) { - goto error_cleanup; - } + if (xtrabackup_incremental) { + if (innodb_init_param()) { +error_cleanup: + xb_filters_free(); + return(false); + } - xb_normalize_init_values(); + xb_normalize_init_values(); + sync_check_init(); + ut_d(sync_check_enable()); + ut_crc32_init(); + recv_sys_init(); + log_sys_init(); + recv_recovery_on = true; - if (xtrabackup_incremental) { - err = xb_data_files_init(); +#ifdef WITH_INNODB_DISALLOW_WRITES + srv_allow_writes_event = os_event_create(0); + os_event_set(srv_allow_writes_event); +#endif + + dberr_t err = xb_data_files_init(); if (err != DB_SUCCESS) { msg("xtrabackup: error: xb_data_files_init() failed " - "with error code %lu\n", err); + "with error %s\n", ut_strerr(err)); goto error_cleanup; } - } - if (xtrabackup_incremental) { + inc_dir_tables_hash = hash_create(1000); - if(!xtrabackup_apply_deltas()) { - xb_data_files_close(); - xb_filter_hash_free(inc_dir_tables_hash); - goto error_cleanup; - } - } - if (xtrabackup_incremental) { + ok = xtrabackup_apply_deltas(); + xb_data_files_close(); - } - if (xtrabackup_incremental) { - /* Cleanup datadir from tablespaces deleted between full and - incremental backups */ - xb_process_datadir("./", ".ibd", rm_if_not_found, NULL); + if (ok) { + /* Cleanup datadir from tablespaces deleted + between full and incremental backups */ + + xb_process_datadir("./", ".ibd", rm_if_not_found); + } xb_filter_hash_free(inc_dir_tables_hash); - } - if (fil_system) { + fil_close(); +#ifdef WITH_INNODB_DISALLOW_WRITES + os_event_destroy(srv_allow_writes_event); +#endif + innodb_free_param(); + log_shutdown(); + sync_check_close(); + if (!ok) goto error_cleanup; } - mem_close(); - ut_free_all_mem(); - - innodb_free_param(); - sync_close(); - sync_initialized = FALSE; - - /* Reset the configuration as it might have been changed by - xb_data_files_init(). */ - if(innodb_init_param()) { + if (innodb_init_param()) { goto error_cleanup; } - srv_apply_log_only = (bool) xtrabackup_apply_log_only; - /* increase IO threads */ - if(srv_n_file_io_threads < 10) { + if (srv_n_file_io_threads < 10) { srv_n_read_io_threads = 4; srv_n_write_io_threads = 4; } @@ -5649,85 +5050,51 @@ skip_check: srv_max_dirty_pages_pct_lwm = srv_max_buf_pool_modified_pct; } - if(innodb_init()) + if (innodb_init()) { goto error_cleanup; - - if (xtrabackup_incremental) { - - it = datafiles_iter_new(fil_system); - if (it == NULL) { - msg("xtrabackup: Error: datafiles_iter_new() failed.\n"); - exit(EXIT_FAILURE); - } - - while ((node = datafiles_iter_next(it)) != NULL) { - byte *header; - ulint size; - ulint actual_size; - mtr_t mtr; - buf_block_t *block; - ulint flags; - - space = node->space; - - /* Align space sizes along with fsp header. We want to process - each space once, so skip all nodes except the first one in a - multi-node space. */ - if (UT_LIST_GET_PREV(chain, node) != NULL) { - continue; - } - - mtr_start(&mtr); - - mtr_s_lock(fil_space_get_latch(space->id, &flags), &mtr); - - block = buf_page_get(space->id, - dict_tf_get_zip_size(flags), - 0, RW_S_LATCH, &mtr); - header = FSP_HEADER_OFFSET + buf_block_get_frame(block); - - size = mtr_read_ulint(header + FSP_SIZE, MLOG_4BYTES, - &mtr); - - mtr_commit(&mtr); - - fil_extend_space_to_desired_size(&actual_size, space->id, size); } - datafiles_iter_free(it); - - } /* if (xtrabackup_incremental) */ - if (xtrabackup_export) { +#if 1 // FIXME: remove the option or fix the logic + /* In MariaDB 10.2, undo log processing would need the + ability to evaluate indexed virtual columns, and we + have not initialized the necessary infrastructure. */ + msg("xtrabackup: --export does not work!\n"); + ok = false; + } else if (xtrabackup_export) { +#endif msg("xtrabackup: export option is specified.\n"); + + /* To allow subsequent MariaDB server startup independent + of the value of --innodb-log-checksums, + unconditionally enable redo log checksums. */ + log_checksum_algorithm_ptr = log_block_calc_checksum_crc32; + pfs_os_file_t info_file; char info_file_path[FN_REFLEN]; - ibool success; + bool success; char table_name[FN_REFLEN]; byte* page; byte* buf = NULL; - buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE * 2)); + buf = static_cast<byte *>(malloc(UNIV_PAGE_SIZE * 2)); page = static_cast<byte *>(ut_align(buf, UNIV_PAGE_SIZE)); - /* flush insert buffer at shutdwon */ - innobase_fast_shutdown = 0; - it = datafiles_iter_new(fil_system); if (it == NULL) { msg("xtrabackup: Error: datafiles_iter_new() " "failed.\n"); - exit(EXIT_FAILURE); - } - while ((node = datafiles_iter_next(it)) != NULL) { + ok = false; + } else + while (fil_node_t *node = datafiles_iter_next(it)) { int len; char *next, *prev, *p; dict_table_t* table; dict_index_t* index; ulint n_index; - space = node->space; + const fil_space_t* space = node->space; /* treat file_per_table only */ if (!fil_is_user_tablespace_id(space->id)) { @@ -5782,15 +5149,14 @@ skip_check: "than 31 indexes, .exp file was not " "generated. Table will fail to import " "on server version prior to 5.6.\n", - table->name); + table->name.m_name); goto next_node; } /* init exp file */ - memset(page, 0, UNIV_PAGE_SIZE); - mach_write_to_4(page , 0x78706f72UL); - mach_write_to_4(page + 4, 0x74696e66UL);/*"xportinf"*/ + memcpy(page, "xportinf", 8); mach_write_to_4(page + 8, n_index); + memset(page + 12, 0, UNIV_PAGE_SIZE - 12); strncpy((char *) page + 12, table_name, 500); @@ -5810,7 +5176,7 @@ skip_check: msg("xtrabackup: name=%s, " "id.low=%lu, page=%lu\n", - index->name, + index->name(), (ulint)(index->id & 0xFFFFFFFFUL), (ulint) index->page); @@ -5818,18 +5184,18 @@ skip_check: n_index++; } - srv_normalize_path_for_win(info_file_path); + os_normalize_path(info_file_path); info_file = os_file_create( 0, info_file_path, OS_FILE_OVERWRITE, OS_FILE_NORMAL, OS_DATA_FILE, - &success,0); + false, &success); if (!success) { os_file_get_last_error(TRUE); goto next_node; } - success = os_file_write(info_file_path, + success = os_file_write(IORequestWrite, info_file_path, info_file, page, 0, UNIV_PAGE_SIZE); if (!success) { @@ -5842,76 +5208,73 @@ skip_check: goto next_node; } next_node: - if (info_file != XB_FILE_UNDEFINED) { + if (info_file != OS_FILE_CLOSED) { os_file_close(info_file); - info_file = XB_FILE_UNDEFINED; + info_file = OS_FILE_CLOSED; } mutex_exit(&(dict_sys->mutex)); } - ut_free(buf); + free(buf); } - /* print the binary log position */ - trx_sys_print_mysql_binlog_offset(); - msg("\n"); - - /* output to xtrabackup_binlog_pos_innodb and (if - backup_safe_binlog_info was available on the server) to - xtrabackup_binlog_info. In the latter case xtrabackup_binlog_pos_innodb - becomes redundant and is created only for compatibility. */ - if (!store_binlog_info("xtrabackup_binlog_pos_innodb") || - (recover_binlog_info && - !store_binlog_info(XTRABACKUP_BINLOG_INFO))) { + if (ok) { + mtr_t mtr; + mtr.start(); + const trx_sysf_t* sys_header = trx_sysf_get(&mtr); + + if (mach_read_from_4(TRX_SYS_MYSQL_LOG_INFO + + TRX_SYS_MYSQL_LOG_MAGIC_N_FLD + + sys_header) + == TRX_SYS_MYSQL_LOG_MAGIC_N) { + ulonglong pos = mach_read_from_8( + TRX_SYS_MYSQL_LOG_INFO + + TRX_SYS_MYSQL_LOG_OFFSET + + sys_header); + const char* name = reinterpret_cast<const char*>( + TRX_SYS_MYSQL_LOG_INFO + TRX_SYS_MYSQL_LOG_NAME + + sys_header); + msg("Last binlog file %s, position %llu\n", name, pos); + + /* output to xtrabackup_binlog_pos_innodb and + (if backup_safe_binlog_info was available on + the server) to xtrabackup_binlog_info. In the + latter case xtrabackup_binlog_pos_innodb + becomes redundant and is created only for + compatibility. */ + ok = store_binlog_info( + "xtrabackup_binlog_pos_innodb", name, pos) + && (!recover_binlog_info || store_binlog_info( + XTRABACKUP_BINLOG_INFO, + name, pos)); + } - exit(EXIT_FAILURE); + mtr.commit(); } /* Check whether the log is applied enough or not. */ - if ((xtrabackup_incremental - && srv_start_lsn < incremental_to_lsn) - ||(!xtrabackup_incremental - && srv_start_lsn < metadata_to_lsn)) { + const lsn_t target_lsn = xtrabackup_incremental + ? incremental_to_lsn : metadata_to_lsn; + if ((srv_start_lsn || fil_space_get(SRV_LOG_SPACE_FIRST_ID)) + && srv_start_lsn < target_lsn) { msg("xtrabackup: error: " - "The transaction log file is corrupted.\n" - "xtrabackup: error: " - "The log was not applied to the intended LSN!\n"); - msg("xtrabackup: Log applied to lsn " LSN_PF "\n", - srv_start_lsn); - if (xtrabackup_incremental) { - msg("xtrabackup: The intended lsn is " LSN_PF "\n", - incremental_to_lsn); - } else { - msg("xtrabackup: The intended lsn is " LSN_PF "\n", - metadata_to_lsn); - } - exit(EXIT_FAILURE); + "The log was only applied up to LSN " LSN_PF + ", instead of " LSN_PF "\n", + srv_start_lsn, target_lsn); + ok = false; } #ifdef WITH_WSREP - xb_write_galera_info(xtrabackup_incremental); + else if (ok) xb_write_galera_info(xtrabackup_incremental); #endif - innodb_end(); - - innodb_free_param(); - - sync_initialized = FALSE; - - /* re-init necessary components */ - ut_mem_init(); - os_sync_init(); - sync_init(); - os_io_init_simple(); - - if(xtrabackup_close_temp_log(TRUE)) - exit(EXIT_FAILURE); + innodb_shutdown(); + innodb_free_param(); /* output to metadata file */ - { + if (ok) { char filename[FN_REFLEN]; - strcpy(metadata_type, srv_apply_log_only ? - "log-applied" : "full-prepared"); + strcpy(metadata_type, "log-applied"); if(xtrabackup_incremental && metadata_to_lsn < incremental_to_lsn) @@ -5925,73 +5288,21 @@ next_node: msg("xtrabackup: Error: failed to write metadata " "to '%s'\n", filename); - exit(EXIT_FAILURE); - } - - if(xtrabackup_extra_lsndir) { + ok = false; + } else if (xtrabackup_extra_lsndir) { sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, XTRABACKUP_METADATA_FILENAME); if (!xtrabackup_write_metadata(filename)) { msg("xtrabackup: Error: failed to write " "metadata to '%s'\n", filename); - exit(EXIT_FAILURE); + ok = false; } } } - if (!apply_log_finish()) { - exit(EXIT_FAILURE); - } - - sync_close(); - sync_initialized = FALSE; - if (fil_system) { - fil_close(); - } - - ut_free_all_mem(); - - /* start InnoDB once again to create log files */ - - if (!xtrabackup_apply_log_only) { - - /* xtrabackup_incremental_dir is used to indicate that - we are going to apply incremental backup. Here we already - applied incremental backup and are about to do final prepare - of the full backup */ - xtrabackup_incremental_dir = NULL; - - if(innodb_init_param()) { - goto error; - } - - srv_apply_log_only = false; - - /* increase IO threads */ - if(srv_n_file_io_threads < 10) { - srv_n_read_io_threads = 4; - srv_n_write_io_threads = 4; - } - - srv_shutdown_state = SRV_SHUTDOWN_NONE; - - if(innodb_init()) - goto error; - - innodb_end(); - innodb_free_param(); - - } - - xb_filters_free(); - - return; + if (ok) ok = apply_log_finish(); -error_cleanup: - xtrabackup_close_temp_log(FALSE); xb_filters_free(); - -error: - exit(EXIT_FAILURE); + return ok; } /************************************************************************** @@ -6084,7 +5395,8 @@ extern void init_signals(void); /* Messages . Avoid loading errmsg.sys file */ void setup_error_messages() { - static const char *all_msgs[ER_ERROR_LAST - ER_ERROR_FIRST +1]; + static const char *my_msgs[ERRORS_PER_RANGE]; + static const char **all_msgs[] = { my_msgs, my_msgs, my_msgs, my_msgs }; my_default_lc_messages = &my_locale_en_US; my_default_lc_messages->errmsgs->errmsgs = all_msgs; @@ -6111,24 +5423,20 @@ void setup_error_messages() }; for (int i = 0; i < (int)array_elements(all_msgs); i++) - all_msgs[i] = "Unknown error"; + all_msgs[0][i] = "Unknown error"; for (int i = 0; i < (int)array_elements(xb_msgs); i++) - all_msgs[xb_msgs[i].id - ER_ERROR_FIRST] = xb_msgs[i].fmt; + all_msgs[0][xb_msgs[i].id - ER_ERROR_FIRST] = xb_msgs[i].fmt; } -extern my_bool(*dict_check_if_skip_table)(const char* name) ; - void handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) { /* Setup some variables for Innodb.*/ - srv_xtrabackup = true; - + srv_operation = SRV_OPERATION_RESTORE; files_charset_info = &my_charset_utf8_general_ci; - dict_check_if_skip_table = check_if_skip_table; setup_error_messages(); sys_var_init(); @@ -6312,25 +5620,21 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) } } -/* ================= main =================== */ -extern my_bool(*fil_check_if_skip_database_by_path)(const char* name); +static int main_low(char** argv); +/* ================= main =================== */ int main(int argc, char **argv) { char **client_defaults, **server_defaults; - char cwd[FN_REFLEN]; - static char INNOBACKUPEX_EXE[]= "innobackupex"; + static char INNOBACKUPEX_EXE[]= "innobackupex"; if (argc > 1 && (strcmp(argv[1], "--innobackupex") == 0)) { argv++; argc--; - argv[0] = INNOBACKUPEX_EXE; + argv[0] = INNOBACKUPEX_EXE; innobackupex_mode = true; } - /* Setup skip fil_load_single_tablespaces callback.*/ - fil_check_if_skip_database_by_path = check_if_skip_database_by_path; - init_signals(); MY_INIT(argv[0]); @@ -6349,30 +5653,69 @@ int main(int argc, char **argv) system_charset_info = &my_charset_utf8_general_ci; key_map_full.set_all(); + logger.init_base(); + logger.set_handlers(LOG_FILE, LOG_NONE, LOG_NONE); + mysql_mutex_init(key_LOCK_error_log, &LOCK_error_log, + MY_MUTEX_INIT_FAST); + handle_options(argc, argv, &client_defaults, &server_defaults); - int argc_server; - for (argc_server = 0; server_defaults[argc_server]; argc_server++) {} +#ifndef DBUG_OFF + if (dbug_option) { + DBUG_SET_INITIAL(dbug_option); + DBUG_SET(dbug_option); + } +#endif - int argc_client; - for (argc_client = 0; client_defaults[argc_client]; argc_client++) {} + int status = main_low(server_defaults); + backup_cleanup(); if (innobackupex_mode) { + ibx_cleanup(); + } + + free_defaults(client_defaults); + free_defaults(server_defaults); + +#ifndef DBUG_OFF + if (dbug_option) { + DBUG_END(); + } +#endif + + if (THR_THD) + (void) pthread_key_delete(THR_THD); + + logger.cleanup_base(); + mysql_mutex_destroy(&LOCK_error_log); + + if (status == EXIT_SUCCESS) { + msg_ts("completed OK!\n"); + } + + return status; +} + +static int main_low(char** argv) +{ + if (innobackupex_mode) { if (!ibx_init()) { - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } } - if ((!xtrabackup_print_param) && (!xtrabackup_prepare) && (strcmp(mysql_data_home, "./") == 0)) { + if (!xtrabackup_print_param && !xtrabackup_prepare + && !strcmp(mysql_data_home, "./")) { if (!xtrabackup_print_param) usage(); msg("\nxtrabackup: Error: Please set parameter 'datadir'\n"); - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } /* Expand target-dir, incremental-basedir, etc. */ + char cwd[FN_REFLEN]; my_getwd(cwd, sizeof(cwd), MYF(0)); my_load_path(xtrabackup_real_target_dir, @@ -6438,7 +5781,7 @@ int main(int argc, char **argv) if (error) { msg("xtrabackup: value '%s' may be wrong format for " "incremental option.\n", xtrabackup_incremental); - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } } else if (xtrabackup_backup && xtrabackup_incremental_basedir) { char filename[FN_REFLEN]; @@ -6448,7 +5791,7 @@ int main(int argc, char **argv) if (!xtrabackup_read_metadata(filename)) { msg("xtrabackup: error: failed to read metadata from " "%s\n", filename); - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } incremental_lsn = metadata_to_lsn; @@ -6461,7 +5804,7 @@ int main(int argc, char **argv) if (!xtrabackup_read_metadata(filename)) { msg("xtrabackup: error: failed to read metadata from " "%s\n", filename); - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } incremental_lsn = metadata_from_lsn; @@ -6478,15 +5821,13 @@ int main(int argc, char **argv) } if (!xb_init()) { - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } /* --print-param */ if (xtrabackup_print_param) { - printf("%s", print_param_str.str().c_str()); - - exit(EXIT_SUCCESS); + return(EXIT_SUCCESS); } print_version(); @@ -6512,7 +5853,7 @@ int main(int argc, char **argv) if (xtrabackup_decrypt_decompress) num++; if (num != 1) { /* !XOR (for now) */ usage(); - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } } @@ -6523,41 +5864,28 @@ int main(int argc, char **argv) #endif /* --backup */ - if (xtrabackup_backup) - xtrabackup_backup_func(); + if (xtrabackup_backup && !xtrabackup_backup_func()) { + return(EXIT_FAILURE); + } /* --prepare */ - if (xtrabackup_prepare) { - xtrabackup_prepare_func(argc_server, server_defaults); + if (xtrabackup_prepare + && !xtrabackup_prepare_func(argv)) { + return(EXIT_FAILURE); } if (xtrabackup_copy_back || xtrabackup_move_back) { if (!check_if_param_set("datadir")) { msg("Error: datadir must be specified.\n"); - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } if (!copy_back()) - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } if (xtrabackup_decrypt_decompress && !decrypt_decompress()) { - exit(EXIT_FAILURE); - } - - backup_cleanup(); - - if (innobackupex_mode) { - ibx_cleanup(); + return(EXIT_FAILURE); } - - free_defaults(client_defaults); - free_defaults(server_defaults); - - if (THR_THD) - (void) pthread_key_delete(THR_THD); - - msg_ts("completed OK!\n"); - - exit(EXIT_SUCCESS); + return(EXIT_SUCCESS); } diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index eafc848fd43..b883194ef86 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -26,26 +26,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include "xbstream.h" #include "changed_page_bitmap.h" -#ifdef __WIN__ -#define XB_FILE_UNDEFINED INVALID_HANDLE_VALUE -#else -#define XB_FILE_UNDEFINED (-1) -#endif +struct xb_delta_info_t +{ + xb_delta_info_t(page_size_t page_size, ulint space_id) + : page_size(page_size), space_id(space_id) {} -typedef struct { - ulint page_size; - ulint zip_size; - ulint space_id; -} xb_delta_info_t; - -/* ======== Datafiles iterator ======== */ -typedef struct { - fil_system_t *system; - fil_space_t *space; - fil_node_t *node; - ibool started; - os_ib_mutex_t mutex; -} datafiles_iter_t; + page_size_t page_size; + ulint space_id; +}; /* value of the --incremental option */ extern lsn_t incremental_lsn; @@ -84,15 +72,11 @@ extern ibool xtrabackup_compress; extern my_bool xtrabackup_backup; extern my_bool xtrabackup_prepare; -extern my_bool xtrabackup_apply_log_only; extern my_bool xtrabackup_copy_back; extern my_bool xtrabackup_move_back; extern my_bool xtrabackup_decrypt_decompress; extern char *innobase_data_file_path; -extern char *innobase_doublewrite_file; -extern longlong innobase_log_file_size; -extern long innobase_log_files_in_group; extern longlong innobase_page_size; extern int xtrabackup_parallel; @@ -108,9 +92,7 @@ extern "C"{ } #endif extern my_bool xtrabackup_export; -extern char *xtrabackup_incremental_basedir; extern char *xtrabackup_extra_lsndir; -extern char *xtrabackup_incremental_dir; extern ulint xtrabackup_log_copy_interval; extern char *xtrabackup_stream_str; extern long xtrabackup_throttle; @@ -166,14 +148,9 @@ void xtrabackup_io_throttling(void); my_bool xb_write_delta_metadata(const char *filename, const xb_delta_info_t *info); -datafiles_iter_t *datafiles_iter_new(fil_system_t *f_system); -fil_node_t *datafiles_iter_next(datafiles_iter_t *it); -void datafiles_iter_free(datafiles_iter_t *it); - -/*********************************************************************** -Reads the space flags from a given data file and returns the compressed -page size, or 0 if the space is not compressed. */ -ulint xb_get_zip_size(pfs_os_file_t file); +/** @return the tablespace flags from a given data file +@retval ULINT_UNDEFINED if the file is not readable */ +ulint xb_get_space_flags(pfs_os_file_t file); /************************************************************************ Checks if a table specified as a name in the form "database/name" (InnoDB 5.6) @@ -204,17 +181,10 @@ bool check_if_param_set(const char *param); #if defined(HAVE_OPENSSL) -extern my_bool opt_use_ssl; extern my_bool opt_ssl_verify_server_cert; -#if !defined(HAVE_YASSL) -extern char *opt_server_public_key; -#endif #endif -void -xtrabackup_backup_func(void); - my_bool xb_get_one_option(int optid, const struct my_option *opt __attribute__((unused)), |