diff options
| author | Marko Mäkelä <marko.makela@mariadb.com> | 2022-02-25 13:00:48 +0200 |
|---|---|---|
| committer | Marko Mäkelä <marko.makela@mariadb.com> | 2022-02-25 13:00:48 +0200 |
| commit | f5ff7d09c73b5527cb6e0540cd470db9d8a82108 (patch) | |
| tree | 23728ff9752891cb78060ed748a3fc8bb7381c40 | |
| parent | 0eabc285a3c0cf0285d569a29c549634238fbdae (diff) | |
| parent | 9ba385a50d0cd611fce61462fc3e03e82b1ffee9 (diff) | |
| download | mariadb-git-f5ff7d09c73b5527cb6e0540cd470db9d8a82108.tar.gz | |
Merge 10.3 into 10.4
45 files changed, 1435 insertions, 613 deletions
diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index cea92f9dbfb..92f6b93dc95 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -578,7 +578,6 @@ datafile_read(datafile_cur_t *cursor) Check to see if a file exists. Takes name of the file to check. @return true if file exists. */ -static bool file_exists(const char *filename) { @@ -1540,13 +1539,14 @@ bool backup_start(CorruptedPages &corrupted_pages) if (!write_galera_info(mysql_connection)) { return(false); } - write_current_binlog_file(mysql_connection); } - if (opt_binlog_info == BINLOG_INFO_ON) { + bool with_binlogs = opt_binlog_info == BINLOG_INFO_ON; - lock_binlog_maybe(mysql_connection); - write_binlog_info(mysql_connection); + if (with_binlogs || opt_galera_info) { + if (!write_current_binlog_file(mysql_connection, with_binlogs)) { + return(false); + } } if (have_flush_engine_logs && !opt_no_lock) { @@ -1580,15 +1580,34 @@ void backup_release() } } +static const char *default_buffer_pool_file = "ib_buffer_pool"; + +static +const char * get_buffer_pool_filename(size_t *length) +{ + /* If mariabackup is run for Galera, then the file + name is changed to the default so that the receiving + node can find this file and rename it according to its + settings, otherwise we keep the original file name: */ + size_t dir_length = 0; + const char *dst_name = default_buffer_pool_file; + if (!opt_galera_info) { + dir_length = dirname_length(buffer_pool_filename); + dst_name = buffer_pool_filename + dir_length; + } + if (length) { + *length=dir_length; + } + return dst_name; +} + /** Finish after backup_start() and backup_release() */ bool backup_finish() { /* Copy buffer pool dump or LRU dump */ if (!opt_rsync) { if (buffer_pool_filename && file_exists(buffer_pool_filename)) { - const char *dst_name; - - dst_name = trim_dotslash(buffer_pool_filename); + const char *dst_name = get_buffer_pool_filename(NULL); copy_file(ds_data, buffer_pool_filename, dst_name, 0); } if (file_exists("ib_lru_dump")) { @@ -1677,17 +1696,14 @@ ibx_copy_incremental_over_full() /* copy buffer pool dump */ if (innobase_buffer_pool_filename) { - const char *src_name; - - src_name = trim_dotslash(innobase_buffer_pool_filename); + const char *src_name = get_buffer_pool_filename(NULL); snprintf(path, sizeof(path), "%s/%s", xtrabackup_incremental_dir, src_name); if (file_exists(path)) { - copy_file(ds_data, path, - innobase_buffer_pool_filename, 0); + copy_file(ds_data, path, src_name, 0); } } @@ -1922,6 +1938,14 @@ copy_back() datadir_node_init(&node); + /* If mariabackup is run for Galera, then the file + name is changed to the default so that the receiving + node can find this file and rename it according to its + settings, otherwise we keep the original file name: */ + size_t dir_length; + const char *src_buffer_pool; + src_buffer_pool = get_buffer_pool_filename(&dir_length); + while (datadir_iter_next(it, &node)) { const char *ext_list[] = {"backup-my.cnf", "xtrabackup_binary", "xtrabackup_binlog_info", @@ -1984,6 +2008,11 @@ copy_back() continue; } + /* skip buffer pool dump */ + if (!strcmp(filename, src_buffer_pool)) { + continue; + } + /* skip innodb data files */ is_ibdata_file = false; for (Tablespace::const_iterator iter(srv_sys_space.begin()), @@ -2006,23 +2035,18 @@ copy_back() /* copy buffer pool dump */ - if (innobase_buffer_pool_filename) { - const char *src_name; - char path[FN_REFLEN]; - - src_name = trim_dotslash(innobase_buffer_pool_filename); - - snprintf(path, sizeof(path), "%s/%s", - mysql_data_home, - src_name); - - /* could be already copied with other files - from data directory */ - if (file_exists(src_name) && - !file_exists(innobase_buffer_pool_filename)) { - copy_or_move_file(src_name, - innobase_buffer_pool_filename, - mysql_data_home, 0); + if (file_exists(src_buffer_pool)) { + char dst_dir[FN_REFLEN]; + while (IS_TRAILING_SLASH(buffer_pool_filename, dir_length)) { + dir_length--; + } + memcpy(dst_dir, buffer_pool_filename, dir_length); + dst_dir[dir_length] = 0; + if (!(ret = copy_or_move_file(src_buffer_pool, + src_buffer_pool, + dst_dir, 1))) + { + goto cleanup; } } diff --git a/extra/mariabackup/backup_copy.h b/extra/mariabackup/backup_copy.h index 62b2b1bc232..858182001ce 100644 --- a/extra/mariabackup/backup_copy.h +++ b/extra/mariabackup/backup_copy.h @@ -32,6 +32,13 @@ copy_file(ds_ctxt_t *datasink, const char *dst_file_path, uint thread_n); +/************************************************************************ +Check to see if a file exists. +Takes name of the file to check. +@return true if file exists. */ +bool +file_exists(const char *filename); + /** Start --backup */ bool backup_start(CorruptedPages &corrupted_pages); /** Release resources after backup_start() */ diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index 1c8305580c7..475d438dca3 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -84,7 +84,6 @@ os_event_t kill_query_thread_stop; bool sql_thread_started = false; char *mysql_slave_position = NULL; char *mysql_binlog_position = NULL; -char *buffer_pool_filename = NULL; /* History on server */ time_t history_start_time; @@ -1224,27 +1223,29 @@ cleanup: } +static +bool +write_binlog_info(MYSQL *connection, char *log_bin_dir, + MYSQL_RES *mysql_result, my_ulonglong n_rows, + my_ulonglong start); + /*********************************************************************//** Flush and copy the current binary log file into the backup, if GTID is enabled */ bool -write_current_binlog_file(MYSQL *connection) +write_current_binlog_file(MYSQL *connection, bool write_binlogs) { + char *log_bin = NULL; + char *filename = NULL; + char *position = NULL; char *executed_gtid_set = NULL; char *gtid_binlog_state = NULL; - char *log_bin_file = NULL; char *log_bin_dir = NULL; bool gtid_exists; bool result = true; - char filepath[FN_REFLEN]; - mysql_variable status[] = { - {"Executed_Gtid_Set", &executed_gtid_set}, - {NULL, NULL} - }; - - mysql_variable status_after_flush[] = { - {"File", &log_bin_file}, + mysql_variable log_bin_var[] = { + {"@@GLOBAL.log_bin", &log_bin}, {NULL, NULL} }; @@ -1254,21 +1255,36 @@ write_current_binlog_file(MYSQL *connection) {NULL, NULL} }; + mysql_variable status[] = { + {"File", &filename}, + {"Position", &position}, + {"Executed_Gtid_Set", &executed_gtid_set}, + {NULL, NULL} + }; + + read_mysql_variables(connection, "SELECT @@GLOBAL.log_bin", log_bin_var, false); + + /* Do not create xtrabackup_binlog_info if binary log is disabled: */ + if (strncmp(log_bin, "1", 2) != 0) { + goto binlog_disabled; + } + + lock_binlog_maybe(connection); + read_mysql_variables(connection, "SHOW MASTER STATUS", status, false); + + /* Do not create xtrabackup_binlog_info if replication + has not started yet: */ + if (filename == NULL || position == NULL) { + goto no_replication; + } + read_mysql_variables(connection, "SHOW VARIABLES", vars, true); gtid_exists = (executed_gtid_set && *executed_gtid_set) || (gtid_binlog_state && *gtid_binlog_state); - if (gtid_exists) { - size_t log_bin_dir_length; - - lock_binlog_maybe(connection); - - xb_mysql_query(connection, "FLUSH BINARY LOGS", false); - - read_mysql_variables(connection, "SHOW MASTER STATUS", - status_after_flush, false); + if (write_binlogs || gtid_exists) { if (opt_log_bin != NULL && strchr(opt_log_bin, FN_LIBCHAR)) { /* If log_bin is set, it has priority */ @@ -1278,34 +1294,89 @@ write_current_binlog_file(MYSQL *connection) log_bin_dir = strdup(opt_log_bin); } else if (log_bin_dir == NULL) { /* Default location is MySQL datadir */ - log_bin_dir = strdup("./"); + log_bin_dir = static_cast<char*>(malloc(3)); + ut_a(log_bin_dir); + log_bin_dir[0] = '.'; + log_bin_dir[1] = FN_LIBCHAR; + log_bin_dir[2] = 0; } + size_t log_bin_dir_length; + dirname_part(log_bin_dir, log_bin_dir, &log_bin_dir_length); /* strip final slash if it is not the only path component */ - if (log_bin_dir_length > 1 && - log_bin_dir[log_bin_dir_length - 1] == FN_LIBCHAR) { - log_bin_dir[log_bin_dir_length - 1] = 0; + while (IS_TRAILING_SLASH(log_bin_dir, log_bin_dir_length)) { + log_bin_dir_length--; } + log_bin_dir[log_bin_dir_length] = 0; - if (log_bin_dir == NULL || log_bin_file == NULL) { - msg("Failed to get master binlog coordinates from " - "SHOW MASTER STATUS"); + if (log_bin_dir == NULL) { + msg("Failed to locate binary log files"); result = false; goto cleanup; } - snprintf(filepath, sizeof(filepath), "%s%c%s", - log_bin_dir, FN_LIBCHAR, log_bin_file); - result = copy_file(ds_data, filepath, log_bin_file, 0); + uint max_binlogs; + max_binlogs = opt_max_binlogs; + if (max_binlogs == 0) { + if (gtid_exists) { + max_binlogs = 1; + } else { + goto cleanup; + } + } + + xb_mysql_query(connection, "FLUSH BINARY LOGS", false); + + MYSQL_RES *mysql_result; + + mysql_result = xb_mysql_query(connection, "SHOW BINARY LOGS", true); + + ut_ad(mysql_num_fields(mysql_result) >= 2); + + my_ulonglong n_rows; + my_ulonglong start; + + n_rows = mysql_num_rows(mysql_result); + + start = 0; + if (max_binlogs < n_rows) { + start = n_rows - max_binlogs; + } + if (start) { + mysql_data_seek(mysql_result, start); + } + + MYSQL_ROW row; + while ((row = mysql_fetch_row(mysql_result))) { + const char *binlog_name = row[0]; + char filepath[FN_REFLEN]; + snprintf(filepath, sizeof(filepath), "%s%c%s", + log_bin_dir, FN_LIBCHAR, binlog_name); + if (file_exists(filepath)) { + result = copy_file(ds_data, filepath, binlog_name, 0); + if (!result) break; + } + } + + if (result) { + write_binlog_info(connection, log_bin_dir, + mysql_result, n_rows, start); + } + + mysql_free_result(mysql_result); } cleanup: - free_mysql_variables(status_after_flush); - free_mysql_variables(status); free_mysql_variables(vars); +no_replication: + free_mysql_variables(status); + +binlog_disabled: + free_mysql_variables(log_bin_var); + return(result); } @@ -1313,8 +1384,11 @@ cleanup: /*********************************************************************//** Retrieves MySQL binlog position and saves it in a file. It also prints it to stdout. */ +static bool -write_binlog_info(MYSQL *connection) +write_binlog_info(MYSQL *connection, char *log_bin_dir, + MYSQL_RES *mysql_result, my_ulonglong n_rows, + my_ulonglong start) { char *filename = NULL; char *position = NULL; @@ -1322,9 +1396,13 @@ write_binlog_info(MYSQL *connection) char *gtid_current_pos = NULL; char *gtid_executed = NULL; char *gtid = NULL; - bool result; + char *buffer; + char *buf; + size_t total; + bool result = true; bool mysql_gtid; bool mariadb_gtid; + bool with_gtid; mysql_variable status[] = { {"File", &filename}, @@ -1342,39 +1420,106 @@ write_binlog_info(MYSQL *connection) read_mysql_variables(connection, "SHOW MASTER STATUS", status, false); read_mysql_variables(connection, "SHOW VARIABLES", vars, true); - if (filename == NULL || position == NULL) { - /* Do not create xtrabackup_binlog_info if binary - log is disabled */ - result = true; - goto cleanup; - } - - mysql_gtid = ((gtid_mode != NULL) && (strcmp(gtid_mode, "ON") == 0)); - mariadb_gtid = (gtid_current_pos != NULL); + mysql_gtid = gtid_mode && (strcmp(gtid_mode, "ON") == 0); + mariadb_gtid = gtid_current_pos && *gtid_current_pos; - gtid = (gtid_executed != NULL ? gtid_executed : gtid_current_pos); + gtid = (gtid_executed && *gtid_executed) ? gtid_executed : gtid_current_pos; - if (mariadb_gtid || mysql_gtid) { + with_gtid = mariadb_gtid || mysql_gtid; + if (with_gtid) { ut_a(asprintf(&mysql_binlog_position, "filename '%s', position '%s', " "GTID of the last change '%s'", filename, position, gtid) != -1); - result = backup_file_printf(XTRABACKUP_BINLOG_INFO, - "%s\t%s\t%s\n", filename, position, - gtid); } else { ut_a(asprintf(&mysql_binlog_position, "filename '%s', position '%s'", filename, position) != -1); - result = backup_file_printf(XTRABACKUP_BINLOG_INFO, - "%s\t%s\n", filename, position); } + mysql_data_seek(mysql_result, start); + + MYSQL_ROW row; + my_ulonglong current; + + total = 1; + current = start; + while ((row = mysql_fetch_row(mysql_result))) { + const char *binlog_name = row[0]; + /* The position in the current binlog is taken from + the global variable, but for the previous ones it is + determined by their length: */ + const char *binlog_pos = + ++current == n_rows ? position : row[1]; + total += strlen(binlog_name) + strlen(binlog_pos) + 2; + if (with_gtid && current != n_rows) { + /* Add the "\t[]" length to the buffer size: */ + total += 3; + } + } + /* For the last of the binray log files, also add + the length of the GTID (+ one character for '\t'): */ + if (with_gtid) { + total += strlen(gtid) + 1; + } + + buffer = static_cast<char*>(malloc(total)); + if (!buffer) { + msg("Failed to allocate memory for temporary buffer"); + result = false; + goto cleanup; + } + + mysql_data_seek(mysql_result, start); + + buf = buffer; + current = start; + while ((row = mysql_fetch_row(mysql_result))) { + const char *binlog_name = row[0]; + char filepath[FN_REFLEN]; + snprintf(filepath, sizeof(filepath), "%s%c%s", + log_bin_dir, FN_LIBCHAR, binlog_name); + current++; + if (file_exists(filepath)) { + /* The position in the current binlog is taken from + the global variable, but for the previous ones it is + determined by their length: */ + char *binlog_pos = + current == n_rows ? position : row[1]; + int bytes; + if (with_gtid) { + bytes = snprintf(buf, total, "%s\t%s\t%s\n", + binlog_name, binlog_pos, + current == n_rows ? gtid : "[]"); + } else { + bytes = snprintf(buf, total, "%s\t%s\n", + binlog_name, binlog_pos); + } + if (bytes <= 0) { + goto buffer_overflow; + } + buf += bytes; + total -= bytes; + } + } + + if (buf != buffer) { + result = backup_file_printf(XTRABACKUP_BINLOG_INFO, "%s", buffer); + } + +cleanup2: + free(buffer); + cleanup: - free_mysql_variables(status); free_mysql_variables(vars); + free_mysql_variables(status); return(result); + +buffer_overflow: + msg("Internal error: buffer overflow in the write_binlog_info()"); + result = false; + goto cleanup2; } struct escape_and_quote @@ -1702,7 +1847,6 @@ backup_cleanup() { free(mysql_slave_position); free(mysql_binlog_position); - free(buffer_pool_filename); if (mysql_connection) { mysql_close(mysql_connection); diff --git a/extra/mariabackup/backup_mysql.h b/extra/mariabackup/backup_mysql.h index b61fa2362c6..5e78281e1cb 100644 --- a/extra/mariabackup/backup_mysql.h +++ b/extra/mariabackup/backup_mysql.h @@ -28,7 +28,6 @@ extern time_t history_lock_time; extern bool sql_thread_started; extern char *mysql_slave_position; extern char *mysql_binlog_position; -extern char *buffer_pool_filename; /** connection to mysql server */ extern MYSQL *mysql_connection; @@ -62,10 +61,7 @@ void unlock_all(MYSQL *connection); bool -write_current_binlog_file(MYSQL *connection); - -bool -write_binlog_info(MYSQL *connection); +write_current_binlog_file(MYSQL *connection, bool write_binlogs); bool write_xtrabackup_info(MYSQL *connection, const char * filename, bool history, diff --git a/extra/mariabackup/common.h b/extra/mariabackup/common.h index 1973512ad82..beb49524608 100644 --- a/extra/mariabackup/common.h +++ b/extra/mariabackup/common.h @@ -187,4 +187,14 @@ xb_read_full(File fd, uchar *buf, size_t len) return tlen; } +#ifdef _WIN32 +#define IS_TRAILING_SLASH(name, length) \ + ((length) > 1 && \ + (name[(length) - 1] == '/' || \ + name[(length) - 1] == '\\')) +#else +#define IS_TRAILING_SLASH(name, length) \ + ((length) > 1 && name[(length) - 1] == FN_LIBCHAR) +#endif + #endif diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 8e202a8cd24..b57dcc86ff2 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -4,7 +4,7 @@ MariaBackup: hot backup tool for InnoDB Originally Created 3/3/2009 Yasufumi Kinoshita Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. -(c) 2017, 2021, MariaDB Corporation. +(c) 2017, 2022, MariaDB Corporation. Portions written by Marko Mäkelä. This program is free software; you can redistribute it and/or modify @@ -245,7 +245,8 @@ long innobase_read_io_threads = 4; long innobase_write_io_threads = 4; longlong innobase_page_size = (1LL << 14); /* 16KB */ -char* innobase_buffer_pool_filename = NULL; +char *innobase_buffer_pool_filename = NULL; +char *buffer_pool_filename = NULL; /* The default values for the following char* start-up parameters are determined in innobase_init below: */ @@ -354,6 +355,7 @@ uint opt_lock_wait_timeout = 0; uint opt_lock_wait_threshold = 0; uint opt_debug_sleep_before_unlock = 0; uint opt_safe_slave_backup_timeout = 0; +uint opt_max_binlogs = UINT_MAX; const char *opt_history = NULL; @@ -1054,7 +1056,8 @@ enum options_xtrabackup OPT_XTRA_CHECK_PRIVILEGES, OPT_XTRA_MYSQLD_ARGS, OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION, - OPT_INNODB_FORCE_RECOVERY + OPT_INNODB_FORCE_RECOVERY, + OPT_MAX_BINLOGS }; struct my_option xb_client_options[]= { @@ -1457,6 +1460,17 @@ struct my_option xb_client_options[]= { &opt_log_innodb_page_corruption, &opt_log_innodb_page_corruption, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"sst_max_binlogs", OPT_MAX_BINLOGS, + "Number of recent binary logs to be included in the backup. " + "Setting this parameter to zero normally disables transmission " + "of binary logs to the joiner nodes during SST using Galera. " + "But sometimes a single current binlog can still be transmitted " + "to the joiner even with sst_max_binlogs=0, because it is " + "required for Galera to work properly with GTIDs support.", + (G_PTR *) &opt_max_binlogs, + (G_PTR *) &opt_max_binlogs, 0, GET_UINT, OPT_ARG, + UINT_MAX, 0, UINT_MAX, 0, 1, 0}, + #define MYSQL_CLIENT #include "sslopt-longopts.h" #undef MYSQL_CLIENT @@ -1505,14 +1519,14 @@ struct my_option xb_server_options[] = (G_PTR*)&opt_encrypted_backup, 0, GET_BOOL, NO_ARG, TRUE, 0, 0, 0, 0, 0}, - {"log", OPT_LOG, "Ignored option for MySQL option compatibility", + {"log", OPT_LOG, "Ignored option for MySQL option compatibility", (G_PTR*) &log_ignored_opt, (G_PTR*) &log_ignored_opt, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, - {"log_bin", OPT_LOG, "Base name for the log sequence", + {"log_bin", OPT_LOG, "Base name for the log sequence", &opt_log_bin, &opt_log_bin, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, - {"innodb", OPT_INNODB, "Ignored option for MySQL option compatibility", + {"innodb", OPT_INNODB, "Ignored option for MySQL option compatibility", (G_PTR*) &innobase_ignored_opt, (G_PTR*) &innobase_ignored_opt, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, #ifdef BTR_CUR_HASH_ADAPT @@ -1634,10 +1648,10 @@ struct my_option xb_server_options[] = 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"plugin-dir", OPT_PLUGIN_DIR, - "Server plugin directory. Used to load encryption plugin during 'prepare' phase." - "Has no effect in the 'backup' phase (plugin directory during backup is the same as server's)", - &xb_plugin_dir, &xb_plugin_dir, - 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + "Server plugin directory. Used to load encryption plugin during 'prepare' phase." + "Has no effect in the 'backup' phase (plugin directory during backup is the same as server's)", + &xb_plugin_dir, &xb_plugin_dir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, {"innodb-log-checksums", OPT_INNODB_LOG_CHECKSUMS, "Whether to require checksums for InnoDB redo log blocks", @@ -1659,12 +1673,12 @@ struct my_option xb_server_options[] = &xb_rocksdb_datadir, &xb_rocksdb_datadir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, - { "rocksdb-backup", OPT_BACKUP_ROCKSDB, "Backup rocksdb data, if rocksdb plugin is installed." + {"rocksdb-backup", OPT_BACKUP_ROCKSDB, "Backup rocksdb data, if rocksdb plugin is installed." "Used only with --backup option. Can be useful for partial backups, to exclude all rocksdb data", &xb_backup_rocksdb, &xb_backup_rocksdb, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 }, - {"check-privileges", OPT_XTRA_CHECK_PRIVILEGES, "Check database user " + {"check-privileges", OPT_XTRA_CHECK_PRIVILEGES, "Check database user " "privileges fro the backup user", &opt_check_privileges, &opt_check_privileges, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 }, @@ -6426,6 +6440,44 @@ static bool check_all_privileges() return true; } +static +void +xb_init_buffer_pool(const char * filename) +{ + if (filename && +#ifdef _WIN32 + (filename[0] == '/' || + filename[0] == '\\' || + strchr(filename, ':'))) +#else + filename[0] == FN_LIBCHAR) +#endif + { + buffer_pool_filename = strdup(filename); + } else { + char filepath[FN_REFLEN]; + char *dst_dir = + (innobase_data_home_dir && *innobase_data_home_dir) ? + innobase_data_home_dir : mysql_data_home; + size_t dir_length; + if (dst_dir && *dst_dir) { + dir_length = strlen(dst_dir); + while (IS_TRAILING_SLASH(dst_dir, dir_length)) { + dir_length--; + } + memcpy(filepath, dst_dir, dir_length); + } + else { + filepath[0] = '.'; + dir_length = 1; + } + snprintf(filepath + dir_length, + sizeof(filepath) - dir_length, "%c%s", FN_LIBCHAR, + filename ? filename : "ib_buffer_pool"); + buffer_pool_filename = strdup(filepath); + } +} + bool xb_init() { @@ -6490,11 +6542,16 @@ xb_init() if (!get_mysql_vars(mysql_connection)) { return(false); } + + xb_init_buffer_pool(buffer_pool_filename); + if (opt_check_privileges && !check_all_privileges()) { return(false); } - history_start_time = time(NULL); + history_start_time = time(NULL); + } else { + xb_init_buffer_pool(innobase_buffer_pool_filename); } return(true); @@ -6884,6 +6941,8 @@ int main(int argc, char **argv) free_error_messages(); mysql_mutex_destroy(&LOCK_error_log); + free(buffer_pool_filename); + if (status == EXIT_SUCCESS) { msg("completed OK!"); } diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index 854456c3afd..d6e9f620424 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -71,6 +71,7 @@ extern char *xtrabackup_incremental_dir; extern char *xtrabackup_incremental_basedir; extern char *innobase_data_home_dir; extern char *innobase_buffer_pool_filename; +extern char *buffer_pool_filename; extern char *xb_plugin_dir; extern char *xb_rocksdb_datadir; extern my_bool xb_backup_rocksdb; @@ -166,6 +167,7 @@ extern uint opt_lock_wait_timeout; extern uint opt_lock_wait_threshold; extern uint opt_debug_sleep_before_unlock; extern uint opt_safe_slave_backup_timeout; +extern uint opt_max_binlogs; extern const char *opt_history; diff --git a/mysql-test/main/subselect2.result b/mysql-test/main/subselect2.result index a3d7fda7abc..c54d635230f 100644 --- a/mysql-test/main/subselect2.result +++ b/mysql-test/main/subselect2.result @@ -262,7 +262,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 2 DEPENDENT SUBQUERY t2c ALL NULL NULL NULL NULL 2 100.00 Using where; Using join buffer (incremental, BNL join) Warnings: Note 1276 Field or reference 'test.t1.a' of SELECT #2 was resolved in SELECT #1 -Note 1003 /* select#1 */ select `test`.`t1`.`a` AS `a`,`test`.`t1`.`c1` AS `c1` from `test`.`t1` where !<expr_cache><`test`.`t1`.`c1`,`test`.`t1`.`a`>(<in_optimizer>(`test`.`t1`.`c1`,<exists>(/* select#2 */ select `test`.`t2a`.`c2` from `test`.`t2` `t2a` join `test`.`t2` `t2b` join `test`.`t2` `t2c` where (`test`.`t2b`.`m` <> `test`.`t1`.`a` or `test`.`t2b`.`m` = `test`.`t2a`.`m`) and trigcond(<cache>(`test`.`t1`.`c1`) = `test`.`t2a`.`c2` or `test`.`t2a`.`c2` is null) and `test`.`t2c`.`c2` = `test`.`t2b`.`c2` and `test`.`t2b`.`n` = `test`.`t2a`.`m` having trigcond(`test`.`t2a`.`c2` is null)))) +Note 1003 /* select#1 */ select `test`.`t1`.`a` AS `a`,`test`.`t1`.`c1` AS `c1` from `test`.`t1` where !<expr_cache><`test`.`t1`.`c1`,`test`.`t1`.`a`>(<in_optimizer>(`test`.`t1`.`c1`,<exists>(/* select#2 */ select `test`.`t2a`.`c2` from `test`.`t2` `t2a` join `test`.`t2` `t2b` join `test`.`t2` `t2c` where `test`.`t2c`.`c2` = `test`.`t2b`.`c2` and `test`.`t2b`.`n` = `test`.`t2a`.`m` and (`test`.`t2b`.`m` <> `test`.`t1`.`a` or `test`.`t2b`.`m` = `test`.`t2a`.`m`) and trigcond(<cache>(`test`.`t1`.`c1`) = `test`.`t2a`.`c2` or `test`.`t2a`.`c2` is null) having trigcond(`test`.`t2a`.`c2` is null)))) DROP TABLE t1,t2; # # MDEV-614, also MDEV-536, also LP:1050806: diff --git a/mysql-test/main/subselect3.result b/mysql-test/main/subselect3.result index 5c4544a1b05..0034f61ac23 100644 --- a/mysql-test/main/subselect3.result +++ b/mysql-test/main/subselect3.result @@ -169,7 +169,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 2 DEPENDENT SUBQUERY t2 ref a a 5 test.t1.b 1 100.00 Using where Warnings: Note 1276 Field or reference 'test.t3.oref' of SELECT #2 was resolved in SELECT #1 -Note 1003 /* select#1 */ select `test`.`t3`.`a` AS `a`,`test`.`t3`.`oref` AS `oref`,<expr_cache><`test`.`t3`.`a`,`test`.`t3`.`oref`>(<in_optimizer>(`test`.`t3`.`a`,<exists>(/* select#2 */ select `test`.`t1`.`a` from `test`.`t1` join `test`.`t2` where `test`.`t2`.`b` = `test`.`t3`.`oref` and trigcond(<cache>(`test`.`t3`.`a`) = `test`.`t1`.`a` or `test`.`t1`.`a` is null) and `test`.`t2`.`a` = `test`.`t1`.`b` having trigcond(`test`.`t1`.`a` is null)))) AS `Z` from `test`.`t3` +Note 1003 /* select#1 */ select `test`.`t3`.`a` AS `a`,`test`.`t3`.`oref` AS `oref`,<expr_cache><`test`.`t3`.`a`,`test`.`t3`.`oref`>(<in_optimizer>(`test`.`t3`.`a`,<exists>(/* select#2 */ select `test`.`t1`.`a` from `test`.`t1` join `test`.`t2` where `test`.`t2`.`a` = `test`.`t1`.`b` and `test`.`t2`.`b` = `test`.`t3`.`oref` and trigcond(<cache>(`test`.`t3`.`a`) = `test`.`t1`.`a` or `test`.`t1`.`a` is null) having trigcond(`test`.`t1`.`a` is null)))) AS `Z` from `test`.`t3` drop table t1, t2, t3; create table t1 (a int NOT NULL, b int NOT NULL, key(a)); insert into t1 values @@ -197,7 +197,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 2 DEPENDENT SUBQUERY t2 ref a a 4 test.t1.b 1 100.00 Using where Warnings: Note 1276 Field or reference 'test.t3.oref' of SELECT #2 was resolved in SELECT #1 -Note 1003 /* select#1 */ select `test`.`t3`.`a` AS `a`,`test`.`t3`.`oref` AS `oref`,<expr_cache><`test`.`t3`.`a`,`test`.`t3`.`oref`>(<in_optimizer>(`test`.`t3`.`a`,<exists>(/* select#2 */ select `test`.`t1`.`a` from `test`.`t1` join `test`.`t2` where `test`.`t2`.`b` = `test`.`t3`.`oref` and trigcond(<cache>(`test`.`t3`.`a`) = `test`.`t1`.`a`) and `test`.`t2`.`a` = `test`.`t1`.`b`))) AS `Z` from `test`.`t3` +Note 1003 /* select#1 */ select `test`.`t3`.`a` AS `a`,`test`.`t3`.`oref` AS `oref`,<expr_cache><`test`.`t3`.`a`,`test`.`t3`.`oref`>(<in_optimizer>(`test`.`t3`.`a`,<exists>(/* select#2 */ select `test`.`t1`.`a` from `test`.`t1` join `test`.`t2` where `test`.`t2`.`a` = `test`.`t1`.`b` and `test`.`t2`.`b` = `test`.`t3`.`oref` and trigcond(<cache>(`test`.`t3`.`a`) = `test`.`t1`.`a`)))) AS `Z` from `test`.`t3` drop table t1,t2,t3; create table t1 (oref int, grp int); insert into t1 (oref, grp) values diff --git a/mysql-test/main/subselect3_jcl6.result b/mysql-test/main/subselect3_jcl6.result index 4260676cc37..b7b18bf80e0 100644 --- a/mysql-test/main/subselect3_jcl6.result +++ b/mysql-test/main/subselect3_jcl6.result @@ -172,7 +172,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 2 DEPENDENT SUBQUERY t2 ref a a 5 test.t1.b 1 100.00 Using where; Using join buffer (flat, BKA join); Key-ordered Rowid-ordered scan Warnings: Note 1276 Field or reference 'test.t3.oref' of SELECT #2 was resolved in SELECT #1 -Note 1003 /* select#1 */ select `test`.`t3`.`a` AS `a`,`test`.`t3`.`oref` AS `oref`,<expr_cache><`test`.`t3`.`a`,`test`.`t3`.`oref`>(<in_optimizer>(`test`.`t3`.`a`,<exists>(/* select#2 */ select `test`.`t1`.`a` from `test`.`t1` join `test`.`t2` where `test`.`t2`.`b` = `test`.`t3`.`oref` and trigcond(<cache>(`test`.`t3`.`a`) = `test`.`t1`.`a` or `test`.`t1`.`a` is null) and `test`.`t2`.`a` = `test`.`t1`.`b` having trigcond(`test`.`t1`.`a` is null)))) AS `Z` from `test`.`t3` +Note 1003 /* select#1 */ select `test`.`t3`.`a` AS `a`,`test`.`t3`.`oref` AS `oref`,<expr_cache><`test`.`t3`.`a`,`test`.`t3`.`oref`>(<in_optimizer>(`test`.`t3`.`a`,<exists>(/* select#2 */ select `test`.`t1`.`a` from `test`.`t1` join `test`.`t2` where `test`.`t2`.`a` = `test`.`t1`.`b` and `test`.`t2`.`b` = `test`.`t3`.`oref` and trigcond(<cache>(`test`.`t3`.`a`) = `test`.`t1`.`a` or `test`.`t1`.`a` is null) having trigcond(`test`.`t1`.`a` is null)))) AS `Z` from `test`.`t3` drop table t1, t2, t3; create table t1 (a int NOT NULL, b int NOT NULL, key(a)); insert into t1 values @@ -200,7 +200,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 2 DEPENDENT SUBQUERY t2 ref a a 4 test.t1.b 1 100.00 Using where; Using join buffer (flat, BKA join); Key-ordered Rowid-ordered scan Warnings: Note 1276 Field or reference 'test.t3.oref' of SELECT #2 was resolved in SELECT #1 -Note 1003 /* select#1 */ select `test`.`t3`.`a` AS `a`,`test`.`t3`.`oref` AS `oref`,<expr_cache><`test`.`t3`.`a`,`test`.`t3`.`oref`>(<in_optimizer>(`test`.`t3`.`a`,<exists>(/* select#2 */ select `test`.`t1`.`a` from `test`.`t1` join `test`.`t2` where `test`.`t2`.`b` = `test`.`t3`.`oref` and trigcond(<cache>(`test`.`t3`.`a`) = `test`.`t1`.`a`) and `test`.`t2`.`a` = `test`.`t1`.`b`))) AS `Z` from `test`.`t3` +Note 1003 /* select#1 */ select `test`.`t3`.`a` AS `a`,`test`.`t3`.`oref` AS `oref`,<expr_cache><`test`.`t3`.`a`,`test`.`t3`.`oref`>(<in_optimizer>(`test`.`t3`.`a`,<exists>(/* select#2 */ select `test`.`t1`.`a` from `test`.`t1` join `test`.`t2` where `test`.`t2`.`a` = `test`.`t1`.`b` and `test`.`t2`.`b` = `test`.`t3`.`oref` and trigcond(<cache>(`test`.`t3`.`a`) = `test`.`t1`.`a`)))) AS `Z` from `test`.`t3` drop table t1,t2,t3; create table t1 (oref int, grp int); insert into t1 (oref, grp) values diff --git a/mysql-test/main/subselect4.result b/mysql-test/main/subselect4.result index eba624c3ae4..600a546e87a 100644 --- a/mysql-test/main/subselect4.result +++ b/mysql-test/main/subselect4.result @@ -2351,7 +2351,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t2 system NULL NULL NULL NULL 1 100.00 2 DEPENDENT SUBQUERY t1 ALL NULL NULL NULL NULL 2 100.00 Using where Warnings: -Note 1003 /* select#1 */ select 3 AS `f` from dual where !<expr_cache><3>(<in_optimizer>(3,<exists>(/* select#2 */ select `test`.`t1`.`b` from `test`.`t1` where (`test`.`t1`.`c` = 'USA' or `test`.`t1`.`c` <> 'USA') and trigcond(<cache>(3) = `test`.`t1`.`b` or `test`.`t1`.`b` is null) and `test`.`t1`.`b` = `test`.`t1`.`a` having trigcond(`test`.`t1`.`b` is null)))) +Note 1003 /* select#1 */ select 3 AS `f` from dual where !<expr_cache><3>(<in_optimizer>(3,<exists>(/* select#2 */ select `test`.`t1`.`b` from `test`.`t1` where `test`.`t1`.`b` = `test`.`t1`.`a` and (`test`.`t1`.`c` = 'USA' or `test`.`t1`.`c` <> 'USA') and trigcond(<cache>(3) = `test`.`t1`.`b` or `test`.`t1`.`b` is null) having trigcond(`test`.`t1`.`b` is null)))) SELECT * FROM t2 WHERE f NOT IN (SELECT b FROM t1 WHERE 0 OR (c IN ('USA') OR c NOT IN ('USA')) AND a = b); @@ -2867,6 +2867,44 @@ FROM (t1 JOIN t1 AS ref_t1 ON (t1.i1 > (SELECT ref_t1.i1 AS c0 FROM t1 b ORDER BY -c0))); ERROR 21000: Subquery returns more than 1 row DROP TABLE t1; +# +# MDEV-22377: Subquery in an UPDATE query uses full scan instead of range +# +CREATE TABLE t1 ( +key1 varchar(30) NOT NULL, +col1 int(11) NOT NULL, +filler char(100) +); +insert into t1 select seq, seq, seq from seq_1_to_100; +CREATE TABLE t10 ( +key1 varchar(30) NOT NULL, +col1 int, +filler char(100), +PRIMARY KEY (key1) +); +insert into t10 select seq, seq, seq from seq_1_to_1000; +CREATE TABLE t11 ( +key1 varchar(30) NOT NULL, +filler char(100), +PRIMARY KEY (key1) +); +insert into t11 select seq, seq from seq_1_to_1000; +set @tmp_os=@@optimizer_switch; +set optimizer_switch='semijoin=off,materialization=off'; +# Must use range access (not full scan) for table tms: +explain select * from t1 hist +WHERE +key1 IN ('1','2','3','4','5','6','7','8','9','10') AND +hist.col1 NOT IN (SELECT tn.col1 +FROM t10 tn JOIN t11 tms ON tms.key1 = tn.key1 +WHERE tn.key1 IN ('1','2','3','4','5','6','7','8','9','10') +); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY hist ALL NULL NULL NULL NULL 100 Using where +2 DEPENDENT SUBQUERY tms range PRIMARY PRIMARY 32 NULL 10 Using where; Using index +2 DEPENDENT SUBQUERY tn eq_ref PRIMARY PRIMARY 32 test.tms.key1 1 Using where +set optimizer_switch=@tmp_os; +drop table t1, t10, t11; # End of 10.2 tests # End of 10.3 tests # diff --git a/mysql-test/main/subselect4.test b/mysql-test/main/subselect4.test index 77c179bda61..192791c4c37 100644 --- a/mysql-test/main/subselect4.test +++ b/mysql-test/main/subselect4.test @@ -2378,6 +2378,49 @@ FROM (t1 JOIN t1 AS ref_t1 ON DROP TABLE t1; +--echo # +--echo # MDEV-22377: Subquery in an UPDATE query uses full scan instead of range +--echo # + +CREATE TABLE t1 ( + key1 varchar(30) NOT NULL, + col1 int(11) NOT NULL, + filler char(100) +); +insert into t1 select seq, seq, seq from seq_1_to_100; + +CREATE TABLE t10 ( + key1 varchar(30) NOT NULL, + col1 int, + filler char(100), + PRIMARY KEY (key1) +); +insert into t10 select seq, seq, seq from seq_1_to_1000; + +CREATE TABLE t11 ( + key1 varchar(30) NOT NULL, + filler char(100), + PRIMARY KEY (key1) +); +insert into t11 select seq, seq from seq_1_to_1000; + + +set @tmp_os=@@optimizer_switch; +set optimizer_switch='semijoin=off,materialization=off'; + +--echo # Must use range access (not full scan) for table tms: +explain select * from t1 hist +WHERE + key1 IN ('1','2','3','4','5','6','7','8','9','10') AND + hist.col1 NOT IN (SELECT tn.col1 + FROM t10 tn JOIN t11 tms ON tms.key1 = tn.key1 + WHERE tn.key1 IN ('1','2','3','4','5','6','7','8','9','10') + ); + +set optimizer_switch=@tmp_os; + +drop table t1, t10, t11; + --echo # End of 10.2 tests --echo # End of 10.3 tests diff --git a/mysql-test/main/subselect_mat_cost_bugs.result b/mysql-test/main/subselect_mat_cost_bugs.result index ecceac27b2d..0ccac8d3dd5 100644 --- a/mysql-test/main/subselect_mat_cost_bugs.result +++ b/mysql-test/main/subselect_mat_cost_bugs.result @@ -100,7 +100,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 2 DEPENDENT SUBQUERY t2 index c3 c3 9 NULL 2 100.00 Using where; Using index; Using join buffer (flat, BNL join) Warnings: Note 1276 Field or reference 'test.t1.pk' of SELECT #2 was resolved in SELECT #1 -Note 1003 /* select#1 */ select `test`.`t1`.`pk` AS `pk` from `test`.`t1` where <expr_cache><`test`.`t1`.`c1`,`test`.`t1`.`pk`>(<in_optimizer>(`test`.`t1`.`c1`,<exists>(/* select#2 */ select `test`.`t1a`.`c1` from `test`.`t1b` join `test`.`t2` left join `test`.`t1a` on(`test`.`t1a`.`c2` = `test`.`t1b`.`pk` and 2) where `test`.`t1`.`pk` <> 0 and <cache>(`test`.`t1`.`c1`) = `test`.`t1a`.`c1` and `test`.`t2`.`c3` = `test`.`t1b`.`c4`))) +Note 1003 /* select#1 */ select `test`.`t1`.`pk` AS `pk` from `test`.`t1` where <expr_cache><`test`.`t1`.`c1`,`test`.`t1`.`pk`>(<in_optimizer>(`test`.`t1`.`c1`,<exists>(/* select#2 */ select `test`.`t1a`.`c1` from `test`.`t1b` join `test`.`t2` left join `test`.`t1a` on(`test`.`t1a`.`c2` = `test`.`t1b`.`pk` and 2) where `test`.`t2`.`c3` = `test`.`t1b`.`c4` and `test`.`t1`.`pk` <> 0 and <cache>(`test`.`t1`.`c1`) = `test`.`t1a`.`c1`))) SELECT pk FROM t1 WHERE c1 IN @@ -364,7 +364,7 @@ AND a = SOME (SELECT b FROM t5)); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t3 ALL NULL NULL NULL NULL 2 Using where 2 DEPENDENT SUBQUERY t5 index c c 10 NULL 2 Using where; Using index; Start temporary -2 DEPENDENT SUBQUERY t4 eq_ref PRIMARY PRIMARY 4 test.t5.b 1 Using index condition; Using where; End temporary +2 DEPENDENT SUBQUERY t4 eq_ref PRIMARY PRIMARY 4 test.t5.b 1 Using where; End temporary SELECT * FROM t3 WHERE t3.b > ALL ( diff --git a/mysql-test/main/win.result b/mysql-test/main/win.result index e52ac635c7b..0174441ed43 100644 --- a/mysql-test/main/win.result +++ b/mysql-test/main/win.result @@ -4230,6 +4230,14 @@ i LAST_VALUE(COUNT(i)) OVER (PARTITION BY i ORDER BY j) 4 2 DROP TABLE t1; # +# MDEV-15208: server crashed, when using ORDER BY with window function and UNION +# +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1),(1),(1),(1),(1),(2),(2),(2),(2),(2),(2); +SELECT 1 UNION SELECT a FROM t1 ORDER BY (row_number() over ()); +ERROR HY000: Expression #1 of ORDER BY contains aggregate function and applies to a UNION +DROP TABLE t1; +# # End of 10.2 tests # # diff --git a/mysql-test/main/win.test b/mysql-test/main/win.test index 171a57180dc..1e0f89b5542 100644 --- a/mysql-test/main/win.test +++ b/mysql-test/main/win.test @@ -2731,6 +2731,16 @@ SELECT i, LAST_VALUE(COUNT(i)) OVER (PARTITION BY i ORDER BY j) FROM t1 GROUP BY DROP TABLE t1; --echo # +--echo # MDEV-15208: server crashed, when using ORDER BY with window function and UNION +--echo # + +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1),(1),(1),(1),(1),(2),(2),(2),(2),(2),(2); +--error ER_AGGREGATE_ORDER_FOR_UNION +SELECT 1 UNION SELECT a FROM t1 ORDER BY (row_number() over ()); +DROP TABLE t1; + +--echo # --echo # End of 10.2 tests --echo # diff --git a/mysql-test/suite/encryption/r/tempfiles_encrypted.result b/mysql-test/suite/encryption/r/tempfiles_encrypted.result index de9820184a5..a92a3a04a1a 100644 --- a/mysql-test/suite/encryption/r/tempfiles_encrypted.result +++ b/mysql-test/suite/encryption/r/tempfiles_encrypted.result @@ -4236,6 +4236,14 @@ i LAST_VALUE(COUNT(i)) OVER (PARTITION BY i ORDER BY j) 4 2 DROP TABLE t1; # +# MDEV-15208: server crashed, when using ORDER BY with window function and UNION +# +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1),(1),(1),(1),(1),(2),(2),(2),(2),(2),(2); +SELECT 1 UNION SELECT a FROM t1 ORDER BY (row_number() over ()); +ERROR HY000: Expression #1 of ORDER BY contains aggregate function and applies to a UNION +DROP TABLE t1; +# # End of 10.2 tests # # diff --git a/mysql-test/suite/galera/include/galera_wsrep_recover.inc b/mysql-test/suite/galera/include/galera_wsrep_recover.inc index d2956ea99e6..aa2f0e2e777 100644 --- a/mysql-test/suite/galera/include/galera_wsrep_recover.inc +++ b/mysql-test/suite/galera/include/galera_wsrep_recover.inc @@ -1,5 +1,12 @@ --echo Performing --wsrep-recover ... +if ($wsrep_recover_additional) +{ +--exec $MYSQLD --defaults-group-suffix=.$galera_wsrep_recover_server_id --defaults-file=$MYSQLTEST_VARDIR/my.cnf --log-error=$MYSQL_TMP_DIR/galera_wsrep_recover.log --innodb --wsrep-recover $wsrep_recover_additional > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1 +} +if (!$wsrep_recover_additional) +{ --exec $MYSQLD --defaults-group-suffix=.$galera_wsrep_recover_server_id --defaults-file=$MYSQLTEST_VARDIR/my.cnf --log-error=$MYSQL_TMP_DIR/galera_wsrep_recover.log --innodb --wsrep-recover > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1 +} --perl use strict; diff --git a/mysql-test/suite/galera/r/galera_log_bin.result b/mysql-test/suite/galera/r/galera_log_bin.result index 160575df412..df7ace9d95a 100644 --- a/mysql-test/suite/galera/r/galera_log_bin.result +++ b/mysql-test/suite/galera/r/galera_log_bin.result @@ -78,3 +78,4 @@ DROP TABLE t2; connection node_1; SET GLOBAL wsrep_on=OFF; RESET MASTER; +SET GLOBAL wsrep_on=ON; diff --git a/mysql-test/suite/galera/r/galera_log_bin_ext.result b/mysql-test/suite/galera/r/galera_log_bin_ext.result index 58e13fb71a0..f0914954e0a 100644 --- a/mysql-test/suite/galera/r/galera_log_bin_ext.result +++ b/mysql-test/suite/galera/r/galera_log_bin_ext.result @@ -1,6 +1,8 @@ connection node_2; connection node_1; connection node_1; +connection node_2; +connection node_1; set global wsrep_on=OFF; reset master; set global wsrep_on=ON; @@ -46,6 +48,12 @@ hostname1-bin.000001 # Xid # # COMMIT /* XID */ hostname1-bin.000001 # Gtid # # GTID #-#-# hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER connection node_2; +Shutting down server ... +connection node_1; +Cleaning var directory ... +connection node_2; +Starting server ... +connection node_2; SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; COUNT(*) = 2 1 @@ -72,9 +80,11 @@ hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F hostname1-bin.000001 # Xid # # COMMIT /* XID */ hostname1-bin.000001 # Gtid # # GTID #-#-# hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER +hostname1-bin.000001 # Rotate # # hostname1-bin.000002;pos=4 DROP TABLE t1; DROP TABLE t2; #cleanup connection node_1; SET GLOBAL wsrep_on=OFF; RESET MASTER; +SET GLOBAL wsrep_on=ON; diff --git a/mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result b/mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result new file mode 100644 index 00000000000..f0914954e0a --- /dev/null +++ b/mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result @@ -0,0 +1,90 @@ +connection node_2; +connection node_1; +connection node_1; +connection node_2; +connection node_1; +set global wsrep_on=OFF; +reset master; +set global wsrep_on=ON; +connection node_2; +set global wsrep_on=OFF; +reset master; +set global wsrep_on=ON; +CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1); +CREATE TABLE t2 (id INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (1); +INSERT INTO t2 VALUES (1); +connection node_2; +SELECT COUNT(*) = 1 FROM t1; +COUNT(*) = 1 +1 +SELECT COUNT(*) = 2 FROM t2; +COUNT(*) = 2 +1 +connection node_1; +ALTER TABLE t1 ADD COLUMN f2 INTEGER; +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t1 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t1) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t2 (id INT) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER +connection node_2; +Shutting down server ... +connection node_1; +Cleaning var directory ... +connection node_2; +Starting server ... +connection node_2; +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +COUNT(*) = 2 +1 +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t1 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t1) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t2 (id INT) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER +hostname1-bin.000001 # Rotate # # hostname1-bin.000002;pos=4 +DROP TABLE t1; +DROP TABLE t2; +#cleanup +connection node_1; +SET GLOBAL wsrep_on=OFF; +RESET MASTER; +SET GLOBAL wsrep_on=ON; diff --git a/mysql-test/suite/galera/r/galera_log_bin_opt.result b/mysql-test/suite/galera/r/galera_log_bin_opt.result index 160575df412..df7ace9d95a 100644 --- a/mysql-test/suite/galera/r/galera_log_bin_opt.result +++ b/mysql-test/suite/galera/r/galera_log_bin_opt.result @@ -78,3 +78,4 @@ DROP TABLE t2; connection node_1; SET GLOBAL wsrep_on=OFF; RESET MASTER; +SET GLOBAL wsrep_on=ON; diff --git a/mysql-test/suite/galera/r/galera_sst_mariabackup,debug.rdiff b/mysql-test/suite/galera/r/galera_sst_mariabackup,debug.rdiff index bad8355b514..3aad611cacb 100644 --- a/mysql-test/suite/galera/r/galera_sst_mariabackup,debug.rdiff +++ b/mysql-test/suite/galera/r/galera_sst_mariabackup,debug.rdiff @@ -1,5 +1,5 @@ ---- r/galera_sst_mariabackup.result 2021-04-10 14:25:04.142716409 +0300 -+++ r/galera_sst_mariabackup,debug.reject 2021-04-10 14:53:30.033162191 +0300 +--- r/galera_sst_mariabackup.result ++++ r/galera_sst_mariabackup.reject @@ -516,5 +516,189 @@ 1 DROP TABLE t1; diff --git a/mysql-test/suite/galera/r/galera_sst_rsync,debug.rdiff b/mysql-test/suite/galera/r/galera_sst_rsync,debug.rdiff index c4937bfb4ca..87f8a463bd0 100644 --- a/mysql-test/suite/galera/r/galera_sst_rsync,debug.rdiff +++ b/mysql-test/suite/galera/r/galera_sst_rsync,debug.rdiff @@ -1,5 +1,5 @@ ---- r/galera_sst_rsync_data_dir.result 2021-04-10 14:35:28.090610315 +0300 -+++ r/galera_sst_rsync_data_dir,debug.reject 2021-04-10 15:41:44.876068411 +0300 +--- galera_sst_rsync.result ++++ galera_sst_rsync.reject @@ -516,3 +516,187 @@ 1 DROP TABLE t1; diff --git a/mysql-test/suite/galera/r/galera_wan_restart_sst.result b/mysql-test/suite/galera/r/galera_wan_restart_sst.result index 05390338160..2433a1d9c48 100644 --- a/mysql-test/suite/galera/r/galera_wan_restart_sst.result +++ b/mysql-test/suite/galera/r/galera_wan_restart_sst.result @@ -1,17 +1,21 @@ connection node_2; connection node_1; -SELECT VARIABLE_VALUE = 4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; -VARIABLE_VALUE = 4 -1 +connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; +connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4; +connection node_1; +connection node_2; +connection node_3; +connection node_4; +SELECT VARIABLE_VALUE AS EXPECT_4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +EXPECT_4 +4 connection node_1; CREATE TABLE t1 (f1 INTEGER); INSERT INTO t1 VALUES (1); connection node_2; INSERT INTO t1 VALUES (2); -connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; connection node_3; INSERT INTO t1 VALUES (3); -connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4; connection node_4; INSERT INTO t1 VALUES (4); connection node_3; @@ -85,3 +89,6 @@ CALL mtr.add_suppression("Action message in non-primary configuration from membe connection node_4; CALL mtr.add_suppression("There are no nodes in the same segment that will ever be able to become donors, yet there is a suitable donor outside"); CALL mtr.add_suppression("Action message in non-primary configuration from member 0"); +connection node_1; +disconnect node_3; +disconnect node_4; diff --git a/mysql-test/suite/galera/t/galera_log_bin.inc b/mysql-test/suite/galera/t/galera_log_bin.inc index cc78367b510..c86de528a08 100644 --- a/mysql-test/suite/galera/t/galera_log_bin.inc +++ b/mysql-test/suite/galera/t/galera_log_bin.inc @@ -44,3 +44,4 @@ DROP TABLE t2; --connection node_1 SET GLOBAL wsrep_on=OFF; RESET MASTER; +SET GLOBAL wsrep_on=ON; diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext.cnf b/mysql-test/suite/galera/t/galera_log_bin_ext.cnf index 012209610ea..ae47de90bb8 100644 --- a/mysql-test/suite/galera/t/galera_log_bin_ext.cnf +++ b/mysql-test/suite/galera/t/galera_log_bin_ext.cnf @@ -9,3 +9,6 @@ log-slave-updates log-bin = hostname2-bin log-bin-index = hostname2.bdx log-slave-updates + +[sst] +sst_max_binlogs= diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext.test b/mysql-test/suite/galera/t/galera_log_bin_ext.test index 923bd623a8a..752073aecdb 100644 --- a/mysql-test/suite/galera/t/galera_log_bin_ext.test +++ b/mysql-test/suite/galera/t/galera_log_bin_ext.test @@ -1 +1 @@ ---source galera_log_bin.inc +--source galera_log_bin_sst.inc diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf new file mode 100644 index 00000000000..c988136b8fb --- /dev/null +++ b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf @@ -0,0 +1,19 @@ +!include ../galera_2nodes.cnf + +[mysqld] +wsrep_sst_method=mariabackup +wsrep_sst_auth="root:" + +[mysqld.1] +log-bin=@ENV.MYSQLTEST_VARDIR/mysqld.1/data/hostname1-bin +log-bin-index = hostname1.bdx +log-slave-updates + +[mysqld.2] +log-bin=@ENV.MYSQLTEST_VARDIR/mysqld.2/data/hostname2-bin +log-bin-index = hostname2.bdx +log-slave-updates + +[sst] +transferfmt=@ENV.MTR_GALERA_TFMT +sst_max_binlogs= diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test new file mode 100644 index 00000000000..47df45b4c71 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test @@ -0,0 +1,2 @@ +--source include/have_mariabackup.inc +--source galera_log_bin_sst.inc diff --git a/mysql-test/suite/galera/t/galera_log_bin_sst.inc b/mysql-test/suite/galera/t/galera_log_bin_sst.inc new file mode 100644 index 00000000000..a5336f4e3ac --- /dev/null +++ b/mysql-test/suite/galera/t/galera_log_bin_sst.inc @@ -0,0 +1,90 @@ +--source include/galera_cluster.inc +--source include/force_restart.inc + +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +--connection node_1 +set global wsrep_on=OFF; +reset master; +set global wsrep_on=ON; +--connection node_2 +set global wsrep_on=OFF; +reset master; +set global wsrep_on=ON; + +# +# Test Galera with --log-bin --log-slave-updates . +# This way the actual MySQL binary log is used, +# rather than Galera's own implementation +# + +CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1); + +CREATE TABLE t2 (id INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (1); +INSERT INTO t2 VALUES (1); + +--connection node_2 +SELECT COUNT(*) = 1 FROM t1; +SELECT COUNT(*) = 2 FROM t2; + +--connection node_1 +ALTER TABLE t1 ADD COLUMN f2 INTEGER; +--let $MASTER_MYPORT=$NODE_MYPORT_1 +--source include/show_binlog_events.inc + +--connection node_2 + +#--connection node_2 +#--source suite/galera/include/galera_stop_replication.inc + +--echo Shutting down server ... +--source include/shutdown_mysqld.inc + +--connection node_1 +--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size' +--source include/wait_condition.inc + +# +# Force SST +# +--echo Cleaning var directory ... +--remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/mtr +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/performance_schema +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/test +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/mysql +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data + +--connection node_2 + +--echo Starting server ... +let $restart_noprint=2; +--source include/start_mysqld.inc + +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready'; +--source include/wait_condition.inc + +--connection node_2 +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +--let $MASTER_MYPORT=$NODE_MYPORT_2 +--source include/show_binlog_events.inc + +DROP TABLE t1; +DROP TABLE t2; + +--echo #cleanup +--connection node_1 +SET GLOBAL wsrep_on=OFF; +RESET MASTER; +SET GLOBAL wsrep_on=ON; + +# Restore original auto_increment_offset values. +--source include/auto_increment_offset_restore.inc diff --git a/mysql-test/suite/galera/t/galera_sst_rsync.test b/mysql-test/suite/galera/t/galera_sst_rsync.test index 5c08707e870..c944c8d84b7 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync.test +++ b/mysql-test/suite/galera/t/galera_sst_rsync.test @@ -10,4 +10,5 @@ --source suite/galera/include/galera_st_kill_slave.inc --source suite/galera/include/galera_st_kill_slave_ddl.inc + --source include/auto_increment_offset_restore.inc diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf b/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf index 4f25af7cd8b..3913ab6660f 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf +++ b/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf @@ -12,4 +12,3 @@ log_bin wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true' log_basename=server2 log_bin - diff --git a/mysql-test/suite/galera/t/galera_wan_restart_sst.test b/mysql-test/suite/galera/t/galera_wan_restart_sst.test index 0ded4032103..7a8c0df4946 100644 --- a/mysql-test/suite/galera/t/galera_wan_restart_sst.test +++ b/mysql-test/suite/galera/t/galera_wan_restart_sst.test @@ -12,7 +12,19 @@ --source include/galera_cluster.inc --source include/have_innodb.inc -SELECT VARIABLE_VALUE = 4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 +--connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4 + +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--let $node_3=node_3 +--let $node_4=node_4 +--source include/auto_increment_offset_save.inc + +--let $wait_condition = SELECT VARIABLE_VALUE = 4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc +SELECT VARIABLE_VALUE AS EXPECT_4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; --connection node_1 CREATE TABLE t1 (f1 INTEGER); @@ -21,11 +33,9 @@ INSERT INTO t1 VALUES (1); --connection node_2 INSERT INTO t1 VALUES (2); ---connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 --connection node_3 INSERT INTO t1 VALUES (3); ---connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4 --connection node_4 INSERT INTO t1 VALUES (4); @@ -136,3 +146,10 @@ CALL mtr.add_suppression("Action message in non-primary configuration from membe --connection node_4 CALL mtr.add_suppression("There are no nodes in the same segment that will ever be able to become donors, yet there is a suitable donor outside"); CALL mtr.add_suppression("Action message in non-primary configuration from member 0"); + +# Restore original auto_increment_offset values. +--source include/auto_increment_offset_restore.inc + +--connection node_1 +--disconnect node_3 +--disconnect node_4 diff --git a/mysql-test/suite/innodb/r/innodb_force_recovery.result b/mysql-test/suite/innodb/r/innodb_force_recovery.result index 838de9844e1..9a2cc60ff68 100644 --- a/mysql-test/suite/innodb/r/innodb_force_recovery.result +++ b/mysql-test/suite/innodb/r/innodb_force_recovery.result @@ -118,7 +118,7 @@ SET TRANSACTION ISOLATION LEVEL REPEATABLE READ; select * from t2; f1 f2 1 2 -SET GLOBAL innodb_lock_wait_timeout=1; +SET SESSION innodb_lock_wait_timeout=1; insert into t2 values(1,2); ERROR HY000: Lock wait timeout exceeded; try restarting transaction insert into t2 values(9,10); diff --git a/mysql-test/suite/innodb/t/innodb_force_recovery.test b/mysql-test/suite/innodb/t/innodb_force_recovery.test index fe070100c08..cfda405e216 100644 --- a/mysql-test/suite/innodb/t/innodb_force_recovery.test +++ b/mysql-test/suite/innodb/t/innodb_force_recovery.test @@ -163,7 +163,7 @@ SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; select * from t2; SET TRANSACTION ISOLATION LEVEL REPEATABLE READ; select * from t2; -SET GLOBAL innodb_lock_wait_timeout=1; +SET SESSION innodb_lock_wait_timeout=1; --error ER_LOCK_WAIT_TIMEOUT insert into t2 values(1,2); insert into t2 values(9,10); diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index deebe7cf820..d4d9a58897d 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2017-2022 MariaDB # Copyright (C) 2012-2015 Codership Oy # # This program is free software; you can redistribute it and/or modify @@ -22,6 +22,62 @@ set -ue # Setting the path for some utilities on CentOS export PATH="$PATH:/usr/sbin:/usr/bin:/sbin:/bin" +trim_string() +{ + if [ -n "$BASH_VERSION" ]; then + local pattern="[![:space:]${2:-}]" + local x="${1#*$pattern}" + local z=${#1} + x=${#x} + if [ $x -ne $z ]; then + local y="${1%$pattern*}" + y=${#y} + x=$(( z-x-1 )) + y=$(( y-x+1 )) + printf '%s' "${1:$x:$y}" + else + printf '' + fi + else + local pattern="[[:space:]${2:-}]" + echo "$1" | sed -E "s/^$pattern+|$pattern+\$//g" + fi +} + +trim_dir() +{ + local t=$(trim_string "$1") + if [ "$t" != '/' ]; then + if [ "${t%/}" != "$t" ]; then + t=$(trim_string "${t%/}") + fi + else + t='.' + fi + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$t" + else + echo "$t" + fi +} + +to_minuses() +{ + local x="$1" + local t="${1#*_}" + local r="" + while [ "$t" != "$x" ]; do + r="$r${x%%_*}-" + x="$t" + t="${t#*_}" + done + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$r$x" + else + echo "$r$x" + fi +} + WSREP_SST_OPT_BYPASS=0 WSREP_SST_OPT_BINLOG="" WSREP_SST_OPT_BINLOG_INDEX="" @@ -43,10 +99,9 @@ WSREP_SST_OPT_ADDR="" WSREP_SST_OPT_ADDR_PORT="" WSREP_SST_OPT_HOST="" WSREP_SST_OPT_HOST_UNESCAPED="" -WSREP_SST_OPT_HOST_ESCAPED="" -INNODB_DATA_HOME_DIR="${INNODB_DATA_HOME_DIR:-}" -INNODB_LOG_GROUP_HOME="${INNODB_LOG_GROUP_HOME:-}" -INNODB_UNDO_DIR="${INNODB_UNDO_DIR:-}" +INNODB_DATA_HOME_DIR=$(trim_dir "${INNODB_DATA_HOME_DIR:-}") +INNODB_LOG_GROUP_HOME=$(trim_dir "${INNODB_LOG_GROUP_HOME:-}") +INNODB_UNDO_DIR=$(trim_dir "${INNODB_UNDO_DIR:-}") INNODB_FORCE_RECOVERY="" INNOEXTRA="" @@ -66,20 +121,22 @@ case "$1" in # without square brackets: readonly WSREP_SST_OPT_HOST_UNESCAPED="${addr_no_bracket%%\]*}" # Square brackets are needed in most cases: - readonly WSREP_SST_OPT_HOST="[${WSREP_SST_OPT_HOST_UNESCAPED}]" - readonly WSREP_SST_OPT_HOST_ESCAPED="\\[${WSREP_SST_OPT_HOST_UNESCAPED}\\]" + readonly WSREP_SST_OPT_HOST="[$WSREP_SST_OPT_HOST_UNESCAPED]" # Mark this address as IPv6: readonly WSREP_SST_OPT_HOST_IPv6=1 + # Let's remove the leading part that contains the host address: + remain="${WSREP_SST_OPT_ADDR#*\]}" ;; *) readonly WSREP_SST_OPT_HOST="${WSREP_SST_OPT_ADDR%%[:/]*}" readonly WSREP_SST_OPT_HOST_UNESCAPED="$WSREP_SST_OPT_HOST" - readonly WSREP_SST_OPT_HOST_ESCAPED="$WSREP_SST_OPT_HOST" readonly WSREP_SST_OPT_HOST_IPv6=0 + # Let's remove the leading part that contains the host address: + remain="${WSREP_SST_OPT_ADDR#*[:/]}" ;; esac - # Let's remove the leading part that contains the host address: - remain="${WSREP_SST_OPT_ADDR#$WSREP_SST_OPT_HOST_ESCAPED}" + # If there is nothing but the address, then the remainder is empty: + [ "$remain" = "$WSREP_SST_OPT_ADDR" ] && remain="" # Let's remove the ":" character that separates the port number # from the hostname: remain="${remain#:}" @@ -87,39 +144,32 @@ case "$1" in # up to "/" (if present): WSREP_SST_OPT_ADDR_PORT="${remain%%/*}" # If the "/" character is present, then the path is not empty: - if [ "${remain#*/}" != "$remain" ]; then + if [ "$WSREP_SST_OPT_ADDR_PORT" != "$remain" ]; then # This operation removes everything up to the "/" character, # effectively removing the port number from the string: readonly WSREP_SST_OPT_PATH="${remain#*/}" else readonly WSREP_SST_OPT_PATH="" fi - # The rest of the string is the same as the path (for now): - remain="$WSREP_SST_OPT_PATH" - # If there is one more "/" in the string, then everything before - # it will be the module name, otherwise the module name is empty: - if [ "${remain%%/*}" != "$remain" ]; then - # This operation removes the tail after the very first - # occurrence of the "/" character (inclusively): - readonly WSREP_SST_OPT_MODULE="${remain%%/*}" - else - readonly WSREP_SST_OPT_MODULE="" - fi # Remove the module name part from the string, which ends with "/": remain="${WSREP_SST_OPT_PATH#*/}" - # If the rest of the string does not match the original, then there - # was something else besides the module name: + # This operation removes the tail after the very first occurrence + # of the "/" character, inclusively: + readonly WSREP_SST_OPT_MODULE="${WSREP_SST_OPT_PATH%%/*}" + # If there is one more "/" in the string, then everything before + # it will be the LSN, otherwise the LSN is empty: if [ "$remain" != "$WSREP_SST_OPT_PATH" ]; then # Extract the part that matches the LSN by removing all # characters starting from the very first "/": readonly WSREP_SST_OPT_LSN="${remain%%/*}" # Exctract everything after the first occurrence of # the "/" character in the string: + source="$remain" remain="${remain#*/}" # If the remainder does not match the original string, # then there is something else (the version number in # our case): - if [ "$remain" != "$WSREP_SST_OPT_LSN" ]; then + if [ "$remain" != "$source" ]; then # Let's extract the version number by removing the tail # after the very first occurence of the "/" character # (inclusively): @@ -138,22 +188,22 @@ case "$1" in ;; '--datadir') # Let's remove the trailing slash: - readonly WSREP_SST_OPT_DATA="${2%/}" + readonly WSREP_SST_OPT_DATA=$(trim_dir "$2") shift ;; '--innodb-data-home-dir') # Let's remove the trailing slash: - readonly INNODB_DATA_HOME_DIR="${2%/}" + readonly INNODB_DATA_HOME_DIR=$(trim_dir "$2") shift ;; '--innodb-log-group-home-dir') # Let's remove the trailing slash: - readonly INNODB_LOG_GROUP_HOME="${2%/}" + readonly INNODB_LOG_GROUP_HOME=$(trim_dir "$2") shift ;; '--innodb-undo-directory') # Let's remove the trailing slash: - readonly INNODB_UNDO_DIR="${2%/}" + readonly INNODB_UNDO_DIR=$(trim_dir "$2") shift ;; '--defaults-file') @@ -182,14 +232,12 @@ case "$1" in readonly WSREP_SST_OPT_HOST_UNESCAPED="${addr_no_bracket%%\]*}" # Square brackets are needed in most cases: readonly WSREP_SST_OPT_HOST="[${WSREP_SST_OPT_HOST_UNESCAPED}]" - readonly WSREP_SST_OPT_HOST_ESCAPED="\\[${WSREP_SST_OPT_HOST_UNESCAPED}\\]" # Mark this address as IPv6: readonly WSREP_SST_OPT_HOST_IPv6=1 ;; *) readonly WSREP_SST_OPT_HOST="$2" readonly WSREP_SST_OPT_HOST_UNESCAPED="$2" - readonly WSREP_SST_OPT_HOST_ESCAPED="$2" readonly WSREP_SST_OPT_HOST_IPv6=0 ;; esac @@ -247,6 +295,7 @@ case "$1" in '--mysqld-args') original_cmd="" shift + cmd_tail=0 while [ $# -gt 0 ]; do lname="${1#--}" # "--" is interpreted as the end of the list of options: @@ -261,7 +310,7 @@ case "$1" in shift done fi - break; + break fi # Make sure the argument does not start with "--", otherwise it # is a long option, which is processed after this "if": @@ -279,7 +328,7 @@ case "$1" in else # If it's not bash, then we need to use slow # external utilities: - option=$(echo "$options" | cut -c1-1) + option=$(echo "$options" | cut -c1) fi # And the subsequent characters consider option value: value="" @@ -301,15 +350,25 @@ case "$1" in if [ "${2#-}" = "$2" ]; then shift value="$1" + elif [ "$2" = '--' ]; then + shift + if [ $# -gt 1 ]; then + cmd_tail=1 + shift + value="$1" + fi fi fi - if [ $option = 'h' ]; then + if [ "$option" = 'h' ]; then if [ -z "$WSREP_SST_OPT_DATA" ]; then - MYSQLD_OPT_DATADIR="${value%/}" + MYSQLD_OPT_DATADIR=$(trim_dir "$value") fi - elif [ $option != 'u' -a \ - $option != 'P' ] + elif [ "$option" != 'u' -a \ + "$option" != 'P' ] then + if [ $cmd_tail -ne 0 ]; then + option="$option --" + fi if [ -z "$value" ]; then slist="$slist$option" elif [ -z "$slist" ]; then @@ -317,9 +376,16 @@ case "$1" in else slist="$slist -$option '$value'" fi + break + fi + if [ $cmd_tail -ne 0 ]; then + if [ -n "$slist" ]; then + slist="$slist --" + else + slist='-' + fi fi break - else slist="$slist$option" fi @@ -329,7 +395,7 @@ case "$1" in original_cmd="$original_cmd -$slist" fi elif [ -z "$options" ]; then - # We found an equal sign without any characters after it: + # We found an minus sign without any characters after it: original_cmd="$original_cmd -" else # We found a value that does not start with a minus - @@ -338,12 +404,25 @@ case "$1" in original_cmd="$original_cmd '$1'" fi shift - continue; + if [ $cmd_tail -ne 0 ]; then + # All other arguments must be copied unchanged: + while [ $# -gt 0 ]; do + original_cmd="$original_cmd '$1'" + shift + done + break + fi + continue fi # Now we are sure that we are working with an option # that has a "long" name, so remove all characters after # the first equal sign: option="${1%%=*}" + # If the option name contains underscores, then replace + # them to minuses: + if [ "${option#*_}" != "$option" ]; then + option=$(to_minuses "$option") + fi # The "--loose-" prefix should not affect the recognition # of the option name: if [ "${option#--loose-}" != "$option" ]; then @@ -370,49 +449,49 @@ case "$1" in case "$option" in '--innodb-data-home-dir') if [ -z "$INNODB_DATA_HOME_DIR" ]; then - MYSQLD_OPT_INNODB_DATA_HOME_DIR="${value%/}" + MYSQLD_OPT_INNODB_DATA_HOME_DIR=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; '--innodb-log-group-home-dir') if [ -z "$INNODB_LOG_GROUP_HOME" ]; then - MYSQLD_OPT_INNODB_LOG_GROUP_HOME="${value%/}" + MYSQLD_OPT_INNODB_LOG_GROUP_HOME=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; '--innodb-undo-directory') if [ -z "$INNODB_UNDO_DIR" ]; then - MYSQLD_OPT_INNODB_UNDO_DIR="${value%/}" + MYSQLD_OPT_INNODB_UNDO_DIR=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; '--innodb-force-recovery') if [ -n "$value" -a "$value" != "0" ]; then - INNODB_FORCE_RECOVERY="$value" + INNODB_FORCE_RECOVERY=$(trim_string "$value") fi skip_mysqld_arg=1 ;; '--log-bin') if [ -z "$WSREP_SST_OPT_BINLOG" ]; then - MYSQLD_OPT_LOG_BIN="$value" + MYSQLD_OPT_LOG_BIN=$(trim_string "$value") fi skip_mysqld_arg=1 ;; '--log-bin-index') if [ -z "$WSREP_SST_OPT_BINLOG_INDEX" ]; then - MYSQLD_OPT_LOG_BIN_INDEX="$value" + MYSQLD_OPT_LOG_BIN_INDEX=$(trim_string "$value") fi skip_mysqld_arg=1 ;; '--log-basename') if [ -z "$WSREP_SST_OPT_LOG_BASENAME" ]; then - MYSQLD_OPT_LOG_BASENAME="$value" + MYSQLD_OPT_LOG_BASENAME=$(trim_string "$value") fi skip_mysqld_arg=1 ;; '--datadir') if [ -z "$WSREP_SST_OPT_DATA" ]; then - MYSQLD_OPT_DATADIR="${value%/}" + MYSQLD_OPT_DATADIR=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; @@ -471,8 +550,8 @@ if [ -z "$WSREP_SST_OPT_BINLOG" -a -n "${MYSQLD_OPT_LOG_BIN+x}" ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then # If the WSREP_SST_OPT_BINLOG variable is not set, but # --log-basename is present among the arguments to mysqld, - # then set WSREP_SST_OPT_BINLOG equal to the base name with - # the "-bin" suffix: + # then set WSREP_SST_OPT_BINLOG equal to the base name + # with the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG="$WSREP_SST_OPT_LOG_BASENAME-bin" else # Take the default name: @@ -525,26 +604,23 @@ get_binlog() WSREP_SST_OPT_BINLOG_INDEX=$(parse_cnf '--mysqld' 'log-bin-index') fi # if no command line argument and WSREP_SST_OPT_LOG_BASENAME is not set, - # try to get it from my.cnf: + # then try to get it from my.cnf: if [ -z "$WSREP_SST_OPT_LOG_BASENAME" ]; then WSREP_SST_OPT_LOG_BASENAME=$(parse_cnf '--mysqld' 'log-basename') fi if [ -z "$WSREP_SST_OPT_BINLOG" ]; then - # If the --log-bin option is specified without a parameter, + # If the log-bin option is specified without a parameter, # then we need to build the name of the index file according # to the rules described in the server documentation: - if [ -n "${MYSQLD_OPT_LOG_BIN+x}" -o \ - $(in_config '--mysqld' 'log-bin') -eq 1 ] - then + if [ $(in_config '--mysqld' 'log-bin') -ne 0 ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then # If the WSREP_SST_OPT_BINLOG variable is not set, but # --log-basename is present among the arguments of mysqld, - # then set WSREP_SST_OPT_BINLOG equal to the base name with - # the "-bin" suffix: + # then set WSREP_SST_OPT_BINLOG equal to the base name + # with the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG="$WSREP_SST_OPT_LOG_BASENAME-bin" else - # If the --log-bin option is present without a value, then - # we take the default name: + # Take the default name: readonly WSREP_SST_OPT_BINLOG='mysql-bin' fi fi @@ -554,13 +630,13 @@ get_binlog() # it according to the specifications for the server: if [ -z "$WSREP_SST_OPT_BINLOG_INDEX" ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then - # If the WSREP_SST_OPT_BINLOG variable is not set, but + # If the WSREP_SST_OPT_BINLOG_INDEX variable is not set, but # --log-basename is present among the arguments of mysqld, - # then set WSREP_SST_OPT_BINLOG equal to the base name with - # the "-bin" suffix: + # then set WSREP_SST_OPT_BINLOG_INDEX equal to the base name + # with the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_LOG_BASENAME-bin.index" else - # the default name (note that base of this name + # Use the default name (note that base of this name # is already defined above): readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_BINLOG.index" fi @@ -594,27 +670,18 @@ if [ -n "$WSREP_SST_OPT_ADDR_PORT" ]; then # the corresponding variable: readonly WSREP_SST_OPT_PORT="$WSREP_SST_OPT_ADDR_PORT" fi -elif [ -n "$WSREP_SST_OPT_ADDR" ]; then +else # If the port is missing, take the default port: if [ -z "$WSREP_SST_OPT_PORT" ]; then readonly WSREP_SST_OPT_PORT=4444 fi WSREP_SST_OPT_ADDR_PORT="$WSREP_SST_OPT_PORT" - # Let's remove the leading part that contains the host address: - remain="${WSREP_SST_OPT_ADDR#$WSREP_SST_OPT_HOST_ESCAPED}" - # Let's remove the ":" character that separates the port number - # from the hostname: - remain="${remain#:}" - # Let's remove all characters upto first "/" character that - # separates the hostname with port number from the path: - remain="${remain#/}" - # Let's construct a new value for the address with the port: - WSREP_SST_OPT_ADDR="$WSREP_SST_OPT_HOST:$WSREP_SST_OPT_PORT" - if [ -n "$remain" ]; then - WSREP_SST_OPT_ADDR="$WSREP_SST_OPT_ADDR/$remain" - fi fi +# Let's construct a new value for the address with the port: +sst_path="${WSREP_SST_OPT_PATH:+/}$WSREP_SST_OPT_PATH" +WSREP_SST_OPT_ADDR="$WSREP_SST_OPT_HOST:$WSREP_SST_OPT_PORT$sst_path" + readonly WSREP_SST_OPT_ADDR readonly WSREP_SST_OPT_ADDR_PORT @@ -632,7 +699,7 @@ commandex() # try to use my_print_defaults, mysql and mysqldump that come # with the sources (for MTR suite): script_binary=$(dirname "$0") -SCRIPTS_DIR=$(cd "$script_binary"; pwd -P) +SCRIPTS_DIR=$(cd "$script_binary"; pwd) EXTRA_DIR="$SCRIPTS_DIR/../extra" CLIENT_DIR="$SCRIPTS_DIR/../client" @@ -725,8 +792,11 @@ parse_cnf() local group="${groups%%\|*}" # Remove the remainder (the group name) from the rest # of the groups list (as if it were a prefix): - groups="${groups#$group}" - groups="${groups#\|}" + if [ "$group" != "$groups" ]; then + groups="${groups#*\|}" + else + groups="" + fi # If the group name is the same as the "mysqld" without "--" prefix, # then try to use it together with the group suffix: if [ "$group" = 'mysqld' -a -n "$WSREP_SST_OPT_SUFFIX_VALUE" ]; then @@ -751,10 +821,16 @@ parse_cnf() done # Use default if we haven't found a value: - if [ -z "$reval" ]; then - [ -n "${3:-}" ] && reval="$3" + [ -z "$reval" ] && reval="${3:-}" + + # Truncate spaces: + [ -n "$reval" ] && reval=$(trim_string "$reval") + + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$reval" + else + echo "$reval" fi - echo "$reval" } # @@ -780,8 +856,11 @@ in_config() local group="${groups%%\|*}" # Remove the remainder (the group name) from the rest # of the groups list (as if it were a prefix): - groups="${groups#$group}" - groups="${groups#\|}" + if [ "$group" != "$groups" ]; then + groups="${groups#*\|}" + else + groups="" + fi # If the group name is the same as the "mysqld" without "--" prefix, # then try to use it together with the group suffix: if [ "$group" = 'mysqld' -a -n "$WSREP_SST_OPT_SUFFIX_VALUE" ]; then @@ -804,7 +883,11 @@ in_config() break fi done - echo $found + if [ -n "$BASH_VERSION" ]; then + printf '%s' $found + else + echo $found + fi } wsrep_auth_not_set() @@ -937,11 +1020,22 @@ wsrep_gen_secret() { get_openssl if [ -n "$OPENSSL_BINARY" ]; then - echo $("$OPENSSL_BINARY" rand -hex 16) - else - printf "%04x%04x%04x%04x%04x%04x%04x%04x" \ + "$OPENSSL_BINARY" rand -hex 16 + elif [ -n "$BASH_VERSION" ]; then + printf '%04x%04x%04x%04x%04x%04x%04x%04x' \ $RANDOM $RANDOM $RANDOM $RANDOM \ $RANDOM $RANDOM $RANDOM $RANDOM + elif [ -n "$(commandex cksum)" -a \ + -n "$(commandex printf)" ] + then + printf '%08x%08x%08x%08x' \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) + else + wsrep_log_error "Unable to generate 16-byte secret" + exit 22 fi } @@ -979,14 +1073,14 @@ is_local_ip() if [ -n "$ip_util" ]; then # ip address show ouput format is " inet[6] <address>/<mask>": "$ip_util" address show \ - | grep -E "^[[:space:]]*inet.? [^[:space:]]+/" -o \ + | grep -E '^[[:space:]]*inet.? [^[:space:]]+/' -o \ | grep -F " $1/" >/dev/null && return 0 else local ifconfig_util=$(commandex 'ifconfig') if [ -n "$ifconfig_util" ]; then # ifconfig output format is " inet[6] <address> ...": "$ifconfig_util" \ - | grep -E "^[[:space:]]*inet.? [^[:space:]]+ " -o \ + | grep -E '^[[:space:]]*inet.? [^[:space:]]+ ' -o \ | grep -F " $1 " >/dev/null && return 0 fi fi @@ -1049,7 +1143,7 @@ check_port() ss -nlpH "( sport = :$port )" 2>/dev/null | \ grep -q -E "users:\\(.*\\(\"($utils)[^[:space:]]*\"[^)]*,pid=$pid(,[^)]*)?\\)" && rc=0 else - wsrep_log_error "unknown sockets utility" + wsrep_log_error "Unknown sockets utility" exit 2 # ENOENT fi @@ -1158,13 +1252,6 @@ verify_cert_matches_key() exit 22 fi - # If the diff utility is not installed, then - # we will not do this certificate check: - if [ -z "$(commandex diff)" ]; then - wsrep_log_info "diff utility not found" - return - fi - # If the openssl utility is not installed, then # we will not do this certificate check: get_openssl @@ -1175,9 +1262,9 @@ verify_cert_matches_key() # Generate the public key from the cert and the key. # They should match (otherwise we can't create an SSL connection). - if ! diff <("$OPENSSL_BINARY" x509 -in "$cert" -pubkey -noout 2>/dev/null) \ - <("$OPENSSL_BINARY" pkey -in "$key" -pubout 2>/dev/null) >/dev/null 2>&1 - then + local pk1=$("$OPENSSL_BINARY" x509 -in "$cert" -pubkey -noout 2>/dev/null || :) + local pk2=$("$OPENSSL_BINARY" pkey -in "$key" -pubout 2>/dev/null || :) + if [ "$pk1" != "$pk2" ]; then wsrep_log_error "******************* FATAL ERROR *****************" wsrep_log_error "* The certificate and private key do not match. *" wsrep_log_error "* Please check your certificate and key files. *" @@ -1220,28 +1307,6 @@ check_for_version() return 0 } -trim_string() -{ - if [ -n "$BASH_VERSION" ]; then - local pattern="[![:space:]${2:-}]" - local x="${1#*$pattern}" - local z=${#1} - x=${#x} - if [ $x -ne $z ]; then - local y="${1%$pattern*}" - y=${#y} - x=$(( z-x-1 )) - y=$(( y-x+1 )) - printf '%s' "${1:$x:$y}" - else - printf '' - fi - else - local pattern="[[:space:]${2:-}]" - echo "$1" | sed -E "s/^$pattern+|$pattern+\$//g" - fi -} - # # Check whether process is still running. # The first parameter contains the name of the PID file. @@ -1272,6 +1337,10 @@ check_pid() rm -f "$pid_file" || : fi fi + local config="${3:-}" + if [ -n "$config" -a -f "$config" ]; then + rm -f "$config" || : + fi CHECK_PID=0 return 1 } @@ -1377,13 +1446,9 @@ check_server_ssl_config() fi fi if [ -n "$tcert" ]; then - tcert=$(trim_string "$tcert") if [ "${tcert%/}" != "$tcert" -o -d "$tcert" ]; then tcap="$tcert" tcert="" fi fi - if [ -n "$tcap" ]; then - tcap=$(trim_string "$tcap") - fi } diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 471b0e5f5b0..ce4001fdc56 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -2,7 +2,7 @@ set -ue -# Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2017-2022 MariaDB # Copyright (C) 2013 Percona Inc # # This program is free software; you can redistribute it and/or modify @@ -40,7 +40,7 @@ tcert="" tcap="" tpem="" tkey="" -tmode="DISABLED" +tmode="" sockopt="" progress="" ttime=0 @@ -85,13 +85,13 @@ backup_threads="" encrypt_threads="" encrypt_chunk="" -readonly SECRET_TAG="secret" +readonly SECRET_TAG='secret' # Required for backup locks # For backup locks it is 1 sent by joiner sst_ver=1 -if [ -n "$(commandex pv)" ] && pv --help | grep -qw -- '-F'; then +if [ -n "$(commandex pv)" ] && pv --help | grep -qw -F -- '-F'; then pvopts="$pvopts $pvformat" fi pcmd="pv $pvopts" @@ -104,17 +104,14 @@ if [ -z "$BACKUP_BIN" ]; then fi DATA="$WSREP_SST_OPT_DATA" -INFO_FILE="xtrabackup_galera_info" -IST_FILE="xtrabackup_ist" +INFO_FILE='xtrabackup_galera_info' +IST_FILE='xtrabackup_ist' MAGIC_FILE="$DATA/$INFO_FILE" INNOAPPLYLOG="$DATA/mariabackup.prepare.log" INNOMOVELOG="$DATA/mariabackup.move.log" INNOBACKUPLOG="$DATA/mariabackup.backup.log" -# Setting the path for ss and ip -export PATH="/usr/sbin:/sbin:$PATH" - timeit() { local stage="$1" @@ -154,7 +151,7 @@ get_keys() return fi - if [ $sfmt = 'tar' ]; then + if [ "$sfmt" = 'tar' ]; then wsrep_log_info "NOTE: key-based encryption (encrypt=1)" \ "cannot be enabled with tar format" encrypt=-1 @@ -184,11 +181,11 @@ get_keys() exit 2 fi ecmd="'$OPENSSL_BINARY' enc -$ealgo" - if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-pbkdf2'; then + if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-pbkdf2'; then ecmd="$ecmd -pbkdf2" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-iter'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-iter'; then ecmd="$ecmd -iter 1" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-md'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-md'; then ecmd="$ecmd -md sha256" fi if [ -z "$ekey" ]; then @@ -229,15 +226,15 @@ get_keys() get_transfer() { - if [ $tfmt = 'nc' ]; then + if [ "$tfmt" = 'nc' ]; then wsrep_log_info "Using netcat as streamer" wsrep_check_programs nc - tcmd="nc" + tcmd='nc' if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - if nc -h 2>&1 | grep -q 'ncat'; then + if nc -h 2>&1 | grep -q -F 'ncat'; then wsrep_log_info "Using Ncat as streamer" tcmd="$tcmd -l" - elif nc -h 2>&1 | grep -qw -- '-d'; then + elif nc -h 2>&1 | grep -qw -F -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" tcmd="$tcmd -dl" if [ $WSREP_SST_OPT_HOST_IPv6 -eq 1 ]; then @@ -259,14 +256,14 @@ get_transfer() # transfer and cause the command to timeout. # Older versions of netcat did not need this flag and will # return an error if the flag is used. - if nc -h 2>&1 | grep -qw -- '-N'; then + if nc -h 2>&1 | grep -qw -F -- '-N'; then tcmd="$tcmd -N" wsrep_log_info "Using nc -N" fi # netcat doesn't understand [] around IPv6 address - if nc -h 2>&1 | grep -q ncat; then + if nc -h 2>&1 | grep -q -F 'ncat'; then wsrep_log_info "Using Ncat as streamer" - elif nc -h 2>&1 | grep -qw -- '-d'; then + elif nc -h 2>&1 | grep -qw -F -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" else wsrep_log_info "Using traditional netcat as streamer" @@ -326,7 +323,8 @@ get_transfer() if [ "${sockopt#*,dhparam=}" != "$sockopt" ]; then if [ -z "$ssl_dhparams" ]; then # Determine the socat version - SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*') + SOCAT_VERSION=$(socat -V 2>&1 | \ + grep -m1 -owE '[0-9]+(\.[0-9]+)+' | head -n1) if [ -z "$SOCAT_VERSION" ]; then wsrep_log_error "******** FATAL ERROR ******************" wsrep_log_error "* Cannot determine the socat version. *" @@ -456,7 +454,7 @@ adjust_progress() fi elif [ -z "$progress" -a -n "$rlimit" ]; then # When rlimit is non-zero - pcmd="pv -q" + pcmd='pv -q' fi if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then @@ -476,6 +474,15 @@ read_cnf() tmode=$(parse_cnf "$encgroups" 'ssl-mode' 'DISABLED' | \ tr [:lower:] [:upper:]) + case "$tmode" in + 'VERIFY_IDENTITY'|'VERIFY_CA'|'REQUIRED'|'DISABLED') + ;; + *) + wsrep_log_error "Unrecognized ssl-mode option: '$tmode'" + exit 22 # EINVAL + ;; + esac + if [ $encrypt -eq 0 -o $encrypt -ge 2 ]; then if [ "$tmode" != 'DISABLED' -o $encrypt -ge 2 ]; then check_server_ssl_config @@ -583,8 +590,14 @@ get_stream() sig_joiner_cleanup() { + local estatus=$? + if [ $estatus -ne 0 ]; then + wsrep_log_error "Cleanup after exit with status: $estatus" + fi wsrep_log_error "Removing $MAGIC_FILE file due to signal" + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" + exit $estatus } cleanup_at_exit() @@ -595,6 +608,8 @@ cleanup_at_exit() wsrep_log_error "Cleanup after exit with status: $estatus" fi + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then wsrep_log_info "Removing the sst_in_progress file" wsrep_cleanup_progress_file @@ -624,7 +639,7 @@ cleanup_at_exit() fi # Final cleanup - pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o '[0-9]*' || :) + pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o -E '[0-9]*' || :) # This means no setsid done in mysqld. # We don't want to kill mysqld here otherwise. @@ -661,16 +676,13 @@ setup_ports() # wait_for_listen() { - local PORT="$1" - local ADDR="$2" - local MODULE="$3" for i in {1..150}; do - if check_port "" "$PORT" 'socat|nc'; then + if check_port "" "$SST_PORT" 'socat|nc'; then break fi sleep 0.2 done - echo "ready $ADDR/$MODULE//$sst_ver" + echo "ready $ADDR:$SST_PORT/$MODULE/$lsn/$sst_ver" } check_extra() @@ -715,7 +727,7 @@ recv_joiner() local ltcmd="$tcmd" if [ $tmt -gt 0 ]; then if [ -n "$(commandex timeout)" ]; then - if timeout --help | grep -qw -- '-k'; then + if timeout --help | grep -qw -F -- '-k'; then ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" else ltcmd="timeout -s9 $tmt $tcmd" @@ -727,7 +739,7 @@ recv_joiner() set +e if [ $wait -ne 0 ]; then - wait_for_listen "$SST_PORT" "$ADDR" "$MODULE" & + wait_for_listen & fi timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" @@ -761,7 +773,7 @@ recv_joiner() # check donor supplied secret SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ - cut -d ' ' -f 2) + cut -d ' ' -f2) if [ "$SECRET" != "$MY_SECRET" ]; then wsrep_log_error "Donor does not know my secret!" wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" @@ -815,7 +827,9 @@ monitor_process() [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" -if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then +if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a \ + "$WSREP_SST_OPT_ROLE" != 'donor' ] +then wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'" exit 22 fi @@ -823,25 +837,17 @@ fi read_cnf setup_ports -if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then +if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -F -- '--version-check'; then disver=' --no-version-check' fi -# if no command line argument and INNODB_DATA_HOME_DIR environment variable -# is not set, try to get it from my.cnf: -if [ -z "$INNODB_DATA_HOME_DIR" ]; then - INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') -fi - OLD_PWD="$(pwd)" -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_DATA_HOME_DIR" ]; then - # handle both relative and absolute paths - [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" - cd "$INNODB_DATA_HOME_DIR" +if [ -n "$DATA" -a "$DATA" != '.' ]; then + [ ! -d "$DATA" ] && mkdir -p "$DATA" + cd "$DATA" fi -INNODB_DATA_HOME_DIR=$(pwd -P) +DATA_DIR="$(pwd)" cd "$OLD_PWD" @@ -869,7 +875,7 @@ if [ $ssyslog -eq 1 ]; then else if [ $sstlogarchive -eq 1 ] then - ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N") + ARCHIVETIMESTAMP=$(date '+%Y.%m.%d-%H.%M.%S.%N') if [ -n "$sstlogarchivedir" ]; then if [ ! -d "$sstlogarchivedir" ]; then @@ -929,7 +935,7 @@ setup_commands() recovery=" --innodb-force-recovery=$INNODB_FORCE_RECOVERY" fi INNOAPPLY="$BACKUP_BIN --prepare$disver$recovery${iapts:+ }$iapts$INNOEXTRA --target-dir='$DATA' --datadir='$DATA'$mysqld_args $INNOAPPLY" - INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" + INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts$INNOEXTRA --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" INNOBACKUP="$BACKUP_BIN$WSREP_SST_OPT_CONF --backup$disver${iopts:+ }$iopts$tmpopts$INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA'$mysqld_args $INNOBACKUP" } @@ -1052,6 +1058,11 @@ then iopts="--parallel=$backup_threads${iopts:+ }$iopts" fi + max_binlogs=$(parse_cnf "$encgroups" 'sst-max-binlogs') + if [ -n "$max_binlogs" ]; then + iopts="--sst-max-binlogs=$max_binlogs${iopts:+ }$iopts" + fi + setup_commands set +e timeit "$stagemsg-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" @@ -1096,6 +1107,7 @@ then echo "done $WSREP_SST_OPT_GTID" wsrep_log_info "Total time on donor: $totime seconds" + wsrep_log_info "mariabackup SST/IST completed on donor" elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then @@ -1103,22 +1115,53 @@ then wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" - ib_home_dir="$INNODB_DATA_HOME_DIR" + # if no command line argument and INNODB_DATA_HOME_DIR environment + # variable is not set, try to get it from the my.cnf: + if [ -z "$INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') + INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") + fi + + if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" + cd "$INNODB_DATA_HOME_DIR" + ib_home_dir="$(pwd)" + cd "$OLD_PWD" + fi # if no command line argument and INNODB_LOG_GROUP_HOME is not set, - # try to get it from my.cnf: + # then try to get it from the my.cnf: if [ -z "$INNODB_LOG_GROUP_HOME" ]; then INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") fi - ib_log_dir="$INNODB_LOG_GROUP_HOME" + if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" + cd "$INNODB_LOG_GROUP_HOME" + ib_log_dir="$(pwd)" + cd "$OLD_PWD" + fi - # if no command line argument then try to get it from my.cnf: + # if no command line argument and INNODB_UNDO_DIR is not set, + # then try to get it from the my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') + INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi - ib_undo_dir="$INNODB_UNDO_DIR" + if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" + cd "$INNODB_UNDO_DIR" + ib_undo_dir="$(pwd)" + cd "$OLD_PWD" + fi if [ -n "$backup_threads" ]; then impts="--parallel=$backup_threads${impts:+ }$impts" @@ -1142,16 +1185,15 @@ then stagemsg='Joiner-Recv' - MODULE="xtrabackup_sst" + MODULE="${WSREP_SST_OPT_MODULE:-xtrabackup_sst}" [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" # May need xtrabackup_checkpoints later on [ -f "$DATA/xtrabackup_binary" ] && rm -f "$DATA/xtrabackup_binary" [ -f "$DATA/xtrabackup_galera_info" ] && rm -f "$DATA/xtrabackup_galera_info" - [ -f "$DATA/ib_logfile0" ] && rm -f "$DATA/ib_logfile0" - ADDR="$WSREP_SST_OPT_ADDR" + ADDR="$WSREP_SST_OPT_HOST" if [ "${tmode#VERIFY}" != "$tmode" ]; then # backward-incompatible behavior: @@ -1165,7 +1207,7 @@ then exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \ - tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d '=' -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -1219,6 +1261,36 @@ then jpid=$! wsrep_log_info "Proceeding with SST" + get_binlog + + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") + binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") + binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" + cd "$DATA" + wsrep_log_info "Cleaning the old binary logs" + # If there is a file with binlogs state, delete it: + [ -f "$binlog_base.state" ] && rm -fv "$binlog_base.state" 1>&2 + # Clean up the old binlog files and index: + if [ -f "$binlog_index" ]; then + while read bin_file || [ -n "$bin_file" ]; do + rm -fv "$bin_file" 1>&2 || : + done < "$binlog_index" + rm -fv "$binlog_index" 1>&2 + fi + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \ + -d "$binlog_dir" ] + then + cd "$binlog_dir" + if [ "$(pwd)" != "$DATA_DIR" ]; then + wsrep_log_info \ + "Cleaning the binlog directory '$binlog_dir' as well" + fi + fi + rm -fv "$binlog_base".[0-9]* 1>&2 || : + cd "$OLD_PWD" + fi + wsrep_log_info \ "Cleaning the existing datadir and innodb-data/log directories" if [ "$OS" = 'FreeBSD' ]; then @@ -1235,20 +1307,6 @@ then -o -exec rm -rfv {} 1>&2 \+ fi - get_binlog - - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") - if [ -d "$binlog_dir" ]; then - cd "$binlog_dir" - wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" - rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : - [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ - rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ - cd "$OLD_PWD" - fi - fi - TDATA="$DATA" DATA="$DATA/.sst" @@ -1282,11 +1340,13 @@ then dcmd="xargs -n 2 qpress -dT$nproc" - if [ -n "$progress" ] && pv --help | grep -qw -- '--line-mode'; then + if [ -n "$progress" ] && \ + pv --help | grep -qw -F -- '--line-mode' + then count=$(find "$DATA" -type f -name '*.qp' | wc -l) count=$(( count*2 )) pvopts="-f -s $count -l -N Decompression" - if pv --help | grep -qw -- '-F'; then + if pv --help | grep -qw -F -- '-F'; then pvopts="$pvopts -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" fi pcmd="pv $pvopts" @@ -1296,8 +1356,9 @@ then # Decompress the qpress files wsrep_log_info "Decompression with $nproc threads" - timeit "Joiner-Decompression" \ - "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + timeit 'Joiner-Decompression' \ + "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | \ + $dcmd" extcode=$? if [ $extcode -eq 0 ]; then @@ -1314,25 +1375,9 @@ then fi fi - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - - BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") - BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") - - # To avoid comparing data directory and BINLOG_DIRNAME - mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || : - - cd "$BINLOG_DIRNAME" - for bfile in $(ls -1 "$BINLOG_FILENAME".[0-9]*); do - echo "$BINLOG_DIRNAME/$bfile" >> "$WSREP_SST_OPT_BINLOG_INDEX" - done - cd "$OLD_PWD" - - fi - wsrep_log_info "Preparing the backup at $DATA" setup_commands - timeit "mariabackup prepare stage" "$INNOAPPLY" + timeit 'mariabackup prepare stage' "$INNOAPPLY" if [ $? -ne 0 ]; then wsrep_log_error "mariabackup apply finished with errors." \ @@ -1340,10 +1385,43 @@ then exit 22 fi + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + cd "$DATA" + binlogs="" + if [ -f 'xtrabackup_binlog_info' ]; then + NL=$'\n' + while read bin_string || [ -n "$bin_string" ]; do + bin_file=$(echo "$bin_string" | cut -f1) + if [ -f "$bin_file" ]; then + binlogs="$binlogs${binlogs:+$NL}$bin_file" + fi + done < 'xtrabackup_binlog_info' + else + binlogs=$(ls -d -1 "$binlog_base".[0-9]* 2>/dev/null || :) + fi + cd "$DATA_DIR" + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then + [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" + fi + index_dir=$(dirname "$binlog_index"); + if [ -n "$index_dir" -a "$index_dir" != '.' ]; then + [ ! -d "$index_dir" ] && mkdir -p "$index_dir" + fi + if [ -n "$binlogs" ]; then + wsrep_log_info "Moving binary logs to $binlog_dir" + echo "$binlogs" | \ + while read bin_file || [ -n "$bin_file" ]; do + mv "$DATA/$bin_file" "$binlog_dir" + echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" + done + fi + cd "$OLD_PWD" + fi + MAGIC_FILE="$TDATA/$INFO_FILE" wsrep_log_info "Moving the backup to $TDATA" - timeit "mariabackup move stage" "$INNOMOVE" + timeit 'mariabackup move stage' "$INNOMOVE" if [ $? -eq 0 ]; then wsrep_log_info "Move successful, removing $DATA" rm -rf "$DATA" @@ -1370,6 +1448,7 @@ then cat "$MAGIC_FILE" # Output : UUID:seqno wsrep_gtid_domain_id wsrep_log_info "Total time on joiner: $totime seconds" + wsrep_log_info "mariabackup SST/IST completed on joiner" fi exit 0 diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index 1c8fc181328..3c92f489cb5 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -3,7 +3,7 @@ set -ue # Copyright (C) 2009-2015 Codership Oy -# Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2017-2022 MariaDB # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -40,8 +40,7 @@ then fi # Check client version -if ! $MYSQL_CLIENT --version | grep 'Distrib 10\.[1-9]' >/dev/null -then +if ! $MYSQL_CLIENT --version | grep -q -E 'Distrib 10\.[1-9]'; then $MYSQL_CLIENT --version >&2 wsrep_log_error "this operation requires MySQL client version 10.1 or newer" exit $EINVAL @@ -95,7 +94,7 @@ DROP PREPARE stmt;" SET_START_POSITION="SET GLOBAL wsrep_start_position='$WSREP_SST_OPT_GTID';" SET_WSREP_GTID_DOMAIN_ID="" -if [ -n $WSREP_SST_OPT_GTID_DOMAIN_ID ]; then +if [ -n "$WSREP_SST_OPT_GTID_DOMAIN_ID" ]; then SET_WSREP_GTID_DOMAIN_ID=" SET @val = (SELECT GLOBAL_VALUE FROM INFORMATION_SCHEMA.SYSTEM_VARIABLES WHERE VARIABLE_NAME = 'WSREP_GTID_STRICT_MODE' AND GLOBAL_VALUE > 0); SET @stmt = IF (@val IS NOT NULL, 'SET GLOBAL WSREP_GTID_DOMAIN_ID=$WSREP_SST_OPT_GTID_DOMAIN_ID', 'SET @dummy = 0'); diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index cc73db5da87..67a7afc638f 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -2,7 +2,7 @@ set -ue -# Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2017-2022 MariaDB # Copyright (C) 2010-2014 Codership Oy # # This program is free software; you can redistribute it and/or modify @@ -36,6 +36,8 @@ cleanup_joiner() { local failure=0 + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" + wsrep_log_info "Joiner cleanup: rsync PID=$RSYNC_REAL_PID," \ "stunnel PID=$STUNNEL_REAL_PID" @@ -58,6 +60,7 @@ cleanup_joiner() if [ $failure -eq 0 ]; then if cleanup_pid $RSYNC_REAL_PID "$RSYNC_PID" "$RSYNC_CONF"; then [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" + [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" else wsrep_log_warning "rsync cleanup failed." fi @@ -140,66 +143,77 @@ STUNNEL_PID="$WSREP_SST_OPT_DATA/stunnel.pid" MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete" -BINLOG_TAR_FILE="$WSREP_SST_OPT_DATA/wsrep_sst_binlog.tar" -BINLOG_N_FILES=1 - get_binlog if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") - BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") + binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") + binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") fi -# if no command line argument and INNODB_LOG_GROUP_HOME is not set, -# try to get it from my.cnf: -if [ -z "$INNODB_LOG_GROUP_HOME" ]; then - INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') +OLD_PWD="$(pwd)" + +DATA="$WSREP_SST_OPT_DATA" +if [ -n "$DATA" -a "$DATA" != '.' ]; then + [ ! -d "$DATA" ] && mkdir -p "$DATA" + cd "$DATA" fi +DATA_DIR="$(pwd)" -OLD_PWD="$(pwd)" +cd "$OLD_PWD" + +BINLOG_TAR_FILE="$DATA_DIR/wsrep_sst_binlog.tar" -WSREP_LOG_DIR="$INNODB_LOG_GROUP_HOME" +ib_log_dir="$DATA_DIR" +ib_home_dir="$DATA_DIR" +ib_undo_dir="$DATA_DIR" -cd "$WSREP_SST_OPT_DATA" -if [ -n "$WSREP_LOG_DIR" ]; then - # handle both relative and absolute paths - [ ! -d "$WSREP_LOG_DIR" ] && mkdir -p "$WSREP_LOG_DIR" - cd "$WSREP_LOG_DIR" +# if no command line argument and INNODB_LOG_GROUP_HOME is not set, +# then try to get it from the my.cnf: +if [ -z "$INNODB_LOG_GROUP_HOME" ]; then + INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") fi -WSREP_LOG_DIR=$(pwd -P) -cd "$OLD_PWD" +if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" + cd "$INNODB_LOG_GROUP_HOME" + ib_log_dir="$(pwd)" + cd "$OLD_PWD" +fi -# if no command line argument and INNODB_DATA_HOME_DIR environment variable -# is not set, try to get it from my.cnf: +# if no command line argument and INNODB_DATA_HOME_DIR environment +# variable is not set, try to get it from the my.cnf: if [ -z "$INNODB_DATA_HOME_DIR" ]; then INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') + INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") fi -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_DATA_HOME_DIR" ]; then - # handle both relative and absolute paths +if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" cd "$INNODB_DATA_HOME_DIR" + ib_home_dir="$(pwd)" + cd "$OLD_PWD" fi -INNODB_DATA_HOME_DIR=$(pwd -P) -cd "$OLD_PWD" - -# if no command line argument then try to get it from my.cnf: +# if no command line argument and INNODB_UNDO_DIR is not set, +# then try to get it from the my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') + INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_UNDO_DIR" ]; then - # handle both relative and absolute paths +if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" cd "$INNODB_UNDO_DIR" + ib_undo_dir="$(pwd)" + cd "$OLD_PWD" fi -INNODB_UNDO_DIR=$(pwd -P) - -cd "$OLD_PWD" encgroups='--mysqld|sst' @@ -224,6 +238,15 @@ if [ -z "$SSLMODE" ]; then elif [ -n "$SSTKEY" -a -n "$SSTCERT" ]; then SSLMODE='REQUIRED' fi +else + case "$SSLMODE" in + 'VERIFY_IDENTITY'|'VERIFY_CA'|'REQUIRED'|'DISABLED') + ;; + *) + wsrep_log_error "Unrecognized ssl-mode option: '$SSLMODE'" + exit 22 # EINVAL + ;; + esac fi if [ -n "$SSTKEY" -a -n "$SSTCERT" ]; then @@ -248,18 +271,11 @@ VERIFY_OPT="" CHECK_OPT="" CHECK_OPT_LOCAL="" if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then - case "$SSLMODE" in - 'VERIFY_IDENTITY') + if [ "$SSLMODE" = 'VERIFY_IDENTITY' ]; then VERIFY_OPT='verifyPeer = yes' - ;; - 'VERIFY_CA') + else VERIFY_OPT='verifyChain = yes' - ;; - *) - wsrep_log_error "Unrecognized ssl-mode option: '$SSLMODE'" - exit 22 # EINVAL - ;; - esac + fi if [ -z "$SSTCA$SSTCAP" ]; then wsrep_log_error "Can't have ssl-mode='$SSLMODE' without CA file or path" exit 22 # EINVAL @@ -276,7 +292,7 @@ if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then CHECK_OPT="checkHost = $WSREP_SST_OPT_HOST" fi if is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then - CHECK_OPT_LOCAL="checkHost = localhost" + CHECK_OPT_LOCAL='checkHost = localhost' fi fi fi @@ -293,14 +309,59 @@ if [ -n "$SSLMODE" -a "$SSLMODE" != 'DISABLED' ]; then fi fi -readonly SECRET_TAG="secret" +readonly SECRET_TAG='secret' -if [ "$WSREP_SST_OPT_ROLE" = 'donor' ] -then +SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid" + +# give some time for previous SST to complete: +check_round=0 +while check_pid "$SST_PID" 0; do + wsrep_log_info "Previous SST is not completed, waiting for it to exit" + check_round=$(( check_round + 1 )) + if [ $check_round -eq 20 ]; then + wsrep_log_error "previous SST script still running." + exit 114 # EALREADY + fi + sleep 1 +done + +echo $$ > "$SST_PID" + +# give some time for stunnel from the previous SST to complete: +check_round=0 +while check_pid "$STUNNEL_PID" 1 "$STUNNEL_CONF"; do + wsrep_log_info "Lingering stunnel daemon found at startup," \ + "waiting for it to exit" + check_round=$(( check_round + 1 )) + if [ $check_round -eq 10 ]; then + wsrep_log_error "stunnel daemon still running." + exit 114 # EALREADY + fi + sleep 1 +done + +MODULE="${WSREP_SST_OPT_MODULE:-rsync_sst}" + +RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid" +RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf" + +# give some time for rsync from the previous SST to complete: +check_round=0 +while check_pid "$RSYNC_PID" 1 "$RSYNC_CONF"; do + wsrep_log_info "Lingering rsync daemon found at startup," \ + "waiting for it to exit" + check_round=$(( check_round + 1 )) + if [ $check_round -eq 10 ]; then + wsrep_log_error "rsync daemon still running." + exit 114 # EALREADY + fi + sleep 1 +done + +[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" +[ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" - [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" - [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" - [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID" +if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then if [ -n "$STUNNEL" ] then @@ -319,8 +380,6 @@ ${VERIFY_OPT} ${CHECK_OPT} ${CHECK_OPT_LOCAL} EOF - else - [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" fi RC=0 @@ -333,7 +392,7 @@ EOF [ -f "$FLUSHED" ] && rm -f "$FLUSHED" [ -f "$ERROR" ] && rm -f "$ERROR" - echo "flush tables" + echo 'flush tables' # Wait for : # (a) Tables to be flushed, AND @@ -357,32 +416,100 @@ EOF sync - if [ -n "$WSREP_SST_OPT_BINLOG" -a -d "${BINLOG_DIRNAME:-}" ] - then - # Prepare binlog files - cd "$BINLOG_DIRNAME" - - binlog_files_full=$(tail -n $BINLOG_N_FILES \ - "$WSREP_SST_OPT_BINLOG_INDEX") - binlog_files="" - for file in $binlog_files_full; do - binlog_file=$(basename "$file") - binlog_files="$binlog_files${binlog_files:+ }'$binlog_file'" - done - - if [ -n "$binlog_files" ]; then - wsrep_log_info "Preparing binlog files for transfer:" - eval tar -cvf "'$BINLOG_TAR_FILE'" $binlog_files >&2 + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + # Change the directory to binlog base (if possible): + cd "$DATA" + # Let's check the existence of the file with the index: + if [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ]; then + # Let's read the binlog index: + max_binlogs=$(parse_cnf "$encgroups" 'sst-max-binlogs') + if [ -n "$max_binlogs" ]; then + binlog_files="" + if [ $max_binlogs -gt 0 ]; then + binlog_files=$(tail -n $max_binlogs \ + "$WSREP_SST_OPT_BINLOG_INDEX") + fi + else + binlog_files=$(cat "$WSREP_SST_OPT_BINLOG_INDEX") + fi + if [ -n "$binlog_files" ]; then + # Preparing binlog files for transfer: + wsrep_log_info "Preparing binlog files for transfer:" + tar_type=0 + if tar --help | grep -qw -F -- '--transform'; then + tar_type=1 + elif tar --version | grep -q -E '^bsdtar\>'; then + tar_type=2 + fi + if [ $tar_type -ne 2 ]; then + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$binlog_files" >&2 + else + echo "$binlog_files" >&2 + fi + fi + if [ $tar_type -ne 0 ]; then + # Preparing list of the binlog file names: + echo "$binlog_files" | { + binlogs="" + while read bin_file || [ -n "$bin_file" ]; do + [ ! -f "$bin_file" ] && continue + if [ -n "$BASH_VERSION" ]; then + first="${bin_file:0:1}" + else + first=$(echo "$bin_file" | cut -c1) + fi + if [ "$first" = '-' -o "$first" = '@' ]; then + bin_file="./$bin_file" + fi + binlogs="$binlogs${binlogs:+ }'$bin_file'" + done + if [ -n "$binlogs" ]; then + if [ $tar_type -eq 1 ]; then + tar_options="--transform='s/^.*\///g'" + else + # bsdtar handles backslash incorrectly: + tar_options="-s '?^.*/??g'" + fi + eval tar -P $tar_options \ + -cvf "'$BINLOG_TAR_FILE'" $binlogs >&2 + fi + } + else + tar_options='-cvf' + echo "$binlog_files" | \ + while read bin_file || [ -n "$bin_file" ]; do + [ ! -f "$bin_file" ] && continue + bin_dir=$(dirname "$bin_file") + bin_base=$(basename "$bin_file") + if [ -n "$BASH_VERSION" ]; then + first="${bin_base:0:1}" + else + first=$(echo "$bin_base" | cut -c1) + fi + if [ "$first" = '-' -o "$first" = '@' ]; then + bin_base="./$bin_base" + fi + if [ -n "$bin_dir" -a "$bin_dir" != '.' ]; then + tar $tar_options "$BINLOG_TAR_FILE" \ + -C "$bin_dir" "$bin_base" >&2 + else + tar $tar_options "$BINLOG_TAR_FILE" \ + "$bin_base" >&2 + fi + tar_options='-rvf' + done + fi + fi fi - cd "$OLD_PWD" fi - # Use deltaxfer only for WAN + # Use deltaxfer only for WAN: inv=$(basename "$0") WHOLE_FILE_OPT="" if [ "${inv%wsrep_sst_rsync_wan*}" = "$inv" ]; then - WHOLE_FILE_OPT="--whole-file" + WHOLE_FILE_OPT='--whole-file' fi # Old filter - include everything except selected @@ -399,9 +526,9 @@ FILTER="-f '- /lost+found' -f '- /.pid' -f '- /.conf' -f '+ /wsrep_sst_binlog.tar' - -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump' - -f '- $INNODB_DATA_HOME_DIR/ibdata*' - -f '+ $INNODB_UNDO_DIR/undo*' + -f '- $ib_home_dir/ib_lru_dump' + -f '- $ib_home_dir/ibdata*' + -f '+ $ib_undo_dir/undo*' -f '+ /*/' -f '- /*'" @@ -435,7 +562,7 @@ FILTER="-f '- /lost+found' --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT -f '+ /ibdata*' -f '+ /ib_lru_dump' \ - -f '- **' "$INNODB_DATA_HOME_DIR/" \ + -f '- **' "$ib_home_dir/" \ "rsync://$WSREP_SST_OPT_ADDR-data_dir" >&2 || RC=$? if [ $RC -ne 0 ]; then @@ -448,7 +575,7 @@ FILTER="-f '- /lost+found' --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' -f '+ /aria_log.*' \ - -f '+ /aria_log_control' -f '- **' "$WSREP_LOG_DIR/" \ + -f '+ /aria_log_control' -f '- **' "$ib_log_dir/" \ "rsync://$WSREP_SST_OPT_ADDR-log_dir" >&2 || RC=$? if [ $RC -ne 0 ]; then @@ -459,7 +586,7 @@ FILTER="-f '- /lost+found' # then, we parallelize the transfer of database directories, # use '.' so that path concatenation works: - cd "$WSREP_SST_OPT_DATA" + cd "$DATA" backup_threads=$(parse_cnf '--mysqld|sst' 'backup-threads') if [ -z "$backup_threads" ]; then @@ -518,68 +645,21 @@ FILTER="-f '- /lost+found' [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID" fi + [ -f "$SST_PID" ] && rm -f "$SST_PID" + + wsrep_log_info "rsync SST/IST completed on donor" + elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then check_sockets_utils - SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid" - - # give some time for previous SST to complete: - check_round=0 - while check_pid "$SST_PID" 0 'wsrep_sst_'; do - wsrep_log_info "previous SST is not completed, waiting for it to exit" - check_round=$(( check_round + 1 )) - if [ $check_round -eq 10 ]; then - wsrep_log_error "previous SST script still running." - exit 114 # EALREADY - fi - sleep 1 - done - - echo $$ > "$SST_PID" - - # give some time for stunnel from the previous SST to complete: - check_round=0 - while check_pid "$STUNNEL_PID" 1; do - wsrep_log_info "Lingering stunnel daemon found at startup," \ - "waiting for it to exit" - check_round=$(( check_round + 1 )) - if [ $check_round -eq 10 ]; then - wsrep_log_error "stunnel daemon already running." - exit 114 # EALREADY - fi - sleep 1 - done - - MODULE="rsync_sst" - RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid" - RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf" - - # give some time for rsync from the previous SST to complete: - check_round=0 - while check_pid "$RSYNC_PID" 1; do - wsrep_log_info "Lingering rsync daemon found at startup," \ - "waiting for it to exit" - check_round=$(( check_round + 1 )) - if [ $check_round -eq 10 ]; then - wsrep_log_error "rsync daemon already running." - exit 114 # EALREADY - fi - sleep 1 - done - - [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" - [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" - - [ -z "$STUNNEL" -a -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" - - ADDR="$WSREP_SST_OPT_ADDR" + ADDR="$WSREP_SST_OPT_HOST" RSYNC_PORT="$WSREP_SST_OPT_PORT" RSYNC_ADDR="$WSREP_SST_OPT_HOST" RSYNC_ADDR_UNESCAPED="$WSREP_SST_OPT_HOST_UNESCAPED" - trap "exit 32" HUP PIPE - trap "exit 3" INT TERM ABRT + trap 'exit 32' HUP PIPE + trap 'exit 3' INT TERM ABRT trap cleanup_joiner EXIT touch "$SST_PROGRESS_FILE" @@ -600,13 +680,11 @@ $SILENT path = $WSREP_SST_OPT_DATA exclude = .zfs [$MODULE-log_dir] - path = $WSREP_LOG_DIR + path = $ib_log_dir [$MODULE-data_dir] - path = $INNODB_DATA_HOME_DIR + path = $ib_home_dir EOF -# rm -rf "$DATA/ib_logfile"* # we don't want old logs around - # If the IP is local, listen only on it: if is_local_ip "$RSYNC_ADDR_UNESCAPED" then @@ -617,7 +695,7 @@ EOF RSYNC_EXTRA_ARGS="" STUNNEL_ACCEPT="$RSYNC_PORT" # Overwrite address with all: - RSYNC_ADDR="*" + RSYNC_ADDR='*' fi if [ -z "$STUNNEL" ]; then @@ -675,11 +753,10 @@ EOF TRANSFER_PID="$STUNNEL_PID" fi - if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ] - then # backward-incompatible behavior + if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then + # backward-incompatible behavior: CN="" - if [ -n "$SSTCERT" ] - then + if [ -n "$SSTCERT" ]; then # find out my Common Name get_openssl if [ -z "$OPENSSL_BINARY" ]; then @@ -688,7 +765,7 @@ EOF exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$SSTCERT" | \ - tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d '=' -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -696,7 +773,6 @@ EOF ADDR="$CN:$MY_SECRET@$WSREP_SST_OPT_HOST" else MY_SECRET="" # for check down in recv_joiner() - ADDR="$WSREP_SST_OPT_HOST" fi until check_pid_and_port "$TRANSFER_PID" $TRANSFER_REAL_PID \ @@ -725,16 +801,53 @@ EOF exit 32 fi - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - if [ -f "$BINLOG_TAR_FILE" ]; then - cd "$BINLOG_DIRNAME" + if [ -r "$MAGIC_FILE" ]; then + if [ -n "$MY_SECRET" ]; then + # Check donor supplied secret: + SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ + cut -d ' ' -f2) + if [ "$SECRET" != "$MY_SECRET" ]; then + wsrep_log_error "Donor does not know my secret!" + wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" + exit 32 + fi + fi + else + # This message should cause joiner to abort: + wsrep_log_info "rsync process ended without creating magic file" + echo "rsync process ended without creating '$MAGIC_FILE'" + exit 32 + fi + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + binlog_tar_present=0 + [ -f "$BINLOG_TAR_FILE" ] && binlog_tar_present=1 + # If it is SST (not an IST) or tar with binlogs is present + # among the transferred files, then we need to remove the + # old binlogs: + if [ $WSREP_SST_OPT_BYPASS -eq 0 -o $binlog_tar_present -ne 0 ]; then + cd "$DATA" + # Clean up the old binlog files and index: binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" - - # Clean up old binlog files first - rm -f "$BINLOG_FILENAME".[0-9]* - [ -f "$binlog_index" ] && rm -f "$binlog_index" - + if [ -f "$binlog_index" ]; then + while read bin_file || [ -n "$bin_file" ]; do + rm -f "$bin_file" || : + done < "$binlog_index" + rm -f "$binlog_index" + fi + binlog_cd=0 + # Change the directory to binlog base (if possible): + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \ + -d "$binlog_dir" ] + then + binlog_cd=1 + cd "$binlog_dir" + fi + # Clean up unindexed binlog files: + rm -f "$binlog_base".[0-9]* || : + [ $binlog_cd -ne 0 ] && cd "$DATA_DIR" + fi + if [ $binlog_tar_present -ne 0 ]; then # Create a temporary file: tmpdir=$(parse_cnf '--mysqld|sst' 'tmpdir') if [ -z "$tmpdir" ]; then @@ -744,46 +857,52 @@ EOF else tmpfile=$(TMPDIR="$tmpdir"; mktemp) fi - + index_dir=$(dirname "$binlog_index"); + if [ -n "$index_dir" -a "$index_dir" != '.' ]; then + [ ! -d "$index_dir" ] && mkdir -p "$index_dir" + fi + binlog_cd=0 + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then + [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" + binlog_cd=1 + cd "$binlog_dir" + fi + # Extracting binlog files: wsrep_log_info "Extracting binlog files:" - if ! tar -xvf "$BINLOG_TAR_FILE" > "$tmpfile"; then - wsrep_log_error "Error unpacking tar file with binlog files" + RC=0 + if tar --version | grep -q -E '^bsdtar\>'; then + tar -tf "$BINLOG_TAR_FILE" > "$tmpfile" && \ + tar -xvf "$BINLOG_TAR_FILE" > /dev/null || RC=$? + else + tar -xvf "$BINLOG_TAR_FILE" > "$tmpfile" && \ + cat "$tmpfile" >&2 || RC=$? + fi + if [ $RC -ne 0 ]; then rm -f "$tmpfile" + wsrep_log_error "Error unpacking tar file with binlog files" exit 32 fi - # Rebuild binlog index: - while read bin_file; do - echo "$BINLOG_DIRNAME/$bin_file" >> "$binlog_index" + [ $binlog_cd -ne 0 ] && cd "$DATA_DIR" + while read bin_file || [ -n "$bin_file" ]; do + echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" done < "$tmpfile" rm -f "$tmpfile" - cd "$OLD_PWD" fi fi - if [ -r "$MAGIC_FILE" ]; then - if [ -n "$MY_SECRET" ]; then - # check donor supplied secret - SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ - cut -d ' ' -f 2) - if [ "$SECRET" != "$MY_SECRET" ]; then - wsrep_log_error "Donor does not know my secret!" - wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" - exit 32 - fi - # remove secret from the magic file, and output - # the UUID:seqno & wsrep_gtid_domain_id: - grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" - else - # Output the UUID:seqno and wsrep_gtid_domain_id: - cat "$MAGIC_FILE" - fi + if [ -n "$MY_SECRET" ]; then + # remove secret from the magic file, and output + # the UUID:seqno & wsrep_gtid_domain_id: + grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" else - # this message should cause joiner to abort - echo "rsync process ended without creating '$MAGIC_FILE'" + # Output the UUID:seqno and wsrep_gtid_domain_id: + cat "$MAGIC_FILE" fi + wsrep_log_info "rsync SST/IST completed on joiner" + # wsrep_cleanup_progress_file # cleanup_joiner else diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index 519896065e5..e12deb22797 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -2808,12 +2808,9 @@ bool Item_in_subselect::inject_in_to_exists_cond(JOIN *join_arg) { /* The argument list of the top-level AND may change after fix fields. */ and_args= ((Item_cond*) join_arg->conds)->argument_list(); - List_iterator<Item_equal> li(join_arg->cond_equal->current_level); - Item_equal *elem; - while ((elem= li++)) - { - and_args->push_back(elem, thd->mem_root); - } + ((Item_cond_and *) (join_arg->conds))->m_cond_equal= + *join_arg->cond_equal; + and_args->append((List<Item> *)&join_arg->cond_equal->current_level); } } diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 4b92c4cb7a6..6775c414402 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2016, Oracle and/or its affiliates. - Copyright (c) 2009, 2021, MariaDB Corporation. + Copyright (c) 2009, 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -24531,20 +24531,22 @@ int setup_order(THD *thd, Ref_ptr_array ref_pointer_array, TABLE_LIST *tables, return 1; } - if (!(*order->item)->with_sum_func()) - continue; - /* UNION queries cannot be used with an aggregate function in an ORDER BY clause */ - if (for_union) + if (for_union && + ((*order->item)->with_sum_func() || + (*order->item)->with_window_func)) { my_error(ER_AGGREGATE_ORDER_FOR_UNION, MYF(0), number); return 1; } + if (!(*order->item)->with_sum_func()) + continue; + if (from_window_spec && (*order->item)->type() != Item::SUM_FUNC_ITEM) (*order->item)->split_sum_func(thd, ref_pointer_array, all_fields, SPLIT_SUM_SELECT); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 9143af5dbbf..07076f0660c 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -2751,7 +2751,7 @@ innobase_trx_init( while holding lock_sys.mutex, by lock_rec_enqueue_waiting(), will not end up acquiring LOCK_global_system_variables in intern_sys_var_ptr(). */ - THDVAR(thd, lock_wait_timeout); + (void) THDVAR(thd, lock_wait_timeout); trx->check_foreigns = !thd_test_options( thd, OPTION_NO_FOREIGN_KEY_CHECKS); @@ -15979,7 +15979,7 @@ struct ShowStatus { }; /** Order by m_waits, in descending order. */ - struct OrderByWaits: public std::binary_function<Value, Value, bool> + struct OrderByWaits { /** @return true if rhs < lhs */ bool operator()( diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 8a166398f66..d21c0cefec0 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -6711,6 +6711,9 @@ static ST_FIELD_INFO innodb_sys_tablespaces_fields_info[]= END_OF_ST_FIELD_INFO }; + +extern size_t os_file_get_fs_block_size(const char *path); + /**********************************************************************//** Function to fill INFORMATION_SCHEMA.INNODB_SYS_TABLESPACES with information collected by scanning SYS_TABLESPACESS table. @@ -6775,11 +6778,10 @@ i_s_dict_fill_sys_tablespaces( OK(fields[SYS_TABLESPACES_ZIP_PAGE_SIZE]->store( fil_space_t::physical_size(cflags), true)); - os_file_stat_t stat; + size_t fs_block_size = 0; os_file_size_t file; memset(&file, 0xff, sizeof(file)); - memset(&stat, 0x0, sizeof(stat)); if (fil_space_t* s = fil_space_acquire_silent(space)) { const char *filepath = s->chain.start @@ -6789,36 +6791,19 @@ i_s_dict_fill_sys_tablespaces( } file = os_file_get_size(filepath); - - /* Get the file system (or Volume) block size. */ - switch (dberr_t err = os_file_get_status(filepath, &stat, - false, false)) { - case DB_FAIL: - ib::warn() - << "File '" << filepath << "', failed to get " - << "stats"; - break; - - case DB_SUCCESS: - case DB_NOT_FOUND: - break; - - default: - ib::error() << "File '" << filepath << "' " << err; - break; - } + fs_block_size= os_file_get_fs_block_size(filepath); file_done: s->release(); } if (file.m_total_size == static_cast<os_offset_t>(~0)) { - stat.block_size = 0; + fs_block_size = 0; file.m_total_size = 0; file.m_alloc_size = 0; } - OK(fields[SYS_TABLESPACES_FS_BLOCK_SIZE]->store(stat.block_size, true)); + OK(fields[SYS_TABLESPACES_FS_BLOCK_SIZE]->store(fs_block_size, true)); OK(fields[SYS_TABLESPACES_FILE_SIZE]->store(file.m_total_size, true)); diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 5903fca86b6..a7b6eb57a22 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2019, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2021, MariaDB Corporation. +Copyright (c) 2013, 2022, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -4351,48 +4351,6 @@ os_file_get_status_win32( CloseHandle(fh); } } - stat_info->block_size = 0; - - /* What follows, is calculation of FS block size, which is not important - (it is just shown in I_S innodb tables). The error to calculate it will be ignored.*/ - char volname[MAX_PATH]; - BOOL result = GetVolumePathName(path, volname, MAX_PATH); - static bool warned_once = false; - if (!result) { - if (!warned_once) { - ib::warn() - << "os_file_get_status_win32: " - << "Failed to get the volume path name for: " - << path - << "- OS error number " << GetLastError(); - warned_once = true; - } - return(DB_SUCCESS); - } - - DWORD sectorsPerCluster; - DWORD bytesPerSector; - DWORD numberOfFreeClusters; - DWORD totalNumberOfClusters; - - result = GetDiskFreeSpace( - (LPCSTR) volname, - §orsPerCluster, - &bytesPerSector, - &numberOfFreeClusters, - &totalNumberOfClusters); - - if (!result) { - if (!warned_once) { - ib::warn() - << "GetDiskFreeSpace(" << volname << ",...) " - << "failed " - << "- OS error number " << GetLastError(); - warned_once = true; - } - return(DB_SUCCESS); - } - stat_info->block_size = bytesPerSector * sectorsPerCluster; } else { stat_info->type = OS_FILE_TYPE_UNKNOWN; } @@ -5284,6 +5242,35 @@ IORequest::punch_hole(os_file_t fh, os_offset_t off, ulint len) return (err); } +/* + Get file system block size, by path. + + This is expensive on Windows, and not very useful in general, + (only shown in some I_S table), so we keep that out of usual + stat. +*/ +size_t os_file_get_fs_block_size(const char *path) +{ +#ifdef _WIN32 + char volname[MAX_PATH]; + if (!GetVolumePathName(path, volname, MAX_PATH)) + return 0; + DWORD sectorsPerCluster; + DWORD bytesPerSector; + DWORD numberOfFreeClusters; + DWORD totalNumberOfClusters; + + if (GetDiskFreeSpace(volname, §orsPerCluster, &bytesPerSector, + &numberOfFreeClusters, &totalNumberOfClusters)) + return ((size_t) bytesPerSector) * sectorsPerCluster; +#else + os_file_stat_t info; + if (os_file_get_status(path, &info, false, false) == DB_SUCCESS) + return info.block_size; +#endif + return 0; +} + /** This function returns information about the specified file @param[in] path pathname of the file @param[out] stat_info information of a file in a directory diff --git a/storage/innobase/sync/sync0debug.cc b/storage/innobase/sync/sync0debug.cc index a00c9cef5af..377adc5b009 100644 --- a/storage/innobase/sync/sync0debug.cc +++ b/storage/innobase/sync/sync0debug.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2014, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2020, MariaDB Corporation. +Copyright (c) 2017, 2022, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -101,10 +101,6 @@ struct LatchDebug { /** Comparator for the ThreadMap. */ struct os_thread_id_less - : public std::binary_function< - os_thread_id_t, - os_thread_id_t, - bool> { /** @return true if lhs < rhs */ bool operator()( @@ -390,10 +386,6 @@ private: private: /** Comparator for the Levels . */ struct latch_level_less - : public std::binary_function< - latch_level_t, - latch_level_t, - bool> { /** @return true if lhs < rhs */ bool operator()( |