diff options
author | Alexander Barkov <bar@mariadb.com> | 2023-04-13 15:42:53 +0400 |
---|---|---|
committer | Alexander Barkov <bar@mariadb.com> | 2023-04-21 19:08:35 +0400 |
commit | 9f98a2acd71dcfbdb32a08e72a4737359ed9be40 (patch) | |
tree | 158e4d8662fdc3668d589c6fbd927d184b69ba71 | |
parent | da1c91fb9232fbea88d3f9a27f81b39f85cfc468 (diff) | |
download | mariadb-git-9f98a2acd71dcfbdb32a08e72a4737359ed9be40.tar.gz |
MDEV-30968 mariadb-backup does not copy Aria logs if aria_log_dir_path is used
- `mariadb-backup --backup` was fixed to fetch the value of the
@@aria_log_dir_path server variable and copy aria_log* files
from @@aria_log_dir_path directory to the backup directory.
Absolute and relative (to --datadir) paths are supported.
Before this change aria_log* files were copied to the backup
only if they were in the default location in @@datadir.
- `mariadb-backup --copy-back` now understands a new my.cnf and command line
parameter --aria-log-dir-path.
`mariadb-backup --copy-back` in the main loop in copy_back()
(when copying back from the backup directory to --datadir)
was fixed to ignore all aria_log* files.
A new function copy_back_aria_logs() was added.
It consists of a separate loop copying back aria_log* files from
the backup directory to the directory specified in --aria-log-dir-path.
Absolute and relative (to --datadir) paths are supported.
If --aria-log-dir-path is not specified,
aria_log* files are copied to --datadir by default.
- The function is_absolute_path() was fixed to understand MTR style
paths on Windows with forward slashes, e.g.
--aria-log-dir-path=D:/Buildbot/amd64-windows/build/mysql-test/var/...
-rw-r--r-- | extra/mariabackup/backup_copy.cc | 70 | ||||
-rw-r--r-- | extra/mariabackup/backup_mysql.cc | 7 | ||||
-rw-r--r-- | extra/mariabackup/xtrabackup.cc | 14 | ||||
-rw-r--r-- | extra/mariabackup/xtrabackup.h | 1 | ||||
-rw-r--r-- | mysql-test/suite/mariabackup/aria_log_dir_path.result | 41 | ||||
-rw-r--r-- | mysql-test/suite/mariabackup/aria_log_dir_path.test | 105 | ||||
-rw-r--r-- | mysql-test/suite/mariabackup/aria_log_dir_path_rel.result | 41 | ||||
-rw-r--r-- | mysql-test/suite/mariabackup/aria_log_dir_path_rel.test | 4 | ||||
-rw-r--r-- | storage/innobase/include/os0file.h | 6 |
9 files changed, 281 insertions, 8 deletions
diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index ffaf6dc98e4..8ab52fa983b 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -130,7 +130,9 @@ struct datadir_thread_ctxt_t { bool ret; }; -static bool backup_files_from_datadir(ds_ctxt *ds_data, const char *dir_path); +static bool backup_files_from_datadir(ds_ctxt_t *ds_data, + const char *dir_path, + const char *prefix); /************************************************************************ Retirn true if character if file separator */ @@ -1499,7 +1501,11 @@ bool backup_start(ds_ctxt *ds_data, ds_ctxt *ds_meta, return(false); } - if (!backup_files_from_datadir(ds_data, fil_path_to_mysql_datadir)) { + if (!backup_files_from_datadir(ds_data, fil_path_to_mysql_datadir, + "aws-kms-key") || + !backup_files_from_datadir(ds_data, + aria_log_dir_path, + "aria_log")) { return false; } @@ -1714,7 +1720,12 @@ ibx_copy_incremental_over_full() } } - if (!(ret = backup_files_from_datadir(ds_data, xtrabackup_incremental_dir))) + if (!(ret = backup_files_from_datadir(ds_data, + xtrabackup_incremental_dir, + "aws-kms-key")) || + !(ret = backup_files_from_datadir(ds_data, + xtrabackup_incremental_dir, + "aria_log"))) goto cleanup; /* copy supplementary files */ @@ -1829,6 +1840,41 @@ public: } }; + +static inline bool +is_aria_log_dir_file(const datadir_node_t &node) +{ + return starts_with(node.filepath_rel, "aria_log"); +} + + +bool +copy_back_aria_logs() +{ + Copy_back_dst_dir dst_dir_buf; + const char *dstdir= dst_dir_buf.make(aria_log_dir_path); + std::unique_ptr<ds_ctxt_t, void (&)(ds_ctxt_t*)> + ds_ctxt_aria_log_dir_path(ds_create(dstdir, DS_TYPE_LOCAL), ds_destroy); + + datadir_node_t node; + datadir_node_init(&node); + datadir_iter_t *it = datadir_iter_new(".", false); + + while (datadir_iter_next(it, &node)) + { + if (!is_aria_log_dir_file(node)) + continue; + if (!copy_or_move_file(ds_ctxt_aria_log_dir_path.get(), + node.filepath, node.filepath_rel, + dstdir, 1)) + return false; + } + datadir_node_free(&node); + datadir_iter_free(it); + return true; +} + + bool copy_back() { @@ -1861,6 +1907,10 @@ copy_back() && !directory_exists(srv_log_group_home_dir, true)) { return(false); } + if (aria_log_dir_path && *aria_log_dir_path + && !directory_exists(aria_log_dir_path, true)) { + return false; + } /* cd to backup directory */ if (my_setwd(xtrabackup_target_dir, MYF(MY_WME))) @@ -1869,6 +1919,9 @@ copy_back() return(false); } + if (!copy_back_aria_logs()) + return false; + /* parse data file path */ if (!innobase_data_file_path) { @@ -1973,6 +2026,10 @@ copy_back() int i_tmp; bool is_ibdata_file; + /* Skip aria log files */ + if (is_aria_log_dir_file(node)) + continue; + if (strstr(node.filepath,"/" ROCKSDB_BACKUP_DIR "/") #ifdef _WIN32 || strstr(node.filepath,"\\" ROCKSDB_BACKUP_DIR "\\") @@ -2209,7 +2266,9 @@ decrypt_decompress() Do not copy the Innodb files (ibdata1, redo log files), as this is done in a separate step. */ -static bool backup_files_from_datadir(ds_ctxt *ds_data, const char *dir_path) +static bool backup_files_from_datadir(ds_ctxt_t *ds_data, + const char *dir_path, + const char *prefix) { os_file_dir_t dir = os_file_opendir(dir_path); if (dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) return false; @@ -2225,8 +2284,7 @@ static bool backup_files_from_datadir(ds_ctxt *ds_data, const char *dir_path) if (!pname) pname = info.name; - if (!starts_with(pname, "aws-kms-key") && - !starts_with(pname, "aria_log")) + if (!starts_with(pname, prefix)) /* For ES exchange the above line with the following code: (!xtrabackup_prepare || !xtrabackup_incremental_dir || !starts_with(pname, "aria_log"))) diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index 6003bfb36c4..1831485e957 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -367,6 +367,7 @@ bool get_mysql_vars(MYSQL *connection) char *innodb_undo_directory_var= NULL; char *innodb_page_size_var= NULL; char *innodb_undo_tablespaces_var= NULL; + char *aria_log_dir_path_var= NULL; char *page_zip_level_var= NULL; char *ignore_db_dirs= NULL; char *endptr; @@ -397,6 +398,7 @@ bool get_mysql_vars(MYSQL *connection) {"innodb_undo_tablespaces", &innodb_undo_tablespaces_var}, {"innodb_compression_level", &page_zip_level_var}, {"ignore_db_dirs", &ignore_db_dirs}, + {"aria_log_dir_path", &aria_log_dir_path_var}, {NULL, NULL}}; read_mysql_variables(connection, "SHOW VARIABLES", mysql_vars, true); @@ -538,6 +540,11 @@ bool get_mysql_vars(MYSQL *connection) ut_ad(*endptr == 0); } + if (aria_log_dir_path_var) + { + aria_log_dir_path= my_strdup(aria_log_dir_path_var, MYF(MY_FAE)); + } + if (page_zip_level_var != NULL) { page_zip_level= strtoul(page_zip_level_var, &endptr, 10); diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index ee12034c910..96c90b5afad 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -266,6 +266,8 @@ my_bool innobase_locks_unsafe_for_binlog; my_bool innobase_rollback_on_timeout; my_bool innobase_create_status_file; +char *aria_log_dir_path; + /* The following counter is used to convey information to InnoDB about server activity: in selects it is not sensible to call srv_active_wake_master_thread after each fetch or search, we only do @@ -1105,7 +1107,8 @@ enum options_xtrabackup OPT_XTRA_CHECK_PRIVILEGES, OPT_XTRA_MYSQLD_ARGS, OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION, - OPT_INNODB_FORCE_RECOVERY + OPT_INNODB_FORCE_RECOVERY, + OPT_ARIA_LOG_DIR_PATH }; struct my_option xb_client_options[]= { @@ -1696,6 +1699,11 @@ struct my_option xb_server_options[] = &innodb_log_checksums, &innodb_log_checksums, 0, GET_BOOL, REQUIRED_ARG, 1, 0, 0, 0, 0, 0 }, + {"aria_log_dir_path", OPT_ARIA_LOG_DIR_PATH, + "Path to individual files and their sizes.", + &aria_log_dir_path, &aria_log_dir_path, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file " "descriptors to reserve with setrlimit().", (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG, @@ -2012,6 +2020,10 @@ xb_get_one_option(int optid, } break; + case OPT_ARIA_LOG_DIR_PATH: + ADD_PRINT_PARAM_OPT(aria_log_dir_path); + break; + case OPT_XTRA_TARGET_DIR: strmake(xtrabackup_real_target_dir,argument, sizeof(xtrabackup_real_target_dir)-1); xtrabackup_target_dir= xtrabackup_real_target_dir; diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index de3a96443a3..df2f766aedb 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -74,6 +74,7 @@ extern char *xtrabackup_incremental_dir; extern char *xtrabackup_incremental_basedir; extern char *innobase_data_home_dir; extern char *innobase_buffer_pool_filename; +extern char *aria_log_dir_path; extern char *xb_plugin_dir; extern char *xb_rocksdb_datadir; extern my_bool xb_backup_rocksdb; diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path.result b/mysql-test/suite/mariabackup/aria_log_dir_path.result new file mode 100644 index 00000000000..1a877321bbe --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_log_dir_path.result @@ -0,0 +1,41 @@ +# +# MDEV-30968 mariadb-backup does not copy Aria logs if aria_log_dir_path is used +# +# Restart mariadbd with the test specific parameters +# restart: --aria-log-file-size=8388608 --aria-log-purge-type=external --loose-aria-log-dir-path=MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path +# Create and populate an Aria table (and Aria logs) +CREATE TABLE t1 (id INT, txt LONGTEXT) ENGINE=Aria; +BEGIN NOT ATOMIC +FOR id IN 0..9 DO +INSERT INTO test.t1 (id, txt) VALUES (id, REPEAT(id,1024*1024)); +END FOR; +END; +$$ +# Testing aria log files before --backup +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +SHOW ENGINE aria logs; +Type Name Status +Aria aria_log.00000001 free +Aria aria_log.00000002 in use +# mariadb-backup --backup +# mariadb-backup --prepare +# shutdown server +# remove datadir +# remove aria-log-dir-path +# mariadb-backup --copy-back +# with parameters: --defaults-file=MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=MYSQLTEST_VARDIR/mysqld.1/data/ --target-dir=MYSQLTEST_VARDIR/tmp/backup --parallel=2 --throttle=1 --aria-log-dir-path=MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path +# starting server +# restart: --aria-log-file-size=8388608 --aria-log-purge-type=external --loose-aria-log-dir-path=MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path +# Check that the table is there after --copy-back +SELECT COUNT(*) from t1; +COUNT(*) +10 +DROP TABLE t1; +# Testing aria log files after --copy-back +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +SHOW ENGINE aria logs; +Type Name Status +Aria aria_log.00000001 free +Aria aria_log.00000002 in use +# Restarting mariadbd with default parameters +# restart diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path.test b/mysql-test/suite/mariabackup/aria_log_dir_path.test new file mode 100644 index 00000000000..0178cd4eae5 --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_log_dir_path.test @@ -0,0 +1,105 @@ +--source include/have_maria.inc + +--echo # +--echo # MDEV-30968 mariadb-backup does not copy Aria logs if aria_log_dir_path is used +--echo # + +--let $datadir=`SELECT @@datadir` +--let $targetdir=$MYSQLTEST_VARDIR/tmp/backup + +if ($ARIA_LOGDIR_MARIADB == '') +{ + --let $ARIA_LOGDIR_MARIADB=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path +} + +if ($ARIA_LOGDIR_FS == '') +{ + --let $ARIA_LOGDIR_FS=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path +} + +--let $server_parameters=--aria-log-file-size=8388608 --aria-log-purge-type=external --loose-aria-log-dir-path=$ARIA_LOGDIR_MARIADB + + +--echo # Restart mariadbd with the test specific parameters +--mkdir $ARIA_LOGDIR_FS +--let $restart_parameters=$server_parameters +--source include/restart_mysqld.inc + + +--echo # Create and populate an Aria table (and Aria logs) +CREATE TABLE t1 (id INT, txt LONGTEXT) ENGINE=Aria; +DELIMITER $$; +BEGIN NOT ATOMIC + FOR id IN 0..9 DO + INSERT INTO test.t1 (id, txt) VALUES (id, REPEAT(id,1024*1024)); + END FOR; +END; +$$ +DELIMITER ;$$ + + +--echo # Testing aria log files before --backup +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +--file_exists $ARIA_LOGDIR_FS/aria_log_control +--file_exists $ARIA_LOGDIR_FS/aria_log.00000001 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000002 +--error 1 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000003 +--replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ +SHOW ENGINE aria logs; + + +--echo # mariadb-backup --backup +--disable_result_log +--mkdir $targetdir +--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir +--enable_result_log + + +--echo # mariadb-backup --prepare +--disable_result_log +--exec $XTRABACKUP --prepare --target-dir=$targetdir +--enable_result_log + + +--echo # shutdown server +--disable_result_log +--source include/shutdown_mysqld.inc +--echo # remove datadir +--rmdir $datadir +--echo # remove aria-log-dir-path +--rmdir $ARIA_LOGDIR_FS + +--echo # mariadb-backup --copy-back +--let $mariadb_backup_parameters=--defaults-file=$MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=$datadir --target-dir=$targetdir --parallel=2 --throttle=1 --aria-log-dir-path=$ARIA_LOGDIR_MARIADB +--replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--exec echo "# with parameters: $mariadb_backup_parameters" +--exec $XTRABACKUP $mariadb_backup_parameters + +--echo # starting server +--let $restart_parameters=$server_parameters +--source include/start_mysqld.inc +--enable_result_log +--rmdir $targetdir + + +--echo # Check that the table is there after --copy-back +SELECT COUNT(*) from t1; +DROP TABLE t1; + + +--echo # Testing aria log files after --copy-back +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +--file_exists $ARIA_LOGDIR_FS/aria_log_control +--file_exists $ARIA_LOGDIR_FS/aria_log.00000001 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000002 +--error 1 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000003 +--replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ +SHOW ENGINE aria logs; + + +--echo # Restarting mariadbd with default parameters +--let $restart_parameters= +--source include/restart_mysqld.inc +--rmdir $ARIA_LOGDIR_FS diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path_rel.result b/mysql-test/suite/mariabackup/aria_log_dir_path_rel.result new file mode 100644 index 00000000000..7fef26096e0 --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_log_dir_path_rel.result @@ -0,0 +1,41 @@ +# +# MDEV-30968 mariadb-backup does not copy Aria logs if aria_log_dir_path is used +# +# Restart mariadbd with the test specific parameters +# restart: --aria-log-file-size=8388608 --aria-log-purge-type=external --loose-aria-log-dir-path=../../tmp/backup_aria_log_dir_path_rel +# Create and populate an Aria table (and Aria logs) +CREATE TABLE t1 (id INT, txt LONGTEXT) ENGINE=Aria; +BEGIN NOT ATOMIC +FOR id IN 0..9 DO +INSERT INTO test.t1 (id, txt) VALUES (id, REPEAT(id,1024*1024)); +END FOR; +END; +$$ +# Testing aria log files before --backup +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +SHOW ENGINE aria logs; +Type Name Status +Aria aria_log.00000001 free +Aria aria_log.00000002 in use +# mariadb-backup --backup +# mariadb-backup --prepare +# shutdown server +# remove datadir +# remove aria-log-dir-path +# mariadb-backup --copy-back +# with parameters: --defaults-file=MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=MYSQLTEST_VARDIR/mysqld.1/data/ --target-dir=MYSQLTEST_VARDIR/tmp/backup --parallel=2 --throttle=1 --aria-log-dir-path=../../tmp/backup_aria_log_dir_path_rel +# starting server +# restart: --aria-log-file-size=8388608 --aria-log-purge-type=external --loose-aria-log-dir-path=../../tmp/backup_aria_log_dir_path_rel +# Check that the table is there after --copy-back +SELECT COUNT(*) from t1; +COUNT(*) +10 +DROP TABLE t1; +# Testing aria log files after --copy-back +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +SHOW ENGINE aria logs; +Type Name Status +Aria aria_log.00000001 free +Aria aria_log.00000002 in use +# Restarting mariadbd with default parameters +# restart diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test b/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test new file mode 100644 index 00000000000..c8169959929 --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test @@ -0,0 +1,4 @@ +--let $ARIA_LOGDIR_MARIADB=../../tmp/backup_aria_log_dir_path_rel +--let $ARIA_LOGDIR_FS=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path_rel + +--source aria_log_dir_path.test diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index 20fcc0b64b8..76d4f465b95 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -1567,7 +1567,11 @@ is_absolute_path( } #ifdef _WIN32 - if (path[1] == ':' && path[2] == OS_PATH_SEPARATOR) { + // This will conflict during a 10.5->10.6 merge. + // Choose the 10.6 version as is. + if (path[1] == ':' && + (path[2] == OS_PATH_SEPARATOR || + path[2] == OS_PATH_SEPARATOR_ALT)) { return(true); } #endif /* _WIN32 */ |