From 66d93a809c538f98336d9bf5e5a60d9bb898fdc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Fri, 6 May 2022 08:42:16 +0300 Subject: MDEV-19959 : Galera test failure on galera_binlog_stmt_autoinc Make sure that nodes have correct auto_increment_offset when they start and when control is turned on. --- .../galera/r/galera_binlog_stmt_autoinc.result | 88 +++++++++++++++------- .../suite/galera/t/galera_binlog_stmt_autoinc.cnf | 2 + .../suite/galera/t/galera_binlog_stmt_autoinc.test | 73 +++++++++--------- 3 files changed, 102 insertions(+), 61 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result b/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result index 78b40228eb0..816f947665c 100644 --- a/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result +++ b/mysql-test/suite/galera/r/galera_binlog_stmt_autoinc.result @@ -1,27 +1,38 @@ connection node_1; +SET GLOBAL auto_increment_offset=1; +connection node_2; +SET GLOBAL auto_increment_offset=2; +connection node_1; connection node_2; connection node_2; SET GLOBAL wsrep_forced_binlog_format='STATEMENT'; connection node_1; SET GLOBAL wsrep_forced_binlog_format='STATEMENT'; CREATE TABLE t1 ( -i int(11) NOT NULL AUTO_INCREMENT, -c char(32) DEFAULT 'dummy_text', -PRIMARY KEY (i) +i int(11) NOT NULL primary key AUTO_INCREMENT, +c char(32) DEFAULT 'dummy_text' ) ENGINE=InnoDB DEFAULT CHARSET=latin1; +show variables like 'auto_increment%'; +Variable_name Value +auto_increment_increment 2 +auto_increment_offset 1 insert into t1(i) values(null); -select * from t1; +select * from t1 order by i; i c 1 dummy_text insert into t1(i) values(null), (null), (null); -select * from t1; +select * from t1 order by i; i c 1 dummy_text 3 dummy_text 5 dummy_text 7 dummy_text connection node_2; -select * from t1; +show variables like 'auto_increment%'; +Variable_name Value +auto_increment_increment 2 +auto_increment_offset 2 +select * from t1 order by i; i c 1 dummy_text 3 dummy_text @@ -39,23 +50,30 @@ SET GLOBAL wsrep_auto_increment_control='OFF'; SET SESSION auto_increment_increment = 3; SET SESSION auto_increment_offset = 1; CREATE TABLE t1 ( -i int(11) NOT NULL AUTO_INCREMENT, -c char(32) DEFAULT 'dummy_text', -PRIMARY KEY (i) +i int(11) NOT NULL primary key AUTO_INCREMENT, +c char(32) DEFAULT 'dummy_text' ) ENGINE=InnoDB DEFAULT CHARSET=latin1; +show variables like 'auto_increment%'; +Variable_name Value +auto_increment_increment 3 +auto_increment_offset 1 insert into t1(i) values(null); -select * from t1; +select * from t1 order by i; i c 1 dummy_text insert into t1(i) values(null), (null), (null); -select * from t1; +select * from t1 order by i; i c 1 dummy_text 4 dummy_text 7 dummy_text 10 dummy_text connection node_2; -select * from t1; +show variables like 'auto_increment%'; +Variable_name Value +auto_increment_increment 2 +auto_increment_offset 2 +select * from t1 order by i; i c 1 dummy_text 4 dummy_text @@ -64,6 +82,7 @@ i c connection node_1; SET GLOBAL wsrep_auto_increment_control='ON'; SET SESSION binlog_format='ROW'; +connection node_1; show variables like 'binlog_format'; Variable_name Value binlog_format ROW @@ -79,29 +98,37 @@ auto_increment_increment 3 auto_increment_offset 1 wsrep_auto_increment_control OFF SET GLOBAL wsrep_auto_increment_control='ON'; +connection node_1; drop table t1; connection node_2; SET GLOBAL wsrep_forced_binlog_format='ROW'; connection node_1; SET GLOBAL wsrep_forced_binlog_format='ROW'; CREATE TABLE t1 ( -i int(11) NOT NULL AUTO_INCREMENT, -c char(32) DEFAULT 'dummy_text', -PRIMARY KEY (i) +i int(11) NOT NULL primary key AUTO_INCREMENT, +c char(32) DEFAULT 'dummy_text' ) ENGINE=InnoDB DEFAULT CHARSET=latin1; +show variables like 'auto_increment%'; +Variable_name Value +auto_increment_increment 2 +auto_increment_offset 1 insert into t1(i) values(null); -select * from t1; +select * from t1 order by i; i c 1 dummy_text insert into t1(i) values(null), (null), (null); -select * from t1; +select * from t1 order by i; i c 1 dummy_text 3 dummy_text 5 dummy_text 7 dummy_text connection node_2; -select * from t1; +show variables like 'auto_increment%'; +Variable_name Value +auto_increment_increment 2 +auto_increment_offset 2 +select * from t1 order by i; i c 1 dummy_text 3 dummy_text @@ -119,23 +146,30 @@ SET GLOBAL wsrep_auto_increment_control='OFF'; SET SESSION auto_increment_increment = 3; SET SESSION auto_increment_offset = 1; CREATE TABLE t1 ( -i int(11) NOT NULL AUTO_INCREMENT, -c char(32) DEFAULT 'dummy_text', -PRIMARY KEY (i) +i int(11) NOT NULL primary key AUTO_INCREMENT, +c char(32) DEFAULT 'dummy_text' ) ENGINE=InnoDB DEFAULT CHARSET=latin1; +show variables like 'auto_increment%'; +Variable_name Value +auto_increment_increment 3 +auto_increment_offset 1 insert into t1(i) values(null); -select * from t1; +select * from t1 order by i; i c 1 dummy_text insert into t1(i) values(null), (null), (null); -select * from t1; +select * from t1 order by i; i c 1 dummy_text 4 dummy_text 7 dummy_text 10 dummy_text connection node_2; -select * from t1; +show variables like 'auto_increment%'; +Variable_name Value +auto_increment_increment 2 +auto_increment_offset 2 +select * from t1 order by i; i c 1 dummy_text 4 dummy_text @@ -149,13 +183,13 @@ binlog_format ROW show variables like '%auto_increment%'; Variable_name Value auto_increment_increment 2 -auto_increment_offset 1 +auto_increment_offset 2 wsrep_auto_increment_control ON SET GLOBAL wsrep_auto_increment_control='OFF'; show variables like '%auto_increment%'; Variable_name Value -auto_increment_increment 3 -auto_increment_offset 1 +auto_increment_increment 1 +auto_increment_offset 2 wsrep_auto_increment_control OFF SET GLOBAL wsrep_auto_increment_control='ON'; drop table t1; diff --git a/mysql-test/suite/galera/t/galera_binlog_stmt_autoinc.cnf b/mysql-test/suite/galera/t/galera_binlog_stmt_autoinc.cnf index 889c81b4a0a..91e9199b092 100644 --- a/mysql-test/suite/galera/t/galera_binlog_stmt_autoinc.cnf +++ b/mysql-test/suite/galera/t/galera_binlog_stmt_autoinc.cnf @@ -2,6 +2,8 @@ [mysqld.1] auto_increment_offset=1 +auto_increment_increment=1 [mysqld.2] auto_increment_offset=2 +auto_increment_increment=1 diff --git a/mysql-test/suite/galera/t/galera_binlog_stmt_autoinc.test b/mysql-test/suite/galera/t/galera_binlog_stmt_autoinc.test index 817f4f82b43..1c9ee9206fe 100644 --- a/mysql-test/suite/galera/t/galera_binlog_stmt_autoinc.test +++ b/mysql-test/suite/galera/t/galera_binlog_stmt_autoinc.test @@ -3,7 +3,12 @@ ## --source include/galera_cluster.inc ---source include/have_innodb.inc +--source include/force_restart.inc + +--connection node_1 +SET GLOBAL auto_increment_offset=1; +--connection node_2 +SET GLOBAL auto_increment_offset=2; --let $node_1=node_1 --let $node_2=node_2 @@ -22,23 +27,24 @@ SET GLOBAL wsrep_forced_binlog_format='STATEMENT'; SET GLOBAL wsrep_forced_binlog_format='STATEMENT'; CREATE TABLE t1 ( - i int(11) NOT NULL AUTO_INCREMENT, - c char(32) DEFAULT 'dummy_text', - PRIMARY KEY (i) + i int(11) NOT NULL primary key AUTO_INCREMENT, + c char(32) DEFAULT 'dummy_text' ) ENGINE=InnoDB DEFAULT CHARSET=latin1; +show variables like 'auto_increment%'; insert into t1(i) values(null); -select * from t1; +select * from t1 order by i; insert into t1(i) values(null), (null), (null); -select * from t1; +select * from t1 order by i; --connection node_2 --let $wait_condition = SELECT COUNT(*) = 4 FROM t1; --source include/wait_condition.inc -select * from t1; +show variables like 'auto_increment%'; +select * from t1 order by i; SET GLOBAL wsrep_forced_binlog_format='none'; @@ -72,36 +78,31 @@ SET SESSION auto_increment_increment = 3; SET SESSION auto_increment_offset = 1; CREATE TABLE t1 ( - i int(11) NOT NULL AUTO_INCREMENT, - c char(32) DEFAULT 'dummy_text', - PRIMARY KEY (i) + i int(11) NOT NULL primary key AUTO_INCREMENT, + c char(32) DEFAULT 'dummy_text' ) ENGINE=InnoDB DEFAULT CHARSET=latin1; +show variables like 'auto_increment%'; insert into t1(i) values(null); -select * from t1; +select * from t1 order by i; insert into t1(i) values(null), (null), (null); -select * from t1; +select * from t1 order by i; --connection node_2 --let $wait_condition = SELECT COUNT(*) = 4 FROM t1; --source include/wait_condition.inc +show variables like 'auto_increment%'; -select * from t1; +select * from t1 order by i; --connection node_1 - -## -## Verify the return to automatic calculation of the step -## and offset of the auto-increment: -## - SET GLOBAL wsrep_auto_increment_control='ON'; - SET SESSION binlog_format='ROW'; - +--source include/auto_increment_offset_restore.inc +--connection node_1 show variables like 'binlog_format'; show variables like '%auto_increment%'; @@ -119,7 +120,8 @@ show variables like '%auto_increment%'; ## SET GLOBAL wsrep_auto_increment_control='ON'; - +--source include/auto_increment_offset_restore.inc +--connection node_1 drop table t1; ## @@ -134,24 +136,25 @@ SET GLOBAL wsrep_forced_binlog_format='ROW'; SET GLOBAL wsrep_forced_binlog_format='ROW'; CREATE TABLE t1 ( - i int(11) NOT NULL AUTO_INCREMENT, - c char(32) DEFAULT 'dummy_text', - PRIMARY KEY (i) + i int(11) NOT NULL primary key AUTO_INCREMENT, + c char(32) DEFAULT 'dummy_text' ) ENGINE=InnoDB DEFAULT CHARSET=latin1; +show variables like 'auto_increment%'; insert into t1(i) values(null); -select * from t1; +select * from t1 order by i; insert into t1(i) values(null), (null), (null); -select * from t1; +select * from t1 order by i; --connection node_2 --let $wait_condition = SELECT COUNT(*) = 4 FROM t1; --source include/wait_condition.inc -select * from t1; +show variables like 'auto_increment%'; +select * from t1 order by i; SET GLOBAL wsrep_forced_binlog_format='none'; @@ -181,24 +184,25 @@ SET SESSION auto_increment_increment = 3; SET SESSION auto_increment_offset = 1; CREATE TABLE t1 ( - i int(11) NOT NULL AUTO_INCREMENT, - c char(32) DEFAULT 'dummy_text', - PRIMARY KEY (i) + i int(11) NOT NULL primary key AUTO_INCREMENT, + c char(32) DEFAULT 'dummy_text' ) ENGINE=InnoDB DEFAULT CHARSET=latin1; +show variables like 'auto_increment%'; insert into t1(i) values(null); -select * from t1; +select * from t1 order by i; insert into t1(i) values(null), (null), (null); -select * from t1; +select * from t1 order by i; --connection node_2 --let $wait_condition = SELECT COUNT(*) = 4 FROM t1; --source include/wait_condition.inc +show variables like 'auto_increment%'; -select * from t1; +select * from t1 order by i; --connection node_1 @@ -208,6 +212,7 @@ select * from t1; ## SET GLOBAL wsrep_auto_increment_control='ON'; +--source include/auto_increment_offset_restore.inc show variables like 'binlog_format'; show variables like '%auto_increment%'; -- cgit v1.2.1 From 09ee95e33ef41a9b7c47eefd0abde5b1a246a93f Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 11 May 2022 13:24:39 +1000 Subject: MDEV-28534: clang-12 compile warnings Errors where: /buildbot/amd64-ubuntu-2004-msan/build/sql/item.h:6478:12: error: 'val_datetime_packed' overrides a member function but is not marked 'override' [-Werror,-Winconsistent-missing-override] longlong val_datetime_packed(THD *thd) ^ /buildbot/amd64-ubuntu-2004-msan/build/sql/item.h:3501:12: note: overridden virtual function is here longlong val_datetime_packed(THD *thd) override; ^ /buildbot/amd64-ubuntu-2004-msan/build/sql/item.h:6480:12: error: 'val_time_packed' overrides a member function but is not marked 'override' [-Werror,-Winconsistent-missing-override] longlong val_time_packed(THD *thd) ^ /buildbot/amd64-ubuntu-2004-msan/build/sql/item.h:3502:12: note: overridden virtual function is here longlong val_time_packed(THD *thd) override; ^ --- sql/item.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/item.h b/sql/item.h index 5af6dc3bf49..a6de33c0c3a 100644 --- a/sql/item.h +++ b/sql/item.h @@ -6475,9 +6475,9 @@ public: bool get_date(THD *thd, MYSQL_TIME *ltime,date_mode_t fuzzydate) override; bool val_native(THD *thd, Native *to) override; bool val_native_result(THD *thd, Native *to) override; - longlong val_datetime_packed(THD *thd) + longlong val_datetime_packed(THD *thd) override { return Item::val_datetime_packed(thd); } - longlong val_time_packed(THD *thd) + longlong val_time_packed(THD *thd) override { return Item::val_time_packed(thd); } /* Result variants */ -- cgit v1.2.1 From 7da0f30ccc5446cf6c30289e51d461b7f0595891 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 11 May 2022 14:48:51 +0400 Subject: MDEV-28446 mariabackup prepare fails for incrementals if a new schema is created after full backup is taken Adding a 10.6 specific test --- .../incremental_backup_newdb_before_inc.result | 18 +++++++++++ .../incremental_backup_newdb_before_inc.test | 35 ++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 mysql-test/suite/mariabackup/incremental_backup_newdb_before_inc.result create mode 100644 mysql-test/suite/mariabackup/incremental_backup_newdb_before_inc.test diff --git a/mysql-test/suite/mariabackup/incremental_backup_newdb_before_inc.result b/mysql-test/suite/mariabackup/incremental_backup_newdb_before_inc.result new file mode 100644 index 00000000000..e3f8bb245e7 --- /dev/null +++ b/mysql-test/suite/mariabackup/incremental_backup_newdb_before_inc.result @@ -0,0 +1,18 @@ +call mtr.add_suppression("InnoDB: New log files created"); +CREATE TABLE t1 (i INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES(0); +CREATE DATABASE test1; +CREATE TABLE test1.t1 (a INT) ENGINE=InnoDB; +INSERT INTO test1.t1 VALUES (1000); +# shutdown server +# remove datadir +# xtrabackup move back +# restart +SELECT * FROM t1; +i +0 +SELECT * FROM test1.t1; +a +1000 +DROP TABLE t1; +DROP DATABASE test1; diff --git a/mysql-test/suite/mariabackup/incremental_backup_newdb_before_inc.test b/mysql-test/suite/mariabackup/incremental_backup_newdb_before_inc.test new file mode 100644 index 00000000000..a79d280b668 --- /dev/null +++ b/mysql-test/suite/mariabackup/incremental_backup_newdb_before_inc.test @@ -0,0 +1,35 @@ +--source include/have_innodb.inc + +call mtr.add_suppression("InnoDB: New log files created"); + +--let basedir=$MYSQLTEST_VARDIR/tmp/backup +--let incremental_dir=$MYSQLTEST_VARDIR/tmp/backup_inc1 + +CREATE TABLE t1 (i INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES(0); + +--disable_result_log +--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$basedir +--enable_result_log + +CREATE DATABASE test1; +CREATE TABLE test1.t1 (a INT) ENGINE=InnoDB; +INSERT INTO test1.t1 VALUES (1000); + +--disable_result_log +--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=2 --ftwrl-wait-timeout=5 --ftwrl-wait-threshold=300 --ftwrl-wait-query-type=all --target-dir=$incremental_dir --incremental-basedir=$basedir +--exec $XTRABACKUP --prepare --target-dir=$basedir +--exec $XTRABACKUP --prepare --target-dir=$basedir --incremental-dir=$incremental_dir +--enable_result_log + +--let $targetdir=$basedir +--source include/restart_and_restore.inc +--enable_result_log + +SELECT * FROM t1; +SELECT * FROM test1.t1; + +DROP TABLE t1; +DROP DATABASE test1; +--rmdir $basedir +--rmdir $incremental_dir -- cgit v1.2.1 From 3fabdc3ca889db0e490df3a9a48ae527f21eed45 Mon Sep 17 00:00:00 2001 From: Vlad Lesin Date: Fri, 6 May 2022 10:49:35 +0300 Subject: MDEV-28473 field_ref_zero is not initialized in xtrabackup_prepare_func() The solution is to initialize field_ref_zero in main_low() before xtrabackup_backup_func() and xtrabackup_prepare_func() calls. --- extra/mariabackup/xtrabackup.cc | 41 +++++++++++----------- .../suite/mariabackup/incremental_backup.test | 16 +++++++++ storage/innobase/buf/buf0buf.cc | 16 +++++---- 3 files changed, 47 insertions(+), 26 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 2b2adf0643c..13319f44969 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -51,6 +51,7 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include #include #include +#include #include #include @@ -4511,13 +4512,6 @@ fail: } - if (auto b = aligned_malloc(UNIV_PAGE_SIZE_MAX, 4096)) { - field_ref_zero = static_cast( - memset_aligned<4096>(b, 0, UNIV_PAGE_SIZE_MAX)); - } else { - goto fail; - } - { /* definition from recv_recovery_from_checkpoint_start() */ ulint max_cp_field; @@ -4534,10 +4528,6 @@ reread_log_header: msg("Error: cannot read redo log header"); unlock_and_fail: mysql_mutex_unlock(&log_sys.mutex); -free_and_fail: - aligned_free(const_cast(field_ref_zero)); - field_ref_zero = nullptr; - goto fail; } if (log_sys.log.format == 0) { @@ -4563,7 +4553,7 @@ free_and_fail: xtrabackup_init_datasinks(); if (!select_history()) { - goto free_and_fail; + goto fail; } /* open the log file */ @@ -4572,7 +4562,7 @@ free_and_fail: if (dst_log_file == NULL) { msg("Error: failed to open the target stream for '%s'.", LOG_FILE_NAME); - goto free_and_fail; + goto fail; } /* label it */ @@ -4610,7 +4600,7 @@ free_and_fail: if (ds_write(dst_log_file, log_hdr_buf, LOG_FILE_HDR_SIZE)) { msg("error: write to logfile failed"); aligned_free(log_hdr_buf); - goto free_and_fail; + goto fail; } aligned_free(log_hdr_buf); @@ -4631,7 +4621,7 @@ free_and_fail: " error %s.", ut_strerr(err)); fail_before_log_copying_thread_start: log_copying_running = false; - goto free_and_fail; + goto fail; } /* copy log file by current position */ @@ -4648,7 +4638,7 @@ fail_before_log_copying_thread_start: /* FLUSH CHANGED_PAGE_BITMAPS call */ if (!flush_changed_page_bitmaps()) { - goto free_and_fail; + goto fail; } ut_a(xtrabackup_parallel > 0); @@ -4670,7 +4660,7 @@ fail_before_log_copying_thread_start: datafiles_iter_t *it = datafiles_iter_new(); if (it == NULL) { msg("mariabackup: Error: datafiles_iter_new() failed."); - goto free_and_fail; + goto fail; } /* Create data copying threads */ @@ -4725,9 +4715,6 @@ fail_before_log_copying_thread_start: if (opt_log_innodb_page_corruption) ok = corrupted_pages.print_to_file(MB_CORRUPTED_PAGES_FILE); - aligned_free(const_cast(field_ref_zero)); - field_ref_zero = nullptr; - if (!ok) { goto fail; } @@ -6932,6 +6919,20 @@ static int main_low(char** argv) } } + ut_ad(!field_ref_zero); + if (auto b = aligned_malloc(UNIV_PAGE_SIZE_MAX, 4096)) { + field_ref_zero = static_cast( + memset_aligned<4096>(b, 0, UNIV_PAGE_SIZE_MAX)); + } else { + msg("Can't allocate memory for field_ref_zero"); + return EXIT_FAILURE; + } + + auto _ = make_scope_exit([]() { + aligned_free(const_cast(field_ref_zero)); + field_ref_zero = nullptr; + }); + /* --backup */ if (xtrabackup_backup && !xtrabackup_backup_func()) { return(EXIT_FAILURE); diff --git a/mysql-test/suite/mariabackup/incremental_backup.test b/mysql-test/suite/mariabackup/incremental_backup.test index 88e277fd95a..ddcdae4637a 100644 --- a/mysql-test/suite/mariabackup/incremental_backup.test +++ b/mysql-test/suite/mariabackup/incremental_backup.test @@ -8,11 +8,20 @@ if (`select @@max_binlog_stmt_cache_size = 4294963200 and @@innodb_page_size = 6 call mtr.add_suppression("InnoDB: New log files created"); +let $test_comp=`select @@innodb_page_size < 32768`; let basedir=$MYSQLTEST_VARDIR/tmp/backup; let incremental_dir=$MYSQLTEST_VARDIR/tmp/backup_inc1; CREATE TABLE t_aria(i INT) ENGINE ARIA; CREATE TABLE t(i INT PRIMARY KEY) ENGINE INNODB; +if ($test_comp) { + # If MDEV-28474 is not fixed, backup preparing will crash with SIGSEGV. + --disable_query_log + --disable_result_log + CREATE TABLE t_comp(i INT PRIMARY KEY) ENGINE INNODB ROW_FORMAT=COMPRESSED; + --enable_query_log + --enable_result_log +} BEGIN; INSERT INTO t VALUES(2); connect (con1,localhost,root,,); @@ -96,6 +105,13 @@ let $targetdir=$basedir; SELECT * FROM t; DROP TABLE t; +if ($test_comp) { + --disable_query_log + --disable_result_log + DROP TABLE t_comp; + --enable_query_log + --enable_result_log +} DROP TABLE t_aria; # Cleanup diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 98a8f552895..0bb697e7f0a 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1346,15 +1346,19 @@ bool buf_pool_t::create() ut_ad(srv_buf_pool_size > 0); ut_ad(!resizing); ut_ad(!chunks_old); - ut_ad(!field_ref_zero); + /* mariabackup loads tablespaces, and it requires field_ref_zero to be + allocated before innodb initialization */ + ut_ad(srv_operation >= SRV_OPERATION_RESTORE || !field_ref_zero); NUMA_MEMPOLICY_INTERLEAVE_IN_SCOPE; - if (auto b= aligned_malloc(UNIV_PAGE_SIZE_MAX, 4096)) - field_ref_zero= static_cast - (memset_aligned<4096>(b, 0, UNIV_PAGE_SIZE_MAX)); - else - return true; + if (!field_ref_zero) { + if (auto b= aligned_malloc(UNIV_PAGE_SIZE_MAX, 4096)) + field_ref_zero= static_cast + (memset_aligned<4096>(b, 0, UNIV_PAGE_SIZE_MAX)); + else + return true; + } chunk_t::map_reg= UT_NEW_NOKEY(chunk_t::map()); -- cgit v1.2.1 From 0a1d9d0681fda7595c0d08038357b56cf3bea9eb Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 18 Feb 2022 16:21:08 +0100 Subject: MDEV-27415 main.json_normalize and main.json_equals fail with UBSAN runtime error UBSAN: out of bound array read in json json_lib.c:847:25: runtime error: index 200 out of bounds for type 'json_string_char_classes [128]' json_lib.c:847:25: runtime error: load of address 0x56286f7175a0 with insufficient space for an object of type 'json_string_char_classes' fixes main.json_equals and main.json_normalize --- strings/json_lib.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/strings/json_lib.c b/strings/json_lib.c index 3803b2649a5..65058587a93 100644 --- a/strings/json_lib.c +++ b/strings/json_lib.c @@ -844,7 +844,7 @@ static int skip_key(json_engine_t *j) { int t_next, c_len; - if (json_instr_chr_map[j->s.c_next] == S_BKSL && + if (j->s.c_next< 128 && json_instr_chr_map[j->s.c_next] == S_BKSL && json_handle_esc(&j->s)) return 1; -- cgit v1.2.1 From 480323f7d6c3f41f52cfda7197dc63c63b423a36 Mon Sep 17 00:00:00 2001 From: Hartmut Holzgraefe Date: Sun, 1 May 2022 14:45:58 +0200 Subject: MDEV-19161: Let galera_new_cluster use "restart" instead of "start" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Jan Lindström --- scripts/galera_new_cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/galera_new_cluster.sh b/scripts/galera_new_cluster.sh index e0763ed516a..ac9dcf42102 100755 --- a/scripts/galera_new_cluster.sh +++ b/scripts/galera_new_cluster.sh @@ -22,7 +22,7 @@ EOF fi systemctl set-environment _WSREP_NEW_CLUSTER='--wsrep-new-cluster' && \ - systemctl start ${1:-mariadb} + systemctl restart ${1:-mariadb} extcode=$? -- cgit v1.2.1 From f7dd8799e78b6877e6069f8c6b787e6c16715b80 Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 5 May 2022 09:51:49 +0300 Subject: Fixed bug in alter_table_lock.result Before this change the test could abort with ER_OPTION_PREVENTS_STATEMENT --- mysql-test/main/alter_table_lock.result | 2 -- mysql-test/main/alter_table_lock.test | 4 ++++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mysql-test/main/alter_table_lock.result b/mysql-test/main/alter_table_lock.result index ce6097c9ada..620fca23315 100644 --- a/mysql-test/main/alter_table_lock.result +++ b/mysql-test/main/alter_table_lock.result @@ -11,8 +11,6 @@ ERROR 42S02: Table 'test.x' doesn't exist SET SESSION max_session_mem_used= 8192; LOCK TABLE t1 WRITE; ALTER TABLE t1 CHANGE COLUMN IF EXISTS b c INT; -Warnings: -Note 1054 Unknown column 'b' in 't1' SET SESSION max_session_mem_used = @max_session_mem_used_save; UNLOCK TABLES; DROP TABLE t1; diff --git a/mysql-test/main/alter_table_lock.test b/mysql-test/main/alter_table_lock.test index f1f60e38b61..bd26c1ac7d0 100644 --- a/mysql-test/main/alter_table_lock.test +++ b/mysql-test/main/alter_table_lock.test @@ -14,9 +14,13 @@ SELECT * FROM t1; ALTER TABLE x MODIFY xx INT; SET SESSION max_session_mem_used= 8192; +--error 0,ER_OPTION_PREVENTS_STATEMENT LOCK TABLE t1 WRITE; +--disable_warnings +--error 0,ER_OPTION_PREVENTS_STATEMENT ALTER TABLE t1 CHANGE COLUMN IF EXISTS b c INT; +--enable_warnings SET SESSION max_session_mem_used = @max_session_mem_used_save; UNLOCK TABLES; -- cgit v1.2.1 From b729896d00e022f6205399376c0cc107e1ee0704 Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 10 May 2022 11:47:20 +0300 Subject: MDEV-28073 Query performance degradation in newer MariaDB versions when using many tables The issue was that best_extension_by_limited_search() had to go through too many plans with the same cost as there where many EQ_REF tables. Fixed by shortcutting EQ_REF (AND REF) when the result only contains one row. This got the optimization time down from hours to sub seconds. The only known downside with this patch is that in some cases a table with ref and 1 record may be used before on EQ_REF table. The faster optimzation phase should compensate for this. --- mysql-test/main/opt_trace.result | 106 +++++--------- mysql-test/main/opt_trace_index_merge.result | 3 +- .../main/opt_trace_index_merge_innodb.result | 2 +- mysql-test/main/opt_trace_security.result | 6 +- mysql-test/main/subselect_sj2.result | 6 +- mysql-test/main/subselect_sj2_jcl6.result | 6 +- mysql-test/main/subselect_sj2_mat.result | 6 +- sql/sql_select.cc | 163 ++++++++++++++------- sql/sql_select.h | 2 + 9 files changed, 164 insertions(+), 136 deletions(-) diff --git a/mysql-test/main/opt_trace.result b/mysql-test/main/opt_trace.result index 044db82b961..5504f4da81e 100644 --- a/mysql-test/main/opt_trace.result +++ b/mysql-test/main/opt_trace.result @@ -145,8 +145,7 @@ select * from v1 { } }, "rows_for_plan": 1, - "cost_for_plan": 2.404394531, - "estimated_join_cardinality": 1 + "cost_for_plan": 2.404394531 } ] }, @@ -296,8 +295,7 @@ select * from (select * from t1 where t1.a=1)q { } }, "rows_for_plan": 1, - "cost_for_plan": 2.404394531, - "estimated_join_cardinality": 1 + "cost_for_plan": 2.404394531 } ] }, @@ -454,8 +452,7 @@ select * from v2 { }, "rows_for_plan": 1, "cost_for_plan": 2.404394531, - "cost_for_sorting": 1, - "estimated_join_cardinality": 1 + "cost_for_sorting": 1 } ] }, @@ -525,8 +522,7 @@ select * from v2 { } }, "rows_for_plan": 2, - "cost_for_plan": 2.4, - "estimated_join_cardinality": 2 + "cost_for_plan": 2.4 } ] }, @@ -662,8 +658,7 @@ explain select * from v2 { } }, "rows_for_plan": 10, - "cost_for_plan": 4.021972656, - "estimated_join_cardinality": 10 + "cost_for_plan": 4.021972656 } ] }, @@ -780,8 +775,7 @@ explain select * from v1 { }, "rows_for_plan": 10, "cost_for_plan": 4.021972656, - "cost_for_sorting": 10, - "estimated_join_cardinality": 10 + "cost_for_sorting": 10 } ] }, @@ -845,8 +839,7 @@ explain select * from v1 { } }, "rows_for_plan": 10, - "cost_for_plan": 12, - "estimated_join_cardinality": 10 + "cost_for_plan": 12 } ] }, @@ -1047,7 +1040,7 @@ explain select * from t1,t2 where t1.a=t2.b+2 and t2.a= t1.b { }, "rows_for_plan": 100, "cost_for_plan": 242.3759623, - "estimated_join_cardinality": 100 + "pruned_by_hanging_leaf": true } ] }, @@ -1278,8 +1271,7 @@ EXPLAIN SELECT DISTINCT a FROM t1 { } }, "rows_for_plan": 5, - "cost_for_plan": 7.25, - "estimated_join_cardinality": 5 + "cost_for_plan": 7.25 } ] }, @@ -1470,8 +1462,7 @@ EXPLAIN SELECT MIN(d) FROM t1 where b=2 and c=3 group by a { }, "rows_for_plan": 8, "cost_for_plan": 3.8, - "cost_for_sorting": 8, - "estimated_join_cardinality": 8 + "cost_for_sorting": 8 } ] }, @@ -1669,8 +1660,7 @@ EXPLAIN SELECT id,MIN(a),MAX(a) FROM t1 WHERE a>=20010104e0 GROUP BY id { }, "rows_for_plan": 9, "cost_for_plan": 4.15, - "cost_for_sorting": 9, - "estimated_join_cardinality": 9 + "cost_for_sorting": 9 } ] }, @@ -1857,8 +1847,7 @@ EXPLAIN SELECT * FROM t1 WHERE a = 20010104e0 GROUP BY id { }, "rows_for_plan": 9, "cost_for_plan": 4.15, - "cost_for_sorting": 9, - "estimated_join_cardinality": 9 + "cost_for_sorting": 9 } ] }, @@ -2140,8 +2129,7 @@ explain select * from t1 where a=1 and b=2 order by c limit 1 { } }, "rows_for_plan": 21, - "cost_for_plan": 25.34242739, - "estimated_join_cardinality": 21 + "cost_for_plan": 25.34242739 } ] }, @@ -2392,8 +2380,7 @@ select t1.a from t1 left join t2 on t1.a=t2.a { } }, "rows_for_plan": 4, - "cost_for_plan": 2.806835937, - "estimated_join_cardinality": 4 + "cost_for_plan": 2.806835937 } ] }, @@ -2561,7 +2548,7 @@ explain select * from t1 left join t2 on t2.a=t1.a { }, "rows_for_plan": 4, "cost_for_plan": 7.606835937, - "estimated_join_cardinality": 4 + "pruned_by_hanging_leaf": true } ] } @@ -2740,8 +2727,7 @@ explain select t1.a from t1 left join (t2 join t3 on t2.b=t3.b) on t2.a=t1.a and } }, "rows_for_plan": 4, - "cost_for_plan": 2.806835937, - "estimated_join_cardinality": 4 + "cost_for_plan": 2.806835937 } ] }, @@ -2948,8 +2934,7 @@ explain extended select * from t1 where a in (select pk from t10) { } }, "rows_for_plan": 10, - "cost_for_plan": 4.021972656, - "estimated_join_cardinality": 10 + "cost_for_plan": 4.021972656 } ] } @@ -3021,8 +3006,7 @@ explain extended select * from t1 where a in (select pk from t10) { { "chosen_strategy": "SJ-Materialization" } - ], - "estimated_join_cardinality": 3 + ] } ] }, @@ -3427,7 +3411,7 @@ explain select * from t1 where pk = 2 and a=5 and b=1 { }, "rows_for_plan": 1, "cost_for_plan": 0.326073957, - "estimated_join_cardinality": 1 + "pruned_by_hanging_leaf": true } ] }, @@ -3555,8 +3539,7 @@ select f1(a) from t1 { } }, "rows_for_plan": 4, - "cost_for_plan": 2.806835937, - "estimated_join_cardinality": 4 + "cost_for_plan": 2.806835937 } ] }, @@ -3652,8 +3635,7 @@ select f2(a) from t1 { } }, "rows_for_plan": 4, - "cost_for_plan": 2.806835937, - "estimated_join_cardinality": 4 + "cost_for_plan": 2.806835937 } ] }, @@ -3699,7 +3681,7 @@ a 2 select length(trace) from INFORMATION_SCHEMA.OPTIMIZER_TRACE; length(trace) -2141 +2092 set optimizer_trace_max_mem_size=100; select * from t1; a @@ -3713,7 +3695,7 @@ select * from t1 { "join_preparation": { "select_id": 1, "steps": [ - 2041 0 + 1992 0 set optimizer_trace_max_mem_size=0; select * from t1; a @@ -3721,7 +3703,7 @@ a 2 select * from INFORMATION_SCHEMA.OPTIMIZER_TRACE; QUERY TRACE MISSING_BYTES_BEYOND_MAX_MEM_SIZE INSUFFICIENT_PRIVILEGES -select * from t1 2141 0 +select * from t1 2092 0 drop table t1; set optimizer_trace='enabled=off'; set @@optimizer_trace_max_mem_size= @save_optimizer_trace_max_mem_size; @@ -4064,7 +4046,7 @@ explain delete t0,t1 from t0, t1 where t0.a=t1.a and t1.a<3 { }, "rows_for_plan": 3, "cost_for_plan": 4.948514767, - "estimated_join_cardinality": 3 + "pruned_by_hanging_leaf": true } ] }, @@ -4263,8 +4245,7 @@ explain select * from (select rand() from t1)q { } }, "rows_for_plan": 3, - "cost_for_plan": 2.605126953, - "estimated_join_cardinality": 3 + "cost_for_plan": 2.605126953 } ] }, @@ -4328,8 +4309,7 @@ explain select * from (select rand() from t1)q { } }, "rows_for_plan": 3, - "cost_for_plan": 3.6, - "estimated_join_cardinality": 3 + "cost_for_plan": 3.6 } ] }, @@ -4557,8 +4537,7 @@ explain select * from t1 where a in (select t_inner_1.a from t1 t_inner_1, t1 t_ } }, "rows_for_plan": 9, - "cost_for_plan": 6.410253906, - "estimated_join_cardinality": 9 + "cost_for_plan": 6.410253906 } ] }, @@ -4678,8 +4657,7 @@ explain select * from t1 where a in (select t_inner_1.a from t1 t_inner_1, t1 t_ { "chosen_strategy": "SJ-Materialization" } - ], - "estimated_join_cardinality": 3 + ] } ] }, @@ -5222,8 +5200,7 @@ t_outer_2.a in (select t_inner_3.a from t2 t_inner_3, t1 t_inner_4) { { "chosen_strategy": "DuplicateWeedout" } - ], - "estimated_join_cardinality": 27 + ] } ] }, @@ -5345,8 +5322,7 @@ t_outer_2.a in (select t_inner_3.a from t2 t_inner_3, t1 t_inner_4) { { "chosen_strategy": "DuplicateWeedout" } - ], - "estimated_join_cardinality": 27 + ] } ] }, @@ -6667,8 +6643,7 @@ t_outer_2.a in (select t_inner_3.a from t2 t_inner_3, t1 t_inner_4) { } }, "rows_for_plan": 27, - "cost_for_plan": 10.02050781, - "estimated_join_cardinality": 27 + "cost_for_plan": 10.02050781 } ] }, @@ -6741,8 +6716,7 @@ t_outer_2.a in (select t_inner_3.a from t2 t_inner_3, t1 t_inner_4) { } }, "rows_for_plan": 27, - "cost_for_plan": 10.02050781, - "estimated_join_cardinality": 27 + "cost_for_plan": 10.02050781 } ] }, @@ -6961,8 +6935,7 @@ t_outer_2.a in (select t_inner_3.a from t2 t_inner_3, t1 t_inner_4) { { "chosen_strategy": "SJ-Materialization" } - ], - "estimated_join_cardinality": 27 + ] } ] }, @@ -8124,8 +8097,7 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.considered_execution_plans')) } }, "rows_for_plan": 4777.832031, - "cost_for_plan": 1216.438354, - "estimated_join_cardinality": 4777.832031 + "cost_for_plan": 1216.438354 } ] }, @@ -8466,7 +8438,7 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.considered_execution_plans')) "rows_for_plan": 10, "cost_for_plan": 26.0278306, "cost_for_sorting": 10, - "estimated_join_cardinality": 10 + "pruned_by_hanging_leaf": true } ] }, @@ -8786,8 +8758,7 @@ select count(*) from seq_1_to_10000000 { } }, "rows_for_plan": 10000000, - "cost_for_plan": 12000000, - "estimated_join_cardinality": 10000000 + "cost_for_plan": 12000000 } ] }, @@ -9202,8 +9173,7 @@ json_detailed(json_extract(trace, '$**.choose_best_splitting')) }, "rows_for_plan": 1.8367, "cost_for_plan": 2.367925794, - "cost_for_sorting": 1.8367, - "estimated_join_cardinality": 1.8367 + "cost_for_sorting": 1.8367 } ] }, diff --git a/mysql-test/main/opt_trace_index_merge.result b/mysql-test/main/opt_trace_index_merge.result index f1e13586eda..335e408bddd 100644 --- a/mysql-test/main/opt_trace_index_merge.result +++ b/mysql-test/main/opt_trace_index_merge.result @@ -221,8 +221,7 @@ explain select * from t1 where a=1 or b=1 { } }, "rows_for_plan": 2, - "cost_for_plan": 2.884903732, - "estimated_join_cardinality": 2 + "cost_for_plan": 2.884903732 } ] }, diff --git a/mysql-test/main/opt_trace_index_merge_innodb.result b/mysql-test/main/opt_trace_index_merge_innodb.result index fbc7faa0d04..b9c59c7a100 100644 --- a/mysql-test/main/opt_trace_index_merge_innodb.result +++ b/mysql-test/main/opt_trace_index_merge_innodb.result @@ -227,7 +227,7 @@ explain select * from t1 where pk1 != 0 and key1 = 1 { }, "rows_for_plan": 1, "cost_for_plan": 1.325146475, - "estimated_join_cardinality": 1 + "pruned_by_hanging_leaf": true } ] }, diff --git a/mysql-test/main/opt_trace_security.result b/mysql-test/main/opt_trace_security.result index e1937e744a4..83d98c4c183 100644 --- a/mysql-test/main/opt_trace_security.result +++ b/mysql-test/main/opt_trace_security.result @@ -107,8 +107,7 @@ select * from db1.t1 { } }, "rows_for_plan": 3, - "cost_for_plan": 2.605126953, - "estimated_join_cardinality": 3 + "cost_for_plan": 2.605126953 } ] }, @@ -229,8 +228,7 @@ select * from db1.v1 { } }, "rows_for_plan": 3, - "cost_for_plan": 2.605126953, - "estimated_join_cardinality": 3 + "cost_for_plan": 2.605126953 } ] }, diff --git a/mysql-test/main/subselect_sj2.result b/mysql-test/main/subselect_sj2.result index 2d0df9a05d0..cdf9707dcbd 100644 --- a/mysql-test/main/subselect_sj2.result +++ b/mysql-test/main/subselect_sj2.result @@ -466,11 +466,11 @@ where t0.a in ( select t1.a from t1,t2 where t2.a=t0.a and t1.b=t2.b); id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t0 ALL NULL NULL NULL NULL 5 100.00 Using where -1 PRIMARY t2 eq_ref PRIMARY PRIMARY 4 test.t0.a 1 100.00 -1 PRIMARY t1 ref a a 5 test.t0.a 1 100.00 Using where; FirstMatch(t2) +1 PRIMARY t1 ref a a 5 test.t0.a 1 100.00 Start temporary +1 PRIMARY t2 eq_ref PRIMARY PRIMARY 4 test.t0.a 1 100.00 Using where; End temporary Warnings: Note 1276 Field or reference 'test.t0.a' of SELECT #2 was resolved in SELECT #1 -Note 1003 select `test`.`t0`.`a` AS `a` from `test`.`t2` semi join (`test`.`t1`) join `test`.`t0` where `test`.`t2`.`a` = `test`.`t0`.`a` and `test`.`t1`.`a` = `test`.`t0`.`a` and `test`.`t1`.`b` = `test`.`t2`.`b` +Note 1003 select `test`.`t0`.`a` AS `a` from `test`.`t2` semi join (`test`.`t1`) join `test`.`t0` where `test`.`t1`.`a` = `test`.`t0`.`a` and `test`.`t2`.`a` = `test`.`t0`.`a` and `test`.`t2`.`b` = `test`.`t1`.`b` update t1 set a=3, b=11 where a=4; update t2 set b=11 where a=3; select * from t0 where t0.a in diff --git a/mysql-test/main/subselect_sj2_jcl6.result b/mysql-test/main/subselect_sj2_jcl6.result index f0e8e9b4881..84317467e8a 100644 --- a/mysql-test/main/subselect_sj2_jcl6.result +++ b/mysql-test/main/subselect_sj2_jcl6.result @@ -477,11 +477,11 @@ where t0.a in ( select t1.a from t1,t2 where t2.a=t0.a and t1.b=t2.b); id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t0 ALL NULL NULL NULL NULL 5 100.00 Using where -1 PRIMARY t2 eq_ref PRIMARY PRIMARY 4 test.t0.a 1 100.00 Using join buffer (flat, BKA join); Key-ordered Rowid-ordered scan -1 PRIMARY t1 ref a a 5 test.t0.a 1 100.00 Using where; FirstMatch(t2); Using join buffer (incremental, BKA join); Key-ordered Rowid-ordered scan +1 PRIMARY t1 ref a a 5 test.t0.a 1 100.00 Start temporary; Using join buffer (flat, BKA join); Key-ordered Rowid-ordered scan +1 PRIMARY t2 eq_ref PRIMARY PRIMARY 4 test.t0.a 1 100.00 Using where; End temporary; Using join buffer (incremental, BKA join); Key-ordered Rowid-ordered scan Warnings: Note 1276 Field or reference 'test.t0.a' of SELECT #2 was resolved in SELECT #1 -Note 1003 select `test`.`t0`.`a` AS `a` from `test`.`t2` semi join (`test`.`t1`) join `test`.`t0` where `test`.`t2`.`a` = `test`.`t0`.`a` and `test`.`t1`.`a` = `test`.`t0`.`a` and `test`.`t1`.`b` = `test`.`t2`.`b` +Note 1003 select `test`.`t0`.`a` AS `a` from `test`.`t2` semi join (`test`.`t1`) join `test`.`t0` where `test`.`t1`.`a` = `test`.`t0`.`a` and `test`.`t2`.`a` = `test`.`t0`.`a` and `test`.`t2`.`b` = `test`.`t1`.`b` update t1 set a=3, b=11 where a=4; update t2 set b=11 where a=3; # Not anymore: diff --git a/mysql-test/main/subselect_sj2_mat.result b/mysql-test/main/subselect_sj2_mat.result index 9b497ade58b..54286f1fa82 100644 --- a/mysql-test/main/subselect_sj2_mat.result +++ b/mysql-test/main/subselect_sj2_mat.result @@ -468,11 +468,11 @@ where t0.a in ( select t1.a from t1,t2 where t2.a=t0.a and t1.b=t2.b); id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t0 ALL NULL NULL NULL NULL 5 100.00 Using where -1 PRIMARY t2 eq_ref PRIMARY PRIMARY 4 test.t0.a 1 100.00 -1 PRIMARY t1 ref a a 5 test.t0.a 1 100.00 Using where; FirstMatch(t2) +1 PRIMARY t1 ref a a 5 test.t0.a 1 100.00 Start temporary +1 PRIMARY t2 eq_ref PRIMARY PRIMARY 4 test.t0.a 1 100.00 Using where; End temporary Warnings: Note 1276 Field or reference 'test.t0.a' of SELECT #2 was resolved in SELECT #1 -Note 1003 select `test`.`t0`.`a` AS `a` from `test`.`t2` semi join (`test`.`t1`) join `test`.`t0` where `test`.`t2`.`a` = `test`.`t0`.`a` and `test`.`t1`.`a` = `test`.`t0`.`a` and `test`.`t1`.`b` = `test`.`t2`.`b` +Note 1003 select `test`.`t0`.`a` AS `a` from `test`.`t2` semi join (`test`.`t1`) join `test`.`t0` where `test`.`t1`.`a` = `test`.`t0`.`a` and `test`.`t2`.`a` = `test`.`t0`.`a` and `test`.`t2`.`b` = `test`.`t1`.`b` update t1 set a=3, b=11 where a=4; update t2 set b=11 where a=3; select * from t0 where t0.a in diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 39673a0aaa7..5f4881b2f54 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -111,12 +111,19 @@ static void optimize_straight_join(JOIN *join, table_map join_tables); static bool greedy_search(JOIN *join, table_map remaining_tables, uint depth, uint prune_level, uint use_cond_selectivity); -static bool best_extension_by_limited_search(JOIN *join, - table_map remaining_tables, - uint idx, double record_count, - double read_time, uint depth, - uint prune_level, - uint use_cond_selectivity); +enum enum_best_search { + SEARCH_ABORT= -2, + SEARCH_ERROR= -1, + SEARCH_OK= 0, + SEARCH_FOUND_EDGE=1 +}; +static enum_best_search +best_extension_by_limited_search(JOIN *join, + table_map remaining_tables, + uint idx, double record_count, + double read_time, uint depth, + uint prune_level, + uint use_cond_selectivity); static uint determine_search_depth(JOIN* join); C_MODE_START static int join_tab_cmp(const void *dummy, const void* ptr1, const void* ptr2); @@ -8446,6 +8453,7 @@ best_access_path(JOIN *join, pos->records_read= records; pos->read_time= best; pos->key= best_key; + pos->type= best_type; pos->table= s; pos->ref_depend_map= best_ref_depends_map; pos->loosescan_picker.loosescan_key= MAX_KEY; @@ -9101,9 +9109,12 @@ greedy_search(JOIN *join, do { /* Find the extension of the current QEP with the lowest cost */ join->best_read= DBL_MAX; - if (best_extension_by_limited_search(join, remaining_tables, idx, record_count, - read_time, search_depth, prune_level, - use_cond_selectivity)) + if ((int) best_extension_by_limited_search(join, remaining_tables, idx, + record_count, + read_time, search_depth, + prune_level, + use_cond_selectivity) < + (int) SEARCH_OK) DBUG_RETURN(TRUE); /* 'best_read < DBL_MAX' means that optimizer managed to find @@ -9739,6 +9750,28 @@ exit: } +/* + Check if the table is an EQ_REF or similar table and there is no cost + to gain by moveing it to a later stage. + We call such a table a edge table (or hanging leaf) as it will read at + most one row and will not add to the number of row combinations in the join. +*/ + +static inline enum_best_search +check_if_edge_table(POSITION *pos, + double pushdown_cond_selectivity) +{ + + if ((pos->type == JT_EQ_REF || + (pos->type == JT_REF && + pos->records_read == 1 && + !pos->range_rowid_filter_info)) && + pushdown_cond_selectivity >= 0.999) + return SEARCH_FOUND_EDGE; + return SEARCH_OK; +} + + /** Find a good, possibly optimal, query execution plan (QEP) by a possibly exhaustive search. @@ -9853,12 +9886,17 @@ exit: pushed to a table should be taken into account @retval - FALSE ok + enum_best_search::SEARCH_OK All fine @retval - TRUE Fatal error + enum_best_search::SEARCH_FOUND_EDGE All remaning tables are edge tables + @retval + enum_best_search::SEARCH_ABORT Killed by user + @retval + enum_best_search::SEARCH_ERROR Fatal error */ -static bool + +static enum_best_search best_extension_by_limited_search(JOIN *join, table_map remaining_tables, uint idx, @@ -9868,9 +9906,17 @@ best_extension_by_limited_search(JOIN *join, uint prune_level, uint use_cond_selectivity) { - DBUG_ENTER("best_extension_by_limited_search"); - THD *thd= join->thd; + /* + 'join' is a partial plan with lower cost than the best plan so far, + so continue expanding it further with the tables in 'remaining_tables'. + */ + JOIN_TAB *s; + double best_record_count= DBL_MAX; + double best_read_time= DBL_MAX; + bool disable_jbuf= join->thd->variables.join_cache_level == 0; + enum_best_search best_res; + DBUG_ENTER("best_extension_by_limited_search"); DBUG_EXECUTE_IF("show_explain_probe_best_ext_lim_search", if (dbug_user_var_equals_int(thd, @@ -9880,19 +9926,7 @@ best_extension_by_limited_search(JOIN *join, ); if (unlikely(thd->check_killed())) // Abort - DBUG_RETURN(TRUE); - - DBUG_EXECUTE("opt", print_plan(join, idx, read_time, record_count, idx, - "SOFAR:");); - - /* - 'join' is a partial plan with lower cost than the best plan so far, - so continue expanding it further with the tables in 'remaining_tables'. - */ - JOIN_TAB *s; - double best_record_count= DBL_MAX; - double best_read_time= DBL_MAX; - bool disable_jbuf= join->thd->variables.join_cache_level == 0; + DBUG_RETURN(SEARCH_ABORT); DBUG_EXECUTE("opt", print_plan(join, idx, record_count, read_time, read_time, "part_plan");); @@ -9914,9 +9948,11 @@ best_extension_by_limited_search(JOIN *join, (!idx || !check_interleaving_with_nj(s))) { double current_record_count, current_read_time; + double partial_join_cardinality; POSITION *position= join->positions + idx; - + POSITION loose_scan_pos; Json_writer_object trace_one_table(thd); + if (unlikely(thd->trace_started())) { trace_plan_prefix(join, idx, remaining_tables); @@ -9924,7 +9960,6 @@ best_extension_by_limited_search(JOIN *join, } /* Find the best access method from 's' to the current partial plan */ - POSITION loose_scan_pos; best_access_path(join, s, remaining_tables, join->positions, idx, disable_jbuf, record_count, position, &loose_scan_pos); @@ -9999,32 +10034,51 @@ best_extension_by_limited_search(JOIN *join, double pushdown_cond_selectivity= 1.0; if (use_cond_selectivity > 1) pushdown_cond_selectivity= table_cond_selectivity(join, idx, s, - remaining_tables & + remaining_tables & ~real_table_bit); join->positions[idx].cond_selectivity= pushdown_cond_selectivity; - if (unlikely(thd->trace_started()) && pushdown_cond_selectivity < 1.0) - trace_one_table.add("selectivity", pushdown_cond_selectivity); + partial_join_cardinality= (current_record_count * + pushdown_cond_selectivity); - double partial_join_cardinality= current_record_count * - pushdown_cond_selectivity; - if ( (search_depth > 1) && (remaining_tables & ~real_table_bit) & allowed_tables ) - { /* Recursively expand the current partial plan */ + if (unlikely(thd->trace_started())) + { + if (pushdown_cond_selectivity < 1.0) + { + trace_one_table.add("selectivity", pushdown_cond_selectivity); + trace_one_table.add("estimated_join_cardinality", + partial_join_cardinality); + } + } + + if ((search_depth > 1) && (remaining_tables & ~real_table_bit) & + allowed_tables) + { + /* Recursively expand the current partial plan */ swap_variables(JOIN_TAB*, join->best_ref[idx], *pos); Json_writer_array trace_rest(thd, "rest_of_plan"); - if (best_extension_by_limited_search(join, - remaining_tables & ~real_table_bit, - idx + 1, - partial_join_cardinality, - current_read_time, - search_depth - 1, - prune_level, - use_cond_selectivity)) - DBUG_RETURN(TRUE); + best_res= + best_extension_by_limited_search(join, + remaining_tables & + ~real_table_bit, + idx + 1, + partial_join_cardinality, + current_read_time, + search_depth - 1, + prune_level, + use_cond_selectivity); + if ((int) best_res < (int) SEARCH_OK) + DBUG_RETURN(best_res); // Abort swap_variables(JOIN_TAB*, join->best_ref[idx], *pos); + if (best_res == SEARCH_FOUND_EDGE && + check_if_edge_table(join->positions+ idx, + pushdown_cond_selectivity) != + SEARCH_FOUND_EDGE) + best_res= SEARCH_OK; } else - { /* + { + /* 'join' is either the best partial QEP with 'search_depth' relations, or the best complete QEP so far, whichever is smaller. */ @@ -10033,15 +10087,13 @@ best_extension_by_limited_search(JOIN *join, join->positions[join->const_tables].table->table) { /* - We may have to make a temp table, note that this is only a - heuristic since we cannot know for sure at this point. - Hence it may be wrong. + We may have to make a temp table, note that this is only a + heuristic since we cannot know for sure at this point. + Hence it may be wrong. */ trace_one_table.add("cost_for_sorting", current_record_count); current_read_time= COST_ADD(current_read_time, current_record_count); } - trace_one_table.add("estimated_join_cardinality", - partial_join_cardinality); if (current_read_time < join->best_read) { memcpy((uchar*) join->best_positions, (uchar*) join->positions, @@ -10054,12 +10106,19 @@ best_extension_by_limited_search(JOIN *join, read_time, current_read_time, "full_plan");); + best_res= check_if_edge_table(join->positions + idx, + pushdown_cond_selectivity); } restore_prev_nj_state(s); restore_prev_sj_state(remaining_tables, s, idx); + if (best_res == SEARCH_FOUND_EDGE) + { + trace_one_table.add("pruned_by_hanging_leaf", true); + DBUG_RETURN(best_res); + } } } - DBUG_RETURN(FALSE); + DBUG_RETURN(SEARCH_OK); } diff --git a/sql/sql_select.h b/sql/sql_select.h index e854792b073..4a2929207a5 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -989,6 +989,8 @@ public: */ enum sj_strategy_enum sj_strategy; + /* Type of join (EQ_REF, REF etc) */ + enum join_type type; /* Valid only after fix_semijoin_strategies_for_picked_join_order() call: if sj_strategy!=SJ_OPT_NONE, this is the number of subsequent tables that -- cgit v1.2.1 From c1063a1bed0008ca188ca3bba8c5125ea15cdc44 Mon Sep 17 00:00:00 2001 From: anel Date: Fri, 22 Apr 2022 06:34:12 -0700 Subject: MDEV-28342: sys.create_synonym_db fails when a temporary table masks a base table - This commit rely on MDEV-28391 - When temporary table shadows the base table, error is raised (it can be changed if needed), since the procedure is relying on creating the views and view cannot be created from the temporary table. - Reviewed by: --- .../suite/sysschema/r/pr_create_synonym_db.result | 14 ++++++++++++++ .../suite/sysschema/t/pr_create_synonym_db.test | 17 +++++++++++++++++ scripts/sys_schema/procedures/create_synonym_db.sql | 20 +++++++++++++++++--- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/sysschema/r/pr_create_synonym_db.result b/mysql-test/suite/sysschema/r/pr_create_synonym_db.result index bd7e4edc6e6..8c0e8dfa957 100644 --- a/mysql-test/suite/sysschema/r/pr_create_synonym_db.result +++ b/mysql-test/suite/sysschema/r/pr_create_synonym_db.result @@ -50,3 +50,17 @@ DROP TABLE test.t1; DROP TABLE test.t2; DROP TABLE `is`; DROP TABLE `ab``c`; +# +# MDEV-28342: sys.create_synonym_db fails +# when a temporary table masks a base table +# +create database db; +use db; +create table a(a int); +create table t (b int); +create table b(a int); +create temporary table b (a int); +call sys.create_synonym_db('db','db_copy'); +ERROR HY000: Table`db`.`b`shadows base table. View cannot be created! Terminating! +drop database db; +drop database db_copy; diff --git a/mysql-test/suite/sysschema/t/pr_create_synonym_db.test b/mysql-test/suite/sysschema/t/pr_create_synonym_db.test index 4fbe8759380..30c6f502e30 100644 --- a/mysql-test/suite/sysschema/t/pr_create_synonym_db.test +++ b/mysql-test/suite/sysschema/t/pr_create_synonym_db.test @@ -50,3 +50,20 @@ DROP TABLE test.t1; DROP TABLE test.t2; DROP TABLE `is`; DROP TABLE `ab``c`; + +--echo # +--echo # MDEV-28342: sys.create_synonym_db fails +--echo # when a temporary table masks a base table +--echo # + +create database db; +use db; +create table a(a int); +create table t (b int); +create table b(a int); +create temporary table b (a int); +--error ER_SIGNAL_EXCEPTION +call sys.create_synonym_db('db','db_copy'); + +drop database db; +drop database db_copy; diff --git a/scripts/sys_schema/procedures/create_synonym_db.sql b/scripts/sys_schema/procedures/create_synonym_db.sql index 703ec9bff3b..e373a9b4ee6 100644 --- a/scripts/sys_schema/procedures/create_synonym_db.sql +++ b/scripts/sys_schema/procedures/create_synonym_db.sql @@ -97,9 +97,8 @@ BEGIN DECLARE v_db_err_msg TEXT; DECLARE v_table VARCHAR(64); DECLARE v_views_created INT DEFAULT 0; - - DECLARE db_doesnt_exist CONDITION FOR SQLSTATE '42000'; - DECLARE db_name_exists CONDITION FOR SQLSTATE 'HY000'; + DECLARE v_table_exists ENUM('', 'BASE TABLE', 'VIEW', 'TEMPORARY') DEFAULT ''; + DECLARE v_temp_table TEXT; DECLARE c_table_names CURSOR FOR SELECT TABLE_NAME @@ -144,6 +143,21 @@ BEGIN LEAVE c_table_names; END IF; + -- Check does temporary table shadows the base table. If it is so, terminate. + CALL sys.table_exists(in_db_name, v_table, v_table_exists); + IF (v_table_exists = 'TEMPORARY') THEN + SET v_temp_table = + CONCAT( + 'Table', + sys.quote_identifier(in_db_name), + '.', + sys.quote_identifier(v_table), + 'shadows base table. View cannot be created! Terminating!'); + SIGNAL SQLSTATE 'HY000' + SET MESSAGE_TEXT = v_temp_table; + LEAVE c_table_names; + END IF; + SET @create_view_stmt = CONCAT( 'CREATE SQL SECURITY INVOKER VIEW ', sys.quote_identifier(in_synonym), -- cgit v1.2.1 From 726bd8c968e1b60a72f2a798a8fc269b78239cdc Mon Sep 17 00:00:00 2001 From: Andrei Date: Thu, 12 May 2022 19:24:26 +0300 Subject: MDEV-28550 improper handling of replication event group that contains GTID_LIST_EVENT or INCIDENT_EVENT. It's legal to have either of the two inside a group. E.g Gtid_event, Gtid_log_list_event, Query_1, ... Xid_log_event is permitted. However, the slave IO thread treated both as the terminal even when the group represents a DDL query. That causes a premature Gtid state update so the slave IO would think the whole group has been collected while in fact Query_1 etc are yet to process. Fixed with correcting a condition to compute the terminal event of the group. Tested with rpl_mysqlbinlog_slave_consistency (of 10.9) and rpl_gtid_errorlog.test. --- sql/slave.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/slave.cc b/sql/slave.cc index ca3397e5e7b..ac9addee7a4 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -6586,8 +6586,9 @@ dbug_gtid_accept: mi->using_gtid != Master_info::USE_GTID_NO && mi->events_queued_since_last_gtid > 0 && ( (mi->last_queued_gtid_standalone && - !Log_event::is_part_of_group((Log_event_type)(uchar) - buf[EVENT_TYPE_OFFSET])) || + (LOG_EVENT_IS_QUERY((Log_event_type)(uchar) + buf[EVENT_TYPE_OFFSET]) || + (uchar)buf[EVENT_TYPE_OFFSET] == INCIDENT_EVENT)) || (!mi->last_queued_gtid_standalone && ((uchar)buf[EVENT_TYPE_OFFSET] == XID_EVENT || ((uchar)buf[EVENT_TYPE_OFFSET] == QUERY_EVENT && /* QUERY_COMPRESSED_EVENT would never be commmit or rollback */ -- cgit v1.2.1 From 2d26f712df8aa8659a012cf31f6aed96af62af46 Mon Sep 17 00:00:00 2001 From: Brandon Nesterenko Date: Thu, 12 May 2022 14:57:01 -0600 Subject: MDEV-28550: improper handling of replication event group that contains Gtid_log_list_event If a slave received a fake GLLE event after a GTID event it would terminate the group. This adds a test for the previous commit which fixed this issue (939672a). Review by Andrei Elkin --- .../suite/rpl/r/rpl_gtid_glle_no_terminate.result | 46 ++++++++++++++ .../suite/rpl/t/rpl_gtid_glle_no_terminate.test | 72 ++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 mysql-test/suite/rpl/r/rpl_gtid_glle_no_terminate.result create mode 100644 mysql-test/suite/rpl/t/rpl_gtid_glle_no_terminate.test diff --git a/mysql-test/suite/rpl/r/rpl_gtid_glle_no_terminate.result b/mysql-test/suite/rpl/r/rpl_gtid_glle_no_terminate.result new file mode 100644 index 00000000000..f4d257c2668 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_gtid_glle_no_terminate.result @@ -0,0 +1,46 @@ +include/master-slave.inc +[connection master] +connection slave; +include/stop_slave.inc +CHANGE MASTER TO MASTER_USE_GTID=slave_pos; +# +# Initialize test data +connection master; +create table t1 (a int); +SET @@session.server_id= 3; +create table t2 (a int); +include/save_master_gtid.inc +# +# Have the replica "reconnect" and the primary will send Gtid, Glle, DDL +connection slave; +set global gtid_slave_pos="0-3-1"; +include/start_slave.inc +include/sync_with_master_gtid.inc +# +# Ensure that the replica did not error +connection slave; +include/sync_with_master_gtid.inc +Last_SQL_Error = +Last_SQL_Errno = 0 +# +# Ensure that the primary sent a Glle after a Gtid event +include/show_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +slave-relay-bin.000002 # Rotate # # master-bin.000001;pos=POS +slave-relay-bin.000002 # Format_desc # # SERVER_VERSION, BINLOG_VERSION +slave-relay-bin.000002 # Gtid_list # # [] +slave-relay-bin.000002 # Binlog_checkpoint # # master-bin.000001 +slave-relay-bin.000002 # Gtid # # GTID #-#-# +slave-relay-bin.000002 # Gtid_list # # [#-#-#] +slave-relay-bin.000002 # Query # # use `test`; create table t2 (a int) +# +# Ensure the DDL was executed on the replica +# +# Cleanup +# t1 does not make it to the replica +connection master; +set sql_log_bin=0; +DROP TABLE t1; +set sql_log_bin=1; +DROP TABLE t2; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_gtid_glle_no_terminate.test b/mysql-test/suite/rpl/t/rpl_gtid_glle_no_terminate.test new file mode 100644 index 00000000000..f0f38a31da6 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_gtid_glle_no_terminate.test @@ -0,0 +1,72 @@ +# +# Purpose: +# If a fake Glle event follows a Gtid event, we need to ensure the rest of +# the group should not terminate at the Glle event. MDEV-28550 revealed that +# a Glle would terminate the event and upon reconnect, the DDL would be lost. +# +# Methodology: +# Force the primary to send a fake GLLE event after a GTID on a "reconnect" +# and ensure that both 1) the replica does not error, and 2) the original +# command within the GTID is executed. +# +# References: +# MDEV-28550: improper handling of replication event group that contains Gtid_log_list_event + +--source include/master-slave.inc + +# Independent of binlog format +--source include/have_binlog_format_statement.inc + +--connection slave +--source include/stop_slave.inc +CHANGE MASTER TO MASTER_USE_GTID=slave_pos; + +--echo # +--echo # Initialize test data +--connection master +create table t1 (a int); +SET @@session.server_id= 3; +create table t2 (a int); +--source include/save_master_gtid.inc + +--echo # +--echo # Have the replica "reconnect" and the primary will send Gtid, Glle, DDL +--connection slave +eval set global gtid_slave_pos="0-3-1"; +--source include/start_slave.inc +--source include/sync_with_master_gtid.inc + +--echo # +--echo # Ensure that the replica did not error +connection slave; +--source include/sync_with_master_gtid.inc +let $error= query_get_value(SHOW SLAVE STATUS, Last_SQL_Error, 1); +--echo Last_SQL_Error = $error +let $errno= query_get_value(SHOW SLAVE STATUS, Last_SQL_Errno, 1); +--echo Last_SQL_Errno = $errno + +--echo # +--echo # Ensure that the primary sent a Glle after a Gtid event +let $binlog_file= query_get_value(SHOW SLAVE STATUS, Relay_Log_File, 1); +let $binlog_start= $relaylog_start; +let $binlog_limit=0,10; +--source include/show_relaylog_events.inc + +--echo # +--echo # Ensure the DDL was executed on the replica +if (!`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't2'`) +{ + die "t2 should exist on slave"; +} + +--echo # +--echo # Cleanup + +--echo # t1 does not make it to the replica +--connection master +set sql_log_bin=0; +DROP TABLE t1; +set sql_log_bin=1; +DROP TABLE t2; + +--source include/rpl_end.inc -- cgit v1.2.1 From f027c1217b5b3ac76e742b6ec4ee75ed79ec5871 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 13 May 2022 13:13:45 +0200 Subject: MDEV-28471 mysql_install_db.exe does not work with --innodb-page-size=64K The error message "InnoDB: innodb_page_size=65536 requires innodb_buffer_pool_size >= 20MiB current 10MiB" is the relevant one. The root cause: mysql_install_db bootstraps with --innodb-buffer-pool-size=10M. Small bufferpool is here by design - bootstrap should succeed, even if there is not much RAM available, bootstrap does not need that much memory. For pagesize 64K specifically, Innodb thinks it needs a larger bufferpool, and thus it lets the bootstrap process die (although the expected behavior in this case would be to adjust value, give warning and continue) The workaround: - pass --innodb-buffer-pool-size=20M, which is suitable for all page sizes. - check the same limit in MSI custom action. Also, the patch adds mtr test for 64K page size. --- mysql-test/main/mysql_install_db_win.test | 7 +++++ sql/mysql_install_db.cc | 2 +- win/packaging/ca/CustomAction.cpp | 49 ++++++++++++++++++------------- 3 files changed, 37 insertions(+), 21 deletions(-) diff --git a/mysql-test/main/mysql_install_db_win.test b/mysql-test/main/mysql_install_db_win.test index ceb7293f611..c400dbf3fb9 100644 --- a/mysql-test/main/mysql_install_db_win.test +++ b/mysql-test/main/mysql_install_db_win.test @@ -42,6 +42,13 @@ remove_file $log; rmdir $ddir; +# MDEV-28471 - mysql_install_db.exe fails with --innodb-page-size=64K +--disable_result_log +exec $MYSQL_INSTALL_DB_EXE --datadir=$ddir --password=foo -R --innodb-page-size=64K --verbose +--enable_result_log +rmdir $ddir; + +# Tests with config file let $restart_parameters=; connection default; diff --git a/sql/mysql_install_db.cc b/sql/mysql_install_db.cc index 713c6ac4c3f..1e79e6444ff 100644 --- a/sql/mysql_install_db.cc +++ b/sql/mysql_install_db.cc @@ -333,7 +333,7 @@ static char *init_bootstrap_command_line(char *cmdline, size_t size) " %s" " --bootstrap" " --datadir=." - " --loose-innodb-buffer-pool-size=10M" + " --loose-innodb-buffer-pool-size=20M" "\"" , mysqld_path, opt_verbose_bootstrap ? "--console" : ""); return cmdline; diff --git a/win/packaging/ca/CustomAction.cpp b/win/packaging/ca/CustomAction.cpp index 328e73accb8..dbd0b36745f 100644 --- a/win/packaging/ca/CustomAction.cpp +++ b/win/packaging/ca/CustomAction.cpp @@ -663,6 +663,12 @@ unsigned long long GetMaxBufferSize(unsigned long long totalPhys) } +/* + Magic undocumented number for bufferpool minimum, + allows innodb to start also for all page sizes. +*/ +static constexpr unsigned long long minBufferpoolMB= 20; + /* Checks SERVICENAME, PORT and BUFFERSIZE parameters */ @@ -791,34 +797,37 @@ extern "C" UINT __stdcall CheckDatabaseProperties (MSIHANDLE hInstall) unsigned long long availableMemory= GetMaxBufferSize(memstatus.ullTotalPhys)/ONE_MB; swprintf_s(invalidValueMsg, - L"Invalid buffer pool size. Please use a number between 1 and %llu", - availableMemory); - if(BufferPoolSizeLen == 0 || BufferPoolSizeLen > 15) + L"Invalid buffer pool size. Please use a number between %llu and %llu", + minBufferpoolMB, availableMemory); + if (BufferPoolSizeLen == 0 || BufferPoolSizeLen > 15 || !BufferPoolSize[0]) { ErrorMsg= invalidValueMsg; goto LExit; } - for (DWORD i=0; i < BufferPoolSizeLen && BufferPoolSize[BufferPoolSizeLen]; - i++) - { - if(BufferPoolSize[i]< '0' || BufferPoolSize[i] > '9') - { - ErrorMsg= invalidValueMsg; - goto LExit; - } - } + BufferPoolSize[BufferPoolSizeLen]=0; MsiSetPropertyW(hInstall, L"BUFFERPOOLSIZE", BufferPoolSize); - long long sz = _wtoi64(BufferPoolSize); - if(sz <= 0 || sz > (long long)availableMemory) + wchar_t *end; + unsigned long long sz = wcstoull(BufferPoolSize, &end, 10); + if (sz > availableMemory || sz < minBufferpoolMB || *end) { - if(sz > 0) + if (*end == 0) { - swprintf_s(invalidValueMsg, - L"Value for buffer pool size is too large." - L"Only approximately %llu MB is available for allocation." - L"Please use a number between 1 and %llu.", - availableMemory, availableMemory); + if(sz > availableMemory) + { + swprintf_s(invalidValueMsg, + L"Value for buffer pool size is too large." + L"Only approximately %llu MB is available for allocation." + L"Please use a number between %llu and %llu.", + availableMemory, minBufferpoolMB, availableMemory); + } + else if(sz < minBufferpoolMB) + { + swprintf_s(invalidValueMsg, + L"Value for buffer pool size is too small." + L"Please use a number between %llu and %llu.", + minBufferpoolMB, availableMemory); + } } ErrorMsg= invalidValueMsg; goto LExit; -- cgit v1.2.1 From 8c28b27f00941b2886f691010f822fe506b15af7 Mon Sep 17 00:00:00 2001 From: Nayuta Yanagisawa Date: Fri, 13 May 2022 21:32:49 +0900 Subject: MDEV-28301 Spider: Fix GCC warnings, comparing the result of pointer addition ... and NULL The condition of the if statements are always true. --- storage/spider/spd_db_mysql.cc | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/storage/spider/spd_db_mysql.cc b/storage/spider/spd_db_mysql.cc index 0fd5112380b..c665aa62100 100644 --- a/storage/spider/spd_db_mysql.cc +++ b/storage/spider/spd_db_mysql.cc @@ -3444,27 +3444,15 @@ int spider_db_mbase::append_lock_tables( conn_link_idx = tmp_spider->conn_link_idx[tmp_link_idx]; spider_mbase_share *db_share = (spider_mbase_share *) tmp_spider->share->dbton_share[conn->dbton_id]; - if (&db_share->db_names_str[conn_link_idx]) - { - db_name = db_share->db_names_str[conn_link_idx].ptr(); - db_name_length = db_share->db_names_str[conn_link_idx].length(); - db_name_charset = tmp_spider->share->access_charset; - } else { - db_name = tmp_spider->share->tgt_dbs[conn_link_idx]; - db_name_length = tmp_spider->share->tgt_dbs_lengths[conn_link_idx]; - db_name_charset = system_charset_info; - } - if (&db_share->table_names_str[conn_link_idx]) - { - table_name = db_share->table_names_str[conn_link_idx].ptr(); - table_name_length = db_share->table_names_str[conn_link_idx].length(); - table_name_charset = tmp_spider->share->access_charset; - } else { - table_name = tmp_spider->share->tgt_table_names[conn_link_idx]; - table_name_length = - tmp_spider->share->tgt_table_names_lengths[conn_link_idx]; - table_name_charset = system_charset_info; - } + + db_name = db_share->db_names_str[conn_link_idx].ptr(); + db_name_length = db_share->db_names_str[conn_link_idx].length(); + db_name_charset = tmp_spider->share->access_charset; + + table_name = db_share->table_names_str[conn_link_idx].ptr(); + table_name_length = db_share->table_names_str[conn_link_idx].length(); + table_name_charset = tmp_spider->share->access_charset; + if ((error_num = spider_db_mbase_utility-> append_lock_table_body( str, -- cgit v1.2.1 From 730eb1c4be84b7f06dc8d4a894536941cbe5da63 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sat, 14 May 2022 20:19:05 +0300 Subject: Code cleanup in/around check_interleaving_with_nj() - In best_extension_by_limited_search(), do not check for "(remaining_tables & real_table_bit)", it is guaranteed to be true. Make it an assert. - In (!idx || check_interleaving_with_nj())", remove the !idx part. This check made sense only in the original version of this function. - "micro optimization" in check_interleaving_with_nj(). --- sql/sql_select.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 6d8d389b81d..ee834f5d806 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -8962,10 +8962,11 @@ best_extension_by_limited_search(JOIN *join, for (JOIN_TAB **pos= join->best_ref + idx ; (s= *pos) ; pos++) { table_map real_table_bit= s->table->map; - if ((remaining_tables & real_table_bit) && - (allowed_tables & real_table_bit) && + DBUG_ASSERT(remaining_tables & real_table_bit); + + if ((allowed_tables & real_table_bit) && !(remaining_tables & s->dependent) && - (!idx || !check_interleaving_with_nj(s))) + !check_interleaving_with_nj(s)) { double current_record_count, current_read_time; POSITION *position= join->positions + idx; @@ -16076,7 +16077,6 @@ static uint reset_nj_counters(JOIN *join, List *join_list) static bool check_interleaving_with_nj(JOIN_TAB *next_tab) { - TABLE_LIST *next_emb= next_tab->table->pos_in_table_list->embedding; JOIN *join= next_tab->join; if (join->cur_embedding_map & ~next_tab->embedding_map) @@ -16088,6 +16088,7 @@ static bool check_interleaving_with_nj(JOIN_TAB *next_tab) return TRUE; } + TABLE_LIST *next_emb= next_tab->table->pos_in_table_list->embedding; /* Do update counters for "pairs of brackets" that we've left (marked as X,Y,Z in the above picture) -- cgit v1.2.1 From e03e72234aa59016c71c51b7adc5a59ae7b126b2 Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Sun, 15 May 2022 17:59:43 +0300 Subject: Fixed warning from UBSAN The warning comes from copying POSITION objects where 'type' is not initialized. This does not affect any production code as when 'type' was compared/used, it was always initialized. Removed by initializing the type variable in the constructor --- sql/sql_select.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 5f4881b2f54..673ce8b1573 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -391,6 +391,7 @@ POSITION::POSITION() range_rowid_filter_info= 0; ref_depend_map= dups_producing_tables= 0; inner_tables_handled_with_other_sjs= 0; + type= JT_UNKNOWN; dups_weedout_picker.set_empty(); firstmatch_picker.set_empty(); loosescan_picker.set_empty(); -- cgit v1.2.1 From 29c07643a162d614dcb3ec6d69992bdd1df37cd6 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 6 May 2022 01:13:05 +0200 Subject: enable -Wenum-compare -Wenum-conversion to make the all headers -std=c++20 clean for those, who need c++20 (some plugins) --- cmake/maintainer.cmake | 2 ++ sql/gcalc_tools.cc | 2 +- sql/gcalc_tools.h | 16 +++++++--------- sql/handler.h | 8 ++++---- sql/sql_class.h | 4 ++-- sql/sql_lex.h | 12 +++++------- 6 files changed, 21 insertions(+), 23 deletions(-) diff --git a/cmake/maintainer.cmake b/cmake/maintainer.cmake index 8dbe7a6df47..5a1d186cf58 100644 --- a/cmake/maintainer.cmake +++ b/cmake/maintainer.cmake @@ -22,6 +22,8 @@ ENDIF() SET(MY_WARNING_FLAGS -Wall -Wdeclaration-after-statement + -Wenum-compare + -Wenum-conversion -Wextra -Wformat-security -Wno-format-truncation diff --git a/sql/gcalc_tools.cc b/sql/gcalc_tools.cc index 307f063fb43..25c80a7a796 100644 --- a/sql/gcalc_tools.cc +++ b/sql/gcalc_tools.cc @@ -132,7 +132,7 @@ int Gcalc_function::count_internal(const char *cur_func, uint set_type, int mask= (c_op & op_not) ? 1:0; uint n_ops= c_op & ~(op_any | op_not | v_mask); uint n_shape= c_op & ~(op_any | op_not | v_mask); /* same as n_ops */ - value v_state= (value) (c_op & v_mask); + op_type v_state= (op_type) (c_op & v_mask); int result= 0; const char *sav_cur_func= cur_func; diff --git a/sql/gcalc_tools.h b/sql/gcalc_tools.h index e625b355d95..bb1f473e180 100644 --- a/sql/gcalc_tools.h +++ b/sql/gcalc_tools.h @@ -52,17 +52,15 @@ private: int count_internal(const char *cur_func, uint set_type, const char **end); public: - enum value - { - v_empty= 0x0000000, - v_find_t= 0x1000000, - v_find_f= 0x2000000, - v_t_found= 0x3000000, - v_f_found= 0x4000000, - v_mask= 0x7000000 - }; enum op_type { + v_empty= 0x00000000, + v_find_t= 0x01000000, + v_find_f= 0x02000000, + v_t_found= 0x03000000, + v_f_found= 0x04000000, + v_mask= 0x07000000, + op_not= 0x80000000, op_shape= 0x00000000, op_union= 0x10000000, diff --git a/sql/handler.h b/sql/handler.h index 47da1b938d7..cd999f30bc0 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -2636,10 +2636,10 @@ public: double import_cost; /* cost of remote operations */ double mem_cost; /* cost of used memory */ - enum { IO_COEFF=1 }; - enum { CPU_COEFF=1 }; - enum { MEM_COEFF=1 }; - enum { IMPORT_COEFF=1 }; + static const int IO_COEFF=1; + static const int CPU_COEFF=1; + static const int MEM_COEFF=1; + static const int IMPORT_COEFF=1; Cost_estimate() { diff --git a/sql/sql_class.h b/sql/sql_class.h index 19466b3de90..0301eeec093 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -5133,8 +5133,8 @@ my_eof(THD *thd) inline date_conv_mode_t sql_mode_for_dates(THD *thd) { - static_assert((date_conv_mode_t::KNOWN_MODES & - time_round_mode_t::KNOWN_MODES) == 0, + static_assert((ulonglong(date_conv_mode_t::KNOWN_MODES) & + ulonglong(time_round_mode_t::KNOWN_MODES)) == 0, "date_conv_mode_t and time_round_mode_t must use different " "bit values"); static_assert(MODE_NO_ZERO_DATE == date_mode_t::NO_ZERO_DATE && diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 3db50222a27..09e0df2edca 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -1995,8 +1995,7 @@ public: @retval nonzero if the statement is a row injection */ inline bool is_stmt_row_injection() const { - return binlog_stmt_flags & - (1U << (BINLOG_STMT_UNSAFE_COUNT + BINLOG_STMT_TYPE_ROW_INJECTION)); + return binlog_stmt_flags & (1U << BINLOG_STMT_TYPE_ROW_INJECTION); } /** @@ -2006,8 +2005,7 @@ public: */ inline void set_stmt_row_injection() { DBUG_ENTER("set_stmt_row_injection"); - binlog_stmt_flags|= - (1U << (BINLOG_STMT_UNSAFE_COUNT + BINLOG_STMT_TYPE_ROW_INJECTION)); + binlog_stmt_flags|= (1U << BINLOG_STMT_TYPE_ROW_INJECTION); DBUG_VOID_RETURN; } @@ -2283,7 +2281,7 @@ private: The statement is a row injection (i.e., either a BINLOG statement or a row event executed by the slave SQL thread). */ - BINLOG_STMT_TYPE_ROW_INJECTION = 0, + BINLOG_STMT_TYPE_ROW_INJECTION = BINLOG_STMT_UNSAFE_COUNT, /** The last element of this enumeration type. */ BINLOG_STMT_TYPE_COUNT @@ -2297,8 +2295,8 @@ private: - The low BINLOG_STMT_UNSAFE_COUNT bits indicate the types of unsafeness that the current statement has. - - The next BINLOG_STMT_TYPE_COUNT bits indicate if the statement - is of some special type. + - The next BINLOG_STMT_TYPE_COUNT-BINLOG_STMT_TYPE_COUNT bits indicate if + the statement is of some special type. This must be a member of LEX, not of THD: each stored procedure needs to remember its unsafeness state between calls and each -- cgit v1.2.1 From a68c698b462196b6226a460c38743629d4f39b16 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 15 May 2022 16:07:02 +0200 Subject: fix occasional failures in --embedded followup for d16c3aca3c3ecd --- sql/mysqld.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index fb3bee9db1a..9f03e4e819a 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -5405,7 +5405,9 @@ static int init_server_components() unireg_abort(1); } +#ifndef EMBEDDED_LIBRARY start_handle_manager(); +#endif if (opt_bin_log) { int error; -- cgit v1.2.1 From b03ab1270d24c1fe011aa50f9e6b495c6d508706 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Sun, 15 May 2022 23:28:06 +0400 Subject: MDEV-28490 Strange result truncation with group_concat_max_len=1GB. Arythmetic can overrun the uint type when possible group_concat_max_len is multiplied to collation.mbmaxlen (can easily be like 4). So use ulonglong there for calculations. --- .../sys_vars/r/group_concat_max_len_func.result | 54 ++++++++++++++++++++++ .../sys_vars/t/group_concat_max_len_func.test | 31 +++++++++++++ sql/item_sum.cc | 6 +-- 3 files changed, 88 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/group_concat_max_len_func.result b/mysql-test/suite/sys_vars/r/group_concat_max_len_func.result index 30e2639af37..01f44ae51be 100644 --- a/mysql-test/suite/sys_vars/r/group_concat_max_len_func.result +++ b/mysql-test/suite/sys_vars/r/group_concat_max_len_func.result @@ -94,4 +94,58 @@ DROP TABLE t1; disconnect test_con2; disconnect test_con1; connection default; +CREATE TABLE t1(val VARCHAR(100) PRIMARY KEY) CHARACTER SET utf8mb4 COLLATE utf8mb4_danish_ci; +INSERT INTO t1 VALUES('bar'); +INSERT INTO t1 VALUES('foo'); +SET group_concat_max_len = 1073741823; +SHOW VARIABLES LIKE 'group_concat_max_len'; +Variable_name Value +group_concat_max_len 1073741823 +SELECT GROUP_CONCAT(val) AS simple FROM t1; +simple +bar,foo +SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp; +nested +bar,foo +SET group_concat_max_len = 1073741824; +SHOW VARIABLES LIKE 'group_concat_max_len'; +Variable_name Value +group_concat_max_len 1073741824 +SELECT GROUP_CONCAT(val) AS simple FROM t1; +simple +bar,foo +SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp; +nested +bar,foo +SET group_concat_max_len = 1073741825; +SHOW VARIABLES LIKE 'group_concat_max_len'; +Variable_name Value +group_concat_max_len 1073741825 +SELECT GROUP_CONCAT(val) AS simple FROM t1; +simple +bar,foo +SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp; +nested +bar,foo +SET group_concat_max_len = 1073741826; +SHOW VARIABLES LIKE 'group_concat_max_len'; +Variable_name Value +group_concat_max_len 1073741826 +SELECT GROUP_CONCAT(val) AS simple FROM t1; +simple +bar,foo +SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp; +nested +bar,foo +SET group_concat_max_len = 2147483649; +SHOW VARIABLES LIKE 'group_concat_max_len'; +Variable_name Value +group_concat_max_len 2147483649 +SELECT GROUP_CONCAT(val) AS simple FROM t1; +simple +bar,foo +SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp; +nested +bar,foo +DROP TABLE t1; SET @@global.group_concat_max_len = @save; diff --git a/mysql-test/suite/sys_vars/t/group_concat_max_len_func.test b/mysql-test/suite/sys_vars/t/group_concat_max_len_func.test index b053ee229d7..d90fc061289 100644 --- a/mysql-test/suite/sys_vars/t/group_concat_max_len_func.test +++ b/mysql-test/suite/sys_vars/t/group_concat_max_len_func.test @@ -132,4 +132,35 @@ disconnect test_con1; connection default; +CREATE TABLE t1(val VARCHAR(100) PRIMARY KEY) CHARACTER SET utf8mb4 COLLATE utf8mb4_danish_ci; +INSERT INTO t1 VALUES('bar'); +INSERT INTO t1 VALUES('foo'); + +SET group_concat_max_len = 1073741823; +SHOW VARIABLES LIKE 'group_concat_max_len'; +SELECT GROUP_CONCAT(val) AS simple FROM t1; +SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp; + +SET group_concat_max_len = 1073741824; +SHOW VARIABLES LIKE 'group_concat_max_len'; +SELECT GROUP_CONCAT(val) AS simple FROM t1; +SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp; + +SET group_concat_max_len = 1073741825; +SHOW VARIABLES LIKE 'group_concat_max_len'; +SELECT GROUP_CONCAT(val) AS simple FROM t1; +SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp; + +SET group_concat_max_len = 1073741826; +SHOW VARIABLES LIKE 'group_concat_max_len'; +SELECT GROUP_CONCAT(val) AS simple FROM t1; +SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp; + +SET group_concat_max_len = 2147483649; +SHOW VARIABLES LIKE 'group_concat_max_len'; +SELECT GROUP_CONCAT(val) AS simple FROM t1; +SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp; + +DROP TABLE t1; + SET @@global.group_concat_max_len = @save; diff --git a/sql/item_sum.cc b/sql/item_sum.cc index 5065aba922a..1c17b5c6409 100644 --- a/sql/item_sum.cc +++ b/sql/item_sum.cc @@ -4242,9 +4242,9 @@ Item_func_group_concat::fix_fields(THD *thd, Item **ref) result.set_charset(collation.collation); result_field= 0; null_value= 1; - max_length= (uint32)MY_MIN(thd->variables.group_concat_max_len - / collation.collation->mbminlen - * collation.collation->mbmaxlen, UINT_MAX32); + max_length= (uint32) MY_MIN((ulonglong) thd->variables.group_concat_max_len + / collation.collation->mbminlen + * collation.collation->mbmaxlen, UINT_MAX32); uint32 offset; if (separator->needs_conversion(separator->length(), separator->charset(), -- cgit v1.2.1 From e2173e8067de79cbb134c484557920da3f93d831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 16 May 2022 09:16:58 +0300 Subject: MDEV-18182 : Galera test failure on galera.galera_many_tables_nopk Decrease the number of tables and operations. --- .../suite/galera/r/galera_many_tables_nopk.result | 6 +++--- .../suite/galera/t/galera_many_tables_nopk.test | 23 +++++++++++----------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_many_tables_nopk.result b/mysql-test/suite/galera/r/galera_many_tables_nopk.result index 2a226defcc7..d341a1816e5 100644 --- a/mysql-test/suite/galera/r/galera_many_tables_nopk.result +++ b/mysql-test/suite/galera/r/galera_many_tables_nopk.result @@ -6,8 +6,8 @@ START TRANSACTION; COMMIT; connection node_2; CREATE TABLE sum_table (f1 INTEGER); -SELECT SUM(f1) = 900 FROM sum_table; -SUM(f1) = 900 +SELECT SUM(f1) = 100 FROM sum_table; +SUM(f1) = 100 1 connection node_1; SET AUTOCOMMIT=OFF; @@ -15,7 +15,7 @@ START TRANSACTION; connection node_2; SET AUTOCOMMIT=OFF; START TRANSACTION; -UPDATE t900 SET f1 = 3; +UPDATE t100 SET f1 = 3; connection node_1; COMMIT; connection node_2; diff --git a/mysql-test/suite/galera/t/galera_many_tables_nopk.test b/mysql-test/suite/galera/t/galera_many_tables_nopk.test index 98a65b7c660..5bfab686726 100644 --- a/mysql-test/suite/galera/t/galera_many_tables_nopk.test +++ b/mysql-test/suite/galera/t/galera_many_tables_nopk.test @@ -7,18 +7,17 @@ if (!`SELECT @@open_files_limit >= 1024`){ } # -# This test forces 900 tables without a PK to participate in a single -# transaction. The reason for 900 is that some linux system has by default -# a limit of 1024 open files / process +# This test forces 100 tables without a PK to participate in a single +# transaction. # # -# First, create 900 tables +# First, create 100 tables # --connection node_1 ---let $count = 900 +--let $count = 100 while ($count) { --disable_query_log @@ -28,7 +27,7 @@ while ($count) --dec $count } ---let $count = 900 +--let $count = 100 while ($count) { --disable_query_log @@ -39,13 +38,13 @@ while ($count) } # -# Second, perform 900 updates +# Second, perform 100 updates # SET AUTOCOMMIT=OFF; START TRANSACTION; ---let $count = 900 +--let $count = 100 while ($count) { --disable_query_log @@ -63,7 +62,7 @@ COMMIT; --connection node_2 CREATE TABLE sum_table (f1 INTEGER); ---let $count = 900 +--let $count = 100 while ($count) { --disable_query_log @@ -73,7 +72,7 @@ while ($count) --dec $count } -SELECT SUM(f1) = 900 FROM sum_table; +SELECT SUM(f1) = 100 FROM sum_table; # # Fourth, create a deadlock @@ -83,7 +82,7 @@ SELECT SUM(f1) = 900 FROM sum_table; SET AUTOCOMMIT=OFF; START TRANSACTION; ---let $count = 900 +--let $count = 100 while ($count) { --disable_query_log @@ -96,7 +95,7 @@ while ($count) --connection node_2 SET AUTOCOMMIT=OFF; START TRANSACTION; -UPDATE t900 SET f1 = 3; +UPDATE t100 SET f1 = 3; --connection node_1 COMMIT; -- cgit v1.2.1 From c79e2bfe9fa961cc5c6b2282210a0c57d8540615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 16 May 2022 09:18:51 +0300 Subject: MDEV-23595 : galera_3nodes.galera_wsrep_schema MTR failed: mysql_shutdown failed Add disconnect. --- mysql-test/suite/galera_3nodes/r/galera_wsrep_schema.result | 1 + mysql-test/suite/galera_3nodes/t/galera_wsrep_schema.test | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/mysql-test/suite/galera_3nodes/r/galera_wsrep_schema.result b/mysql-test/suite/galera_3nodes/r/galera_wsrep_schema.result index f51eb815cd5..deef311b1e4 100644 --- a/mysql-test/suite/galera_3nodes/r/galera_wsrep_schema.result +++ b/mysql-test/suite/galera_3nodes/r/galera_wsrep_schema.result @@ -74,3 +74,4 @@ cluster_uuid = (SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHER SELECT COUNT(*) AS EXPECT_3 FROM mysql.wsrep_cluster_members; EXPECT_3 3 +disconnect node_3; diff --git a/mysql-test/suite/galera_3nodes/t/galera_wsrep_schema.test b/mysql-test/suite/galera_3nodes/t/galera_wsrep_schema.test index 52bbf3a652c..d6e20d3bbfb 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_wsrep_schema.test +++ b/mysql-test/suite/galera_3nodes/t/galera_wsrep_schema.test @@ -17,6 +17,9 @@ # Make the test fail if table structure has changed +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + SHOW CREATE TABLE mysql.wsrep_cluster; SHOW CREATE TABLE mysql.wsrep_cluster_members; @@ -74,3 +77,5 @@ SELECT cluster_uuid = (SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STAT SELECT COUNT(*) AS EXPECT_3 FROM mysql.wsrep_cluster_members; --source ../galera/include/auto_increment_offset_restore.inc + +--disconnect node_3 -- cgit v1.2.1 From 65eea2315ffda0c2d5b48ef09069e10e777b8619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 16 May 2022 09:22:16 +0300 Subject: Update disabled.def --- mysql-test/suite/galera/disabled.def | 2 -- 1 file changed, 2 deletions(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 0263c57c264..9f360c52ada 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -18,10 +18,8 @@ galera_as_slave_replication_bundle : MDEV-15785 OPTION_GTID_BEGIN is set in Gtid galera_bf_abort_group_commit : MDEV-18282 Galera test failure on galera.galera_bf_abort_group_commit galera_bf_lock_wait : MDEV-21597 wsrep::transaction::start_transaction(): Assertion `active() == false' failed galera_encrypt_tmp_files : Get error failed to enable encryption of temporary files -galera_ftwrl : MDEV-21525 galera.galera_ftwrl galera_gcache_recover_manytrx : MDEV-18834 Galera test failure galera_kill_largechanges : MDEV-18179 Galera test failure on galera.galera_kill_largechanges -galera_many_tables_nopk : MDEV-18182 Galera test failure on galera.galera_many_tables_nopk galera_mdl_race : MDEV-21524 galera.galera_mdl_race galera_parallel_simple : MDEV-20318 galera.galera_parallel_simple fails galera_pc_ignore_sb : MDEV-20888 galera.galera_pc_ignore_sb -- cgit v1.2.1 From 8d12dd8f503282179a078f2f883b88f6ccee5ebd Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Wed, 11 May 2022 14:33:20 +0200 Subject: MDEV-28053 Sysbench data load crashes Galera secondary node in async master slave setup This patch fixes a problem that arises when a Galera node acts as a replica for native replication. When parallel applying is enabled, it is possible to end up with attempts to write binlog events with gtids out of order. This happens because when multiple events are delivered from the native replication stream and applied in concurrently, it is for them to be replicated to the Galera cluster in an order which is different from the original order in which they were committed in the aync replication master. To correct this behavior we now wait_for_prior_commit() before replicating changes though galera. As a consequence, parallel appliers may apply events in parallel until the galera replication step, which is now serialized. --- mysql-test/suite/galera/r/MDEV-28053.result | 14 +++++++ mysql-test/suite/galera/t/MDEV-28053.cnf | 6 +++ mysql-test/suite/galera/t/MDEV-28053.test | 61 +++++++++++++++++++++++++++++ sql/wsrep_trans_observer.h | 4 ++ 4 files changed, 85 insertions(+) create mode 100644 mysql-test/suite/galera/r/MDEV-28053.result create mode 100644 mysql-test/suite/galera/t/MDEV-28053.cnf create mode 100644 mysql-test/suite/galera/t/MDEV-28053.test diff --git a/mysql-test/suite/galera/r/MDEV-28053.result b/mysql-test/suite/galera/r/MDEV-28053.result new file mode 100644 index 00000000000..b3f93688dd0 --- /dev/null +++ b/mysql-test/suite/galera/r/MDEV-28053.result @@ -0,0 +1,14 @@ +connection node_2; +connection node_1; +connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; +connection node_3; +CREATE TABLE t1 (f1 INTEGER PRIMARY KEY AUTO_INCREMENT) ENGINE=InnoDB; +connection node_2; +connection node_3; +DROP TABLE t1; +connection node_2; +connection node_2; +STOP SLAVE; +RESET SLAVE ALL; +connection node_3; +RESET MASTER; diff --git a/mysql-test/suite/galera/t/MDEV-28053.cnf b/mysql-test/suite/galera/t/MDEV-28053.cnf new file mode 100644 index 00000000000..2a500639d1d --- /dev/null +++ b/mysql-test/suite/galera/t/MDEV-28053.cnf @@ -0,0 +1,6 @@ +!include ../galera_2nodes_as_slave.cnf + +[mysqld] +slave_parallel_threads=4 +slave_parallel_mode=optimistic +gtid_strict_mode=1 diff --git a/mysql-test/suite/galera/t/MDEV-28053.test b/mysql-test/suite/galera/t/MDEV-28053.test new file mode 100644 index 00000000000..85cb20c7e10 --- /dev/null +++ b/mysql-test/suite/galera/t/MDEV-28053.test @@ -0,0 +1,61 @@ +# +# MDEV-28053 - Sysbench data load crashes Galera secondary node in +# async master slave setup +# +# Setup: node 3 is a regular MariaDB server, nodes 1 and 2 are members +# of a Galera cluster. Node 2 connects to node 3 through async replication. +# +# Test uses multiple parallel async applier threads (see MDEV-28053.cnf) +# + +--source include/have_innodb.inc +--source include/galera_cluster.inc + +--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 + +--connection node_3 +CREATE TABLE t1 (f1 INTEGER PRIMARY KEY AUTO_INCREMENT) ENGINE=InnoDB; + +# +# Execute a few INSERTs, to simulate sysbench data load phase +# +--let $counter=100 +--disable_query_log +while ($counter) { + --connection node_3 + INSERT INTO t1 VALUES(); + --dec $counter +} +--enable_query_log +--let gtid = `SELECT @@last_gtid` + +# +# Start async replication on node 2. +# If bug is present, expect a crash when applying +# events concurrently. +# +--connection node_2 +--disable_query_log +--disable_result_log +--eval CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_USER='root', MASTER_PORT=$NODE_MYPORT_3; +START SLAVE; +--eval SELECT MASTER_GTID_WAIT('$gtid', 600) +--enable_result_log +--enable_query_log + +# +# Cleanup +# +--connection node_3 +DROP TABLE t1; + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc + +--connection node_2 +STOP SLAVE; +RESET SLAVE ALL; + +--connection node_3 +RESET MASTER; diff --git a/sql/wsrep_trans_observer.h b/sql/wsrep_trans_observer.h index 837bad4c50f..cde8163cf4f 100644 --- a/sql/wsrep_trans_observer.h +++ b/sql/wsrep_trans_observer.h @@ -229,6 +229,10 @@ static inline int wsrep_before_prepare(THD* thd, bool all) WSREP_DEBUG("wsrep_before_prepare: %d", wsrep_is_real(thd, all)); int ret= 0; DBUG_ASSERT(wsrep_run_commit_hook(thd, all)); + if ((ret= thd->wsrep_parallel_slave_wait_for_prior_commit())) + { + DBUG_RETURN(ret); + } if ((ret= thd->wsrep_cs().before_prepare()) == 0) { DBUG_ASSERT(!thd->wsrep_trx().ws_meta().gtid().is_undefined()); -- cgit v1.2.1 From 4e1bf2bb2368b0ebb98e20563f4834c1061ed498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 16 May 2022 13:41:53 +0300 Subject: MDEV-28537 Unused or useless InnoDB counters num_index_pages_written, num_non_index_pages_written The counters were added in commit 5e55d1ced52c52fb2f0508e1346059901a85960f and any code to update them was inadvertently removed in commit 2e814d4702d71a04388386a9f591d14a35980bfe when applying InnoDB changes from MySQL 5.7. Let us remove these counters that never reported anything useful. If such statistics are really needed in a special case, they can be obtained by instrumenting the code by some means, such as eBPF or a source code patch. --- .../innodb/r/innodb_skip_innodb_is_tables.result | 2 -- mysql-test/suite/innodb/r/monitor.result | 2 -- storage/innobase/handler/ha_innodb.cc | 4 ---- storage/innobase/include/srv0mon.h | 4 +--- storage/innobase/include/srv0srv.h | 8 -------- storage/innobase/srv/srv0mon.cc | 24 +--------------------- storage/innobase/srv/srv0srv.cc | 2 -- .../rocksdb/r/innodb_i_s_tables_disabled.result | 2 -- 8 files changed, 2 insertions(+), 46 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result b/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result index 015bfe9dcef..fe851bc3b25 100644 --- a/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result +++ b/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result @@ -71,8 +71,6 @@ buffer_pool_bytes_dirty buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL buffer_pool_pages_free buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled value Buffer pages currently free (innodb_buffer_pool_pages_free) buffer_pages_created buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of pages created (innodb_pages_created) buffer_pages_written buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of pages written (innodb_pages_written) -buffer_index_pages_written buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of index pages written (innodb_index_pages_written) -buffer_non_index_pages_written buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of non index pages written (innodb_non_index_pages_written) buffer_pages_read buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of pages read (innodb_pages_read) buffer_pages0_read buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of page 0 read (innodb_pages0_read) buffer_index_sec_rec_cluster_reads buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of secondary record reads triggered cluster read diff --git a/mysql-test/suite/innodb/r/monitor.result b/mysql-test/suite/innodb/r/monitor.result index a40bfdac0d0..b65bba276f0 100644 --- a/mysql-test/suite/innodb/r/monitor.result +++ b/mysql-test/suite/innodb/r/monitor.result @@ -36,8 +36,6 @@ buffer_pool_bytes_dirty disabled buffer_pool_pages_free disabled buffer_pages_created disabled buffer_pages_written disabled -buffer_index_pages_written disabled -buffer_non_index_pages_written disabled buffer_pages_read disabled buffer_pages0_read disabled buffer_index_sec_rec_cluster_reads disabled diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 40754a131ee..8c610b0fc3d 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1142,10 +1142,6 @@ static SHOW_VAR innodb_status_variables[]= { /* Status variables for page compression */ {"page_compression_saved", (char*) &export_vars.innodb_page_compression_saved, SHOW_LONGLONG}, - {"num_index_pages_written", - (char*) &export_vars.innodb_index_pages_written, SHOW_LONGLONG}, - {"num_non_index_pages_written", - (char*) &export_vars.innodb_non_index_pages_written, SHOW_LONGLONG}, {"num_pages_page_compressed", (char*) &export_vars.innodb_pages_page_compressed, SHOW_LONGLONG}, {"num_page_compressed_trim_op", diff --git a/storage/innobase/include/srv0mon.h b/storage/innobase/include/srv0mon.h index 17e72b6b218..1a825c419dd 100644 --- a/storage/innobase/include/srv0mon.h +++ b/storage/innobase/include/srv0mon.h @@ -2,7 +2,7 @@ Copyright (c) 2010, 2015, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2020, MariaDB Corporation. +Copyright (c) 2013, 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the @@ -174,8 +174,6 @@ enum monitor_id_t { MONITOR_OVLD_BUF_POOL_PAGES_FREE, MONITOR_OVLD_PAGE_CREATED, MONITOR_OVLD_PAGES_WRITTEN, - MONITOR_OVLD_INDEX_PAGES_WRITTEN, - MONITOR_OVLD_NON_INDEX_PAGES_WRITTEN, MONITOR_OVLD_PAGES_READ, MONITOR_OVLD_PAGES0_READ, MONITOR_OVLD_INDEX_SEC_REC_CLUSTER_READS, diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index a7bed269fa4..8cf568a0105 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -109,10 +109,6 @@ struct srv_stats_t /** Number of bytes saved by page compression */ ulint_ctr_64_t page_compression_saved; - /* Number of index pages written */ - ulint_ctr_64_t index_pages_written; - /* Number of non index pages written */ - ulint_ctr_64_t non_index_pages_written; /* Number of pages compressed with page compression */ ulint_ctr_64_t pages_page_compressed; /* Number of TRIM operations induced by page compression */ @@ -1030,10 +1026,6 @@ struct export_var_t{ int64_t innodb_page_compression_saved;/*!< Number of bytes saved by page compression */ - int64_t innodb_index_pages_written; /*!< Number of index pages - written */ - int64_t innodb_non_index_pages_written; /*!< Number of non index pages - written */ int64_t innodb_pages_page_compressed;/*!< Number of pages compressed by page compression */ int64_t innodb_page_compressed_trim_op;/*!< Number of TRIM operations diff --git a/storage/innobase/srv/srv0mon.cc b/storage/innobase/srv/srv0mon.cc index 0404335574a..39f495c4ced 100644 --- a/storage/innobase/srv/srv0mon.cc +++ b/storage/innobase/srv/srv0mon.cc @@ -2,7 +2,7 @@ Copyright (c) 2010, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2018, MariaDB Corporation. +Copyright (c) 2013, 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -280,18 +280,6 @@ static monitor_info_t innodb_counter_info[] = MONITOR_EXISTING | MONITOR_DEFAULT_ON), MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES_WRITTEN}, - {"buffer_index_pages_written", "buffer", - "Number of index pages written (innodb_index_pages_written)", - static_cast( - MONITOR_EXISTING | MONITOR_DEFAULT_ON), - MONITOR_DEFAULT_START, MONITOR_OVLD_INDEX_PAGES_WRITTEN}, - - {"buffer_non_index_pages_written", "buffer", - "Number of non index pages written (innodb_non_index_pages_written)", - static_cast( - MONITOR_EXISTING | MONITOR_DEFAULT_ON), - MONITOR_DEFAULT_START, MONITOR_OVLD_NON_INDEX_PAGES_WRITTEN}, - {"buffer_pages_read", "buffer", "Number of pages read (innodb_pages_read)", static_cast( @@ -1732,16 +1720,6 @@ srv_mon_process_existing_counter( value = stat.n_pages_written; break; - /* innodb_index_pages_written, the number of index pages written */ - case MONITOR_OVLD_INDEX_PAGES_WRITTEN: - value = srv_stats.index_pages_written; - break; - - /* innodb_non_index_pages_written, the number of non index pages written */ - case MONITOR_OVLD_NON_INDEX_PAGES_WRITTEN: - value = srv_stats.non_index_pages_written; - break; - /* innodb_pages_read */ case MONITOR_OVLD_PAGES_READ: buf_get_total_stat(&stat); diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 2bc5500ef91..811892b3cab 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -1575,8 +1575,6 @@ srv_export_innodb_status(void) export_vars.innodb_available_undo_logs = srv_available_undo_logs; export_vars.innodb_page_compression_saved = srv_stats.page_compression_saved; - export_vars.innodb_index_pages_written = srv_stats.index_pages_written; - export_vars.innodb_non_index_pages_written = srv_stats.non_index_pages_written; export_vars.innodb_pages_page_compressed = srv_stats.pages_page_compressed; export_vars.innodb_page_compressed_trim_op = srv_stats.page_compressed_trim_op; export_vars.innodb_pages_page_decompressed = srv_stats.pages_page_decompressed; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/innodb_i_s_tables_disabled.result b/storage/rocksdb/mysql-test/rocksdb/r/innodb_i_s_tables_disabled.result index 61f005c1758..a7679e7cc3e 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/innodb_i_s_tables_disabled.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/innodb_i_s_tables_disabled.result @@ -53,8 +53,6 @@ buffer_pool_bytes_dirty buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL buffer_pool_pages_free buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled value Buffer pages currently free (innodb_buffer_pool_pages_free) buffer_pages_created buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of pages created (innodb_pages_created) buffer_pages_written buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of pages written (innodb_pages_written) -buffer_index_pages_written buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of index pages written (innodb_index_pages_written) -buffer_non_index_pages_written buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of non index pages written (innodb_non_index_pages_written) buffer_pages_read buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of pages read (innodb_pages_read) buffer_pages0_read buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of page 0 read (innodb_pages0_read) buffer_index_sec_rec_cluster_reads buffer 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL disabled status_counter Number of secondary record reads triggered cluster read -- cgit v1.2.1 From 3e564d468d69b52e8e4a48af6980d40005205864 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 16 May 2022 13:45:17 +0300 Subject: MDEV-28541 Unused counter Innodb_encryption_key_rotation_list_length The counter srv_stats.key_rotation_list_length is never updated, and therefore Innodb_encryption_key_rotation_list_length will always be 0. The view INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION comes close to reporting this information. --- storage/innobase/handler/ha_innodb.cc | 3 --- storage/innobase/include/srv0srv.h | 4 ---- storage/innobase/srv/srv0srv.cc | 2 -- 3 files changed, 9 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 8c610b0fc3d..514e6fa31a2 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1209,9 +1209,6 @@ static SHOW_VAR innodb_status_variables[]= { {"encryption_rotation_estimated_iops", (char*) &export_vars.innodb_encryption_rotation_estimated_iops, SHOW_LONG}, - {"encryption_key_rotation_list_length", - (char*)&export_vars.innodb_key_rotation_list_length, - SHOW_LONGLONG}, {"encryption_n_merge_blocks_encrypted", (char*)&export_vars.innodb_n_merge_blocks_encrypted, SHOW_LONGLONG}, diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 8cf568a0105..f9d3701ec0b 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -181,9 +181,6 @@ struct srv_stats_t /** Number of log scrub operations */ ulint_ctr_64_t n_log_scrubs; - /** Number of spaces in keyrotation list */ - ulint_ctr_64_t key_rotation_list_length; - /** Number of temporary tablespace blocks encrypted */ ulint_ctr_64_t n_temp_blocks_encrypted; @@ -1064,7 +1061,6 @@ struct export_var_t{ ulint innodb_encryption_rotation_pages_flushed; ulint innodb_encryption_rotation_estimated_iops; int64_t innodb_encryption_key_requests; - int64_t innodb_key_rotation_list_length; ulint innodb_scrub_page_reorganizations; ulint innodb_scrub_page_splits; diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 811892b3cab..9f02e31277b 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -1619,8 +1619,6 @@ srv_export_innodb_status(void) crypt_stat.estimated_iops; export_vars.innodb_encryption_key_requests = srv_stats.n_key_requests; - export_vars.innodb_key_rotation_list_length = - srv_stats.key_rotation_list_length; export_vars.innodb_scrub_page_reorganizations = scrub_stat.page_reorganizations; -- cgit v1.2.1 From a2bcfa64fee99bd6892a198ebb20043a39523d40 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 10 May 2022 20:24:42 +0200 Subject: galera.MDEV-26575 and galera_sr.galera_sr_shutdown_slave failures --- mysql-test/suite/galera/r/MDEV-26575.result | 3 +++ mysql-test/suite/galera/t/MDEV-26575.test | 4 ++++ mysql-test/suite/galera_sr/r/galera_sr_shutdown_slave.result | 2 ++ mysql-test/suite/galera_sr/t/galera_sr_shutdown_slave.test | 2 ++ 4 files changed, 11 insertions(+) diff --git a/mysql-test/suite/galera/r/MDEV-26575.result b/mysql-test/suite/galera/r/MDEV-26575.result index b8d5d431aaa..5b447e1ae20 100644 --- a/mysql-test/suite/galera/r/MDEV-26575.result +++ b/mysql-test/suite/galera/r/MDEV-26575.result @@ -1,5 +1,8 @@ connection node_2; connection node_1; +connection node_2; +call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:.*"); +connection node_1; connection node_1; connection node_2; connection node_2; diff --git a/mysql-test/suite/galera/t/MDEV-26575.test b/mysql-test/suite/galera/t/MDEV-26575.test index e714f9fc430..4554f632c6f 100644 --- a/mysql-test/suite/galera/t/MDEV-26575.test +++ b/mysql-test/suite/galera/t/MDEV-26575.test @@ -5,6 +5,10 @@ --source include/galera_cluster.inc +--connection node_2 +call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:.*"); +--connection node_1 + # Save original auto_increment_offset values. --let $node_1=node_1 --let $node_2=node_2 diff --git a/mysql-test/suite/galera_sr/r/galera_sr_shutdown_slave.result b/mysql-test/suite/galera_sr/r/galera_sr_shutdown_slave.result index 902aa27d5aa..34995d35a4f 100644 --- a/mysql-test/suite/galera_sr/r/galera_sr_shutdown_slave.result +++ b/mysql-test/suite/galera_sr/r/galera_sr_shutdown_slave.result @@ -2,6 +2,8 @@ connection node_2; connection node_1; connection node_1; connection node_2; +connection node_2; +call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:.*"); connection node_1; CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE = InnoDB; connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1; diff --git a/mysql-test/suite/galera_sr/t/galera_sr_shutdown_slave.test b/mysql-test/suite/galera_sr/t/galera_sr_shutdown_slave.test index d1b0d4b8c88..5d4a58b2d03 100644 --- a/mysql-test/suite/galera_sr/t/galera_sr_shutdown_slave.test +++ b/mysql-test/suite/galera_sr/t/galera_sr_shutdown_slave.test @@ -8,6 +8,8 @@ --let $node_1=node_1 --let $node_2=node_2 --source ../galera/include/auto_increment_offset_save.inc +--connection node_2 +call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:.*"); --connection node_1 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE = InnoDB; -- cgit v1.2.1 From 4a8a6f605d3f9c8cbca14a04cb75c6d2a29fb6f6 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 16 May 2022 17:07:50 +0200 Subject: MDEV-28578 Server crashes in Item_field::fix_outer_field after CREATE SELECT same as MDEV-26412, but in CREATE...SELECT. fix: apply 39feab3cd31b to create rule too. --- mysql-test/main/insert.result | 8 ++++++++ mysql-test/main/insert.test | 9 +++++++++ sql/sql_yacc.yy | 1 + 3 files changed, 18 insertions(+) diff --git a/mysql-test/main/insert.result b/mysql-test/main/insert.result index af7dcbedd1f..8423319f557 100644 --- a/mysql-test/main/insert.result +++ b/mysql-test/main/insert.result @@ -770,4 +770,12 @@ create table t (a int); select 1 in (select count(*) from t t1 join (t t2 join t t3 on (t1.a != 0))); ERROR 42S22: Unknown column 't1.a' in 'on clause' drop table t; +# +# MDEV-28578 Server crashes in Item_field::fix_outer_field after CREATE SELECT +# +create table t1 (i int) ; +create table t2 (j int) ; +create table t4 select * from t1 join t2 on (select t3.i); +ERROR 42S22: Unknown column 't3.i' in 'field list' +drop table t1, t2; # End of 10.4 tests diff --git a/mysql-test/main/insert.test b/mysql-test/main/insert.test index 27d44918bbb..b27290bc91c 100644 --- a/mysql-test/main/insert.test +++ b/mysql-test/main/insert.test @@ -638,4 +638,13 @@ create table t (a int); select 1 in (select count(*) from t t1 join (t t2 join t t3 on (t1.a != 0))); drop table t; +--echo # +--echo # MDEV-28578 Server crashes in Item_field::fix_outer_field after CREATE SELECT +--echo # +create table t1 (i int) ; +create table t2 (j int) ; +--error ER_BAD_FIELD_ERROR +create table t4 select * from t1 join t2 on (select t3.i); +drop table t1, t2; + --echo # End of 10.4 tests diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index e2e554f6d82..ff91a649310 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -2693,6 +2693,7 @@ create: lex->create_info.default_table_charset= NULL; lex->name= null_clex_str; lex->create_last_non_select_table= lex->last_table(); + lex->inc_select_stack_outer_barrier(); } create_body { -- cgit v1.2.1 From 4dffa7b5c5baadf95e01d7d595b4ae0dab5965cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 12 May 2022 15:17:37 +0300 Subject: MDEV-28546 : Possible to write/update with read_only=ON and not a SUPER privilege Function wsrep_read_only_option was already removed in commit d54bc3c0d1 because it could cause race condition on variable opt_readonly so that value OFF can become permanent. Removed function again and added test case. Note that writes to TEMPORARY tables are still allowed when read_only=ON. --- mysql-test/suite/galera/r/galera_read_only.result | 12 ++++++++++-- mysql-test/suite/galera/t/galera_read_only.test | 13 ++++++++++++- sql/sql_parse.cc | 18 +----------------- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_read_only.result b/mysql-test/suite/galera/r/galera_read_only.result index fe8b45fa596..e7e18ca8145 100644 --- a/mysql-test/suite/galera/r/galera_read_only.result +++ b/mysql-test/suite/galera/r/galera_read_only.result @@ -17,10 +17,18 @@ connect foo_node_2,127.0.0.1,foo,,test,$port_2,; connection foo_node_2; INSERT INTO t1 VALUES (2); ERROR HY000: The MariaDB server is running with the --read-only option so it cannot execute this statement +CREATE TEMPORARY TABLE t2(id int not null primary key) engine=innodb; +INSERT INTO t2 values (1); +DROP TABLE t2; connection node_2; -SELECT COUNT(*) = 1 FROM t1; -COUNT(*) = 1 +SELECT COUNT(*) AS EXPECT_1 FROM t1; +EXPECT_1 1 +connection node_2; +SET GLOBAL read_only=TRUE; +CREATE TEMPORARY TABLE t2(id int not null primary key) engine=innodb; +INSERT INTO t2 values (1); +DROP TABLE t2; SET GLOBAL read_only=FALSE; DROP TABLE t1; DROP USER foo@localhost; diff --git a/mysql-test/suite/galera/t/galera_read_only.test b/mysql-test/suite/galera/t/galera_read_only.test index c0fa4af07e0..56fe2fdd910 100644 --- a/mysql-test/suite/galera/t/galera_read_only.test +++ b/mysql-test/suite/galera/t/galera_read_only.test @@ -28,9 +28,20 @@ CREATE USER foo@localhost; --connection foo_node_2 --error ER_OPTION_PREVENTS_STATEMENT INSERT INTO t1 VALUES (2); +# Writes to temporary tables are allowed +CREATE TEMPORARY TABLE t2(id int not null primary key) engine=innodb; +INSERT INTO t2 values (1); +DROP TABLE t2; --connection node_2 -SELECT COUNT(*) = 1 FROM t1; +SELECT COUNT(*) AS EXPECT_1 FROM t1; + +--connection node_2 +SET GLOBAL read_only=TRUE; +# Writes to temporary tables are allowed +CREATE TEMPORARY TABLE t2(id int not null primary key) engine=innodb; +INSERT INTO t2 values (1); +DROP TABLE t2; # Cleanup SET GLOBAL read_only=FALSE; diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index b5e6b6540c5..04c3450dd98 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -1468,22 +1468,6 @@ static bool deny_updates_if_read_only_option(THD *thd, TABLE_LIST *all_tables) } #ifdef WITH_WSREP -static my_bool wsrep_read_only_option(THD *thd, TABLE_LIST *all_tables) -{ - int opt_readonly_saved = opt_readonly; - ulong flag_saved = (ulong)(thd->security_ctx->master_access & SUPER_ACL); - - opt_readonly = 0; - thd->security_ctx->master_access &= ~SUPER_ACL; - - my_bool ret = !deny_updates_if_read_only_option(thd, all_tables); - - opt_readonly = opt_readonly_saved; - thd->security_ctx->master_access |= flag_saved; - - return ret; -} - static void wsrep_copy_query(THD *thd) { thd->wsrep_retry_command = thd->get_command(); @@ -7807,7 +7791,7 @@ static bool wsrep_mysql_parse(THD *thd, char *rawbuf, uint length, { bool is_autocommit= !thd->in_multi_stmt_transaction_mode() && - wsrep_read_only_option(thd, thd->lex->query_tables); + !thd->wsrep_applier; bool retry_autocommit; do { -- cgit v1.2.1 From c9b5a05341d7342db5f369493ea200b5fb9db243 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 17 May 2022 15:34:58 +0400 Subject: MDEV-28588 SIGSEGV in __memmove_avx_unaligned_erms, strmake_root --- mysql-test/suite/compat/oracle/r/sp.result | 18 ++++++++++++++++++ mysql-test/suite/compat/oracle/t/sp.test | 23 +++++++++++++++++++++++ sql/sql_lex.cc | 2 +- 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/compat/oracle/r/sp.result b/mysql-test/suite/compat/oracle/r/sp.result index 2d4244110f9..5756db4fc5b 100644 --- a/mysql-test/suite/compat/oracle/r/sp.result +++ b/mysql-test/suite/compat/oracle/r/sp.result @@ -2560,3 +2560,21 @@ idx idx 1 DROP PROCEDURE p1; +# +# MDEV-28588 SIGSEGV in __memmove_avx_unaligned_erms, strmake_root +# +SET sql_mode=ORACLE; +BEGIN END; +SET sql_mode=ORACLE; +CREATE EVENT ev ON SCHEDULE AT CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO BEGIN END; +Warnings: +Warning 1105 Event scheduler is switched off, use SET GLOBAL event_scheduler=ON to enable it. +SELECT EVENT_DEFINITION FROM INFORMATION_SCHEMA.EVENTS WHERE EVENT_SCHEMA='test' AND EVENT_NAME='ev'; +EVENT_DEFINITION BEGIN END +DROP EVENT ev; +CREATE TABLE t1 (a INT); +CREATE TRIGGER tr AFTER INSERT ON t1 FOR EACH ROW BEGIN END; +SELECT ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS WHERE TRIGGER_SCHEMA='test' AND TRIGGER_NAME='tr'; +ACTION_STATEMENT BEGIN END +DROP TRIGGER tr; +DROP TABLE t1; diff --git a/mysql-test/suite/compat/oracle/t/sp.test b/mysql-test/suite/compat/oracle/t/sp.test index 31657aa4aef..ca99739533f 100644 --- a/mysql-test/suite/compat/oracle/t/sp.test +++ b/mysql-test/suite/compat/oracle/t/sp.test @@ -2397,3 +2397,26 @@ $$ DELIMITER ;$$ CALL p1(); DROP PROCEDURE p1; + + +--echo # +--echo # MDEV-28588 SIGSEGV in __memmove_avx_unaligned_erms, strmake_root +--echo # + +SET sql_mode=ORACLE; +BEGIN END; + +SET sql_mode=ORACLE; +CREATE EVENT ev ON SCHEDULE AT CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO BEGIN END; +--vertical_results +SELECT EVENT_DEFINITION FROM INFORMATION_SCHEMA.EVENTS WHERE EVENT_SCHEMA='test' AND EVENT_NAME='ev'; +--horizontal_results +DROP EVENT ev; + +CREATE TABLE t1 (a INT); +CREATE TRIGGER tr AFTER INSERT ON t1 FOR EACH ROW BEGIN END; +--vertical_results +SELECT ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS WHERE TRIGGER_SCHEMA='test' AND TRIGGER_NAME='tr'; +--horizontal_results +DROP TRIGGER tr; +DROP TABLE t1; diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index f3951628513..5a00518b806 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -6674,7 +6674,7 @@ bool LEX::maybe_start_compound_statement(THD *thd) if (!make_sp_head(thd, NULL, &sp_handler_procedure)) return true; sphead->set_suid(SP_IS_NOT_SUID); - sphead->set_body_start(thd, thd->m_parser_state->m_lip.get_cpp_ptr()); + sphead->set_body_start(thd, thd->m_parser_state->m_lip.get_cpp_tok_start()); } return false; } -- cgit v1.2.1 From 84984b79f27399d015c43a51d2b1967838119d34 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 17 May 2022 11:36:09 +0200 Subject: Revert "MDEV-27524: Incorrect binlogs after Galera SST using rsync and mariabackup" This reverts commit 17e0f5224c8339ec08707a6ad0397bbf8c19bbd3. --- extra/mariabackup/backup_copy.cc | 84 ++-- extra/mariabackup/backup_copy.h | 7 - extra/mariabackup/backup_mysql.cc | 248 +++-------- extra/mariabackup/backup_mysql.h | 6 +- extra/mariabackup/common.h | 10 - extra/mariabackup/xtrabackup.cc | 62 +-- extra/mariabackup/xtrabackup.h | 2 - .../suite/galera/include/galera_wsrep_recover.inc | 7 - .../suite/galera/r/galera_log_bin_ext.result | 9 - .../galera/r/galera_log_bin_ext_mariabackup.result | 82 ---- .../suite/galera/r/galera_sst_rsync,debug.rdiff | 4 +- .../suite/galera/r/galera_sst_rsync2,debug.rdiff | 4 +- .../galera/r/galera_sst_xtrabackup-v2,debug.rdiff | 4 +- .../galera_sst_xtrabackup-v2_data_dir,debug.rdiff | 19 +- .../r/galera_sst_xtrabackup-v2_data_dir.result | 2 - .../suite/galera/r/galera_wan_restart_sst.result | 11 +- mysql-test/suite/galera/t/galera_log_bin_ext.cnf | 3 - mysql-test/suite/galera/t/galera_log_bin_ext.test | 2 +- .../galera/t/galera_log_bin_ext_mariabackup.cnf | 19 - .../galera/t/galera_log_bin_ext_mariabackup.test | 2 - mysql-test/suite/galera/t/galera_log_bin_sst.inc | 84 ---- mysql-test/suite/galera/t/galera_sst_rsync.test | 1 - .../galera/t/galera_sst_rsync_logbasename.cnf | 1 + .../suite/galera/t/galera_wan_restart_sst.test | 19 +- scripts/wsrep_sst_common.sh | 185 +++------ scripts/wsrep_sst_mariabackup.sh | 234 ++++------- scripts/wsrep_sst_mysqldump.sh | 7 +- scripts/wsrep_sst_rsync.sh | 451 ++++++++------------- scripts/wsrep_sst_xtrabackup-v2.sh | 235 ++++------- 29 files changed, 492 insertions(+), 1312 deletions(-) delete mode 100644 mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result delete mode 100644 mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf delete mode 100644 mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test delete mode 100644 mysql-test/suite/galera/t/galera_log_bin_sst.inc diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index 9c6f3665a6a..c46c32213d6 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -582,6 +582,7 @@ datafile_read(datafile_cur_t *cursor) Check to see if a file exists. Takes name of the file to check. @return true if file exists. */ +static bool file_exists(const char *filename) { @@ -1546,14 +1547,13 @@ bool backup_start(CorruptedPages &corrupted_pages) if (!write_galera_info(mysql_connection)) { return(false); } + write_current_binlog_file(mysql_connection); } - bool with_binlogs = opt_binlog_info == BINLOG_INFO_ON; + if (opt_binlog_info == BINLOG_INFO_ON) { - if (with_binlogs || opt_galera_info) { - if (!write_current_binlog_file(mysql_connection, with_binlogs)) { - return(false); - } + lock_binlog_maybe(mysql_connection); + write_binlog_info(mysql_connection); } if (have_flush_engine_logs && !opt_no_lock) { @@ -1587,34 +1587,15 @@ void backup_release() } } -static const char *default_buffer_pool_file = "ib_buffer_pool"; - -static -const char * get_buffer_pool_filename(size_t *length) -{ - /* If mariabackup is run for Galera, then the file - name is changed to the default so that the receiving - node can find this file and rename it according to its - settings, otherwise we keep the original file name: */ - size_t dir_length = 0; - const char *dst_name = default_buffer_pool_file; - if (!opt_galera_info) { - dir_length = dirname_length(buffer_pool_filename); - dst_name = buffer_pool_filename + dir_length; - } - if (length) { - *length=dir_length; - } - return dst_name; -} - /** Finish after backup_start() and backup_release() */ bool backup_finish() { /* Copy buffer pool dump or LRU dump */ if (!opt_rsync) { if (buffer_pool_filename && file_exists(buffer_pool_filename)) { - const char *dst_name = get_buffer_pool_filename(NULL); + const char *dst_name; + + dst_name = trim_dotslash(buffer_pool_filename); copy_file(ds_data, buffer_pool_filename, dst_name, 0); } if (file_exists("ib_lru_dump")) { @@ -1703,14 +1684,17 @@ ibx_copy_incremental_over_full() /* copy buffer pool dump */ if (innobase_buffer_pool_filename) { - const char *src_name = get_buffer_pool_filename(NULL); + const char *src_name; + + src_name = trim_dotslash(innobase_buffer_pool_filename); snprintf(path, sizeof(path), "%s/%s", xtrabackup_incremental_dir, src_name); if (file_exists(path)) { - copy_file(ds_data, path, src_name, 0); + copy_file(ds_data, path, + innobase_buffer_pool_filename, 0); } } @@ -1945,14 +1929,6 @@ copy_back() datadir_node_init(&node); - /* If mariabackup is run for Galera, then the file - name is changed to the default so that the receiving - node can find this file and rename it according to its - settings, otherwise we keep the original file name: */ - size_t dir_length; - const char *src_buffer_pool; - src_buffer_pool = get_buffer_pool_filename(&dir_length); - while (datadir_iter_next(it, &node)) { const char *ext_list[] = {"backup-my.cnf", "xtrabackup_binary", "xtrabackup_binlog_info", @@ -2015,11 +1991,6 @@ copy_back() continue; } - /* skip buffer pool dump */ - if (!strcmp(filename, src_buffer_pool)) { - continue; - } - /* skip innodb data files */ is_ibdata_file = false; for (Tablespace::const_iterator iter(srv_sys_space.begin()), @@ -2042,18 +2013,23 @@ copy_back() /* copy buffer pool dump */ - if (file_exists(src_buffer_pool)) { - char dst_dir[FN_REFLEN]; - while (IS_TRAILING_SLASH(buffer_pool_filename, dir_length)) { - dir_length--; - } - memcpy(dst_dir, buffer_pool_filename, dir_length); - dst_dir[dir_length] = 0; - if (!(ret = copy_or_move_file(src_buffer_pool, - src_buffer_pool, - dst_dir, 1))) - { - goto cleanup; + if (innobase_buffer_pool_filename) { + const char *src_name; + char path[FN_REFLEN]; + + src_name = trim_dotslash(innobase_buffer_pool_filename); + + snprintf(path, sizeof(path), "%s/%s", + mysql_data_home, + src_name); + + /* could be already copied with other files + from data directory */ + if (file_exists(src_name) && + !file_exists(innobase_buffer_pool_filename)) { + copy_or_move_file(src_name, + innobase_buffer_pool_filename, + mysql_data_home, 0); } } diff --git a/extra/mariabackup/backup_copy.h b/extra/mariabackup/backup_copy.h index 858182001ce..62b2b1bc232 100644 --- a/extra/mariabackup/backup_copy.h +++ b/extra/mariabackup/backup_copy.h @@ -32,13 +32,6 @@ copy_file(ds_ctxt_t *datasink, const char *dst_file_path, uint thread_n); -/************************************************************************ -Check to see if a file exists. -Takes name of the file to check. -@return true if file exists. */ -bool -file_exists(const char *filename); - /** Start --backup */ bool backup_start(CorruptedPages &corrupted_pages); /** Release resources after backup_start() */ diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index bddf069f4e9..be979a27457 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -83,6 +83,7 @@ os_event_t kill_query_thread_stop; bool sql_thread_started = false; char *mysql_slave_position = NULL; char *mysql_binlog_position = NULL; +char *buffer_pool_filename = NULL; /* History on server */ time_t history_start_time; @@ -1572,68 +1573,51 @@ cleanup: } -static -bool -write_binlog_info(MYSQL *connection, char *log_bin_dir, - MYSQL_RES *mysql_result, my_ulonglong n_rows, - my_ulonglong start); - /*********************************************************************//** Flush and copy the current binary log file into the backup, if GTID is enabled */ bool -write_current_binlog_file(MYSQL *connection, bool write_binlogs) +write_current_binlog_file(MYSQL *connection) { - char *log_bin = NULL; - char *filename = NULL; - char *position = NULL; char *executed_gtid_set = NULL; char *gtid_binlog_state = NULL; + char *log_bin_file = NULL; char *log_bin_dir = NULL; bool gtid_exists; bool result = true; + char filepath[FN_REFLEN]; - mysql_variable log_bin_var[] = { - {"@@GLOBAL.log_bin", &log_bin}, + mysql_variable status[] = { + {"Executed_Gtid_Set", &executed_gtid_set}, {NULL, NULL} }; - mysql_variable vars[] = { - {"gtid_binlog_state", >id_binlog_state}, - {"log_bin_basename", &log_bin_dir}, + mysql_variable status_after_flush[] = { + {"File", &log_bin_file}, {NULL, NULL} }; - mysql_variable status[] = { - {"File", &filename}, - {"Position", &position}, - {"Executed_Gtid_Set", &executed_gtid_set}, + mysql_variable vars[] = { + {"gtid_binlog_state", >id_binlog_state}, + {"log_bin_basename", &log_bin_dir}, {NULL, NULL} }; - read_mysql_variables(connection, "SELECT @@GLOBAL.log_bin", log_bin_var, false); - - /* Do not create xtrabackup_binlog_info if binary log is disabled: */ - if (strncmp(log_bin, "1", 2) != 0) { - goto binlog_disabled; - } - - lock_binlog_maybe(connection); - read_mysql_variables(connection, "SHOW MASTER STATUS", status, false); - - /* Do not create xtrabackup_binlog_info if replication - has not started yet: */ - if (filename == NULL || position == NULL) { - goto no_replication; - } - read_mysql_variables(connection, "SHOW VARIABLES", vars, true); gtid_exists = (executed_gtid_set && *executed_gtid_set) || (gtid_binlog_state && *gtid_binlog_state); - if (write_binlogs || gtid_exists) { + if (gtid_exists) { + size_t log_bin_dir_length; + + lock_binlog_maybe(connection); + + xb_mysql_query(connection, "FLUSH BINARY LOGS", false); + + read_mysql_variables(connection, "SHOW MASTER STATUS", + status_after_flush, false); if (opt_log_bin != NULL && strchr(opt_log_bin, FN_LIBCHAR)) { /* If log_bin is set, it has priority */ @@ -1643,88 +1627,33 @@ write_current_binlog_file(MYSQL *connection, bool write_binlogs) log_bin_dir = strdup(opt_log_bin); } else if (log_bin_dir == NULL) { /* Default location is MySQL datadir */ - log_bin_dir = static_cast(malloc(3)); - ut_a(log_bin_dir); - log_bin_dir[0] = '.'; - log_bin_dir[1] = FN_LIBCHAR; - log_bin_dir[2] = 0; + log_bin_dir = strdup("./"); } - size_t log_bin_dir_length; - dirname_part(log_bin_dir, log_bin_dir, &log_bin_dir_length); /* strip final slash if it is not the only path component */ - while (IS_TRAILING_SLASH(log_bin_dir, log_bin_dir_length)) { - log_bin_dir_length--; + if (log_bin_dir_length > 1 && + log_bin_dir[log_bin_dir_length - 1] == FN_LIBCHAR) { + log_bin_dir[log_bin_dir_length - 1] = 0; } - log_bin_dir[log_bin_dir_length] = 0; - if (log_bin_dir == NULL) { - msg("Failed to locate binary log files"); + if (log_bin_dir == NULL || log_bin_file == NULL) { + msg("Failed to get master binlog coordinates from " + "SHOW MASTER STATUS"); result = false; goto cleanup; } - uint max_binlogs; - max_binlogs = opt_max_binlogs; - if (max_binlogs == 0) { - if (gtid_exists) { - max_binlogs = 1; - } else { - goto cleanup; - } - } - - xb_mysql_query(connection, "FLUSH BINARY LOGS", false); - - MYSQL_RES *mysql_result; - - mysql_result = xb_mysql_query(connection, "SHOW BINARY LOGS", true); - - ut_ad(mysql_num_fields(mysql_result) >= 2); - - my_ulonglong n_rows; - my_ulonglong start; - - n_rows = mysql_num_rows(mysql_result); - - start = 0; - if (max_binlogs < n_rows) { - start = n_rows - max_binlogs; - } - if (start) { - mysql_data_seek(mysql_result, start); - } - - MYSQL_ROW row; - while ((row = mysql_fetch_row(mysql_result))) { - const char *binlog_name = row[0]; - char filepath[FN_REFLEN]; - snprintf(filepath, sizeof(filepath), "%s%c%s", - log_bin_dir, FN_LIBCHAR, binlog_name); - if (file_exists(filepath)) { - result = copy_file(ds_data, filepath, binlog_name, 0); - if (!result) break; - } - } - - if (result) { - write_binlog_info(connection, log_bin_dir, - mysql_result, n_rows, start); - } - - mysql_free_result(mysql_result); + snprintf(filepath, sizeof(filepath), "%s%c%s", + log_bin_dir, FN_LIBCHAR, log_bin_file); + result = copy_file(ds_data, filepath, log_bin_file, 0); } cleanup: - free_mysql_variables(vars); - -no_replication: + free_mysql_variables(status_after_flush); free_mysql_variables(status); - -binlog_disabled: - free_mysql_variables(log_bin_var); + free_mysql_variables(vars); return(result); } @@ -1733,11 +1662,8 @@ binlog_disabled: /*********************************************************************//** Retrieves MySQL binlog position and saves it in a file. It also prints it to stdout. */ -static bool -write_binlog_info(MYSQL *connection, char *log_bin_dir, - MYSQL_RES *mysql_result, my_ulonglong n_rows, - my_ulonglong start) +write_binlog_info(MYSQL *connection) { char *filename = NULL; char *position = NULL; @@ -1745,13 +1671,9 @@ write_binlog_info(MYSQL *connection, char *log_bin_dir, char *gtid_current_pos = NULL; char *gtid_executed = NULL; char *gtid = NULL; - char *buffer; - char *buf; - size_t total; - bool result = true; + bool result; bool mysql_gtid; bool mariadb_gtid; - bool with_gtid; mysql_variable status[] = { {"File", &filename}, @@ -1769,106 +1691,39 @@ write_binlog_info(MYSQL *connection, char *log_bin_dir, read_mysql_variables(connection, "SHOW MASTER STATUS", status, false); read_mysql_variables(connection, "SHOW VARIABLES", vars, true); - mysql_gtid = gtid_mode && (strcmp(gtid_mode, "ON") == 0); - mariadb_gtid = gtid_current_pos && *gtid_current_pos; + if (filename == NULL || position == NULL) { + /* Do not create xtrabackup_binlog_info if binary + log is disabled */ + result = true; + goto cleanup; + } + + mysql_gtid = ((gtid_mode != NULL) && (strcmp(gtid_mode, "ON") == 0)); + mariadb_gtid = (gtid_current_pos != NULL); - gtid = (gtid_executed && *gtid_executed) ? gtid_executed : gtid_current_pos; + gtid = (gtid_executed != NULL ? gtid_executed : gtid_current_pos); - with_gtid = mariadb_gtid || mysql_gtid; - if (with_gtid) { + if (mariadb_gtid || mysql_gtid) { ut_a(asprintf(&mysql_binlog_position, "filename '%s', position '%s', " "GTID of the last change '%s'", filename, position, gtid) != -1); + result = backup_file_printf(XTRABACKUP_BINLOG_INFO, + "%s\t%s\t%s\n", filename, position, + gtid); } else { ut_a(asprintf(&mysql_binlog_position, "filename '%s', position '%s'", filename, position) != -1); + result = backup_file_printf(XTRABACKUP_BINLOG_INFO, + "%s\t%s\n", filename, position); } - mysql_data_seek(mysql_result, start); - - MYSQL_ROW row; - my_ulonglong current; - - total = 1; - current = start; - while ((row = mysql_fetch_row(mysql_result))) { - const char *binlog_name = row[0]; - /* The position in the current binlog is taken from - the global variable, but for the previous ones it is - determined by their length: */ - const char *binlog_pos = - ++current == n_rows ? position : row[1]; - total += strlen(binlog_name) + strlen(binlog_pos) + 2; - if (with_gtid && current != n_rows) { - /* Add the "\t[]" length to the buffer size: */ - total += 3; - } - } - /* For the last of the binray log files, also add - the length of the GTID (+ one character for '\t'): */ - if (with_gtid) { - total += strlen(gtid) + 1; - } - - buffer = static_cast(malloc(total)); - if (!buffer) { - msg("Failed to allocate memory for temporary buffer"); - result = false; - goto cleanup; - } - - mysql_data_seek(mysql_result, start); - - buf = buffer; - current = start; - while ((row = mysql_fetch_row(mysql_result))) { - const char *binlog_name = row[0]; - char filepath[FN_REFLEN]; - snprintf(filepath, sizeof(filepath), "%s%c%s", - log_bin_dir, FN_LIBCHAR, binlog_name); - current++; - if (file_exists(filepath)) { - /* The position in the current binlog is taken from - the global variable, but for the previous ones it is - determined by their length: */ - char *binlog_pos = - current == n_rows ? position : row[1]; - int bytes; - if (with_gtid) { - bytes = snprintf(buf, total, "%s\t%s\t%s\n", - binlog_name, binlog_pos, - current == n_rows ? gtid : "[]"); - } else { - bytes = snprintf(buf, total, "%s\t%s\n", - binlog_name, binlog_pos); - } - if (bytes <= 0) { - goto buffer_overflow; - } - buf += bytes; - total -= bytes; - } - } - - if (buf != buffer) { - result = backup_file_printf(XTRABACKUP_BINLOG_INFO, "%s", buffer); - } - -cleanup2: - free(buffer); - cleanup: - free_mysql_variables(vars); free_mysql_variables(status); + free_mysql_variables(vars); return(result); - -buffer_overflow: - msg("Internal error: buffer overflow in the write_binlog_info()"); - result = false; - goto cleanup2; } struct escape_and_quote @@ -2196,6 +2051,7 @@ backup_cleanup() { free(mysql_slave_position); free(mysql_binlog_position); + free(buffer_pool_filename); if (mysql_connection) { mysql_close(mysql_connection); diff --git a/extra/mariabackup/backup_mysql.h b/extra/mariabackup/backup_mysql.h index 5e78281e1cb..b61fa2362c6 100644 --- a/extra/mariabackup/backup_mysql.h +++ b/extra/mariabackup/backup_mysql.h @@ -28,6 +28,7 @@ extern time_t history_lock_time; extern bool sql_thread_started; extern char *mysql_slave_position; extern char *mysql_binlog_position; +extern char *buffer_pool_filename; /** connection to mysql server */ extern MYSQL *mysql_connection; @@ -61,7 +62,10 @@ void unlock_all(MYSQL *connection); bool -write_current_binlog_file(MYSQL *connection, bool write_binlogs); +write_current_binlog_file(MYSQL *connection); + +bool +write_binlog_info(MYSQL *connection); bool write_xtrabackup_info(MYSQL *connection, const char * filename, bool history, diff --git a/extra/mariabackup/common.h b/extra/mariabackup/common.h index beb49524608..1973512ad82 100644 --- a/extra/mariabackup/common.h +++ b/extra/mariabackup/common.h @@ -187,14 +187,4 @@ xb_read_full(File fd, uchar *buf, size_t len) return tlen; } -#ifdef _WIN32 -#define IS_TRAILING_SLASH(name, length) \ - ((length) > 1 && \ - (name[(length) - 1] == '/' || \ - name[(length) - 1] == '\\')) -#else -#define IS_TRAILING_SLASH(name, length) \ - ((length) > 1 && name[(length) - 1] == FN_LIBCHAR) -#endif - #endif diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index ada63e1e882..4680d20bfa7 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -240,8 +240,7 @@ long innobase_log_buffer_size = 1024*1024L; long innobase_open_files = 300L; longlong innobase_page_size = (1LL << 14); /* 16KB */ -char *innobase_buffer_pool_filename = NULL; -char *buffer_pool_filename = NULL; +char* innobase_buffer_pool_filename = NULL; /* The default values for the following char* start-up parameters are determined in innobase_init below: */ @@ -348,7 +347,6 @@ uint opt_lock_wait_timeout = 0; uint opt_lock_wait_threshold = 0; uint opt_debug_sleep_before_unlock = 0; uint opt_safe_slave_backup_timeout = 0; -uint opt_max_binlogs = UINT_MAX; const char *opt_history = NULL; @@ -1053,8 +1051,7 @@ enum options_xtrabackup OPT_BACKUP_ROCKSDB, OPT_XTRA_CHECK_PRIVILEGES, OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION, - OPT_INNODB_FORCE_RECOVERY, - OPT_MAX_BINLOGS + OPT_INNODB_FORCE_RECOVERY }; struct my_option xb_client_options[]= { @@ -1457,17 +1454,6 @@ struct my_option xb_client_options[]= { &opt_log_innodb_page_corruption, &opt_log_innodb_page_corruption, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"sst_max_binlogs", OPT_MAX_BINLOGS, - "Number of recent binary logs to be included in the backup. " - "Setting this parameter to zero normally disables transmission " - "of binary logs to the joiner nodes during SST using Galera. " - "But sometimes a single current binlog can still be transmitted " - "to the joiner even with sst_max_binlogs=0, because it is " - "required for Galera to work properly with GTIDs support.", - (G_PTR *) &opt_max_binlogs, - (G_PTR *) &opt_max_binlogs, 0, GET_UINT, OPT_ARG, - UINT_MAX, 0, UINT_MAX, 0, 1, 0}, - #define MYSQL_CLIENT #include "sslopt-longopts.h" #undef MYSQL_CLIENT @@ -6298,44 +6284,6 @@ check_all_privileges() } } -static -void -xb_init_buffer_pool(const char * filename) -{ - if (filename && -#ifdef _WIN32 - (filename[0] == '/' || - filename[0] == '\\' || - strchr(filename, ':'))) -#else - filename[0] == FN_LIBCHAR) -#endif - { - buffer_pool_filename = strdup(filename); - } else { - char filepath[FN_REFLEN]; - char *dst_dir = - (innobase_data_home_dir && *innobase_data_home_dir) ? - innobase_data_home_dir : mysql_data_home; - size_t dir_length; - if (dst_dir && *dst_dir) { - dir_length = strlen(dst_dir); - while (IS_TRAILING_SLASH(dst_dir, dir_length)) { - dir_length--; - } - memcpy(filepath, dst_dir, dir_length); - } - else { - filepath[0] = '.'; - dir_length = 1; - } - snprintf(filepath + dir_length, - sizeof(filepath) - dir_length, "%c%s", FN_LIBCHAR, - filename ? filename : "ib_buffer_pool"); - buffer_pool_filename = strdup(filepath); - } -} - bool xb_init() { @@ -6400,15 +6348,11 @@ xb_init() if (!get_mysql_vars(mysql_connection)) { return(false); } - xb_init_buffer_pool(buffer_pool_filename); - if (opt_check_privileges) { check_all_privileges(); } history_start_time = time(NULL); - } else { - xb_init_buffer_pool(innobase_buffer_pool_filename); } return(true); @@ -6702,8 +6646,6 @@ int main(int argc, char **argv) free_error_messages(); mysql_mutex_destroy(&LOCK_error_log); - free(buffer_pool_filename); - if (status == EXIT_SUCCESS) { msg("completed OK!"); } diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index dbcd0d83f9b..e2955f58d6a 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -69,7 +69,6 @@ extern char *xtrabackup_incremental_dir; extern char *xtrabackup_incremental_basedir; extern char *innobase_data_home_dir; extern char *innobase_buffer_pool_filename; -extern char *buffer_pool_filename; extern char *xb_plugin_dir; extern char *xb_rocksdb_datadir; extern my_bool xb_backup_rocksdb; @@ -165,7 +164,6 @@ extern uint opt_lock_wait_timeout; extern uint opt_lock_wait_threshold; extern uint opt_debug_sleep_before_unlock; extern uint opt_safe_slave_backup_timeout; -extern uint opt_max_binlogs; extern const char *opt_history; diff --git a/mysql-test/suite/galera/include/galera_wsrep_recover.inc b/mysql-test/suite/galera/include/galera_wsrep_recover.inc index aa2f0e2e777..d2956ea99e6 100644 --- a/mysql-test/suite/galera/include/galera_wsrep_recover.inc +++ b/mysql-test/suite/galera/include/galera_wsrep_recover.inc @@ -1,12 +1,5 @@ --echo Performing --wsrep-recover ... -if ($wsrep_recover_additional) -{ ---exec $MYSQLD --defaults-group-suffix=.$galera_wsrep_recover_server_id --defaults-file=$MYSQLTEST_VARDIR/my.cnf --log-error=$MYSQL_TMP_DIR/galera_wsrep_recover.log --innodb --wsrep-recover $wsrep_recover_additional > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1 -} -if (!$wsrep_recover_additional) -{ --exec $MYSQLD --defaults-group-suffix=.$galera_wsrep_recover_server_id --defaults-file=$MYSQLTEST_VARDIR/my.cnf --log-error=$MYSQL_TMP_DIR/galera_wsrep_recover.log --innodb --wsrep-recover > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1 -} --perl use strict; diff --git a/mysql-test/suite/galera/r/galera_log_bin_ext.result b/mysql-test/suite/galera/r/galera_log_bin_ext.result index c5ff66d17b3..f5276b7d1ac 100644 --- a/mysql-test/suite/galera/r/galera_log_bin_ext.result +++ b/mysql-test/suite/galera/r/galera_log_bin_ext.result @@ -1,6 +1,4 @@ connection node_1; -connection node_2; -connection node_1; reset master; connection node_2; reset master; @@ -42,12 +40,6 @@ hostname1-bin.000001 # Xid # # COMMIT /* XID */ hostname1-bin.000001 # Gtid # # GTID #-#-# hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER connection node_2; -Shutting down server ... -connection node_1; -Cleaning var directory ... -connection node_2; -Starting server ... -connection node_2; SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; COUNT(*) = 2 1 @@ -74,7 +66,6 @@ hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F hostname1-bin.000001 # Xid # # COMMIT /* XID */ hostname1-bin.000001 # Gtid # # GTID #-#-# hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER -hostname1-bin.000001 # Rotate # # hostname1-bin.000002;pos=4 DROP TABLE t1; DROP TABLE t2; #cleanup diff --git a/mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result b/mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result deleted file mode 100644 index c5ff66d17b3..00000000000 --- a/mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result +++ /dev/null @@ -1,82 +0,0 @@ -connection node_1; -connection node_2; -connection node_1; -reset master; -connection node_2; -reset master; -CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB; -INSERT INTO t1 VALUES (1); -CREATE TABLE t2 (id INT) ENGINE=InnoDB; -INSERT INTO t2 VALUES (1); -INSERT INTO t2 VALUES (1); -connection node_2; -SELECT COUNT(*) = 1 FROM t1; -COUNT(*) = 1 -1 -SELECT COUNT(*) = 2 FROM t2; -COUNT(*) = 2 -1 -connection node_1; -ALTER TABLE t1 ADD COLUMN f2 INTEGER; -include/show_binlog_events.inc -Log_name Pos Event_type Server_id End_log_pos Info -hostname1-bin.000001 # Gtid # # GTID #-#-# -hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB -hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# -hostname1-bin.000001 # Annotate_rows # # INSERT INTO t1 VALUES (1) -hostname1-bin.000001 # Table_map # # table_id: # (test.t1) -hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F -hostname1-bin.000001 # Xid # # COMMIT /* XID */ -hostname1-bin.000001 # Gtid # # GTID #-#-# -hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t2 (id INT) ENGINE=InnoDB -hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# -hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) -hostname1-bin.000001 # Table_map # # table_id: # (test.t2) -hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F -hostname1-bin.000001 # Xid # # COMMIT /* XID */ -hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# -hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) -hostname1-bin.000001 # Table_map # # table_id: # (test.t2) -hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F -hostname1-bin.000001 # Xid # # COMMIT /* XID */ -hostname1-bin.000001 # Gtid # # GTID #-#-# -hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER -connection node_2; -Shutting down server ... -connection node_1; -Cleaning var directory ... -connection node_2; -Starting server ... -connection node_2; -SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; -COUNT(*) = 2 -1 -include/show_binlog_events.inc -Log_name Pos Event_type Server_id End_log_pos Info -hostname1-bin.000001 # Gtid # # GTID #-#-# -hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB -hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# -hostname1-bin.000001 # Annotate_rows # # INSERT INTO t1 VALUES (1) -hostname1-bin.000001 # Table_map # # table_id: # (test.t1) -hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F -hostname1-bin.000001 # Xid # # COMMIT /* XID */ -hostname1-bin.000001 # Gtid # # GTID #-#-# -hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t2 (id INT) ENGINE=InnoDB -hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# -hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) -hostname1-bin.000001 # Table_map # # table_id: # (test.t2) -hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F -hostname1-bin.000001 # Xid # # COMMIT /* XID */ -hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# -hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) -hostname1-bin.000001 # Table_map # # table_id: # (test.t2) -hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F -hostname1-bin.000001 # Xid # # COMMIT /* XID */ -hostname1-bin.000001 # Gtid # # GTID #-#-# -hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER -hostname1-bin.000001 # Rotate # # hostname1-bin.000002;pos=4 -DROP TABLE t1; -DROP TABLE t2; -#cleanup -connection node_1; -RESET MASTER; diff --git a/mysql-test/suite/galera/r/galera_sst_rsync,debug.rdiff b/mysql-test/suite/galera/r/galera_sst_rsync,debug.rdiff index 7b10cd23b60..94dd8c2e502 100644 --- a/mysql-test/suite/galera/r/galera_sst_rsync,debug.rdiff +++ b/mysql-test/suite/galera/r/galera_sst_rsync,debug.rdiff @@ -1,6 +1,6 @@ --- galera_sst_rsync.result -+++ galera_sst_rsync.reject -@@ -286,3 +286,111 @@ ++++ galera_sst_rsync,debug.reject +@@ -284,3 +284,111 @@ DROP TABLE t1; COMMIT; SET AUTOCOMMIT=ON; diff --git a/mysql-test/suite/galera/r/galera_sst_rsync2,debug.rdiff b/mysql-test/suite/galera/r/galera_sst_rsync2,debug.rdiff index 23ed9862ea4..525156d88da 100644 --- a/mysql-test/suite/galera/r/galera_sst_rsync2,debug.rdiff +++ b/mysql-test/suite/galera/r/galera_sst_rsync2,debug.rdiff @@ -1,5 +1,5 @@ ---- suite/galera/r/galera_sst_rsync2.result -+++ suite/galera/r/galera_sst_rsync2.reject +--- suite/galera/r/galera_sst_rsync2.result 2018-09-12 13:09:35.352229478 +0200 ++++ suite/galera/r/galera_sst_rsync2,debug.reject 2018-09-12 17:00:51.601974979 +0200 @@ -286,3 +286,111 @@ DROP TABLE t1; COMMIT; diff --git a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2,debug.rdiff b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2,debug.rdiff index 412f8996f15..8b091eb370a 100644 --- a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2,debug.rdiff +++ b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2,debug.rdiff @@ -1,5 +1,5 @@ ---- galera_sst_xtrabackup-v2.result -+++ galera_sst_xtrabackup-v2.reject +--- galera_sst_mariabackup.result ++++ galera_sst_mariabackup,debug.reject @@ -286,5 +286,113 @@ DROP TABLE t1; COMMIT; diff --git a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir,debug.rdiff b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir,debug.rdiff index 1cb0725177b..ac232020037 100644 --- a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir,debug.rdiff +++ b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir,debug.rdiff @@ -1,12 +1,11 @@ ---- galera_sst_xtrabackup-v2_data_dir.result -+++ galera_sst_xtrabackup-v2_data_dir.reject -@@ -286,5 +286,113 @@ +--- r/galera_sst_xtrabackup-v2_data_dir.result 2018-11-19 12:27:24.795221479 +0200 ++++ r/galera_sst_xtrabackup-v2_data_dir.reject 2018-11-19 19:15:38.774008404 +0200 +@@ -260,3 +260,100 @@ DROP TABLE t1; COMMIT; SET AUTOCOMMIT=ON; +Performing State Transfer on a server that has been killed and restarted +while a DDL was in progress on it -+connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; @@ -15,7 +14,6 @@ +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); -+connection node_2; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); @@ -24,12 +22,9 @@ +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +SET GLOBAL debug_dbug = 'd,sync.alter_opened_table'; -+connection node_1; +ALTER TABLE t1 ADD COLUMN f2 INTEGER; -+connection node_2; +SET wsrep_sync_wait = 0; +Killing server ... -+connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node1_committed_during'); @@ -44,7 +39,6 @@ +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); -+connect node_1a_galera_st_kill_slave_ddl, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); @@ -52,9 +46,7 @@ +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); -+connection node_2; +Performing --wsrep-recover ... -+connection node_2; +Starting server ... +Using --wsrep-start-position when starting mysqld ... +SET AUTOCOMMIT=OFF; @@ -65,7 +57,6 @@ +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +COMMIT; -+connection node_1; +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); @@ -80,7 +71,6 @@ +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +COMMIT; -+connection node_1a_galera_st_kill_slave_ddl; +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); @@ -98,7 +88,6 @@ +1 +COMMIT; +SET AUTOCOMMIT=ON; -+connection node_1; +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +COUNT(*) = 2 +1 @@ -112,5 +101,3 @@ +COMMIT; +SET AUTOCOMMIT=ON; +SET GLOBAL debug_dbug = $debug_orig; - disconnect node_2; - disconnect node_1; diff --git a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result index fdb5883b590..ff85a7d6c0f 100644 --- a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result +++ b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result @@ -286,5 +286,3 @@ COUNT(*) = 0 DROP TABLE t1; COMMIT; SET AUTOCOMMIT=ON; -disconnect node_2; -disconnect node_1; diff --git a/mysql-test/suite/galera/r/galera_wan_restart_sst.result b/mysql-test/suite/galera/r/galera_wan_restart_sst.result index 2744c91a06b..1adcbfd1d50 100644 --- a/mysql-test/suite/galera/r/galera_wan_restart_sst.result +++ b/mysql-test/suite/galera/r/galera_wan_restart_sst.result @@ -1,9 +1,3 @@ -connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; -connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4; -connection node_1; -connection node_2; -connection node_3; -connection node_4; SELECT VARIABLE_VALUE AS EXPECT_4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; EXPECT_4 4 @@ -12,8 +6,10 @@ CREATE TABLE t1 (f1 INTEGER); INSERT INTO t1 VALUES (1); connection node_2; INSERT INTO t1 VALUES (2); +connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; connection node_3; INSERT INTO t1 VALUES (3); +connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4; connection node_4; INSERT INTO t1 VALUES (4); connection node_3; @@ -85,6 +81,3 @@ CALL mtr.add_suppression("There are no nodes in the same segment that will ever CALL mtr.add_suppression("Action message in non-primary configuration from member 0"); connection node_4; CALL mtr.add_suppression("Action message in non-primary configuration from member 0"); -connection node_1; -disconnect node_3; -disconnect node_4; diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext.cnf b/mysql-test/suite/galera/t/galera_log_bin_ext.cnf index ae47de90bb8..012209610ea 100644 --- a/mysql-test/suite/galera/t/galera_log_bin_ext.cnf +++ b/mysql-test/suite/galera/t/galera_log_bin_ext.cnf @@ -9,6 +9,3 @@ log-slave-updates log-bin = hostname2-bin log-bin-index = hostname2.bdx log-slave-updates - -[sst] -sst_max_binlogs= diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext.test b/mysql-test/suite/galera/t/galera_log_bin_ext.test index 752073aecdb..923bd623a8a 100644 --- a/mysql-test/suite/galera/t/galera_log_bin_ext.test +++ b/mysql-test/suite/galera/t/galera_log_bin_ext.test @@ -1 +1 @@ ---source galera_log_bin_sst.inc +--source galera_log_bin.inc diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf deleted file mode 100644 index c988136b8fb..00000000000 --- a/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf +++ /dev/null @@ -1,19 +0,0 @@ -!include ../galera_2nodes.cnf - -[mysqld] -wsrep_sst_method=mariabackup -wsrep_sst_auth="root:" - -[mysqld.1] -log-bin=@ENV.MYSQLTEST_VARDIR/mysqld.1/data/hostname1-bin -log-bin-index = hostname1.bdx -log-slave-updates - -[mysqld.2] -log-bin=@ENV.MYSQLTEST_VARDIR/mysqld.2/data/hostname2-bin -log-bin-index = hostname2.bdx -log-slave-updates - -[sst] -transferfmt=@ENV.MTR_GALERA_TFMT -sst_max_binlogs= diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test deleted file mode 100644 index 47df45b4c71..00000000000 --- a/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test +++ /dev/null @@ -1,2 +0,0 @@ ---source include/have_mariabackup.inc ---source galera_log_bin_sst.inc diff --git a/mysql-test/suite/galera/t/galera_log_bin_sst.inc b/mysql-test/suite/galera/t/galera_log_bin_sst.inc deleted file mode 100644 index 5d543f6f8b6..00000000000 --- a/mysql-test/suite/galera/t/galera_log_bin_sst.inc +++ /dev/null @@ -1,84 +0,0 @@ ---source include/galera_cluster.inc ---source include/force_restart.inc - -# Save original auto_increment_offset values. ---let $node_1=node_1 ---let $node_2=node_2 ---source include/auto_increment_offset_save.inc - ---connection node_1 -reset master; ---connection node_2 -reset master; - -# -# Test Galera with --log-bin --log-slave-updates . -# This way the actual MySQL binary log is used, -# rather than Galera's own implementation -# - -CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB; -INSERT INTO t1 VALUES (1); - -CREATE TABLE t2 (id INT) ENGINE=InnoDB; -INSERT INTO t2 VALUES (1); -INSERT INTO t2 VALUES (1); - ---connection node_2 -SELECT COUNT(*) = 1 FROM t1; -SELECT COUNT(*) = 2 FROM t2; - ---connection node_1 -ALTER TABLE t1 ADD COLUMN f2 INTEGER; ---let $MASTER_MYPORT=$NODE_MYPORT_1 ---source include/show_binlog_events.inc - ---connection node_2 - -#--connection node_2 -#--source suite/galera/include/galera_stop_replication.inc - ---echo Shutting down server ... ---source include/shutdown_mysqld.inc - ---connection node_1 ---let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size' ---source include/wait_condition.inc - -# -# Force SST -# ---echo Cleaning var directory ... ---remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat ---remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/mtr ---remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/performance_schema ---remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/test ---remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/mysql ---remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data - ---connection node_2 - ---echo Starting server ... -let $restart_noprint=2; ---source include/start_mysqld.inc - ---let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; ---source include/wait_condition.inc - ---let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready'; ---source include/wait_condition.inc - ---connection node_2 -SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; ---let $MASTER_MYPORT=$NODE_MYPORT_2 ---source include/show_binlog_events.inc - -DROP TABLE t1; -DROP TABLE t2; - ---echo #cleanup ---connection node_1 -RESET MASTER; - -# Restore original auto_increment_offset values. ---source include/auto_increment_offset_restore.inc diff --git a/mysql-test/suite/galera/t/galera_sst_rsync.test b/mysql-test/suite/galera/t/galera_sst_rsync.test index c944c8d84b7..5c08707e870 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync.test +++ b/mysql-test/suite/galera/t/galera_sst_rsync.test @@ -10,5 +10,4 @@ --source suite/galera/include/galera_st_kill_slave.inc --source suite/galera/include/galera_st_kill_slave_ddl.inc - --source include/auto_increment_offset_restore.inc diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf b/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf index 3913ab6660f..4f25af7cd8b 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf +++ b/mysql-test/suite/galera/t/galera_sst_rsync_logbasename.cnf @@ -12,3 +12,4 @@ log_bin wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true' log_basename=server2 log_bin + diff --git a/mysql-test/suite/galera/t/galera_wan_restart_sst.test b/mysql-test/suite/galera/t/galera_wan_restart_sst.test index 2048977b4ef..16e073e7164 100644 --- a/mysql-test/suite/galera/t/galera_wan_restart_sst.test +++ b/mysql-test/suite/galera/t/galera_wan_restart_sst.test @@ -12,16 +12,6 @@ --source include/galera_cluster.inc --source include/force_restart.inc ---connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 ---connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4 - -# Save original auto_increment_offset values. ---let $node_1=node_1 ---let $node_2=node_2 ---let $node_3=node_3 ---let $node_4=node_4 ---source include/auto_increment_offset_save.inc - --let $wait_condition = SELECT VARIABLE_VALUE = 4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; --source include/wait_condition.inc SELECT VARIABLE_VALUE AS EXPECT_4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; @@ -33,9 +23,11 @@ INSERT INTO t1 VALUES (1); --connection node_2 INSERT INTO t1 VALUES (2); +--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 --connection node_3 INSERT INTO t1 VALUES (3); +--connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4 --connection node_4 INSERT INTO t1 VALUES (4); @@ -164,10 +156,3 @@ CALL mtr.add_suppression("Action message in non-primary configuration from membe --connection node_4 CALL mtr.add_suppression("Action message in non-primary configuration from member 0"); - -# Restore original auto_increment_offset values. ---source include/auto_increment_offset_restore.inc - ---connection node_1 ---disconnect node_3 ---disconnect node_4 diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index d4d9a58897d..1b5f49f1191 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2022 MariaDB +# Copyright (C) 2017-2021 MariaDB # Copyright (C) 2012-2015 Codership Oy # # This program is free software; you can redistribute it and/or modify @@ -44,39 +44,6 @@ trim_string() fi } -trim_dir() -{ - local t=$(trim_string "$1") - if [ "$t" != '/' ]; then - if [ "${t%/}" != "$t" ]; then - t=$(trim_string "${t%/}") - fi - else - t='.' - fi - if [ -n "$BASH_VERSION" ]; then - printf '%s' "$t" - else - echo "$t" - fi -} - -to_minuses() -{ - local x="$1" - local t="${1#*_}" - local r="" - while [ "$t" != "$x" ]; do - r="$r${x%%_*}-" - x="$t" - t="${t#*_}" - done - if [ -n "$BASH_VERSION" ]; then - printf '%s' "$r$x" - else - echo "$r$x" - fi -} WSREP_SST_OPT_BYPASS=0 WSREP_SST_OPT_BINLOG="" @@ -99,9 +66,9 @@ WSREP_SST_OPT_ADDR="" WSREP_SST_OPT_ADDR_PORT="" WSREP_SST_OPT_HOST="" WSREP_SST_OPT_HOST_UNESCAPED="" -INNODB_DATA_HOME_DIR=$(trim_dir "${INNODB_DATA_HOME_DIR:-}") -INNODB_LOG_GROUP_HOME=$(trim_dir "${INNODB_LOG_GROUP_HOME:-}") -INNODB_UNDO_DIR=$(trim_dir "${INNODB_UNDO_DIR:-}") +INNODB_DATA_HOME_DIR="${INNODB_DATA_HOME_DIR:-}" +INNODB_LOG_GROUP_HOME="${INNODB_LOG_GROUP_HOME:-}" +INNODB_UNDO_DIR="${INNODB_UNDO_DIR:-}" INNODB_FORCE_RECOVERY="" INNOEXTRA="" @@ -188,22 +155,22 @@ case "$1" in ;; '--datadir') # Let's remove the trailing slash: - readonly WSREP_SST_OPT_DATA=$(trim_dir "$2") + readonly WSREP_SST_OPT_DATA="${2%/}" shift ;; '--innodb-data-home-dir') # Let's remove the trailing slash: - readonly INNODB_DATA_HOME_DIR=$(trim_dir "$2") + readonly INNODB_DATA_HOME_DIR="${2%/}" shift ;; '--innodb-log-group-home-dir') # Let's remove the trailing slash: - readonly INNODB_LOG_GROUP_HOME=$(trim_dir "$2") + readonly INNODB_LOG_GROUP_HOME="${2%/}" shift ;; '--innodb-undo-directory') # Let's remove the trailing slash: - readonly INNODB_UNDO_DIR=$(trim_dir "$2") + readonly INNODB_UNDO_DIR="${2%/}" shift ;; '--defaults-file') @@ -295,7 +262,6 @@ case "$1" in '--mysqld-args') original_cmd="" shift - cmd_tail=0 while [ $# -gt 0 ]; do lname="${1#--}" # "--" is interpreted as the end of the list of options: @@ -310,7 +276,7 @@ case "$1" in shift done fi - break + break; fi # Make sure the argument does not start with "--", otherwise it # is a long option, which is processed after this "if": @@ -350,25 +316,15 @@ case "$1" in if [ "${2#-}" = "$2" ]; then shift value="$1" - elif [ "$2" = '--' ]; then - shift - if [ $# -gt 1 ]; then - cmd_tail=1 - shift - value="$1" - fi fi fi - if [ "$option" = 'h' ]; then + if [ $option = 'h' ]; then if [ -z "$WSREP_SST_OPT_DATA" ]; then - MYSQLD_OPT_DATADIR=$(trim_dir "$value") + MYSQLD_OPT_DATADIR="${value%/}" fi - elif [ "$option" != 'u' -a \ - "$option" != 'P' ] + elif [ $option != 'u' -a \ + $option != 'P' ] then - if [ $cmd_tail -ne 0 ]; then - option="$option --" - fi if [ -z "$value" ]; then slist="$slist$option" elif [ -z "$slist" ]; then @@ -376,16 +332,9 @@ case "$1" in else slist="$slist -$option '$value'" fi - break - fi - if [ $cmd_tail -ne 0 ]; then - if [ -n "$slist" ]; then - slist="$slist --" - else - slist='-' - fi fi break + else slist="$slist$option" fi @@ -395,7 +344,7 @@ case "$1" in original_cmd="$original_cmd -$slist" fi elif [ -z "$options" ]; then - # We found an minus sign without any characters after it: + # We found an equal sign without any characters after it: original_cmd="$original_cmd -" else # We found a value that does not start with a minus - @@ -404,25 +353,12 @@ case "$1" in original_cmd="$original_cmd '$1'" fi shift - if [ $cmd_tail -ne 0 ]; then - # All other arguments must be copied unchanged: - while [ $# -gt 0 ]; do - original_cmd="$original_cmd '$1'" - shift - done - break - fi - continue + continue; fi # Now we are sure that we are working with an option # that has a "long" name, so remove all characters after # the first equal sign: option="${1%%=*}" - # If the option name contains underscores, then replace - # them to minuses: - if [ "${option#*_}" != "$option" ]; then - option=$(to_minuses "$option") - fi # The "--loose-" prefix should not affect the recognition # of the option name: if [ "${option#--loose-}" != "$option" ]; then @@ -449,19 +385,19 @@ case "$1" in case "$option" in '--innodb-data-home-dir') if [ -z "$INNODB_DATA_HOME_DIR" ]; then - MYSQLD_OPT_INNODB_DATA_HOME_DIR=$(trim_dir "$value") + MYSQLD_OPT_INNODB_DATA_HOME_DIR="${value%/}" fi skip_mysqld_arg=1 ;; '--innodb-log-group-home-dir') if [ -z "$INNODB_LOG_GROUP_HOME" ]; then - MYSQLD_OPT_INNODB_LOG_GROUP_HOME=$(trim_dir "$value") + MYSQLD_OPT_INNODB_LOG_GROUP_HOME="${value%/}" fi skip_mysqld_arg=1 ;; '--innodb-undo-directory') if [ -z "$INNODB_UNDO_DIR" ]; then - MYSQLD_OPT_INNODB_UNDO_DIR=$(trim_dir "$value") + MYSQLD_OPT_INNODB_UNDO_DIR="${value%/}" fi skip_mysqld_arg=1 ;; @@ -491,7 +427,7 @@ case "$1" in ;; '--datadir') if [ -z "$WSREP_SST_OPT_DATA" ]; then - MYSQLD_OPT_DATADIR=$(trim_dir "$value") + MYSQLD_OPT_DATADIR="${value%/}" fi skip_mysqld_arg=1 ;; @@ -550,8 +486,8 @@ if [ -z "$WSREP_SST_OPT_BINLOG" -a -n "${MYSQLD_OPT_LOG_BIN+x}" ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then # If the WSREP_SST_OPT_BINLOG variable is not set, but # --log-basename is present among the arguments to mysqld, - # then set WSREP_SST_OPT_BINLOG equal to the base name - # with the "-bin" suffix: + # then set WSREP_SST_OPT_BINLOG equal to the base name with + # the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG="$WSREP_SST_OPT_LOG_BASENAME-bin" else # Take the default name: @@ -604,23 +540,26 @@ get_binlog() WSREP_SST_OPT_BINLOG_INDEX=$(parse_cnf '--mysqld' 'log-bin-index') fi # if no command line argument and WSREP_SST_OPT_LOG_BASENAME is not set, - # then try to get it from my.cnf: + # try to get it from my.cnf: if [ -z "$WSREP_SST_OPT_LOG_BASENAME" ]; then WSREP_SST_OPT_LOG_BASENAME=$(parse_cnf '--mysqld' 'log-basename') fi if [ -z "$WSREP_SST_OPT_BINLOG" ]; then - # If the log-bin option is specified without a parameter, + # If the --log-bin option is specified without a parameter, # then we need to build the name of the index file according # to the rules described in the server documentation: - if [ $(in_config '--mysqld' 'log-bin') -ne 0 ]; then + if [ -n "${MYSQLD_OPT_LOG_BIN+x}" -o \ + $(in_config '--mysqld' 'log-bin') -eq 1 ] + then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then # If the WSREP_SST_OPT_BINLOG variable is not set, but # --log-basename is present among the arguments of mysqld, - # then set WSREP_SST_OPT_BINLOG equal to the base name - # with the "-bin" suffix: + # then set WSREP_SST_OPT_BINLOG equal to the base name with + # the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG="$WSREP_SST_OPT_LOG_BASENAME-bin" else - # Take the default name: + # If the --log-bin option is present without a value, then + # we take the default name: readonly WSREP_SST_OPT_BINLOG='mysql-bin' fi fi @@ -630,13 +569,13 @@ get_binlog() # it according to the specifications for the server: if [ -z "$WSREP_SST_OPT_BINLOG_INDEX" ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then - # If the WSREP_SST_OPT_BINLOG_INDEX variable is not set, but + # If the WSREP_SST_OPT_BINLOG variable is not set, but # --log-basename is present among the arguments of mysqld, - # then set WSREP_SST_OPT_BINLOG_INDEX equal to the base name - # with the "-bin" suffix: + # then set WSREP_SST_OPT_BINLOG equal to the base name with + # the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_LOG_BASENAME-bin.index" else - # Use the default name (note that base of this name + # the default name (note that base of this name # is already defined above): readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_BINLOG.index" fi @@ -699,7 +638,7 @@ commandex() # try to use my_print_defaults, mysql and mysqldump that come # with the sources (for MTR suite): script_binary=$(dirname "$0") -SCRIPTS_DIR=$(cd "$script_binary"; pwd) +SCRIPTS_DIR=$(cd "$script_binary"; pwd -P) EXTRA_DIR="$SCRIPTS_DIR/../extra" CLIENT_DIR="$SCRIPTS_DIR/../client" @@ -826,11 +765,7 @@ parse_cnf() # Truncate spaces: [ -n "$reval" ] && reval=$(trim_string "$reval") - if [ -n "$BASH_VERSION" ]; then - printf '%s' "$reval" - else - echo "$reval" - fi + echo "$reval" } # @@ -883,11 +818,7 @@ in_config() break fi done - if [ -n "$BASH_VERSION" ]; then - printf '%s' $found - else - echo $found - fi + echo $found } wsrep_auth_not_set() @@ -1020,22 +951,11 @@ wsrep_gen_secret() { get_openssl if [ -n "$OPENSSL_BINARY" ]; then - "$OPENSSL_BINARY" rand -hex 16 - elif [ -n "$BASH_VERSION" ]; then - printf '%04x%04x%04x%04x%04x%04x%04x%04x' \ + echo $("$OPENSSL_BINARY" rand -hex 16) + else + printf "%04x%04x%04x%04x%04x%04x%04x%04x" \ $RANDOM $RANDOM $RANDOM $RANDOM \ $RANDOM $RANDOM $RANDOM $RANDOM - elif [ -n "$(commandex cksum)" -a \ - -n "$(commandex printf)" ] - then - printf '%08x%08x%08x%08x' \ - $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ - $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ - $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ - $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) - else - wsrep_log_error "Unable to generate 16-byte secret" - exit 22 fi } @@ -1073,14 +993,14 @@ is_local_ip() if [ -n "$ip_util" ]; then # ip address show ouput format is " inet[6]
/": "$ip_util" address show \ - | grep -E '^[[:space:]]*inet.? [^[:space:]]+/' -o \ + | grep -E "^[[:space:]]*inet.? [^[:space:]]+/" -o \ | grep -F " $1/" >/dev/null && return 0 else local ifconfig_util=$(commandex 'ifconfig') if [ -n "$ifconfig_util" ]; then # ifconfig output format is " inet[6]
...": "$ifconfig_util" \ - | grep -E '^[[:space:]]*inet.? [^[:space:]]+ ' -o \ + | grep -E "^[[:space:]]*inet.? [^[:space:]]+ " -o \ | grep -F " $1 " >/dev/null && return 0 fi fi @@ -1143,7 +1063,7 @@ check_port() ss -nlpH "( sport = :$port )" 2>/dev/null | \ grep -q -E "users:\\(.*\\(\"($utils)[^[:space:]]*\"[^)]*,pid=$pid(,[^)]*)?\\)" && rc=0 else - wsrep_log_error "Unknown sockets utility" + wsrep_log_error "unknown sockets utility" exit 2 # ENOENT fi @@ -1252,6 +1172,13 @@ verify_cert_matches_key() exit 22 fi + # If the diff utility is not installed, then + # we will not do this certificate check: + if [ -z "$(commandex diff)" ]; then + wsrep_log_info "diff utility not found" + return + fi + # If the openssl utility is not installed, then # we will not do this certificate check: get_openssl @@ -1262,9 +1189,9 @@ verify_cert_matches_key() # Generate the public key from the cert and the key. # They should match (otherwise we can't create an SSL connection). - local pk1=$("$OPENSSL_BINARY" x509 -in "$cert" -pubkey -noout 2>/dev/null || :) - local pk2=$("$OPENSSL_BINARY" pkey -in "$key" -pubout 2>/dev/null || :) - if [ "$pk1" != "$pk2" ]; then + if ! diff <("$OPENSSL_BINARY" x509 -in "$cert" -pubkey -noout 2>/dev/null) \ + <("$OPENSSL_BINARY" pkey -in "$key" -pubout 2>/dev/null) >/dev/null 2>&1 + then wsrep_log_error "******************* FATAL ERROR *****************" wsrep_log_error "* The certificate and private key do not match. *" wsrep_log_error "* Please check your certificate and key files. *" @@ -1337,10 +1264,6 @@ check_pid() rm -f "$pid_file" || : fi fi - local config="${3:-}" - if [ -n "$config" -a -f "$config" ]; then - rm -f "$config" || : - fi CHECK_PID=0 return 1 } diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index ce4001fdc56..fd44836e7d3 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -2,7 +2,7 @@ set -ue -# Copyright (C) 2017-2022 MariaDB +# Copyright (C) 2017-2021 MariaDB # Copyright (C) 2013 Percona Inc # # This program is free software; you can redistribute it and/or modify @@ -85,13 +85,13 @@ backup_threads="" encrypt_threads="" encrypt_chunk="" -readonly SECRET_TAG='secret' +readonly SECRET_TAG="secret" # Required for backup locks # For backup locks it is 1 sent by joiner sst_ver=1 -if [ -n "$(commandex pv)" ] && pv --help | grep -qw -F -- '-F'; then +if [ -n "$(commandex pv)" ] && pv --help | grep -qw -- '-F'; then pvopts="$pvopts $pvformat" fi pcmd="pv $pvopts" @@ -104,14 +104,17 @@ if [ -z "$BACKUP_BIN" ]; then fi DATA="$WSREP_SST_OPT_DATA" -INFO_FILE='xtrabackup_galera_info' -IST_FILE='xtrabackup_ist' +INFO_FILE="xtrabackup_galera_info" +IST_FILE="xtrabackup_ist" MAGIC_FILE="$DATA/$INFO_FILE" INNOAPPLYLOG="$DATA/mariabackup.prepare.log" INNOMOVELOG="$DATA/mariabackup.move.log" INNOBACKUPLOG="$DATA/mariabackup.backup.log" +# Setting the path for ss and ip +export PATH="/usr/sbin:/sbin:$PATH" + timeit() { local stage="$1" @@ -151,7 +154,7 @@ get_keys() return fi - if [ "$sfmt" = 'tar' ]; then + if [ $sfmt = 'tar' ]; then wsrep_log_info "NOTE: key-based encryption (encrypt=1)" \ "cannot be enabled with tar format" encrypt=-1 @@ -181,11 +184,11 @@ get_keys() exit 2 fi ecmd="'$OPENSSL_BINARY' enc -$ealgo" - if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-pbkdf2'; then + if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-pbkdf2'; then ecmd="$ecmd -pbkdf2" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-iter'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-iter'; then ecmd="$ecmd -iter 1" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-md'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-md'; then ecmd="$ecmd -md sha256" fi if [ -z "$ekey" ]; then @@ -226,15 +229,15 @@ get_keys() get_transfer() { - if [ "$tfmt" = 'nc' ]; then + if [ $tfmt = 'nc' ]; then wsrep_log_info "Using netcat as streamer" wsrep_check_programs nc - tcmd='nc' + tcmd="nc" if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - if nc -h 2>&1 | grep -q -F 'ncat'; then + if nc -h 2>&1 | grep -q 'ncat'; then wsrep_log_info "Using Ncat as streamer" tcmd="$tcmd -l" - elif nc -h 2>&1 | grep -qw -F -- '-d'; then + elif nc -h 2>&1 | grep -qw -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" tcmd="$tcmd -dl" if [ $WSREP_SST_OPT_HOST_IPv6 -eq 1 ]; then @@ -256,14 +259,14 @@ get_transfer() # transfer and cause the command to timeout. # Older versions of netcat did not need this flag and will # return an error if the flag is used. - if nc -h 2>&1 | grep -qw -F -- '-N'; then + if nc -h 2>&1 | grep -qw -- '-N'; then tcmd="$tcmd -N" wsrep_log_info "Using nc -N" fi # netcat doesn't understand [] around IPv6 address - if nc -h 2>&1 | grep -q -F 'ncat'; then + if nc -h 2>&1 | grep -q ncat; then wsrep_log_info "Using Ncat as streamer" - elif nc -h 2>&1 | grep -qw -F -- '-d'; then + elif nc -h 2>&1 | grep -qw -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" else wsrep_log_info "Using traditional netcat as streamer" @@ -454,7 +457,7 @@ adjust_progress() fi elif [ -z "$progress" -a -n "$rlimit" ]; then # When rlimit is non-zero - pcmd='pv -q' + pcmd="pv -q" fi if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then @@ -590,14 +593,8 @@ get_stream() sig_joiner_cleanup() { - local estatus=$? - if [ $estatus -ne 0 ]; then - wsrep_log_error "Cleanup after exit with status: $estatus" - fi wsrep_log_error "Removing $MAGIC_FILE file due to signal" - [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" - exit $estatus } cleanup_at_exit() @@ -608,8 +605,6 @@ cleanup_at_exit() wsrep_log_error "Cleanup after exit with status: $estatus" fi - [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" - if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then wsrep_log_info "Removing the sst_in_progress file" wsrep_cleanup_progress_file @@ -639,7 +634,7 @@ cleanup_at_exit() fi # Final cleanup - pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o -E '[0-9]*' || :) + pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o '[0-9]*' || :) # This means no setsid done in mysqld. # We don't want to kill mysqld here otherwise. @@ -727,7 +722,7 @@ recv_joiner() local ltcmd="$tcmd" if [ $tmt -gt 0 ]; then if [ -n "$(commandex timeout)" ]; then - if timeout --help | grep -qw -F -- '-k'; then + if timeout --help | grep -qw -- '-k'; then ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" else ltcmd="timeout -s9 $tmt $tcmd" @@ -827,9 +822,7 @@ monitor_process() [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" -if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a \ - "$WSREP_SST_OPT_ROLE" != 'donor' ] -then +if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'" exit 22 fi @@ -837,17 +830,25 @@ fi read_cnf setup_ports -if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -F -- '--version-check'; then +if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then disver=' --no-version-check' fi +# if no command line argument and INNODB_DATA_HOME_DIR environment variable +# is not set, try to get it from my.cnf: +if [ -z "$INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') +fi + OLD_PWD="$(pwd)" -if [ -n "$DATA" -a "$DATA" != '.' ]; then - [ ! -d "$DATA" ] && mkdir -p "$DATA" - cd "$DATA" +cd "$WSREP_SST_OPT_DATA" +if [ -n "$INNODB_DATA_HOME_DIR" ]; then + # handle both relative and absolute paths + [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" + cd "$INNODB_DATA_HOME_DIR" fi -DATA_DIR="$(pwd)" +INNODB_DATA_HOME_DIR=$(pwd -P) cd "$OLD_PWD" @@ -875,7 +876,7 @@ if [ $ssyslog -eq 1 ]; then else if [ $sstlogarchive -eq 1 ] then - ARCHIVETIMESTAMP=$(date '+%Y.%m.%d-%H.%M.%S.%N') + ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N") if [ -n "$sstlogarchivedir" ]; then if [ ! -d "$sstlogarchivedir" ]; then @@ -935,7 +936,7 @@ setup_commands() recovery=" --innodb-force-recovery=$INNODB_FORCE_RECOVERY" fi INNOAPPLY="$BACKUP_BIN --prepare$disver$recovery${iapts:+ }$iapts$INNOEXTRA --target-dir='$DATA' --datadir='$DATA'$mysqld_args $INNOAPPLY" - INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts$INNOEXTRA --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" + INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" INNOBACKUP="$BACKUP_BIN$WSREP_SST_OPT_CONF --backup$disver${iopts:+ }$iopts$tmpopts$INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA'$mysqld_args $INNOBACKUP" } @@ -1058,11 +1059,6 @@ then iopts="--parallel=$backup_threads${iopts:+ }$iopts" fi - max_binlogs=$(parse_cnf "$encgroups" 'sst-max-binlogs') - if [ -n "$max_binlogs" ]; then - iopts="--sst-max-binlogs=$max_binlogs${iopts:+ }$iopts" - fi - setup_commands set +e timeit "$stagemsg-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" @@ -1107,7 +1103,6 @@ then echo "done $WSREP_SST_OPT_GTID" wsrep_log_info "Total time on donor: $totime seconds" - wsrep_log_info "mariabackup SST/IST completed on donor" elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then @@ -1115,53 +1110,22 @@ then wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" - # if no command line argument and INNODB_DATA_HOME_DIR environment - # variable is not set, try to get it from the my.cnf: - if [ -z "$INNODB_DATA_HOME_DIR" ]; then - INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') - INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") - fi - - if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" - cd "$INNODB_DATA_HOME_DIR" - ib_home_dir="$(pwd)" - cd "$OLD_PWD" - fi + ib_home_dir="$INNODB_DATA_HOME_DIR" # if no command line argument and INNODB_LOG_GROUP_HOME is not set, - # then try to get it from the my.cnf: + # try to get it from my.cnf: if [ -z "$INNODB_LOG_GROUP_HOME" ]; then INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') - INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") fi - if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" - cd "$INNODB_LOG_GROUP_HOME" - ib_log_dir="$(pwd)" - cd "$OLD_PWD" - fi + ib_log_dir="$INNODB_LOG_GROUP_HOME" - # if no command line argument and INNODB_UNDO_DIR is not set, - # then try to get it from the my.cnf: + # if no command line argument then try to get it from my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') - INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi - if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" - cd "$INNODB_UNDO_DIR" - ib_undo_dir="$(pwd)" - cd "$OLD_PWD" - fi + ib_undo_dir="$INNODB_UNDO_DIR" if [ -n "$backup_threads" ]; then impts="--parallel=$backup_threads${impts:+ }$impts" @@ -1192,6 +1156,7 @@ then # May need xtrabackup_checkpoints later on [ -f "$DATA/xtrabackup_binary" ] && rm -f "$DATA/xtrabackup_binary" [ -f "$DATA/xtrabackup_galera_info" ] && rm -f "$DATA/xtrabackup_galera_info" + [ -f "$DATA/ib_logfile0" ] && rm -f "$DATA/ib_logfile0" ADDR="$WSREP_SST_OPT_HOST" @@ -1207,7 +1172,7 @@ then exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \ - tr ',' '\n' | grep -F 'CN =' | cut -d '=' -f2 | sed s/^\ // | \ + tr "," "\n" | grep -F 'CN =' | cut -d '=' -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -1261,36 +1226,6 @@ then jpid=$! wsrep_log_info "Proceeding with SST" - get_binlog - - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") - binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") - binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" - cd "$DATA" - wsrep_log_info "Cleaning the old binary logs" - # If there is a file with binlogs state, delete it: - [ -f "$binlog_base.state" ] && rm -fv "$binlog_base.state" 1>&2 - # Clean up the old binlog files and index: - if [ -f "$binlog_index" ]; then - while read bin_file || [ -n "$bin_file" ]; do - rm -fv "$bin_file" 1>&2 || : - done < "$binlog_index" - rm -fv "$binlog_index" 1>&2 - fi - if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \ - -d "$binlog_dir" ] - then - cd "$binlog_dir" - if [ "$(pwd)" != "$DATA_DIR" ]; then - wsrep_log_info \ - "Cleaning the binlog directory '$binlog_dir' as well" - fi - fi - rm -fv "$binlog_base".[0-9]* 1>&2 || : - cd "$OLD_PWD" - fi - wsrep_log_info \ "Cleaning the existing datadir and innodb-data/log directories" if [ "$OS" = 'FreeBSD' ]; then @@ -1307,6 +1242,20 @@ then -o -exec rm -rfv {} 1>&2 \+ fi + get_binlog + + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") + if [ -d "$binlog_dir" ]; then + cd "$binlog_dir" + wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" + rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : + [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ + rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ + cd "$OLD_PWD" + fi + fi + TDATA="$DATA" DATA="$DATA/.sst" @@ -1340,13 +1289,11 @@ then dcmd="xargs -n 2 qpress -dT$nproc" - if [ -n "$progress" ] && \ - pv --help | grep -qw -F -- '--line-mode' - then + if [ -n "$progress" ] && pv --help | grep -qw -- '--line-mode'; then count=$(find "$DATA" -type f -name '*.qp' | wc -l) count=$(( count*2 )) pvopts="-f -s $count -l -N Decompression" - if pv --help | grep -qw -F -- '-F'; then + if pv --help | grep -qw -- '-F'; then pvopts="$pvopts -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" fi pcmd="pv $pvopts" @@ -1356,9 +1303,8 @@ then # Decompress the qpress files wsrep_log_info "Decompression with $nproc threads" - timeit 'Joiner-Decompression' \ - "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | \ - $dcmd" + timeit "Joiner-Decompression" \ + "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" extcode=$? if [ $extcode -eq 0 ]; then @@ -1375,9 +1321,25 @@ then fi fi + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + + BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") + BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") + + # To avoid comparing data directory and BINLOG_DIRNAME + mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || : + + cd "$BINLOG_DIRNAME" + for bfile in $(ls -1 "$BINLOG_FILENAME".[0-9]*); do + echo "$BINLOG_DIRNAME/$bfile" >> "$WSREP_SST_OPT_BINLOG_INDEX" + done + cd "$OLD_PWD" + + fi + wsrep_log_info "Preparing the backup at $DATA" setup_commands - timeit 'mariabackup prepare stage' "$INNOAPPLY" + timeit "mariabackup prepare stage" "$INNOAPPLY" if [ $? -ne 0 ]; then wsrep_log_error "mariabackup apply finished with errors." \ @@ -1385,43 +1347,10 @@ then exit 22 fi - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - cd "$DATA" - binlogs="" - if [ -f 'xtrabackup_binlog_info' ]; then - NL=$'\n' - while read bin_string || [ -n "$bin_string" ]; do - bin_file=$(echo "$bin_string" | cut -f1) - if [ -f "$bin_file" ]; then - binlogs="$binlogs${binlogs:+$NL}$bin_file" - fi - done < 'xtrabackup_binlog_info' - else - binlogs=$(ls -d -1 "$binlog_base".[0-9]* 2>/dev/null || :) - fi - cd "$DATA_DIR" - if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then - [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" - fi - index_dir=$(dirname "$binlog_index"); - if [ -n "$index_dir" -a "$index_dir" != '.' ]; then - [ ! -d "$index_dir" ] && mkdir -p "$index_dir" - fi - if [ -n "$binlogs" ]; then - wsrep_log_info "Moving binary logs to $binlog_dir" - echo "$binlogs" | \ - while read bin_file || [ -n "$bin_file" ]; do - mv "$DATA/$bin_file" "$binlog_dir" - echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" - done - fi - cd "$OLD_PWD" - fi - MAGIC_FILE="$TDATA/$INFO_FILE" wsrep_log_info "Moving the backup to $TDATA" - timeit 'mariabackup move stage' "$INNOMOVE" + timeit "mariabackup move stage" "$INNOMOVE" if [ $? -eq 0 ]; then wsrep_log_info "Move successful, removing $DATA" rm -rf "$DATA" @@ -1448,7 +1377,6 @@ then cat "$MAGIC_FILE" # Output : UUID:seqno wsrep_gtid_domain_id wsrep_log_info "Total time on joiner: $totime seconds" - wsrep_log_info "mariabackup SST/IST completed on joiner" fi exit 0 diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index 3c92f489cb5..1c8fc181328 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -3,7 +3,7 @@ set -ue # Copyright (C) 2009-2015 Codership Oy -# Copyright (C) 2017-2022 MariaDB +# Copyright (C) 2017-2021 MariaDB # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -40,7 +40,8 @@ then fi # Check client version -if ! $MYSQL_CLIENT --version | grep -q -E 'Distrib 10\.[1-9]'; then +if ! $MYSQL_CLIENT --version | grep 'Distrib 10\.[1-9]' >/dev/null +then $MYSQL_CLIENT --version >&2 wsrep_log_error "this operation requires MySQL client version 10.1 or newer" exit $EINVAL @@ -94,7 +95,7 @@ DROP PREPARE stmt;" SET_START_POSITION="SET GLOBAL wsrep_start_position='$WSREP_SST_OPT_GTID';" SET_WSREP_GTID_DOMAIN_ID="" -if [ -n "$WSREP_SST_OPT_GTID_DOMAIN_ID" ]; then +if [ -n $WSREP_SST_OPT_GTID_DOMAIN_ID ]; then SET_WSREP_GTID_DOMAIN_ID=" SET @val = (SELECT GLOBAL_VALUE FROM INFORMATION_SCHEMA.SYSTEM_VARIABLES WHERE VARIABLE_NAME = 'WSREP_GTID_STRICT_MODE' AND GLOBAL_VALUE > 0); SET @stmt = IF (@val IS NOT NULL, 'SET GLOBAL WSREP_GTID_DOMAIN_ID=$WSREP_SST_OPT_GTID_DOMAIN_ID', 'SET @dummy = 0'); diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 67a7afc638f..7365e3f247c 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -2,7 +2,7 @@ set -ue -# Copyright (C) 2017-2022 MariaDB +# Copyright (C) 2017-2021 MariaDB # Copyright (C) 2010-2014 Codership Oy # # This program is free software; you can redistribute it and/or modify @@ -36,8 +36,6 @@ cleanup_joiner() { local failure=0 - [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" - wsrep_log_info "Joiner cleanup: rsync PID=$RSYNC_REAL_PID," \ "stunnel PID=$STUNNEL_REAL_PID" @@ -60,7 +58,6 @@ cleanup_joiner() if [ $failure -eq 0 ]; then if cleanup_pid $RSYNC_REAL_PID "$RSYNC_PID" "$RSYNC_CONF"; then [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" - [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" else wsrep_log_warning "rsync cleanup failed." fi @@ -143,77 +140,66 @@ STUNNEL_PID="$WSREP_SST_OPT_DATA/stunnel.pid" MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete" +BINLOG_TAR_FILE="$WSREP_SST_OPT_DATA/wsrep_sst_binlog.tar" +BINLOG_N_FILES=1 + get_binlog if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") - binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") + BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") + BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") fi -OLD_PWD="$(pwd)" - -DATA="$WSREP_SST_OPT_DATA" -if [ -n "$DATA" -a "$DATA" != '.' ]; then - [ ! -d "$DATA" ] && mkdir -p "$DATA" - cd "$DATA" -fi -DATA_DIR="$(pwd)" - -cd "$OLD_PWD" - -BINLOG_TAR_FILE="$DATA_DIR/wsrep_sst_binlog.tar" - -ib_log_dir="$DATA_DIR" -ib_home_dir="$DATA_DIR" -ib_undo_dir="$DATA_DIR" - # if no command line argument and INNODB_LOG_GROUP_HOME is not set, -# then try to get it from the my.cnf: +# try to get it from my.cnf: if [ -z "$INNODB_LOG_GROUP_HOME" ]; then INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') - INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") fi -if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" - cd "$INNODB_LOG_GROUP_HOME" - ib_log_dir="$(pwd)" - cd "$OLD_PWD" +OLD_PWD="$(pwd)" + +WSREP_LOG_DIR="$INNODB_LOG_GROUP_HOME" + +cd "$WSREP_SST_OPT_DATA" +if [ -n "$WSREP_LOG_DIR" ]; then + # handle both relative and absolute paths + [ ! -d "$WSREP_LOG_DIR" ] && mkdir -p "$WSREP_LOG_DIR" + cd "$WSREP_LOG_DIR" fi +WSREP_LOG_DIR=$(pwd -P) -# if no command line argument and INNODB_DATA_HOME_DIR environment -# variable is not set, try to get it from the my.cnf: +cd "$OLD_PWD" + +# if no command line argument and INNODB_DATA_HOME_DIR environment variable +# is not set, try to get it from my.cnf: if [ -z "$INNODB_DATA_HOME_DIR" ]; then INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') - INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") fi -if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then - # handle both relative and absolute paths: - cd "$DATA" +cd "$WSREP_SST_OPT_DATA" +if [ -n "$INNODB_DATA_HOME_DIR" ]; then + # handle both relative and absolute paths [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" cd "$INNODB_DATA_HOME_DIR" - ib_home_dir="$(pwd)" - cd "$OLD_PWD" fi +INNODB_DATA_HOME_DIR=$(pwd -P) + +cd "$OLD_PWD" -# if no command line argument and INNODB_UNDO_DIR is not set, -# then try to get it from the my.cnf: +# if no command line argument then try to get it from my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') - INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi -if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then - # handle both relative and absolute paths: - cd "$DATA" +cd "$WSREP_SST_OPT_DATA" +if [ -n "$INNODB_UNDO_DIR" ]; then + # handle both relative and absolute paths [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" cd "$INNODB_UNDO_DIR" - ib_undo_dir="$(pwd)" - cd "$OLD_PWD" fi +INNODB_UNDO_DIR=$(pwd -P) + +cd "$OLD_PWD" encgroups='--mysqld|sst' @@ -292,7 +278,7 @@ if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then CHECK_OPT="checkHost = $WSREP_SST_OPT_HOST" fi if is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then - CHECK_OPT_LOCAL='checkHost = localhost' + CHECK_OPT_LOCAL="checkHost = localhost" fi fi fi @@ -309,59 +295,14 @@ if [ -n "$SSLMODE" -a "$SSLMODE" != 'DISABLED' ]; then fi fi -readonly SECRET_TAG='secret' +readonly SECRET_TAG="secret" -SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid" - -# give some time for previous SST to complete: -check_round=0 -while check_pid "$SST_PID" 0; do - wsrep_log_info "Previous SST is not completed, waiting for it to exit" - check_round=$(( check_round + 1 )) - if [ $check_round -eq 20 ]; then - wsrep_log_error "previous SST script still running." - exit 114 # EALREADY - fi - sleep 1 -done - -echo $$ > "$SST_PID" - -# give some time for stunnel from the previous SST to complete: -check_round=0 -while check_pid "$STUNNEL_PID" 1 "$STUNNEL_CONF"; do - wsrep_log_info "Lingering stunnel daemon found at startup," \ - "waiting for it to exit" - check_round=$(( check_round + 1 )) - if [ $check_round -eq 10 ]; then - wsrep_log_error "stunnel daemon still running." - exit 114 # EALREADY - fi - sleep 1 -done - -MODULE="${WSREP_SST_OPT_MODULE:-rsync_sst}" - -RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid" -RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf" - -# give some time for rsync from the previous SST to complete: -check_round=0 -while check_pid "$RSYNC_PID" 1 "$RSYNC_CONF"; do - wsrep_log_info "Lingering rsync daemon found at startup," \ - "waiting for it to exit" - check_round=$(( check_round + 1 )) - if [ $check_round -eq 10 ]; then - wsrep_log_error "rsync daemon still running." - exit 114 # EALREADY - fi - sleep 1 -done - -[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" -[ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" +if [ "$WSREP_SST_OPT_ROLE" = 'donor' ] +then -if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then + [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" + [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" + [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID" if [ -n "$STUNNEL" ] then @@ -380,6 +321,8 @@ ${VERIFY_OPT} ${CHECK_OPT} ${CHECK_OPT_LOCAL} EOF + else + [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" fi RC=0 @@ -392,7 +335,7 @@ EOF [ -f "$FLUSHED" ] && rm -f "$FLUSHED" [ -f "$ERROR" ] && rm -f "$ERROR" - echo 'flush tables' + echo "flush tables" # Wait for : # (a) Tables to be flushed, AND @@ -416,100 +359,32 @@ EOF sync - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - # Change the directory to binlog base (if possible): - cd "$DATA" - # Let's check the existence of the file with the index: - if [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ]; then - # Let's read the binlog index: - max_binlogs=$(parse_cnf "$encgroups" 'sst-max-binlogs') - if [ -n "$max_binlogs" ]; then - binlog_files="" - if [ $max_binlogs -gt 0 ]; then - binlog_files=$(tail -n $max_binlogs \ - "$WSREP_SST_OPT_BINLOG_INDEX") - fi - else - binlog_files=$(cat "$WSREP_SST_OPT_BINLOG_INDEX") - fi - if [ -n "$binlog_files" ]; then - # Preparing binlog files for transfer: - wsrep_log_info "Preparing binlog files for transfer:" - tar_type=0 - if tar --help | grep -qw -F -- '--transform'; then - tar_type=1 - elif tar --version | grep -q -E '^bsdtar\>'; then - tar_type=2 - fi - if [ $tar_type -ne 2 ]; then - if [ -n "$BASH_VERSION" ]; then - printf '%s' "$binlog_files" >&2 - else - echo "$binlog_files" >&2 - fi - fi - if [ $tar_type -ne 0 ]; then - # Preparing list of the binlog file names: - echo "$binlog_files" | { - binlogs="" - while read bin_file || [ -n "$bin_file" ]; do - [ ! -f "$bin_file" ] && continue - if [ -n "$BASH_VERSION" ]; then - first="${bin_file:0:1}" - else - first=$(echo "$bin_file" | cut -c1) - fi - if [ "$first" = '-' -o "$first" = '@' ]; then - bin_file="./$bin_file" - fi - binlogs="$binlogs${binlogs:+ }'$bin_file'" - done - if [ -n "$binlogs" ]; then - if [ $tar_type -eq 1 ]; then - tar_options="--transform='s/^.*\///g'" - else - # bsdtar handles backslash incorrectly: - tar_options="-s '?^.*/??g'" - fi - eval tar -P $tar_options \ - -cvf "'$BINLOG_TAR_FILE'" $binlogs >&2 - fi - } - else - tar_options='-cvf' - echo "$binlog_files" | \ - while read bin_file || [ -n "$bin_file" ]; do - [ ! -f "$bin_file" ] && continue - bin_dir=$(dirname "$bin_file") - bin_base=$(basename "$bin_file") - if [ -n "$BASH_VERSION" ]; then - first="${bin_base:0:1}" - else - first=$(echo "$bin_base" | cut -c1) - fi - if [ "$first" = '-' -o "$first" = '@' ]; then - bin_base="./$bin_base" - fi - if [ -n "$bin_dir" -a "$bin_dir" != '.' ]; then - tar $tar_options "$BINLOG_TAR_FILE" \ - -C "$bin_dir" "$bin_base" >&2 - else - tar $tar_options "$BINLOG_TAR_FILE" \ - "$bin_base" >&2 - fi - tar_options='-rvf' - done - fi - fi + if [ -n "$WSREP_SST_OPT_BINLOG" -a -d "${BINLOG_DIRNAME:-}" ] + then + # Prepare binlog files + cd "$BINLOG_DIRNAME" + + binlog_files_full=$(tail -n $BINLOG_N_FILES \ + "$WSREP_SST_OPT_BINLOG_INDEX") + binlog_files="" + for file in $binlog_files_full; do + binlog_file=$(basename "$file") + binlog_files="$binlog_files${binlog_files:+ }'$binlog_file'" + done + + if [ -n "$binlog_files" ]; then + wsrep_log_info "Preparing binlog files for transfer:" + eval tar -cvf "'$BINLOG_TAR_FILE'" $binlog_files >&2 fi + cd "$OLD_PWD" fi - # Use deltaxfer only for WAN: + # Use deltaxfer only for WAN inv=$(basename "$0") WHOLE_FILE_OPT="" if [ "${inv%wsrep_sst_rsync_wan*}" = "$inv" ]; then - WHOLE_FILE_OPT='--whole-file' + WHOLE_FILE_OPT="--whole-file" fi # Old filter - include everything except selected @@ -526,9 +401,9 @@ FILTER="-f '- /lost+found' -f '- /.pid' -f '- /.conf' -f '+ /wsrep_sst_binlog.tar' - -f '- $ib_home_dir/ib_lru_dump' - -f '- $ib_home_dir/ibdata*' - -f '+ $ib_undo_dir/undo*' + -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump' + -f '- $INNODB_DATA_HOME_DIR/ibdata*' + -f '+ $INNODB_UNDO_DIR/undo*' -f '+ /*/' -f '- /*'" @@ -562,7 +437,7 @@ FILTER="-f '- /lost+found' --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT -f '+ /ibdata*' -f '+ /ib_lru_dump' \ - -f '- **' "$ib_home_dir/" \ + -f '- **' "$INNODB_DATA_HOME_DIR/" \ "rsync://$WSREP_SST_OPT_ADDR-data_dir" >&2 || RC=$? if [ $RC -ne 0 ]; then @@ -575,7 +450,7 @@ FILTER="-f '- /lost+found' --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' -f '+ /aria_log.*' \ - -f '+ /aria_log_control' -f '- **' "$ib_log_dir/" \ + -f '+ /aria_log_control' -f '- **' "$WSREP_LOG_DIR/" \ "rsync://$WSREP_SST_OPT_ADDR-log_dir" >&2 || RC=$? if [ $RC -ne 0 ]; then @@ -586,7 +461,7 @@ FILTER="-f '- /lost+found' # then, we parallelize the transfer of database directories, # use '.' so that path concatenation works: - cd "$DATA" + cd "$WSREP_SST_OPT_DATA" backup_threads=$(parse_cnf '--mysqld|sst' 'backup-threads') if [ -z "$backup_threads" ]; then @@ -645,21 +520,69 @@ FILTER="-f '- /lost+found' [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID" fi - [ -f "$SST_PID" ] && rm -f "$SST_PID" - - wsrep_log_info "rsync SST/IST completed on donor" - elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then check_sockets_utils + SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid" + + # give some time for previous SST to complete: + check_round=0 + while check_pid "$SST_PID" 0 'wsrep_sst_'; do + wsrep_log_info "previous SST is not completed, waiting for it to exit" + check_round=$(( check_round + 1 )) + if [ $check_round -eq 10 ]; then + wsrep_log_error "previous SST script still running." + exit 114 # EALREADY + fi + sleep 1 + done + + echo $$ > "$SST_PID" + + # give some time for stunnel from the previous SST to complete: + check_round=0 + while check_pid "$STUNNEL_PID" 1; do + wsrep_log_info "Lingering stunnel daemon found at startup," \ + "waiting for it to exit" + check_round=$(( check_round + 1 )) + if [ $check_round -eq 10 ]; then + wsrep_log_error "stunnel daemon already running." + exit 114 # EALREADY + fi + sleep 1 + done + + MODULE="${WSREP_SST_OPT_MODULE:-rsync_sst}" + + RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid" + RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf" + + # give some time for rsync from the previous SST to complete: + check_round=0 + while check_pid "$RSYNC_PID" 1; do + wsrep_log_info "Lingering rsync daemon found at startup," \ + "waiting for it to exit" + check_round=$(( check_round + 1 )) + if [ $check_round -eq 10 ]; then + wsrep_log_error "rsync daemon already running." + exit 114 # EALREADY + fi + sleep 1 + done + + [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" + [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" + + [ -z "$STUNNEL" -a -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" + ADDR="$WSREP_SST_OPT_HOST" RSYNC_PORT="$WSREP_SST_OPT_PORT" RSYNC_ADDR="$WSREP_SST_OPT_HOST" RSYNC_ADDR_UNESCAPED="$WSREP_SST_OPT_HOST_UNESCAPED" - trap 'exit 32' HUP PIPE - trap 'exit 3' INT TERM ABRT + trap "exit 32" HUP PIPE + trap "exit 3" INT TERM ABRT trap cleanup_joiner EXIT touch "$SST_PROGRESS_FILE" @@ -680,11 +603,13 @@ $SILENT path = $WSREP_SST_OPT_DATA exclude = .zfs [$MODULE-log_dir] - path = $ib_log_dir + path = $WSREP_LOG_DIR [$MODULE-data_dir] - path = $ib_home_dir + path = $INNODB_DATA_HOME_DIR EOF +# rm -rf "$DATA/ib_logfile"* # we don't want old logs around + # If the IP is local, listen only on it: if is_local_ip "$RSYNC_ADDR_UNESCAPED" then @@ -695,7 +620,7 @@ EOF RSYNC_EXTRA_ARGS="" STUNNEL_ACCEPT="$RSYNC_PORT" # Overwrite address with all: - RSYNC_ADDR='*' + RSYNC_ADDR="*" fi if [ -z "$STUNNEL" ]; then @@ -753,10 +678,11 @@ EOF TRANSFER_PID="$STUNNEL_PID" fi - if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then - # backward-incompatible behavior: + if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ] + then # backward-incompatible behavior: CN="" - if [ -n "$SSTCERT" ]; then + if [ -n "$SSTCERT" ] + then # find out my Common Name get_openssl if [ -z "$OPENSSL_BINARY" ]; then @@ -765,7 +691,7 @@ EOF exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$SSTCERT" | \ - tr ',' '\n' | grep -F 'CN =' | cut -d '=' -f2 | sed s/^\ // | \ + tr "," "\n" | grep -F 'CN =' | cut -d '=' -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -801,53 +727,16 @@ EOF exit 32 fi - if [ -r "$MAGIC_FILE" ]; then - if [ -n "$MY_SECRET" ]; then - # Check donor supplied secret: - SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ - cut -d ' ' -f2) - if [ "$SECRET" != "$MY_SECRET" ]; then - wsrep_log_error "Donor does not know my secret!" - wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" - exit 32 - fi - fi - else - # This message should cause joiner to abort: - wsrep_log_info "rsync process ended without creating magic file" - echo "rsync process ended without creating '$MAGIC_FILE'" - exit 32 - fi - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - binlog_tar_present=0 - [ -f "$BINLOG_TAR_FILE" ] && binlog_tar_present=1 - # If it is SST (not an IST) or tar with binlogs is present - # among the transferred files, then we need to remove the - # old binlogs: - if [ $WSREP_SST_OPT_BYPASS -eq 0 -o $binlog_tar_present -ne 0 ]; then - cd "$DATA" - # Clean up the old binlog files and index: + if [ -f "$BINLOG_TAR_FILE" ]; then + cd "$BINLOG_DIRNAME" + binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" - if [ -f "$binlog_index" ]; then - while read bin_file || [ -n "$bin_file" ]; do - rm -f "$bin_file" || : - done < "$binlog_index" - rm -f "$binlog_index" - fi - binlog_cd=0 - # Change the directory to binlog base (if possible): - if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \ - -d "$binlog_dir" ] - then - binlog_cd=1 - cd "$binlog_dir" - fi - # Clean up unindexed binlog files: - rm -f "$binlog_base".[0-9]* || : - [ $binlog_cd -ne 0 ] && cd "$DATA_DIR" - fi - if [ $binlog_tar_present -ne 0 ]; then + + # Clean up old binlog files first + rm -f "$BINLOG_FILENAME".[0-9]* + [ -f "$binlog_index" ] && rm -f "$binlog_index" + # Create a temporary file: tmpdir=$(parse_cnf '--mysqld|sst' 'tmpdir') if [ -z "$tmpdir" ]; then @@ -857,52 +746,46 @@ EOF else tmpfile=$(TMPDIR="$tmpdir"; mktemp) fi - index_dir=$(dirname "$binlog_index"); - if [ -n "$index_dir" -a "$index_dir" != '.' ]; then - [ ! -d "$index_dir" ] && mkdir -p "$index_dir" - fi - binlog_cd=0 - if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then - [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" - binlog_cd=1 - cd "$binlog_dir" - fi - # Extracting binlog files: + wsrep_log_info "Extracting binlog files:" - RC=0 - if tar --version | grep -q -E '^bsdtar\>'; then - tar -tf "$BINLOG_TAR_FILE" > "$tmpfile" && \ - tar -xvf "$BINLOG_TAR_FILE" > /dev/null || RC=$? - else - tar -xvf "$BINLOG_TAR_FILE" > "$tmpfile" && \ - cat "$tmpfile" >&2 || RC=$? - fi - if [ $RC -ne 0 ]; then - rm -f "$tmpfile" + if ! tar -xvf "$BINLOG_TAR_FILE" > "$tmpfile"; then wsrep_log_error "Error unpacking tar file with binlog files" + rm -f "$tmpfile" exit 32 fi + # Rebuild binlog index: - [ $binlog_cd -ne 0 ] && cd "$DATA_DIR" - while read bin_file || [ -n "$bin_file" ]; do - echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" + while read bin_file; do + echo "$BINLOG_DIRNAME/$bin_file" >> "$binlog_index" done < "$tmpfile" rm -f "$tmpfile" + cd "$OLD_PWD" fi fi - if [ -n "$MY_SECRET" ]; then - # remove secret from the magic file, and output - # the UUID:seqno & wsrep_gtid_domain_id: - grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" + if [ -r "$MAGIC_FILE" ]; then + if [ -n "$MY_SECRET" ]; then + # check donor supplied secret + SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ + cut -d ' ' -f 2) + if [ "$SECRET" != "$MY_SECRET" ]; then + wsrep_log_error "Donor does not know my secret!" + wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" + exit 32 + fi + # remove secret from the magic file, and output + # the UUID:seqno & wsrep_gtid_domain_id: + grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" + else + # Output the UUID:seqno and wsrep_gtid_domain_id: + cat "$MAGIC_FILE" + fi else - # Output the UUID:seqno and wsrep_gtid_domain_id: - cat "$MAGIC_FILE" + # this message should cause joiner to abort + echo "rsync process ended without creating '$MAGIC_FILE'" fi - wsrep_log_info "rsync SST/IST completed on joiner" - # wsrep_cleanup_progress_file # cleanup_joiner else diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index 747378aced8..6e2edb114e0 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -1,8 +1,5 @@ -#!/usr/bin/env bash - -set -ue - -# Copyright (C) 2017-2022 MariaDB +#!/bin/bash -ue +# Copyright (C) 2017-2021 MariaDB # Copyright (C) 2013 Percona Inc # # This program is free software; you can redistribute it and/or modify @@ -88,11 +85,11 @@ backup_threads="" encrypt_threads="" encrypt_chunk="" -readonly SECRET_TAG='secret' +readonly SECRET_TAG="secret" sst_ver=-1 -if [ -n "$(commandex pv)" ] && pv --help | grep -qw -F -- '-F'; then +if [ -n "$(commandex pv)" ] && pv --help | grep -qw -- '-F'; then pvopts="$pvopts $pvformat" fi pcmd="pv $pvopts" @@ -105,14 +102,17 @@ if [ -z "$BACKUP_BIN" ]; then fi DATA="$WSREP_SST_OPT_DATA" -INFO_FILE='xtrabackup_galera_info' -IST_FILE='xtrabackup_ist' +INFO_FILE="xtrabackup_galera_info" +IST_FILE="xtrabackup_ist" MAGIC_FILE="$DATA/$INFO_FILE" INNOAPPLYLOG="$DATA/innobackupex.prepare.log" INNOMOVELOG="$DATA/innobackupex.move.log" INNOBACKUPLOG="$DATA/innobackupex.backup.log" +# Setting the path for ss and ip +export PATH="/usr/sbin:/sbin:$PATH" + timeit() { local stage="$1" @@ -152,7 +152,7 @@ get_keys() return fi - if [ "$sfmt" = 'tar' ]; then + if [ $sfmt = 'tar' ]; then wsrep_log_info "NOTE: key-based encryption (encrypt=1)" \ "cannot be enabled with tar format" encrypt=-1 @@ -182,11 +182,11 @@ get_keys() exit 2 fi ecmd="'$OPENSSL_BINARY' enc -$ealgo" - if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-pbkdf2'; then + if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-pbkdf2'; then ecmd="$ecmd -pbkdf2" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-iter'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-iter'; then ecmd="$ecmd -iter 1" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-md'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-md'; then ecmd="$ecmd -md sha256" fi if [ -z "$ekey" ]; then @@ -231,15 +231,15 @@ get_keys() get_transfer() { - if [ "$tfmt" = 'nc' ]; then + if [ $tfmt = 'nc' ]; then wsrep_log_info "Using netcat as streamer" wsrep_check_programs nc - tcmd='nc' + tcmd="nc" if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - if nc -h 2>&1 | grep -q -F 'ncat'; then + if nc -h 2>&1 | grep -q 'ncat'; then wsrep_log_info "Using Ncat as streamer" tcmd="$tcmd -l" - elif nc -h 2>&1 | grep -qw -F -- '-d'; then + elif nc -h 2>&1 | grep -qw -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" tcmd="$tcmd -dl" if [ $WSREP_SST_OPT_HOST_IPv6 -eq 1 ]; then @@ -261,14 +261,14 @@ get_transfer() # transfer and cause the command to timeout. # Older versions of netcat did not need this flag and will # return an error if the flag is used. - if nc -h 2>&1 | grep -qw -F -- '-N'; then + if nc -h 2>&1 | grep -qw -- '-N'; then tcmd="$tcmd -N" wsrep_log_info "Using nc -N" fi # netcat doesn't understand [] around IPv6 address - if nc -h 2>&1 | grep -q -F 'ncat'; then + if nc -h 2>&1 | grep -q ncat; then wsrep_log_info "Using Ncat as streamer" - elif nc -h 2>&1 | grep -qw -F -- '-d'; then + elif nc -h 2>&1 | grep -qw -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" else wsrep_log_info "Using traditional netcat as streamer" @@ -459,7 +459,7 @@ adjust_progress() fi elif [ -z "$progress" -a -n "$rlimit" ]; then # When rlimit is non-zero - pcmd='pv -q' + pcmd="pv -q" fi if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then @@ -602,14 +602,8 @@ get_stream() sig_joiner_cleanup() { - local estatus=$? - if [ $estatus -ne 0 ]; then - wsrep_log_error "Cleanup after exit with status: $estatus" - fi wsrep_log_error "Removing $MAGIC_FILE file due to signal" - [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" - exit $estatus } cleanup_at_exit() @@ -620,8 +614,6 @@ cleanup_at_exit() wsrep_log_error "Cleanup after exit with status: $estatus" fi - [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" - if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then wsrep_log_info "Removing the sst_in_progress file" wsrep_cleanup_progress_file @@ -651,7 +643,7 @@ cleanup_at_exit() fi # Final cleanup - pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o -E '[0-9]*' || :) + pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o '[0-9]*' || :) # This means no setsid done in mysqld. # We don't want to kill mysqld here otherwise. @@ -739,7 +731,7 @@ recv_joiner() local ltcmd="$tcmd" if [ $tmt -gt 0 ]; then if [ -n "$(commandex timeout)" ]; then - if timeout --help | grep -qw -F -- '-k'; then + if timeout --help | grep -qw -- '-k'; then ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" else ltcmd="timeout -s9 $tmt $tcmd" @@ -839,7 +831,7 @@ monitor_process() # check the version, we require XB-2.3.5 to ensure that we can pass the # datadir via the command-line option -XB_REQUIRED_VERSION='2.3.5' +XB_REQUIRED_VERSION="2.3.5" XB_VERSION=$($BACKUP_BIN --version 2>&1 | \ grep -m1 -owE '[0-9]+(\.[0-9]+)+' | head -n1) @@ -859,9 +851,7 @@ fi [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" -if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a \ - "$WSREP_SST_OPT_ROLE" != 'donor' ] -then +if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'" exit 22 fi @@ -869,17 +859,25 @@ fi read_cnf setup_ports -if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -F -- '--version-check'; then +if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then disver=' --no-version-check' fi +# if no command line argument and INNODB_DATA_HOME_DIR environment variable +# is not set, try to get it from my.cnf: +if [ -z "$INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') +fi + OLD_PWD="$(pwd)" -if [ -n "$DATA" -a "$DATA" != '.' ]; then - [ ! -d "$DATA" ] && mkdir -p "$DATA" - cd "$DATA" +cd "$WSREP_SST_OPT_DATA" +if [ -n "$INNODB_DATA_HOME_DIR" ]; then + # handle both relative and absolute paths + [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" + cd "$INNODB_DATA_HOME_DIR" fi -DATA_DIR="$(pwd)" +INNODB_DATA_HOME_DIR=$(pwd -P) cd "$OLD_PWD" @@ -907,7 +905,7 @@ if [ $ssyslog -eq 1 ]; then else if [ $sstlogarchive -eq 1 ] then - ARCHIVETIMESTAMP=$(date '+%Y.%m.%d-%H.%M.%S.%N') + ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N") if [ -n "$sstlogarchivedir" ]; then if [ ! -d "$sstlogarchivedir" ]; then @@ -963,7 +961,7 @@ setup_commands() recovery=" --innodb-force-recovery=$INNODB_FORCE_RECOVERY" fi INNOAPPLY="$BACKUP_BIN$disver$recovery${iapts:+ }$iapts$INNOEXTRA --apply-log${rebuildcmd:+ }$rebuildcmd --datadir='$DATA' '$DATA' $INNOAPPLY" - INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts$INNOEXTRA --force-non-empty-directories --datadir='${TDATA:-$DATA}' '$DATA' $INNOMOVE" + INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts --force-non-empty-directories --datadir='${TDATA:-$DATA}' '$DATA' $INNOMOVE" local sfmt_work="$sfmt" if [ "$sfmt" = 'mbstream' ]; then sfmt_work='xbstream' @@ -1134,7 +1132,6 @@ then echo "done $WSREP_SST_OPT_GTID" wsrep_log_info "Total time on donor: $totime seconds" - wsrep_log_info "xtrabackup SST/IST completed on donor" elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then @@ -1142,53 +1139,22 @@ then wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" - # if no command line argument and INNODB_DATA_HOME_DIR environment - # variable is not set, try to get it from the my.cnf: - if [ -z "$INNODB_DATA_HOME_DIR" ]; then - INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') - INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") - fi - - if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" - cd "$INNODB_DATA_HOME_DIR" - ib_home_dir="$(pwd)" - cd "$OLD_PWD" - fi + ib_home_dir="$INNODB_DATA_HOME_DIR" # if no command line argument and INNODB_LOG_GROUP_HOME is not set, - # then try to get it from the my.cnf: + # try to get it from my.cnf: if [ -z "$INNODB_LOG_GROUP_HOME" ]; then INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') - INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") fi - if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" - cd "$INNODB_LOG_GROUP_HOME" - ib_log_dir="$(pwd)" - cd "$OLD_PWD" - fi + ib_log_dir="$INNODB_LOG_GROUP_HOME" - # if no command line argument and INNODB_UNDO_DIR is not set, - # then try to get it from the my.cnf: + # if no command line argument then try to get it from my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') - INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi - if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then - # handle both relative and absolute paths: - cd "$DATA" - [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" - cd "$INNODB_UNDO_DIR" - ib_undo_dir="$(pwd)" - cd "$OLD_PWD" - fi + ib_undo_dir="$INNODB_UNDO_DIR" if [ -n "$backup_threads" ]; then impts="--parallel=$backup_threads${impts:+ }$impts" @@ -1234,7 +1200,7 @@ then exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \ - tr ',' '\n' | grep -F 'CN =' | cut -d '=' -f2 | sed s/^\ // | \ + tr "," "\n" | grep -F 'CN =' | cut -d '=' -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -1288,36 +1254,6 @@ then jpid=$! wsrep_log_info "Proceeding with SST" - get_binlog - - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") - binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") - binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" - cd "$DATA" - wsrep_log_info "Cleaning the old binary logs" - # If there is a file with binlogs state, delete it: - [ -f "$binlog_base.state" ] && rm -fv "$binlog_base.state" 1>&2 - # Clean up the old binlog files and index: - if [ -f "$binlog_index" ]; then - while read bin_file || [ -n "$bin_file" ]; do - rm -fv "$bin_file" 1>&2 || : - done < "$binlog_index" - rm -fv "$binlog_index" 1>&2 - fi - if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \ - -d "$binlog_dir" ] - then - cd "$binlog_dir" - if [ "$(pwd)" != "$DATA_DIR" ]; then - wsrep_log_info \ - "Cleaning the binlog directory '$binlog_dir' as well" - fi - fi - rm -fv "$binlog_base".[0-9]* 1>&2 || : - cd "$OLD_PWD" - fi - wsrep_log_info \ "Cleaning the existing datadir and innodb-data/log directories" if [ "$OS" = 'FreeBSD' ]; then @@ -1334,6 +1270,20 @@ then -o -exec rm -rfv {} 1>&2 \+ fi + get_binlog + + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") + if [ -d "$binlog_dir" ]; then + cd "$binlog_dir" + wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" + rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : + [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ + rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ + cd "$OLD_PWD" + fi + fi + TDATA="$DATA" DATA="$DATA/.sst" @@ -1369,13 +1319,11 @@ then dcmd="xargs -n 2 qpress -dT$nproc" - if [ -n "$progress" ] && \ - pv --help | grep -qw -F -- '--line-mode' - then + if [ -n "$progress" ] && pv --help | grep -qw -- '--line-mode'; then count=$(find "$DATA" -type f -name '*.qp' | wc -l) count=$(( count*2 )) pvopts="-f -s $count -l -N Decompression" - if pv --help | grep -qw -F -- '-F'; then + if pv --help | grep -qw -- '-F'; then pvopts="$pvopts -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" fi pcmd="pv $pvopts" @@ -1385,9 +1333,8 @@ then # Decompress the qpress files wsrep_log_info "Decompression with $nproc threads" - timeit 'Joiner-Decompression' \ - "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | \ - $dcmd" + timeit "Joiner-Decompression" \ + "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" extcode=$? if [ $extcode -eq 0 ]; then @@ -1404,9 +1351,25 @@ then fi fi + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + + BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") + BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") + + # To avoid comparing data directory and BINLOG_DIRNAME + mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || : + + cd "$BINLOG_DIRNAME" + for bfile in $(ls -1 "$BINLOG_FILENAME".[0-9]*); do + echo "$BINLOG_DIRNAME/$bfile" >> "$WSREP_SST_OPT_BINLOG_INDEX" + done + cd "$OLD_PWD" + + fi + wsrep_log_info "Preparing the backup at $DATA" setup_commands - timeit 'Xtrabackup prepare stage' "$INNOAPPLY" + timeit "Xtrabackup prepare stage" "$INNOAPPLY" if [ $? -ne 0 ]; then wsrep_log_error "xtrabackup apply finished with errors." \ @@ -1414,43 +1377,10 @@ then exit 22 fi - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - cd "$DATA" - binlogs="" - if [ -f 'xtrabackup_binlog_info' ]; then - NL=$'\n' - while read bin_string || [ -n "$bin_string" ]; do - bin_file=$(echo "$bin_string" | cut -f1) - if [ -f "$bin_file" ]; then - binlogs="$binlogs${binlogs:+$NL}$bin_file" - fi - done < 'xtrabackup_binlog_info' - else - binlogs=$(ls -d -1 "$binlog_base".[0-9]* 2>/dev/null || :) - fi - cd "$DATA_DIR" - if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then - [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" - fi - index_dir=$(dirname "$binlog_index"); - if [ -n "$index_dir" -a "$index_dir" != '.' ]; then - [ ! -d "$index_dir" ] && mkdir -p "$index_dir" - fi - if [ -n "$binlogs" ]; then - wsrep_log_info "Moving binary logs to $binlog_dir" - echo "$binlogs" | \ - while read bin_file || [ -n "$bin_file" ]; do - mv "$DATA/$bin_file" "$binlog_dir" - echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" - done - fi - cd "$OLD_PWD" - fi - MAGIC_FILE="$TDATA/$INFO_FILE" wsrep_log_info "Moving the backup to $TDATA" - timeit 'Xtrabackup move stage' "$INNOMOVE" + timeit "Xtrabackup move stage" "$INNOMOVE" if [ $? -eq 0 ]; then wsrep_log_info "Move successful, removing $DATA" rm -rf "$DATA" @@ -1477,7 +1407,6 @@ then cat "$MAGIC_FILE" # Output : UUID:seqno wsrep_gtid_domain_id wsrep_log_info "Total time on joiner: $totime seconds" - wsrep_log_info "xtrabackup SST/IST completed on joiner" fi exit 0 -- cgit v1.2.1 From 74068dd2ac9e2cc221e226d6412825f2d4f3dd40 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 18 May 2022 01:20:23 +0200 Subject: fix tests for embedded followup for c9b5a05341d7 --- mysql-test/suite/compat/oracle/r/events.result | 9 +++++++++ mysql-test/suite/compat/oracle/r/sp.result | 7 ------- mysql-test/suite/compat/oracle/t/events.test | 10 +++++++++- mysql-test/suite/compat/oracle/t/sp.test | 7 ------- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/mysql-test/suite/compat/oracle/r/events.result b/mysql-test/suite/compat/oracle/r/events.result index ef9c50115ae..1f62e2e5ee2 100644 --- a/mysql-test/suite/compat/oracle/r/events.result +++ b/mysql-test/suite/compat/oracle/r/events.result @@ -16,3 +16,12 @@ COUNT(*) 1 DROP TABLE t1; SET GLOBAL event_scheduler=off; +# +# MDEV-28588 SIGSEGV in __memmove_avx_unaligned_erms, strmake_root +# +CREATE EVENT ev ON SCHEDULE AT CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO BEGIN END; +Warnings: +Warning 1105 Event scheduler is switched off, use SET GLOBAL event_scheduler=ON to enable it. +SELECT EVENT_DEFINITION FROM INFORMATION_SCHEMA.EVENTS WHERE EVENT_SCHEMA='test' AND EVENT_NAME='ev'; +EVENT_DEFINITION BEGIN END +DROP EVENT ev; diff --git a/mysql-test/suite/compat/oracle/r/sp.result b/mysql-test/suite/compat/oracle/r/sp.result index 5756db4fc5b..e11e0404dd1 100644 --- a/mysql-test/suite/compat/oracle/r/sp.result +++ b/mysql-test/suite/compat/oracle/r/sp.result @@ -2565,13 +2565,6 @@ DROP PROCEDURE p1; # SET sql_mode=ORACLE; BEGIN END; -SET sql_mode=ORACLE; -CREATE EVENT ev ON SCHEDULE AT CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO BEGIN END; -Warnings: -Warning 1105 Event scheduler is switched off, use SET GLOBAL event_scheduler=ON to enable it. -SELECT EVENT_DEFINITION FROM INFORMATION_SCHEMA.EVENTS WHERE EVENT_SCHEMA='test' AND EVENT_NAME='ev'; -EVENT_DEFINITION BEGIN END -DROP EVENT ev; CREATE TABLE t1 (a INT); CREATE TRIGGER tr AFTER INSERT ON t1 FOR EACH ROW BEGIN END; SELECT ACTION_STATEMENT FROM INFORMATION_SCHEMA.TRIGGERS WHERE TRIGGER_SCHEMA='test' AND TRIGGER_NAME='tr'; diff --git a/mysql-test/suite/compat/oracle/t/events.test b/mysql-test/suite/compat/oracle/t/events.test index e898528636a..fb56af51d87 100644 --- a/mysql-test/suite/compat/oracle/t/events.test +++ b/mysql-test/suite/compat/oracle/t/events.test @@ -25,5 +25,13 @@ let $wait_condition = SELECT COUNT(*) FROM t1; DROP TABLE t1; - SET GLOBAL event_scheduler=off; + +--echo # +--echo # MDEV-28588 SIGSEGV in __memmove_avx_unaligned_erms, strmake_root +--echo # +CREATE EVENT ev ON SCHEDULE AT CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO BEGIN END; +--vertical_results +SELECT EVENT_DEFINITION FROM INFORMATION_SCHEMA.EVENTS WHERE EVENT_SCHEMA='test' AND EVENT_NAME='ev'; +--horizontal_results +DROP EVENT ev; diff --git a/mysql-test/suite/compat/oracle/t/sp.test b/mysql-test/suite/compat/oracle/t/sp.test index ca99739533f..945968d9512 100644 --- a/mysql-test/suite/compat/oracle/t/sp.test +++ b/mysql-test/suite/compat/oracle/t/sp.test @@ -2406,13 +2406,6 @@ DROP PROCEDURE p1; SET sql_mode=ORACLE; BEGIN END; -SET sql_mode=ORACLE; -CREATE EVENT ev ON SCHEDULE AT CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO BEGIN END; ---vertical_results -SELECT EVENT_DEFINITION FROM INFORMATION_SCHEMA.EVENTS WHERE EVENT_SCHEMA='test' AND EVENT_NAME='ev'; ---horizontal_results -DROP EVENT ev; - CREATE TABLE t1 (a INT); CREATE TRIGGER tr AFTER INSERT ON t1 FOR EACH ROW BEGIN END; --vertical_results -- cgit v1.2.1 From 8609254f4f299cac7f047cd170ae09af17feed0b Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 12 May 2022 13:23:24 +0200 Subject: cleanup:have_log_bin.inc prefer if/skip over require (works better with debugging, not affected by query log) --- mysql-test/include/have_log_bin.inc | 7 +++---- mysql-test/include/have_log_bin.require | 2 -- 2 files changed, 3 insertions(+), 6 deletions(-) delete mode 100644 mysql-test/include/have_log_bin.require diff --git a/mysql-test/include/have_log_bin.inc b/mysql-test/include/have_log_bin.inc index eb2529dae3b..fd5dc66e416 100644 --- a/mysql-test/include/have_log_bin.inc +++ b/mysql-test/include/have_log_bin.inc @@ -8,7 +8,6 @@ source include/not_embedded.inc; --- require include/have_log_bin.require -disable_query_log; -show variables like 'log_bin'; -enable_query_log; +if (`select not @@log_bin`) { + skip Test requires: 'have_log_bin'; +} diff --git a/mysql-test/include/have_log_bin.require b/mysql-test/include/have_log_bin.require deleted file mode 100644 index d4fd77e4f8d..00000000000 --- a/mysql-test/include/have_log_bin.require +++ /dev/null @@ -1,2 +0,0 @@ -Variable_name Value -log_bin ON -- cgit v1.2.1 From 107623c5c57fa81fd2942ba43ac3677f32ae230c Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Thu, 12 May 2022 20:19:33 +0300 Subject: MDEV-28552 Assertion `inited==RND' failed in handler::ha_rnd_end We cannot permanently change bits in read_partitions in the middle of processing because ha_rnd_init()/ha_rnd_end() depends on that. --- mysql-test/suite/versioning/r/partition.result | 12 ++++++++++++ mysql-test/suite/versioning/t/partition.test | 18 ++++++++++++++++++ sql/ha_partition.h | 1 - sql/partition_info.cc | 2 -- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/versioning/r/partition.result b/mysql-test/suite/versioning/r/partition.result index a5bc8284f8c..eae7d276fde 100644 --- a/mysql-test/suite/versioning/r/partition.result +++ b/mysql-test/suite/versioning/r/partition.result @@ -1133,5 +1133,17 @@ select count(*) from t1 partition (p1); count(*) 100 drop table t1; +# +# MDEV-28552 Assertion `inited==RND' failed in handler::ha_rnd_end +# +create table tcount (c int unsigned); +insert into tcount values (0); +create table t (f int) with system versioning +partition by system_time limit 1000 +(partition p1 history, partition pn current); +insert into t values (1),(2); +create trigger tr before insert on t for each row update tcount set c = c + 1; +insert into t select * from t; +drop table tcount, t; # End of 10.3 tests set global innodb_stats_persistent= @save_persistent; diff --git a/mysql-test/suite/versioning/t/partition.test b/mysql-test/suite/versioning/t/partition.test index f65f544147f..b18493ae91c 100644 --- a/mysql-test/suite/versioning/t/partition.test +++ b/mysql-test/suite/versioning/t/partition.test @@ -1080,6 +1080,24 @@ replace t1 select * from t1; select count(*) from t1 partition (p0); select count(*) from t1 partition (p1); drop table t1; + +--echo # +--echo # MDEV-28552 Assertion `inited==RND' failed in handler::ha_rnd_end +--echo # +create table tcount (c int unsigned); +insert into tcount values (0); + +create table t (f int) with system versioning +partition by system_time limit 1000 +(partition p1 history, partition pn current); +insert into t values (1),(2); +create trigger tr before insert on t for each row update tcount set c = c + 1; + +insert into t select * from t; + +# cleanup +drop table tcount, t; + --echo # End of 10.3 tests set global innodb_stats_persistent= @save_persistent; diff --git a/sql/ha_partition.h b/sql/ha_partition.h index f5c12318b2d..e85c61b839d 100644 --- a/sql/ha_partition.h +++ b/sql/ha_partition.h @@ -1608,7 +1608,6 @@ public: for (; part_id < part_id_end; ++part_id) { handler *file= m_file[part_id]; - DBUG_ASSERT(bitmap_is_set(&(m_part_info->read_partitions), part_id)); file->info(HA_STATUS_VARIABLE | HA_STATUS_NO_LOCK | HA_STATUS_OPEN); part_recs+= file->stats.records; } diff --git a/sql/partition_info.cc b/sql/partition_info.cc index 90ef388f3b9..fb55091f05e 100644 --- a/sql/partition_info.cc +++ b/sql/partition_info.cc @@ -904,8 +904,6 @@ void partition_info::vers_check_limit(THD *thd) uint32 part_id= vers_info->hist_part->id * sub_factor; const uint32 part_id_end= part_id + sub_factor; DBUG_ASSERT(part_id_end <= num_parts * sub_factor); - for (; part_id < part_id_end; ++part_id) - bitmap_set_bit(&read_partitions, part_id); ha_partition *hp= (ha_partition*)(table->file); ha_rows hist_rows= hp->part_records(vers_info->hist_part); -- cgit v1.2.1 From b081ad8c65d3a94210841477cb5f0683ce64a7e3 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Fri, 6 May 2022 02:38:36 +0200 Subject: MDEV-28423: Galera IST is failing on Joiner node This commit fixes an issue with IST handling in version 10.9 which is a regression after MDEV-26971 and related to trying to get a non-existent "total" tag on the IST branch (this tag is only defined in SST mode). --- .../galera/r/galera_ist_MDEV-28423,debug.rdiff | 190 ++++++++ .../suite/galera/r/galera_ist_MDEV-28423.result | 519 +++++++++++++++++++++ .../suite/galera/t/galera_ist_MDEV-28423.cnf | 44 ++ .../suite/galera/t/galera_ist_MDEV-28423.test | 18 + scripts/wsrep_sst_mariabackup.sh | 22 +- scripts/wsrep_sst_rsync.sh | 33 +- 6 files changed, 799 insertions(+), 27 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_ist_MDEV-28423,debug.rdiff create mode 100644 mysql-test/suite/galera/r/galera_ist_MDEV-28423.result create mode 100644 mysql-test/suite/galera/t/galera_ist_MDEV-28423.cnf create mode 100644 mysql-test/suite/galera/t/galera_ist_MDEV-28423.test diff --git a/mysql-test/suite/galera/r/galera_ist_MDEV-28423,debug.rdiff b/mysql-test/suite/galera/r/galera_ist_MDEV-28423,debug.rdiff new file mode 100644 index 00000000000..8c84321e774 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_ist_MDEV-28423,debug.rdiff @@ -0,0 +1,190 @@ +--- r/galera_ist_MDEV-28423.result ++++ r/galera_ist_MDEV-28423,debug.reject +@@ -517,3 +517,187 @@ + 1 + DROP TABLE t1; + COMMIT; ++Performing State Transfer on a server that has been killed and restarted ++while a DDL was in progress on it ++connection node_1; ++CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB; ++SET AUTOCOMMIT=OFF; ++START TRANSACTION; ++INSERT INTO t1 VALUES (1,'node1_committed_before'); ++INSERT INTO t1 VALUES (2,'node1_committed_before'); ++INSERT INTO t1 VALUES (3,'node1_committed_before'); ++INSERT INTO t1 VALUES (4,'node1_committed_before'); ++INSERT INTO t1 VALUES (5,'node1_committed_before'); ++connection node_2; ++START TRANSACTION; ++INSERT INTO t1 VALUES (6,'node2_committed_before'); ++INSERT INTO t1 VALUES (7,'node2_committed_before'); ++INSERT INTO t1 VALUES (8,'node2_committed_before'); ++INSERT INTO t1 VALUES (9,'node2_committed_before'); ++INSERT INTO t1 VALUES (10,'node2_committed_before'); ++COMMIT; ++SET GLOBAL debug_dbug = 'd,sync.alter_opened_table'; ++connection node_1; ++ALTER TABLE t1 ADD COLUMN f2 INTEGER; ++connection node_2; ++SET wsrep_sync_wait = 0; ++Killing server ... ++connection node_1; ++SET AUTOCOMMIT=OFF; ++START TRANSACTION; ++INSERT INTO t1 (id,f1) VALUES (11,'node1_committed_during'); ++INSERT INTO t1 (id,f1) VALUES (12,'node1_committed_during'); ++INSERT INTO t1 (id,f1) VALUES (13,'node1_committed_during'); ++INSERT INTO t1 (id,f1) VALUES (14,'node1_committed_during'); ++INSERT INTO t1 (id,f1) VALUES (15,'node1_committed_during'); ++COMMIT; ++START TRANSACTION; ++INSERT INTO t1 (id,f1) VALUES (16,'node1_to_be_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (17,'node1_to_be_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (18,'node1_to_be_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (19,'node1_to_be_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (20,'node1_to_be_committed_after'); ++connect node_1a_galera_st_kill_slave_ddl, 127.0.0.1, root, , test, $NODE_MYPORT_1; ++SET AUTOCOMMIT=OFF; ++START TRANSACTION; ++INSERT INTO t1 (id,f1) VALUES (21,'node1_to_be_rollbacked_after'); ++INSERT INTO t1 (id,f1) VALUES (22,'node1_to_be_rollbacked_after'); ++INSERT INTO t1 (id,f1) VALUES (23,'node1_to_be_rollbacked_after'); ++INSERT INTO t1 (id,f1) VALUES (24,'node1_to_be_rollbacked_after'); ++INSERT INTO t1 (id,f1) VALUES (25,'node1_to_be_rollbacked_after'); ++connection node_2; ++Performing --wsrep-recover ... ++connection node_2; ++Starting server ... ++Using --wsrep-start-position when starting mysqld ... ++SET AUTOCOMMIT=OFF; ++START TRANSACTION; ++INSERT INTO t1 (id,f1) VALUES (26,'node2_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (27,'node2_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (28,'node2_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (29,'node2_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (30,'node2_committed_after'); ++COMMIT; ++connection node_1; ++INSERT INTO t1 (id,f1) VALUES (31,'node1_to_be_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (32,'node1_to_be_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (33,'node1_to_be_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (34,'node1_to_be_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (35,'node1_to_be_committed_after'); ++COMMIT; ++SET AUTOCOMMIT=OFF; ++START TRANSACTION; ++INSERT INTO t1 (id,f1) VALUES (36,'node1_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (37,'node1_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (38,'node1_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (39,'node1_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (40,'node1_committed_after'); ++COMMIT; ++connection node_1a_galera_st_kill_slave_ddl; ++INSERT INTO t1 (id,f1) VALUES (41,'node1_to_be_rollbacked_after'); ++INSERT INTO t1 (id,f1) VALUES (42,'node1_to_be_rollbacked_after'); ++INSERT INTO t1 (id,f1) VALUES (43,'node1_to_be_rollbacked_after'); ++INSERT INTO t1 (id,f1) VALUES (44,'node1_to_be_rollbacked_after'); ++INSERT INTO t1 (id,f1) VALUES (45,'node1_to_be_rollbacked_after'); ++ROLLBACK; ++SET AUTOCOMMIT=ON; ++SET SESSION wsrep_sync_wait=15; ++SELECT COUNT(*) AS EXPECT_3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; ++EXPECT_3 ++3 ++SELECT COUNT(*) AS EXPECT_35 FROM t1; ++EXPECT_35 ++35 ++SELECT * FROM t1; ++id f1 f2 ++1 node1_committed_before NULL ++2 node1_committed_before NULL ++3 node1_committed_before NULL ++4 node1_committed_before NULL ++5 node1_committed_before NULL ++6 node2_committed_before NULL ++7 node2_committed_before NULL ++8 node2_committed_before NULL ++9 node2_committed_before NULL ++10 node2_committed_before NULL ++11 node1_committed_during NULL ++12 node1_committed_during NULL ++13 node1_committed_during NULL ++14 node1_committed_during NULL ++15 node1_committed_during NULL ++16 node1_to_be_committed_after NULL ++17 node1_to_be_committed_after NULL ++18 node1_to_be_committed_after NULL ++19 node1_to_be_committed_after NULL ++20 node1_to_be_committed_after NULL ++26 node2_committed_after NULL ++27 node2_committed_after NULL ++28 node2_committed_after NULL ++29 node2_committed_after NULL ++30 node2_committed_after NULL ++31 node1_to_be_committed_after NULL ++32 node1_to_be_committed_after NULL ++33 node1_to_be_committed_after NULL ++34 node1_to_be_committed_after NULL ++35 node1_to_be_committed_after NULL ++36 node1_committed_after NULL ++37 node1_committed_after NULL ++38 node1_committed_after NULL ++39 node1_committed_after NULL ++40 node1_committed_after NULL ++SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; ++COUNT(*) = 0 ++1 ++COMMIT; ++connection node_1; ++SET AUTOCOMMIT=ON; ++SET SESSION wsrep_sync_wait=15; ++SELECT COUNT(*) AS EXPECT_3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; ++EXPECT_3 ++3 ++SELECT COUNT(*) AS EXPECT_35 FROM t1; ++EXPECT_35 ++35 ++SELECT * FROM t1; ++id f1 f2 ++1 node1_committed_before NULL ++2 node1_committed_before NULL ++3 node1_committed_before NULL ++4 node1_committed_before NULL ++5 node1_committed_before NULL ++6 node2_committed_before NULL ++7 node2_committed_before NULL ++8 node2_committed_before NULL ++9 node2_committed_before NULL ++10 node2_committed_before NULL ++11 node1_committed_during NULL ++12 node1_committed_during NULL ++13 node1_committed_during NULL ++14 node1_committed_during NULL ++15 node1_committed_during NULL ++16 node1_to_be_committed_after NULL ++17 node1_to_be_committed_after NULL ++18 node1_to_be_committed_after NULL ++19 node1_to_be_committed_after NULL ++20 node1_to_be_committed_after NULL ++26 node2_committed_after NULL ++27 node2_committed_after NULL ++28 node2_committed_after NULL ++29 node2_committed_after NULL ++30 node2_committed_after NULL ++31 node1_to_be_committed_after NULL ++32 node1_to_be_committed_after NULL ++33 node1_to_be_committed_after NULL ++34 node1_to_be_committed_after NULL ++35 node1_to_be_committed_after NULL ++36 node1_committed_after NULL ++37 node1_committed_after NULL ++38 node1_committed_after NULL ++39 node1_committed_after NULL ++40 node1_committed_after NULL ++SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; ++COUNT(*) = 0 ++1 ++DROP TABLE t1; ++COMMIT; ++SET GLOBAL debug_dbug = $debug_orig; diff --git a/mysql-test/suite/galera/r/galera_ist_MDEV-28423.result b/mysql-test/suite/galera/r/galera_ist_MDEV-28423.result new file mode 100644 index 00000000000..5a71b490a80 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_ist_MDEV-28423.result @@ -0,0 +1,519 @@ +connection node_2; +connection node_1; +connection node_1; +connection node_2; +Performing State Transfer on a server that has been temporarily disconnected +connection node_1; +CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (1,'node1_committed_before'); +INSERT INTO t1 VALUES (2,'node1_committed_before'); +INSERT INTO t1 VALUES (3,'node1_committed_before'); +INSERT INTO t1 VALUES (4,'node1_committed_before'); +INSERT INTO t1 VALUES (5,'node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (6,'node2_committed_before'); +INSERT INTO t1 VALUES (7,'node2_committed_before'); +INSERT INTO t1 VALUES (8,'node2_committed_before'); +INSERT INTO t1 VALUES (9,'node2_committed_before'); +INSERT INTO t1 VALUES (10,'node2_committed_before'); +COMMIT; +Unloading wsrep provider ... +SET GLOBAL wsrep_cluster_address = ''; +connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (11,'node1_committed_during'); +INSERT INTO t1 VALUES (12,'node1_committed_during'); +INSERT INTO t1 VALUES (13,'node1_committed_during'); +INSERT INTO t1 VALUES (14,'node1_committed_during'); +INSERT INTO t1 VALUES (15,'node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES (16,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (17,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (18,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (19,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (20,'node1_to_be_committed_after'); +connect node_1a_galera_st_disconnect_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after'); +connection node_2; +Loading wsrep provider ... +disconnect node_2; +connect node_2, 127.0.0.1, root, , test, $NODE_MYPORT_2; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (26,'node2_committed_after'); +INSERT INTO t1 VALUES (27,'node2_committed_after'); +INSERT INTO t1 VALUES (28,'node2_committed_after'); +INSERT INTO t1 VALUES (29,'node2_committed_after'); +INSERT INTO t1 VALUES (30,'node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES (31,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (32,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (33,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (34,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (35,'node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (36,'node1_committed_after'); +INSERT INTO t1 VALUES (37,'node1_committed_after'); +INSERT INTO t1 VALUES (38,'node1_committed_after'); +INSERT INTO t1 VALUES (39,'node1_committed_after'); +INSERT INTO t1 VALUES (40,'node1_committed_after'); +COMMIT; +connection node_1a_galera_st_disconnect_slave; +INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (44,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after'); +ROLLBACK; +SET AUTOCOMMIT=ON; +SET SESSION wsrep_sync_wait=15; +SELECT COUNT(*) AS EXPECT_35 FROM t1; +EXPECT_35 +35 +SELECT * FROM t1; +id f1 +1 node1_committed_before +2 node1_committed_before +3 node1_committed_before +4 node1_committed_before +5 node1_committed_before +6 node2_committed_before +7 node2_committed_before +8 node2_committed_before +9 node2_committed_before +10 node2_committed_before +11 node1_committed_during +12 node1_committed_during +13 node1_committed_during +14 node1_committed_during +15 node1_committed_during +16 node1_to_be_committed_after +17 node1_to_be_committed_after +18 node1_to_be_committed_after +19 node1_to_be_committed_after +20 node1_to_be_committed_after +26 node2_committed_after +27 node2_committed_after +28 node2_committed_after +29 node2_committed_after +30 node2_committed_after +31 node1_to_be_committed_after +32 node1_to_be_committed_after +33 node1_to_be_committed_after +34 node1_to_be_committed_after +35 node1_to_be_committed_after +36 node1_committed_after +37 node1_committed_after +38 node1_committed_after +39 node1_committed_after +40 node1_committed_after +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +connection node_1; +SET AUTOCOMMIT=ON; +SET SESSION wsrep_sync_wait=15; +SELECT COUNT(*) AS EXPECT_35 FROM t1; +EXPECT_35 +35 +SELECT * FROM t1; +id f1 +1 node1_committed_before +2 node1_committed_before +3 node1_committed_before +4 node1_committed_before +5 node1_committed_before +6 node2_committed_before +7 node2_committed_before +8 node2_committed_before +9 node2_committed_before +10 node2_committed_before +11 node1_committed_during +12 node1_committed_during +13 node1_committed_during +14 node1_committed_during +15 node1_committed_during +16 node1_to_be_committed_after +17 node1_to_be_committed_after +18 node1_to_be_committed_after +19 node1_to_be_committed_after +20 node1_to_be_committed_after +26 node2_committed_after +27 node2_committed_after +28 node2_committed_after +29 node2_committed_after +30 node2_committed_after +31 node1_to_be_committed_after +32 node1_to_be_committed_after +33 node1_to_be_committed_after +34 node1_to_be_committed_after +35 node1_to_be_committed_after +36 node1_committed_after +37 node1_committed_after +38 node1_committed_after +39 node1_committed_after +40 node1_committed_after +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +Performing State Transfer on a server that has been shut down cleanly and restarted +connection node_1; +CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (1,'node1_committed_before'); +INSERT INTO t1 VALUES (2,'node1_committed_before'); +INSERT INTO t1 VALUES (3,'node1_committed_before'); +INSERT INTO t1 VALUES (4,'node1_committed_before'); +INSERT INTO t1 VALUES (5,'node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (6,'node2_committed_before'); +INSERT INTO t1 VALUES (7,'node2_committed_before'); +INSERT INTO t1 VALUES (8,'node2_committed_before'); +INSERT INTO t1 VALUES (9,'node2_committed_before'); +INSERT INTO t1 VALUES (10,'node2_committed_before'); +COMMIT; +Shutting down server ... +connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (11,'node1_committed_during'); +INSERT INTO t1 VALUES (12,'node1_committed_during'); +INSERT INTO t1 VALUES (13,'node1_committed_during'); +INSERT INTO t1 VALUES (14,'node1_committed_during'); +INSERT INTO t1 VALUES (15,'node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES (16,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (17,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (18,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (19,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (20,'node1_to_be_committed_after'); +connect node_1a_galera_st_shutdown_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after'); +connection node_2; +Starting server ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (26,'node2_committed_after'); +INSERT INTO t1 VALUES (27,'node2_committed_after'); +INSERT INTO t1 VALUES (28,'node2_committed_after'); +INSERT INTO t1 VALUES (29,'node2_committed_after'); +INSERT INTO t1 VALUES (30,'node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES (31,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (32,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (33,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (34,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (35,'node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (36,'node1_committed_after'); +INSERT INTO t1 VALUES (37,'node1_committed_after'); +INSERT INTO t1 VALUES (38,'node1_committed_after'); +INSERT INTO t1 VALUES (39,'node1_committed_after'); +INSERT INTO t1 VALUES (40,'node1_committed_after'); +COMMIT; +connection node_1a_galera_st_shutdown_slave; +INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (44,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after'); +ROLLBACK; +SET AUTOCOMMIT=ON; +SET SESSION wsrep_sync_wait=15; +SELECT COUNT(*) AS EXPECT_15 FROM t1; +EXPECT_15 +35 +SELECT * from t1; +id f1 +1 node1_committed_before +2 node1_committed_before +3 node1_committed_before +4 node1_committed_before +5 node1_committed_before +6 node2_committed_before +7 node2_committed_before +8 node2_committed_before +9 node2_committed_before +10 node2_committed_before +11 node1_committed_during +12 node1_committed_during +13 node1_committed_during +14 node1_committed_during +15 node1_committed_during +16 node1_to_be_committed_after +17 node1_to_be_committed_after +18 node1_to_be_committed_after +19 node1_to_be_committed_after +20 node1_to_be_committed_after +26 node2_committed_after +27 node2_committed_after +28 node2_committed_after +29 node2_committed_after +30 node2_committed_after +31 node1_to_be_committed_after +32 node1_to_be_committed_after +33 node1_to_be_committed_after +34 node1_to_be_committed_after +35 node1_to_be_committed_after +36 node1_committed_after +37 node1_committed_after +38 node1_committed_after +39 node1_committed_after +40 node1_committed_after +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +connection node_1; +SET AUTOCOMMIT=ON; +SET SESSION wsrep_sync_wait=15; +SELECT COUNT(*) AS EXPECT_15 FROM t1; +EXPECT_15 +35 +SELECT * from t1; +id f1 +1 node1_committed_before +2 node1_committed_before +3 node1_committed_before +4 node1_committed_before +5 node1_committed_before +6 node2_committed_before +7 node2_committed_before +8 node2_committed_before +9 node2_committed_before +10 node2_committed_before +11 node1_committed_during +12 node1_committed_during +13 node1_committed_during +14 node1_committed_during +15 node1_committed_during +16 node1_to_be_committed_after +17 node1_to_be_committed_after +18 node1_to_be_committed_after +19 node1_to_be_committed_after +20 node1_to_be_committed_after +26 node2_committed_after +27 node2_committed_after +28 node2_committed_after +29 node2_committed_after +30 node2_committed_after +31 node1_to_be_committed_after +32 node1_to_be_committed_after +33 node1_to_be_committed_after +34 node1_to_be_committed_after +35 node1_to_be_committed_after +36 node1_committed_after +37 node1_committed_after +38 node1_committed_after +39 node1_committed_after +40 node1_committed_after +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +Performing State Transfer on a server that has been killed and restarted +connection node_1; +CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (1,'node1_committed_before'); +INSERT INTO t1 VALUES (2,'node1_committed_before'); +INSERT INTO t1 VALUES (3,'node1_committed_before'); +INSERT INTO t1 VALUES (4,'node1_committed_before'); +INSERT INTO t1 VALUES (5,'node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (6,'node2_committed_before'); +INSERT INTO t1 VALUES (7,'node2_committed_before'); +INSERT INTO t1 VALUES (8,'node2_committed_before'); +INSERT INTO t1 VALUES (9,'node2_committed_before'); +INSERT INTO t1 VALUES (10,'node2_committed_before'); +COMMIT; +Killing server ... +connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (11,'node1_committed_during'); +INSERT INTO t1 VALUES (12,'node1_committed_during'); +INSERT INTO t1 VALUES (13,'node1_committed_during'); +INSERT INTO t1 VALUES (14,'node1_committed_during'); +INSERT INTO t1 VALUES (15,'node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES (16,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (17,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (18,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (19,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (20,'node1_to_be_committed_after'); +connect node_1a_galera_st_kill_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after'); +connection node_2; +Performing --wsrep-recover ... +Starting server ... +Using --wsrep-start-position when starting mysqld ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (26,'node2_committed_after'); +INSERT INTO t1 VALUES (27,'node2_committed_after'); +INSERT INTO t1 VALUES (28,'node2_committed_after'); +INSERT INTO t1 VALUES (29,'node2_committed_after'); +INSERT INTO t1 VALUES (30,'node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES (31,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (32,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (33,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (34,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (35,'node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (36,'node1_committed_after'); +INSERT INTO t1 VALUES (37,'node1_committed_after'); +INSERT INTO t1 VALUES (38,'node1_committed_after'); +INSERT INTO t1 VALUES (39,'node1_committed_after'); +INSERT INTO t1 VALUES (40,'node1_committed_after'); +COMMIT; +connection node_1a_galera_st_kill_slave; +INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (46,'node1_to_be_rollbacked_after'); +ROLLBACK; +SET AUTOCOMMIT=ON; +SET SESSION wsrep_sync_wait=15; +SELECT COUNT(*) AS EXPECT_35 FROM t1; +EXPECT_35 +35 +SELECT * FROM t1; +id f1 +1 node1_committed_before +2 node1_committed_before +3 node1_committed_before +4 node1_committed_before +5 node1_committed_before +6 node2_committed_before +7 node2_committed_before +8 node2_committed_before +9 node2_committed_before +10 node2_committed_before +11 node1_committed_during +12 node1_committed_during +13 node1_committed_during +14 node1_committed_during +15 node1_committed_during +16 node1_to_be_committed_after +17 node1_to_be_committed_after +18 node1_to_be_committed_after +19 node1_to_be_committed_after +20 node1_to_be_committed_after +26 node2_committed_after +27 node2_committed_after +28 node2_committed_after +29 node2_committed_after +30 node2_committed_after +31 node1_to_be_committed_after +32 node1_to_be_committed_after +33 node1_to_be_committed_after +34 node1_to_be_committed_after +35 node1_to_be_committed_after +36 node1_committed_after +37 node1_committed_after +38 node1_committed_after +39 node1_committed_after +40 node1_committed_after +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +connection node_1; +SET AUTOCOMMIT=ON; +SET SESSION wsrep_sync_wait=15; +SELECT COUNT(*) AS EXPECT_35 FROM t1; +EXPECT_35 +35 +SELECT * FROM t1; +id f1 +1 node1_committed_before +2 node1_committed_before +3 node1_committed_before +4 node1_committed_before +5 node1_committed_before +6 node2_committed_before +7 node2_committed_before +8 node2_committed_before +9 node2_committed_before +10 node2_committed_before +11 node1_committed_during +12 node1_committed_during +13 node1_committed_during +14 node1_committed_during +15 node1_committed_during +16 node1_to_be_committed_after +17 node1_to_be_committed_after +18 node1_to_be_committed_after +19 node1_to_be_committed_after +20 node1_to_be_committed_after +26 node2_committed_after +27 node2_committed_after +28 node2_committed_after +29 node2_committed_after +30 node2_committed_after +31 node1_to_be_committed_after +32 node1_to_be_committed_after +33 node1_to_be_committed_after +34 node1_to_be_committed_after +35 node1_to_be_committed_after +36 node1_committed_after +37 node1_committed_after +38 node1_committed_after +39 node1_committed_after +40 node1_committed_after +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; diff --git a/mysql-test/suite/galera/t/galera_ist_MDEV-28423.cnf b/mysql-test/suite/galera/t/galera_ist_MDEV-28423.cnf new file mode 100644 index 00000000000..691e52208b1 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_ist_MDEV-28423.cnf @@ -0,0 +1,44 @@ +!include ../galera_2nodes.cnf + +[mysqld.1] +# server-id=101 +#wsrep-debug=1 +innodb_file_per_table +innodb_autoinc_lock_mode=2 +#wsrep_sst_method=rsync +wsrep_sst_method=mariabackup +wsrep_sst_auth=root: +binlog_format=ROW +core-file +log-output=none +wsrep_slave_threads=2 +wsrep_on=1 +gtid_strict_mode=1 +log_slave_updates=ON +log_bin=binlog + +[mysqld.2] +# server-id=102 +#wsrep-debug=1 +innodb_file_per_table +innodb_autoinc_lock_mode=2 +#wsrep_sst_method=rsync +wsrep_sst_method=mariabackup +wsrep_sst_auth=root: +binlog_format=ROW +core-file +log-output=none +wsrep_slave_threads=2 +wsrep_on=1 +gtid_strict_mode=1 +log_slave_updates=ON +log_bin=binlog + +[mysqld.1] +wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true' + +[mysqld.2] +wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true' + +[sst] +transferfmt=@ENV.MTR_GALERA_TFMT diff --git a/mysql-test/suite/galera/t/galera_ist_MDEV-28423.test b/mysql-test/suite/galera/t/galera_ist_MDEV-28423.test new file mode 100644 index 00000000000..8668c4ce158 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_ist_MDEV-28423.test @@ -0,0 +1,18 @@ +# MDEV-28423: Galera IST is failing on Joiner node + +--source include/big_test.inc +--source include/galera_cluster.inc +--source include/have_innodb.inc +--source include/have_mariabackup.inc + +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +--source suite/galera/include/galera_st_disconnect_slave.inc +--source suite/galera/include/galera_st_shutdown_slave.inc + +--source suite/galera/include/galera_st_kill_slave.inc +--source suite/galera/include/galera_st_kill_slave_ddl.inc + +--source include/auto_increment_offset_restore.inc diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 87ed8392475..314d2349bcc 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -86,6 +86,7 @@ encrypt_threads="" encrypt_chunk="" readonly SECRET_TAG='secret' +readonly TOTAL_TAG='total' # Required for backup locks # For backup locks it is 1 sent by joiner @@ -324,7 +325,8 @@ get_transfer() if [ -z "$ssl_dhparams" ]; then # Determine the socat version SOCAT_VERSION=$(socat -V 2>&1 | \ - grep -m1 -owE '[0-9]+(\.[0-9]+)+' | head -n1) + grep -m1 -owE '[0-9]+(\.[0-9]+)+' | \ + head -n1 || :) if [ -z "$SOCAT_VERSION" ]; then wsrep_log_error "******** FATAL ERROR ******************" wsrep_log_error "* Cannot determine the socat version. *" @@ -770,25 +772,26 @@ recv_joiner() if [ $checkf -eq 1 ]; then if [ ! -r "$MAGIC_FILE" ]; then - # this message should cause joiner to abort + # this message should cause joiner to abort: wsrep_log_error "receiving process ended without creating" \ - "'$MAGIC_FILE'" - wsrep_log_info "Contents of datadir" + "magic file ($MAGIC_FILE)" + wsrep_log_info "Contents of datadir:" wsrep_log_info $(ls -l "$dir/"*) exit 32 fi # check donor supplied secret - SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ - cut -d ' ' -f2) + SECRET=$(grep -m1 -E "^$SECRET_TAG[[:space:]]" -- "$MAGIC_FILE" || :) + SECRET=$(trim_string "${SECRET#$SECRET_TAG}") if [ "$SECRET" != "$MY_SECRET" ]; then wsrep_log_error "Donor does not know my secret!" wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" exit 32 fi - # remove secret from the magic file - grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" > "$MAGIC_FILE.new" + # remove secret and total from the magic file + grep -v -E "^($SECRET_TAG|$TOTAL_TAG)[[:space:]]" -- \ + "$MAGIC_FILE" > "$MAGIC_FILE.new" mv "$MAGIC_FILE.new" "$MAGIC_FILE" fi } @@ -1256,8 +1259,7 @@ then recv_joiner "$STATDIR" "$stagemsg-gtid" $stimeout 1 1 - if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1 - then + if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1; then wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT)" \ "terminated unexpectedly." exit 32 diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 29e2b390e27..27d4e875674 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -804,8 +804,7 @@ EOF sleep 1 done - if ! ps -p $MYSQLD_PID >/dev/null 2>&1 - then + if ! ps -p $MYSQLD_PID >/dev/null 2>&1; then wsrep_log_error \ "Parent mysqld process (PID: $MYSQLD_PID) terminated unexpectedly." kill -- -$MYSQLD_PID @@ -813,24 +812,24 @@ EOF exit 32 fi - if [ -r "$MAGIC_FILE" ]; then - if [ -n "$MY_SECRET" ]; then - # Check donor supplied secret: - SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ - cut -d ' ' -f2) - if [ "$SECRET" != "$MY_SECRET" ]; then - wsrep_log_error "Donor does not know my secret!" - wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" - exit 32 - fi - fi - else + if [ ! -r "$MAGIC_FILE" ]; then # This message should cause joiner to abort: - wsrep_log_info "rsync process ended without creating magic file" - echo "rsync process ended without creating '$MAGIC_FILE'" + wsrep_log_info "rsync process ended without creating" \ + "magic file ($MAGIC_FILE)" exit 32 fi + if [ -n "$MY_SECRET" ]; then + # Check donor supplied secret: + SECRET=$(grep -m1 -E "^$SECRET_TAG[[:space:]]" -- "$MAGIC_FILE" || :) + SECRET=$(trim_string "${SECRET#$SECRET_TAG}") + if [ "$SECRET" != "$MY_SECRET" ]; then + wsrep_log_error "Donor does not know my secret!" + wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" + exit 32 + fi + fi + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then binlog_tar_present=0 [ -f "$BINLOG_TAR_FILE" ] && binlog_tar_present=1 @@ -907,7 +906,7 @@ EOF if [ -n "$MY_SECRET" ]; then # remove secret from the magic file, and output # the UUID:seqno & wsrep_gtid_domain_id: - grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" + grep -v -E "^$SECRET_TAG[[:space:]]" -- "$MAGIC_FILE" else # Output the UUID:seqno and wsrep_gtid_domain_id: cat "$MAGIC_FILE" -- cgit v1.2.1 From d388e7eb864a3ba677f832dff8353c55275a7942 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Tue, 17 May 2022 11:04:04 +0200 Subject: MDEV-28583: Galera: binlogs disappear after rsync IST This commit sends a flag indicating the presence of the "--bypass" option from the donor node to the joiner nodes during rsync IST, because without such a flag it is impossible to distinguish IST from the SST on the joiner nodes (in IST/SST scripts, because the "--bypass" option is still not passed to scripts from server code). Specifically, this fixes an issue with binary logs disappearing after IST (via rsync). There are also changes to diagnostic messages here that will make it easier to diagnose script-related problems in the future when debugging and when checking the logs. This commit also adds more robust signal handlers - to handle exceptions during script execution. These handlers won't mask some crashes and it also unifies exit codes between different scripts. These changes have already been helpful to debugging "bypass" flag handling. --- .../galera/r/galera_ist_MDEV-28583,debug.rdiff | 190 ++++++++ .../suite/galera/r/galera_ist_MDEV-28583.result | 519 +++++++++++++++++++++ .../suite/galera/t/galera_ist_MDEV-28583.cnf | 44 ++ .../suite/galera/t/galera_ist_MDEV-28583.test | 18 + scripts/wsrep_sst_backup.sh | 14 +- scripts/wsrep_sst_common.sh | 58 ++- scripts/wsrep_sst_mariabackup.sh | 109 ++--- scripts/wsrep_sst_mysqldump.sh | 7 +- scripts/wsrep_sst_rsync.sh | 131 +++--- 9 files changed, 945 insertions(+), 145 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_ist_MDEV-28583,debug.rdiff create mode 100644 mysql-test/suite/galera/r/galera_ist_MDEV-28583.result create mode 100644 mysql-test/suite/galera/t/galera_ist_MDEV-28583.cnf create mode 100644 mysql-test/suite/galera/t/galera_ist_MDEV-28583.test diff --git a/mysql-test/suite/galera/r/galera_ist_MDEV-28583,debug.rdiff b/mysql-test/suite/galera/r/galera_ist_MDEV-28583,debug.rdiff new file mode 100644 index 00000000000..51d2a6bf157 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_ist_MDEV-28583,debug.rdiff @@ -0,0 +1,190 @@ +--- r/galera_ist_MDEV-28583.result ++++ r/galera_ist_MDEV-28583,debug.reject +@@ -517,3 +517,187 @@ + 1 + DROP TABLE t1; + COMMIT; ++Performing State Transfer on a server that has been killed and restarted ++while a DDL was in progress on it ++connection node_1; ++CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB; ++SET AUTOCOMMIT=OFF; ++START TRANSACTION; ++INSERT INTO t1 VALUES (1,'node1_committed_before'); ++INSERT INTO t1 VALUES (2,'node1_committed_before'); ++INSERT INTO t1 VALUES (3,'node1_committed_before'); ++INSERT INTO t1 VALUES (4,'node1_committed_before'); ++INSERT INTO t1 VALUES (5,'node1_committed_before'); ++connection node_2; ++START TRANSACTION; ++INSERT INTO t1 VALUES (6,'node2_committed_before'); ++INSERT INTO t1 VALUES (7,'node2_committed_before'); ++INSERT INTO t1 VALUES (8,'node2_committed_before'); ++INSERT INTO t1 VALUES (9,'node2_committed_before'); ++INSERT INTO t1 VALUES (10,'node2_committed_before'); ++COMMIT; ++SET GLOBAL debug_dbug = 'd,sync.alter_opened_table'; ++connection node_1; ++ALTER TABLE t1 ADD COLUMN f2 INTEGER; ++connection node_2; ++SET wsrep_sync_wait = 0; ++Killing server ... ++connection node_1; ++SET AUTOCOMMIT=OFF; ++START TRANSACTION; ++INSERT INTO t1 (id,f1) VALUES (11,'node1_committed_during'); ++INSERT INTO t1 (id,f1) VALUES (12,'node1_committed_during'); ++INSERT INTO t1 (id,f1) VALUES (13,'node1_committed_during'); ++INSERT INTO t1 (id,f1) VALUES (14,'node1_committed_during'); ++INSERT INTO t1 (id,f1) VALUES (15,'node1_committed_during'); ++COMMIT; ++START TRANSACTION; ++INSERT INTO t1 (id,f1) VALUES (16,'node1_to_be_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (17,'node1_to_be_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (18,'node1_to_be_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (19,'node1_to_be_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (20,'node1_to_be_committed_after'); ++connect node_1a_galera_st_kill_slave_ddl, 127.0.0.1, root, , test, $NODE_MYPORT_1; ++SET AUTOCOMMIT=OFF; ++START TRANSACTION; ++INSERT INTO t1 (id,f1) VALUES (21,'node1_to_be_rollbacked_after'); ++INSERT INTO t1 (id,f1) VALUES (22,'node1_to_be_rollbacked_after'); ++INSERT INTO t1 (id,f1) VALUES (23,'node1_to_be_rollbacked_after'); ++INSERT INTO t1 (id,f1) VALUES (24,'node1_to_be_rollbacked_after'); ++INSERT INTO t1 (id,f1) VALUES (25,'node1_to_be_rollbacked_after'); ++connection node_2; ++Performing --wsrep-recover ... ++connection node_2; ++Starting server ... ++Using --wsrep-start-position when starting mysqld ... ++SET AUTOCOMMIT=OFF; ++START TRANSACTION; ++INSERT INTO t1 (id,f1) VALUES (26,'node2_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (27,'node2_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (28,'node2_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (29,'node2_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (30,'node2_committed_after'); ++COMMIT; ++connection node_1; ++INSERT INTO t1 (id,f1) VALUES (31,'node1_to_be_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (32,'node1_to_be_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (33,'node1_to_be_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (34,'node1_to_be_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (35,'node1_to_be_committed_after'); ++COMMIT; ++SET AUTOCOMMIT=OFF; ++START TRANSACTION; ++INSERT INTO t1 (id,f1) VALUES (36,'node1_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (37,'node1_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (38,'node1_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (39,'node1_committed_after'); ++INSERT INTO t1 (id,f1) VALUES (40,'node1_committed_after'); ++COMMIT; ++connection node_1a_galera_st_kill_slave_ddl; ++INSERT INTO t1 (id,f1) VALUES (41,'node1_to_be_rollbacked_after'); ++INSERT INTO t1 (id,f1) VALUES (42,'node1_to_be_rollbacked_after'); ++INSERT INTO t1 (id,f1) VALUES (43,'node1_to_be_rollbacked_after'); ++INSERT INTO t1 (id,f1) VALUES (44,'node1_to_be_rollbacked_after'); ++INSERT INTO t1 (id,f1) VALUES (45,'node1_to_be_rollbacked_after'); ++ROLLBACK; ++SET AUTOCOMMIT=ON; ++SET SESSION wsrep_sync_wait=15; ++SELECT COUNT(*) AS EXPECT_3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; ++EXPECT_3 ++3 ++SELECT COUNT(*) AS EXPECT_35 FROM t1; ++EXPECT_35 ++35 ++SELECT * FROM t1; ++id f1 f2 ++1 node1_committed_before NULL ++2 node1_committed_before NULL ++3 node1_committed_before NULL ++4 node1_committed_before NULL ++5 node1_committed_before NULL ++6 node2_committed_before NULL ++7 node2_committed_before NULL ++8 node2_committed_before NULL ++9 node2_committed_before NULL ++10 node2_committed_before NULL ++11 node1_committed_during NULL ++12 node1_committed_during NULL ++13 node1_committed_during NULL ++14 node1_committed_during NULL ++15 node1_committed_during NULL ++16 node1_to_be_committed_after NULL ++17 node1_to_be_committed_after NULL ++18 node1_to_be_committed_after NULL ++19 node1_to_be_committed_after NULL ++20 node1_to_be_committed_after NULL ++26 node2_committed_after NULL ++27 node2_committed_after NULL ++28 node2_committed_after NULL ++29 node2_committed_after NULL ++30 node2_committed_after NULL ++31 node1_to_be_committed_after NULL ++32 node1_to_be_committed_after NULL ++33 node1_to_be_committed_after NULL ++34 node1_to_be_committed_after NULL ++35 node1_to_be_committed_after NULL ++36 node1_committed_after NULL ++37 node1_committed_after NULL ++38 node1_committed_after NULL ++39 node1_committed_after NULL ++40 node1_committed_after NULL ++SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; ++COUNT(*) = 0 ++1 ++COMMIT; ++connection node_1; ++SET AUTOCOMMIT=ON; ++SET SESSION wsrep_sync_wait=15; ++SELECT COUNT(*) AS EXPECT_3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; ++EXPECT_3 ++3 ++SELECT COUNT(*) AS EXPECT_35 FROM t1; ++EXPECT_35 ++35 ++SELECT * FROM t1; ++id f1 f2 ++1 node1_committed_before NULL ++2 node1_committed_before NULL ++3 node1_committed_before NULL ++4 node1_committed_before NULL ++5 node1_committed_before NULL ++6 node2_committed_before NULL ++7 node2_committed_before NULL ++8 node2_committed_before NULL ++9 node2_committed_before NULL ++10 node2_committed_before NULL ++11 node1_committed_during NULL ++12 node1_committed_during NULL ++13 node1_committed_during NULL ++14 node1_committed_during NULL ++15 node1_committed_during NULL ++16 node1_to_be_committed_after NULL ++17 node1_to_be_committed_after NULL ++18 node1_to_be_committed_after NULL ++19 node1_to_be_committed_after NULL ++20 node1_to_be_committed_after NULL ++26 node2_committed_after NULL ++27 node2_committed_after NULL ++28 node2_committed_after NULL ++29 node2_committed_after NULL ++30 node2_committed_after NULL ++31 node1_to_be_committed_after NULL ++32 node1_to_be_committed_after NULL ++33 node1_to_be_committed_after NULL ++34 node1_to_be_committed_after NULL ++35 node1_to_be_committed_after NULL ++36 node1_committed_after NULL ++37 node1_committed_after NULL ++38 node1_committed_after NULL ++39 node1_committed_after NULL ++40 node1_committed_after NULL ++SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; ++COUNT(*) = 0 ++1 ++DROP TABLE t1; ++COMMIT; ++SET GLOBAL debug_dbug = $debug_orig; diff --git a/mysql-test/suite/galera/r/galera_ist_MDEV-28583.result b/mysql-test/suite/galera/r/galera_ist_MDEV-28583.result new file mode 100644 index 00000000000..5a71b490a80 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_ist_MDEV-28583.result @@ -0,0 +1,519 @@ +connection node_2; +connection node_1; +connection node_1; +connection node_2; +Performing State Transfer on a server that has been temporarily disconnected +connection node_1; +CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (1,'node1_committed_before'); +INSERT INTO t1 VALUES (2,'node1_committed_before'); +INSERT INTO t1 VALUES (3,'node1_committed_before'); +INSERT INTO t1 VALUES (4,'node1_committed_before'); +INSERT INTO t1 VALUES (5,'node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (6,'node2_committed_before'); +INSERT INTO t1 VALUES (7,'node2_committed_before'); +INSERT INTO t1 VALUES (8,'node2_committed_before'); +INSERT INTO t1 VALUES (9,'node2_committed_before'); +INSERT INTO t1 VALUES (10,'node2_committed_before'); +COMMIT; +Unloading wsrep provider ... +SET GLOBAL wsrep_cluster_address = ''; +connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (11,'node1_committed_during'); +INSERT INTO t1 VALUES (12,'node1_committed_during'); +INSERT INTO t1 VALUES (13,'node1_committed_during'); +INSERT INTO t1 VALUES (14,'node1_committed_during'); +INSERT INTO t1 VALUES (15,'node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES (16,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (17,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (18,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (19,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (20,'node1_to_be_committed_after'); +connect node_1a_galera_st_disconnect_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after'); +connection node_2; +Loading wsrep provider ... +disconnect node_2; +connect node_2, 127.0.0.1, root, , test, $NODE_MYPORT_2; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (26,'node2_committed_after'); +INSERT INTO t1 VALUES (27,'node2_committed_after'); +INSERT INTO t1 VALUES (28,'node2_committed_after'); +INSERT INTO t1 VALUES (29,'node2_committed_after'); +INSERT INTO t1 VALUES (30,'node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES (31,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (32,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (33,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (34,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (35,'node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (36,'node1_committed_after'); +INSERT INTO t1 VALUES (37,'node1_committed_after'); +INSERT INTO t1 VALUES (38,'node1_committed_after'); +INSERT INTO t1 VALUES (39,'node1_committed_after'); +INSERT INTO t1 VALUES (40,'node1_committed_after'); +COMMIT; +connection node_1a_galera_st_disconnect_slave; +INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (44,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after'); +ROLLBACK; +SET AUTOCOMMIT=ON; +SET SESSION wsrep_sync_wait=15; +SELECT COUNT(*) AS EXPECT_35 FROM t1; +EXPECT_35 +35 +SELECT * FROM t1; +id f1 +1 node1_committed_before +2 node1_committed_before +3 node1_committed_before +4 node1_committed_before +5 node1_committed_before +6 node2_committed_before +7 node2_committed_before +8 node2_committed_before +9 node2_committed_before +10 node2_committed_before +11 node1_committed_during +12 node1_committed_during +13 node1_committed_during +14 node1_committed_during +15 node1_committed_during +16 node1_to_be_committed_after +17 node1_to_be_committed_after +18 node1_to_be_committed_after +19 node1_to_be_committed_after +20 node1_to_be_committed_after +26 node2_committed_after +27 node2_committed_after +28 node2_committed_after +29 node2_committed_after +30 node2_committed_after +31 node1_to_be_committed_after +32 node1_to_be_committed_after +33 node1_to_be_committed_after +34 node1_to_be_committed_after +35 node1_to_be_committed_after +36 node1_committed_after +37 node1_committed_after +38 node1_committed_after +39 node1_committed_after +40 node1_committed_after +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +connection node_1; +SET AUTOCOMMIT=ON; +SET SESSION wsrep_sync_wait=15; +SELECT COUNT(*) AS EXPECT_35 FROM t1; +EXPECT_35 +35 +SELECT * FROM t1; +id f1 +1 node1_committed_before +2 node1_committed_before +3 node1_committed_before +4 node1_committed_before +5 node1_committed_before +6 node2_committed_before +7 node2_committed_before +8 node2_committed_before +9 node2_committed_before +10 node2_committed_before +11 node1_committed_during +12 node1_committed_during +13 node1_committed_during +14 node1_committed_during +15 node1_committed_during +16 node1_to_be_committed_after +17 node1_to_be_committed_after +18 node1_to_be_committed_after +19 node1_to_be_committed_after +20 node1_to_be_committed_after +26 node2_committed_after +27 node2_committed_after +28 node2_committed_after +29 node2_committed_after +30 node2_committed_after +31 node1_to_be_committed_after +32 node1_to_be_committed_after +33 node1_to_be_committed_after +34 node1_to_be_committed_after +35 node1_to_be_committed_after +36 node1_committed_after +37 node1_committed_after +38 node1_committed_after +39 node1_committed_after +40 node1_committed_after +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +Performing State Transfer on a server that has been shut down cleanly and restarted +connection node_1; +CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (1,'node1_committed_before'); +INSERT INTO t1 VALUES (2,'node1_committed_before'); +INSERT INTO t1 VALUES (3,'node1_committed_before'); +INSERT INTO t1 VALUES (4,'node1_committed_before'); +INSERT INTO t1 VALUES (5,'node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (6,'node2_committed_before'); +INSERT INTO t1 VALUES (7,'node2_committed_before'); +INSERT INTO t1 VALUES (8,'node2_committed_before'); +INSERT INTO t1 VALUES (9,'node2_committed_before'); +INSERT INTO t1 VALUES (10,'node2_committed_before'); +COMMIT; +Shutting down server ... +connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (11,'node1_committed_during'); +INSERT INTO t1 VALUES (12,'node1_committed_during'); +INSERT INTO t1 VALUES (13,'node1_committed_during'); +INSERT INTO t1 VALUES (14,'node1_committed_during'); +INSERT INTO t1 VALUES (15,'node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES (16,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (17,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (18,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (19,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (20,'node1_to_be_committed_after'); +connect node_1a_galera_st_shutdown_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after'); +connection node_2; +Starting server ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (26,'node2_committed_after'); +INSERT INTO t1 VALUES (27,'node2_committed_after'); +INSERT INTO t1 VALUES (28,'node2_committed_after'); +INSERT INTO t1 VALUES (29,'node2_committed_after'); +INSERT INTO t1 VALUES (30,'node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES (31,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (32,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (33,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (34,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (35,'node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (36,'node1_committed_after'); +INSERT INTO t1 VALUES (37,'node1_committed_after'); +INSERT INTO t1 VALUES (38,'node1_committed_after'); +INSERT INTO t1 VALUES (39,'node1_committed_after'); +INSERT INTO t1 VALUES (40,'node1_committed_after'); +COMMIT; +connection node_1a_galera_st_shutdown_slave; +INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (44,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after'); +ROLLBACK; +SET AUTOCOMMIT=ON; +SET SESSION wsrep_sync_wait=15; +SELECT COUNT(*) AS EXPECT_15 FROM t1; +EXPECT_15 +35 +SELECT * from t1; +id f1 +1 node1_committed_before +2 node1_committed_before +3 node1_committed_before +4 node1_committed_before +5 node1_committed_before +6 node2_committed_before +7 node2_committed_before +8 node2_committed_before +9 node2_committed_before +10 node2_committed_before +11 node1_committed_during +12 node1_committed_during +13 node1_committed_during +14 node1_committed_during +15 node1_committed_during +16 node1_to_be_committed_after +17 node1_to_be_committed_after +18 node1_to_be_committed_after +19 node1_to_be_committed_after +20 node1_to_be_committed_after +26 node2_committed_after +27 node2_committed_after +28 node2_committed_after +29 node2_committed_after +30 node2_committed_after +31 node1_to_be_committed_after +32 node1_to_be_committed_after +33 node1_to_be_committed_after +34 node1_to_be_committed_after +35 node1_to_be_committed_after +36 node1_committed_after +37 node1_committed_after +38 node1_committed_after +39 node1_committed_after +40 node1_committed_after +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +connection node_1; +SET AUTOCOMMIT=ON; +SET SESSION wsrep_sync_wait=15; +SELECT COUNT(*) AS EXPECT_15 FROM t1; +EXPECT_15 +35 +SELECT * from t1; +id f1 +1 node1_committed_before +2 node1_committed_before +3 node1_committed_before +4 node1_committed_before +5 node1_committed_before +6 node2_committed_before +7 node2_committed_before +8 node2_committed_before +9 node2_committed_before +10 node2_committed_before +11 node1_committed_during +12 node1_committed_during +13 node1_committed_during +14 node1_committed_during +15 node1_committed_during +16 node1_to_be_committed_after +17 node1_to_be_committed_after +18 node1_to_be_committed_after +19 node1_to_be_committed_after +20 node1_to_be_committed_after +26 node2_committed_after +27 node2_committed_after +28 node2_committed_after +29 node2_committed_after +30 node2_committed_after +31 node1_to_be_committed_after +32 node1_to_be_committed_after +33 node1_to_be_committed_after +34 node1_to_be_committed_after +35 node1_to_be_committed_after +36 node1_committed_after +37 node1_committed_after +38 node1_committed_after +39 node1_committed_after +40 node1_committed_after +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; +Performing State Transfer on a server that has been killed and restarted +connection node_1; +CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (1,'node1_committed_before'); +INSERT INTO t1 VALUES (2,'node1_committed_before'); +INSERT INTO t1 VALUES (3,'node1_committed_before'); +INSERT INTO t1 VALUES (4,'node1_committed_before'); +INSERT INTO t1 VALUES (5,'node1_committed_before'); +COMMIT; +connection node_2; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (6,'node2_committed_before'); +INSERT INTO t1 VALUES (7,'node2_committed_before'); +INSERT INTO t1 VALUES (8,'node2_committed_before'); +INSERT INTO t1 VALUES (9,'node2_committed_before'); +INSERT INTO t1 VALUES (10,'node2_committed_before'); +COMMIT; +Killing server ... +connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (11,'node1_committed_during'); +INSERT INTO t1 VALUES (12,'node1_committed_during'); +INSERT INTO t1 VALUES (13,'node1_committed_during'); +INSERT INTO t1 VALUES (14,'node1_committed_during'); +INSERT INTO t1 VALUES (15,'node1_committed_during'); +COMMIT; +START TRANSACTION; +INSERT INTO t1 VALUES (16,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (17,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (18,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (19,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (20,'node1_to_be_committed_after'); +connect node_1a_galera_st_kill_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after'); +connection node_2; +Performing --wsrep-recover ... +Starting server ... +Using --wsrep-start-position when starting mysqld ... +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (26,'node2_committed_after'); +INSERT INTO t1 VALUES (27,'node2_committed_after'); +INSERT INTO t1 VALUES (28,'node2_committed_after'); +INSERT INTO t1 VALUES (29,'node2_committed_after'); +INSERT INTO t1 VALUES (30,'node2_committed_after'); +COMMIT; +connection node_1; +INSERT INTO t1 VALUES (31,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (32,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (33,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (34,'node1_to_be_committed_after'); +INSERT INTO t1 VALUES (35,'node1_to_be_committed_after'); +COMMIT; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 VALUES (36,'node1_committed_after'); +INSERT INTO t1 VALUES (37,'node1_committed_after'); +INSERT INTO t1 VALUES (38,'node1_committed_after'); +INSERT INTO t1 VALUES (39,'node1_committed_after'); +INSERT INTO t1 VALUES (40,'node1_committed_after'); +COMMIT; +connection node_1a_galera_st_kill_slave; +INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after'); +INSERT INTO t1 VALUES (46,'node1_to_be_rollbacked_after'); +ROLLBACK; +SET AUTOCOMMIT=ON; +SET SESSION wsrep_sync_wait=15; +SELECT COUNT(*) AS EXPECT_35 FROM t1; +EXPECT_35 +35 +SELECT * FROM t1; +id f1 +1 node1_committed_before +2 node1_committed_before +3 node1_committed_before +4 node1_committed_before +5 node1_committed_before +6 node2_committed_before +7 node2_committed_before +8 node2_committed_before +9 node2_committed_before +10 node2_committed_before +11 node1_committed_during +12 node1_committed_during +13 node1_committed_during +14 node1_committed_during +15 node1_committed_during +16 node1_to_be_committed_after +17 node1_to_be_committed_after +18 node1_to_be_committed_after +19 node1_to_be_committed_after +20 node1_to_be_committed_after +26 node2_committed_after +27 node2_committed_after +28 node2_committed_after +29 node2_committed_after +30 node2_committed_after +31 node1_to_be_committed_after +32 node1_to_be_committed_after +33 node1_to_be_committed_after +34 node1_to_be_committed_after +35 node1_to_be_committed_after +36 node1_committed_after +37 node1_committed_after +38 node1_committed_after +39 node1_committed_after +40 node1_committed_after +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +COMMIT; +connection node_1; +SET AUTOCOMMIT=ON; +SET SESSION wsrep_sync_wait=15; +SELECT COUNT(*) AS EXPECT_35 FROM t1; +EXPECT_35 +35 +SELECT * FROM t1; +id f1 +1 node1_committed_before +2 node1_committed_before +3 node1_committed_before +4 node1_committed_before +5 node1_committed_before +6 node2_committed_before +7 node2_committed_before +8 node2_committed_before +9 node2_committed_before +10 node2_committed_before +11 node1_committed_during +12 node1_committed_during +13 node1_committed_during +14 node1_committed_during +15 node1_committed_during +16 node1_to_be_committed_after +17 node1_to_be_committed_after +18 node1_to_be_committed_after +19 node1_to_be_committed_after +20 node1_to_be_committed_after +26 node2_committed_after +27 node2_committed_after +28 node2_committed_after +29 node2_committed_after +30 node2_committed_after +31 node1_to_be_committed_after +32 node1_to_be_committed_after +33 node1_to_be_committed_after +34 node1_to_be_committed_after +35 node1_to_be_committed_after +36 node1_committed_after +37 node1_committed_after +38 node1_committed_after +39 node1_committed_after +40 node1_committed_after +SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1; +COUNT(*) = 0 +1 +DROP TABLE t1; +COMMIT; diff --git a/mysql-test/suite/galera/t/galera_ist_MDEV-28583.cnf b/mysql-test/suite/galera/t/galera_ist_MDEV-28583.cnf new file mode 100644 index 00000000000..3835cd02a41 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_ist_MDEV-28583.cnf @@ -0,0 +1,44 @@ +!include ../galera_2nodes.cnf + +[mysqld.1] +# server-id=101 +#wsrep-debug=1 +innodb_file_per_table +innodb_autoinc_lock_mode=2 +wsrep_sst_method=rsync +#wsrep_sst_method=mariabackup +wsrep_sst_auth=root: +binlog_format=ROW +core-file +log-output=none +wsrep_slave_threads=2 +wsrep_on=1 +gtid_strict_mode=1 +log_slave_updates=ON +log_bin=binlog + +[mysqld.2] +# server-id=102 +#wsrep-debug=1 +innodb_file_per_table +innodb_autoinc_lock_mode=2 +wsrep_sst_method=rsync +#wsrep_sst_method=mariabackup +wsrep_sst_auth=root: +binlog_format=ROW +core-file +log-output=none +wsrep_slave_threads=2 +wsrep_on=1 +gtid_strict_mode=1 +log_slave_updates=ON +log_bin=binlog + +[mysqld.1] +wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true' + +[mysqld.2] +wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true' + +[sst] +transferfmt=@ENV.MTR_GALERA_TFMT diff --git a/mysql-test/suite/galera/t/galera_ist_MDEV-28583.test b/mysql-test/suite/galera/t/galera_ist_MDEV-28583.test new file mode 100644 index 00000000000..2c8c0bd80da --- /dev/null +++ b/mysql-test/suite/galera/t/galera_ist_MDEV-28583.test @@ -0,0 +1,18 @@ +# MDEV-28583: Galera: binlogs disappear after rsync IST + +--source include/big_test.inc +--source include/galera_cluster.inc +--source include/have_innodb.inc +--source include/have_mariabackup.inc + +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +--source suite/galera/include/galera_st_disconnect_slave.inc +--source suite/galera/include/galera_st_shutdown_slave.inc + +--source suite/galera/include/galera_st_kill_slave.inc +--source suite/galera/include/galera_st_kill_slave_ddl.inc + +--source include/auto_increment_offset_restore.inc diff --git a/scripts/wsrep_sst_backup.sh b/scripts/wsrep_sst_backup.sh index 55e11ddffc0..301739905b6 100644 --- a/scripts/wsrep_sst_backup.sh +++ b/scripts/wsrep_sst_backup.sh @@ -64,7 +64,7 @@ then [ -f "$FLUSHED" ] && rm -f "$FLUSHED" [ -f "$ERROR" ] && rm -f "$ERROR" - echo "flush tables" + echo "flush tables" # Wait for : # (a) Tables to be flushed, AND @@ -72,7 +72,7 @@ then # (c) ERROR file, in case flush tables operation failed. while [ ! -r "$FLUSHED" ] && \ - ! grep -q -F ':' '--' "$FLUSHED" >/dev/null 2>&1 + ! grep -q -F ':' -- "$FLUSHED" >/dev/null 2>&1 do # Check whether ERROR file exists. if [ -f "$ERROR" ]; then @@ -98,15 +98,11 @@ then echo "done $STATE" -elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] -then - wsrep_log_error "Unrecognized role: '$WSREP_SST_OPT_ROLE'" - exit 22 # EINVAL - +else # joiner -else - wsrep_log_error "Unrecognized role: '$WSREP_SST_OPT_ROLE'" + wsrep_log_error "Unsupported role: '$WSREP_SST_OPT_ROLE'" exit 22 # EINVAL + fi exit 0 diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index d4d9a58897d..3d0a132f3fc 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -17,7 +17,8 @@ # This is a common command line parser to be sourced by other SST scripts -set -ue +trap 'exit 32' HUP PIPE +trap 'exit 3' INT QUIT TERM # Setting the path for some utilities on CentOS export PATH="$PATH:/usr/sbin:/usr/bin:/sbin:/bin" @@ -184,7 +185,7 @@ case "$1" in shift ;; '--bypass') - WSREP_SST_OPT_BYPASS=1 + readonly WSREP_SST_OPT_BYPASS=1 ;; '--datadir') # Let's remove the trailing slash: @@ -511,7 +512,24 @@ case "$1" in esac shift done -readonly WSREP_SST_OPT_BYPASS + +WSREP_TRANSFER_TYPE='SST' +[ $WSREP_SST_OPT_BYPASS -ne 0 ] && readonly WSREP_TRANSFER_TYPE='IST' +# Let's take the name of the current script as a base, +# removing the directory, extension and "wsrep_sst_" prefix: +WSREP_METHOD="${0##*/}" +WSREP_METHOD="${WSREP_METHOD%.*}" +readonly WSREP_METHOD="${WSREP_METHOD#wsrep_sst_}" +if [ -n "${WSREP_SST_OPT_ROLE+x}" ]; then + if [ "$WSREP_SST_OPT_ROLE" != 'donor' -a \ + "$WSREP_SST_OPT_ROLE" != 'joiner' ] + then + wsrep_log_error "Unrecognized role: '$WSREP_SST_OPT_ROLE'" + exit 22 # EINVAL + fi +else + readonly WSREP_SST_OPT_ROLE='donor' +fi # The same argument can be present on the command line several # times, in this case we must take its last value: @@ -719,7 +737,7 @@ wsrep_log() { # echo everything to stderr so that it gets into common error log # deliberately made to look different from the rest of the log - local readonly tst="$(date +%Y%m%d\ %H:%M:%S.%N | cut -b -21)" + local readonly tst=$(date "+%Y%m%d %H:%M:%S.%N" | cut -b -21) echo "WSREP_SST: $* ($tst)" >&2 } @@ -1050,7 +1068,7 @@ is_local_ip() [ "$1" = '127.0.0.1' -o \ "$1" = '127.0.0.2' -o \ "$1" = 'localhost' -o \ - "$1" = '[::1]' ] && return 0 + "$1" = '::1' ] && return 0 # If the address starts with "127." this is probably a local # address, but we need to clarify what follows this prefix: if [ "${1#127.}" != "$1" ]; then @@ -1067,21 +1085,25 @@ is_local_ip() "$1" = "$(hostname -f)" -o \ "$1" = "$(hostname -d)" ] && return 0 fi + # If the address contains anything other than digits + # and separators, it is not a local address: + [ "${1#*[!0-9.]}" != "$1" ] && \ + [ "${1#*[!0-9A-Fa-f:\[\]]}" != "$1" ] && return 1 # Now let's check if the given address is assigned to # one of the network cards: local ip_util=$(commandex 'ip') if [ -n "$ip_util" ]; then # ip address show ouput format is " inet[6]
/": "$ip_util" address show \ - | grep -E '^[[:space:]]*inet.? [^[:space:]]+/' -o \ - | grep -F " $1/" >/dev/null && return 0 + | grep -o -E '^[[:space:]]*inet.?[[:space:]]+[^[:space:]]+/' \ + | grep -qw -F -- "$1/" && return 0 else local ifconfig_util=$(commandex 'ifconfig') if [ -n "$ifconfig_util" ]; then # ifconfig output format is " inet[6]
...": "$ifconfig_util" \ - | grep -E '^[[:space:]]*inet.? [^[:space:]]+ ' -o \ - | grep -F " $1 " >/dev/null && return 0 + | grep -o -E '^[[:space:]]*inet.?[[:space:]]+[^[:space:]]+' \ + | grep -qw -F -- "$1" && return 0 fi fi return 1 @@ -1403,7 +1425,7 @@ get_proc() if [ -z "$nproc" ]; then set +e if [ "$OS" = 'Linux' ]; then - nproc=$(grep -c processor /proc/cpuinfo 2>/dev/null) + nproc=$(grep -cw -E '^processor' /proc/cpuinfo 2>/dev/null) elif [ "$OS" = 'Darwin' -o "$OS" = 'FreeBSD' ]; then nproc=$(sysctl -n hw.ncpu) fi @@ -1452,3 +1474,19 @@ check_server_ssl_config() fi fi } + +simple_cleanup() +{ + # Since this is invoked just after exit NNN + local estatus=$? + if [ $estatus -ne 0 ]; then + wsrep_log_error "Cleanup after exit with status: $estatus" + fi + if [ -n "${SST_PID:-}" ]; then + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" + [ -f "$SST_PID" ] && rm -f "$SST_PID" || : + fi + exit $estatus +} + +wsrep_log_info "$WSREP_METHOD $WSREP_TRANSFER_TYPE started on $WSREP_SST_OPT_ROLE" diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 314d2349bcc..a3fc4b78718 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -86,7 +86,7 @@ encrypt_threads="" encrypt_chunk="" readonly SECRET_TAG='secret' -readonly TOTAL_TAG='total' +readonly TOTAL_TAG='secret /total' # Required for backup locks # For backup locks it is 1 sent by joiner @@ -166,12 +166,10 @@ get_keys() exit 3 fi - if [ -z "$ekey" ]; then - if [ ! -r "$ekeyfile" ]; then - wsrep_log_error "FATAL: Either key must be specified" \ - "or keyfile must be readable" - exit 3 - fi + if [ -z "$ekey" -a ! -r "$ekeyfile" ]; then + wsrep_log_error "FATAL: Either key must be specified" \ + "or keyfile must be readable" + exit 3 fi if [ "$eformat" = 'openssl' ]; then @@ -218,9 +216,7 @@ get_keys() exit 2 fi - if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - ecmd="$ecmd -d" - fi + [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] && ecmd="$ecmd -d" stagemsg="$stagemsg-XB-Encrypted" } @@ -597,18 +593,6 @@ get_stream() wsrep_log_info "Streaming with $sfmt" } -sig_joiner_cleanup() -{ - local estatus=$? - if [ $estatus -ne 0 ]; then - wsrep_log_error "Cleanup after exit with status: $estatus" - fi - wsrep_log_error "Removing $MAGIC_FILE file due to signal" - [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" - [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" - exit $estatus -} - cleanup_at_exit() { # Since this is invoked just after exit NNN @@ -619,6 +603,11 @@ cleanup_at_exit() [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" + if [ $estatus -ne 0 ]; then + wsrep_log_error "Removing $MAGIC_FILE file due to signal" + [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" || : + fi + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then wsrep_log_info "Removing the sst_in_progress file" wsrep_cleanup_progress_file @@ -648,7 +637,7 @@ cleanup_at_exit() fi # Final cleanup - pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o -E '[0-9]*' || :) + pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o -E '[0-9]+' || :) # This means no setsid done in mysqld. # We don't want to kill mysqld here otherwise. @@ -744,17 +733,15 @@ recv_joiner() fi fi - pushd "$dir" 1>/dev/null - set +e - if [ $wait -ne 0 ]; then wait_for_listen & fi + cd "$dir" + set +e timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" - set -e - popd 1>/dev/null + cd "$OLD_PWD" if [ ${RC[0]} -eq 124 ]; then wsrep_log_error "Possible timeout in receiving first data from" \ @@ -779,20 +766,19 @@ recv_joiner() wsrep_log_info $(ls -l "$dir/"*) exit 32 fi - - # check donor supplied secret - SECRET=$(grep -m1 -E "^$SECRET_TAG[[:space:]]" -- "$MAGIC_FILE" || :) + # Select the "secret" tag whose value does not start + # with a slash symbol. All new tags must to start with + # the space and the slash symbol after the word "secret" - + # to be removed by older versions of the SST scripts: + SECRET=$(grep -m1 -E "^$SECRET_TAG[[:space:]]+[^/]" \ + -- "$MAGIC_FILE" || :) + # Check donor supplied secret: SECRET=$(trim_string "${SECRET#$SECRET_TAG}") if [ "$SECRET" != "$MY_SECRET" ]; then wsrep_log_error "Donor does not know my secret!" wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" exit 32 fi - - # remove secret and total from the magic file - grep -v -E "^($SECRET_TAG|$TOTAL_TAG)[[:space:]]" -- \ - "$MAGIC_FILE" > "$MAGIC_FILE.new" - mv "$MAGIC_FILE.new" "$MAGIC_FILE" fi } @@ -801,11 +787,11 @@ send_donor() local dir="$1" local msg="$2" - pushd "$dir" 1>/dev/null + cd "$dir" set +e timeit "$msg" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )" set -e - popd 1>/dev/null + cd "$OLD_PWD" for ecode in "${RC[@]}"; do if [ $ecode -ne 0 ]; then @@ -820,7 +806,7 @@ monitor_process() { local sst_stream_pid=$1 - while true ; do + while :; do if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1; then wsrep_log_error \ "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT)" \ @@ -837,13 +823,6 @@ monitor_process() [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" -if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a \ - "$WSREP_SST_OPT_ROLE" != 'donor' ] -then - wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'" - exit 22 -fi - read_cnf setup_ports @@ -960,8 +939,8 @@ setup_commands() get_stream get_transfer -if [ "$WSREP_SST_OPT_ROLE" = 'donor' ] -then +if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then + trap cleanup_at_exit EXIT if [ $WSREP_SST_OPT_BYPASS -eq 0 ] @@ -1082,6 +1061,7 @@ then fi setup_commands + set +e timeit "$stagemsg-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" set -e @@ -1125,10 +1105,9 @@ then echo "done $WSREP_SST_OPT_GTID" wsrep_log_info "Total time on donor: $totime seconds" - wsrep_log_info "mariabackup SST/IST completed on donor" -elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] -then +else # joiner + [ -e "$SST_PROGRESS_FILE" ] && \ wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" @@ -1199,6 +1178,7 @@ then sleep 1 done + trap simple_cleanup EXIT echo $$ > "$SST_PID" stagemsg='Joiner-Recv' @@ -1208,7 +1188,7 @@ then [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" # May need xtrabackup_checkpoints later on - [ -f "$DATA/xtrabackup_binary" ] && rm -f "$DATA/xtrabackup_binary" + [ -f "$DATA/xtrabackup_binary" ] && rm -f "$DATA/xtrabackup_binary" [ -f "$DATA/xtrabackup_galera_info" ] && rm -f "$DATA/xtrabackup_galera_info" ADDR="$WSREP_SST_OPT_HOST" @@ -1235,7 +1215,6 @@ then MY_SECRET="" # for check down in recv_joiner() fi - trap sig_joiner_cleanup HUP PIPE INT TERM trap cleanup_at_exit EXIT if [ -n "$progress" ]; then @@ -1287,13 +1266,13 @@ then cd "$DATA" wsrep_log_info "Cleaning the old binary logs" # If there is a file with binlogs state, delete it: - [ -f "$binlog_base.state" ] && rm -fv "$binlog_base.state" 1>&2 + [ -f "$binlog_base.state" ] && rm -f "$binlog_base.state" >&2 # Clean up the old binlog files and index: if [ -f "$binlog_index" ]; then while read bin_file || [ -n "$bin_file" ]; do - rm -fv "$bin_file" 1>&2 || : + rm -f "$bin_file" >&2 || : done < "$binlog_index" - rm -fv "$binlog_index" 1>&2 + rm -f "$binlog_index" >&2 fi if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \ -d "$binlog_dir" ] @@ -1304,7 +1283,7 @@ then "Cleaning the binlog directory '$binlog_dir' as well" fi fi - rm -fv "$binlog_base".[0-9]* 1>&2 || : + rm -f "$binlog_base".[0-9]* >&2 || : cd "$OLD_PWD" fi @@ -1315,13 +1294,13 @@ then ${ib_undo_dir:+"$ib_undo_dir"} \ ${ib_log_dir:+"$ib_log_dir"} \ "$DATA" -mindepth 1 -prune -regex "$cpat" \ - -o -exec rm -rfv {} 1>&2 \+ + -o -exec rm -rf {} >&2 \+ else find ${ib_home_dir:+"$ib_home_dir"} \ ${ib_undo_dir:+"$ib_undo_dir"} \ ${ib_log_dir:+"$ib_log_dir"} \ "$DATA" -mindepth 1 -prune -regex "$cpat" \ - -o -exec rm -rfv {} 1>&2 \+ + -o -exec rm -rf {} >&2 \+ fi TDATA="$DATA" @@ -1395,7 +1374,6 @@ then wsrep_log_info "Preparing the backup at $DATA" setup_commands timeit 'mariabackup prepare stage' "$INNOAPPLY" - if [ $? -ne 0 ]; then wsrep_log_error "mariabackup apply finished with errors." \ "Check syslog or '$INNOAPPLYLOG' for details." @@ -1452,6 +1430,10 @@ then else wsrep_log_info "'$IST_FILE' received from donor: Running IST" + if [ $WSREP_SST_OPT_BYPASS -eq 0 ]; then + readonly WSREP_SST_OPT_BYPASS=1 + readonly WSREP_TRANSFER_TYPE='IST' + fi fi @@ -1460,12 +1442,13 @@ then exit 2 fi - coords=$(cat "$MAGIC_FILE") + # Remove special tags from the magic file, and from the output: + coords=$(grep -v -E "^$SECRET_TAG[[:space:]]" -- "$MAGIC_FILE") wsrep_log_info "Galera co-ords from recovery: $coords" - cat "$MAGIC_FILE" # Output : UUID:seqno wsrep_gtid_domain_id + echo "$coords" # Output : UUID:seqno wsrep_gtid_domain_id wsrep_log_info "Total time on joiner: $totime seconds" - wsrep_log_info "mariabackup SST/IST completed on joiner" fi +wsrep_log_info "$WSREP_METHOD $WSREP_TRANSFER_TYPE completed on $WSREP_SST_OPT_ROLE" exit 0 diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index 3c92f489cb5..e1efcbf11ad 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -163,10 +163,11 @@ then echo "$STOP_WSREP" && $MYSQLDUMP && echo "$CSV_TABLES_FIX" && \ echo "$RESTORE_GENERAL_LOG" && echo "$RESTORE_SLOW_QUERY_LOG" && \ echo "$SET_START_POSITION" && echo "$SET_WSREP_GTID_DOMAIN_ID" \ - || echo "SST failed to complete;") | $MYSQL + || echo "SST failed to complete;") | $MYSQL || exit $? else wsrep_log_info "Bypassing state dump." - echo "$SET_START_POSITION" | $MYSQL + echo "$SET_START_POSITION" | $MYSQL || exit $? fi -# +wsrep_log_info "$WSREP_METHOD $WSREP_TRANSFER_TYPE completed on $WSREP_SST_OPT_ROLE" +exit 0 diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 27d4e875674..1775281e634 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -34,6 +34,12 @@ wsrep_check_programs rsync cleanup_joiner() { + # Since this is invoked just after exit NNN + local estatus=$? + if [ $estatus -ne 0 ]; then + wsrep_log_error "Cleanup after exit with status: $estatus" + fi + local failure=0 [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" @@ -72,7 +78,9 @@ cleanup_joiner() wsrep_cleanup_progress_file fi - [ -f "$SST_PID" ] && rm -f "$SST_PID" + [ -f "$SST_PID" ] && rm -f "$SST_PID" || : + + exit $estatus } check_pid_and_port() @@ -310,6 +318,7 @@ if [ -n "$SSLMODE" -a "$SSLMODE" != 'DISABLED' ]; then fi readonly SECRET_TAG='secret' +readonly BYPASS_TAG='secret /bypass' SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid" @@ -325,6 +334,7 @@ while check_pid "$SST_PID" 0; do sleep 1 done +trap simple_cleanup EXIT echo $$ > "$SST_PID" # give some time for stunnel from the previous SST to complete: @@ -358,7 +368,7 @@ while check_pid "$RSYNC_PID" 1 "$RSYNC_CONF"; do sleep 1 done -[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" +[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then @@ -390,7 +400,7 @@ EOF ERROR="$WSREP_SST_OPT_DATA/sst_error" [ -f "$FLUSHED" ] && rm -f "$FLUSHED" - [ -f "$ERROR" ] && rm -f "$ERROR" + [ -f "$ERROR" ] && rm -f "$ERROR" echo 'flush tables' @@ -400,7 +410,7 @@ EOF # (c) ERROR file, in case flush tables operation failed. while [ ! -r "$FLUSHED" ] && \ - ! grep -q -F ':' '--' "$FLUSHED" >/dev/null 2>&1 + ! grep -q -F ':' -- "$FLUSHED" 2>/dev/null do # Check whether ERROR file exists. if [ -f "$ERROR" ]; then @@ -440,10 +450,10 @@ EOF tar_type=0 if tar --help | grep -qw -F -- '--transform'; then tar_type=1 - elif tar --version | grep -q -E '^bsdtar\>'; then + elif tar --version | grep -qw -E '^bsdtar'; then tar_type=2 fi - if [ $tar_type -ne 2 ]; then + if [ $tar_type -eq 2 ]; then if [ -n "$BASH_VERSION" ]; then printf '%s' "$binlog_files" >&2 else @@ -508,9 +518,8 @@ EOF fi # Use deltaxfer only for WAN: - inv=$(basename "$0") WHOLE_FILE_OPT="" - if [ "${inv%wsrep_sst_rsync_wan*}" = "$inv" ]; then + if [ "${WSREP_METHOD%_wan}" = "$WSREP_METHOD" ]; then WHOLE_FILE_OPT='--whole-file' fi @@ -620,7 +629,6 @@ FILTER="-f '- /lost+found' wsrep_log_info "Transfer of data done" - else # BYPASS wsrep_log_info "Bypassing state dump." @@ -641,6 +649,10 @@ FILTER="-f '- /lost+found' echo "$SECRET_TAG $WSREP_SST_OPT_REMOTE_PSWD" >> "$MAGIC_FILE" fi + if [ $WSREP_SST_OPT_BYPASS -ne 0 ]; then + echo "$BYPASS_TAG" >> "$MAGIC_FILE" + fi + rsync ${STUNNEL:+--rsh="$STUNNEL"} \ --archive --quiet --checksum "$MAGIC_FILE" \ "rsync://$WSREP_SST_OPT_ADDR" >&2 || RC=$? @@ -654,15 +666,11 @@ FILTER="-f '- /lost+found' if [ -n "$STUNNEL" ]; then [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" - [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID" + [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID" fi - [ -f "$SST_PID" ] && rm -f "$SST_PID" - - wsrep_log_info "rsync SST/IST completed on donor" +else # joiner -elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] -then check_sockets_utils ADDR="$WSREP_SST_OPT_HOST" @@ -670,8 +678,6 @@ then RSYNC_ADDR="$WSREP_SST_OPT_HOST" RSYNC_ADDR_UNESCAPED="$WSREP_SST_OPT_HOST_UNESCAPED" - trap 'exit 32' HUP PIPE - trap 'exit 3' INT TERM ABRT trap cleanup_joiner EXIT touch "$SST_PROGRESS_FILE" @@ -820,8 +826,13 @@ EOF fi if [ -n "$MY_SECRET" ]; then + # Select the "secret" tag whose value does not start + # with a slash symbol. All new tags must to start with + # the space and the slash symbol after the word "secret" - + # to be removed by older versions of the SST scripts: + SECRET=$(grep -m1 -E "^$SECRET_TAG[[:space:]]+[^/]" \ + -- "$MAGIC_FILE" || :) # Check donor supplied secret: - SECRET=$(grep -m1 -E "^$SECRET_TAG[[:space:]]" -- "$MAGIC_FILE" || :) SECRET=$(trim_string "${SECRET#$SECRET_TAG}") if [ "$SECRET" != "$MY_SECRET" ]; then wsrep_log_error "Donor does not know my secret!" @@ -830,34 +841,45 @@ EOF fi fi - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - binlog_tar_present=0 - [ -f "$BINLOG_TAR_FILE" ] && binlog_tar_present=1 + if [ $WSREP_SST_OPT_BYPASS -eq 0 ]; then + if grep -m1 -qE "^$BYPASS_TAG([[space]]+.*)?\$" -- "$MAGIC_FILE"; then + readonly WSREP_SST_OPT_BYPASS=1 + readonly WSREP_TRANSFER_TYPE='IST' + fi + fi + + binlog_tar_present=0 + if [ -f "$BINLOG_TAR_FILE" ]; then + if [ $WSREP_SST_OPT_BYPASS -ne 0 ]; then + wsrep_log_warning "tar with binlogs transferred in the IST mode" + fi + binlog_tar_present=1 + fi + + if [ $WSREP_SST_OPT_BYPASS -eq 0 -a -n "$WSREP_SST_OPT_BINLOG" ]; then # If it is SST (not an IST) or tar with binlogs is present # among the transferred files, then we need to remove the # old binlogs: - if [ $WSREP_SST_OPT_BYPASS -eq 0 -o $binlog_tar_present -ne 0 ]; then - cd "$DATA" - # Clean up the old binlog files and index: - binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" - if [ -f "$binlog_index" ]; then - while read bin_file || [ -n "$bin_file" ]; do - rm -f "$bin_file" || : - done < "$binlog_index" - rm -f "$binlog_index" - fi - binlog_cd=0 - # Change the directory to binlog base (if possible): - if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \ - -d "$binlog_dir" ] - then - binlog_cd=1 - cd "$binlog_dir" - fi - # Clean up unindexed binlog files: - rm -f "$binlog_base".[0-9]* || : - [ $binlog_cd -ne 0 ] && cd "$DATA_DIR" + cd "$DATA" + # Clean up the old binlog files and index: + binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" + if [ -f "$binlog_index" ]; then + while read bin_file || [ -n "$bin_file" ]; do + rm -f "$bin_file" || : + done < "$binlog_index" + rm -f "$binlog_index" + fi + binlog_cd=0 + # Change the directory to binlog base (if possible): + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \ + -d "$binlog_dir" ] + then + binlog_cd=1 + cd "$binlog_dir" fi + # Clean up unindexed binlog files: + rm -f "$binlog_base".[0-9]* || : + [ $binlog_cd -ne 0 ] && cd "$DATA_DIR" if [ $binlog_tar_present -ne 0 ]; then # Create a temporary file: tmpdir=$(parse_cnf '--mysqld|sst' 'tmpdir') @@ -881,7 +903,7 @@ EOF # Extracting binlog files: wsrep_log_info "Extracting binlog files:" RC=0 - if tar --version | grep -q -E '^bsdtar\>'; then + if tar --version | grep -qw -E '^bsdtar'; then tar -tf "$BINLOG_TAR_FILE" > "$tmpfile" && \ tar -xvf "$BINLOG_TAR_FILE" > /dev/null || RC=$? else @@ -889,8 +911,8 @@ EOF cat "$tmpfile" >&2 || RC=$? fi if [ $RC -ne 0 ]; then - rm -f "$tmpfile" wsrep_log_error "Error unpacking tar file with binlog files" + rm -f "$tmpfile" exit 32 fi # Rebuild binlog index: @@ -903,24 +925,13 @@ EOF fi fi - if [ -n "$MY_SECRET" ]; then - # remove secret from the magic file, and output - # the UUID:seqno & wsrep_gtid_domain_id: - grep -v -E "^$SECRET_TAG[[:space:]]" -- "$MAGIC_FILE" - else - # Output the UUID:seqno and wsrep_gtid_domain_id: - cat "$MAGIC_FILE" - fi - - wsrep_log_info "rsync SST/IST completed on joiner" - -# wsrep_cleanup_progress_file -# cleanup_joiner -else - wsrep_log_error "Unrecognized role: '$WSREP_SST_OPT_ROLE'" - exit 22 # EINVAL + # Remove special tags from the magic file, and from the output: + coords=$(grep -v -E "^$SECRET_TAG[[:space:]]" -- "$MAGIC_FILE") + wsrep_log_info "Galera co-ords from recovery: $coords" + echo "$coords" # Output : UUID:seqno wsrep_gtid_domain_id fi [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" +wsrep_log_info "$WSREP_METHOD $WSREP_TRANSFER_TYPE completed on $WSREP_SST_OPT_ROLE" exit 0 -- cgit v1.2.1 From 98ca71ab28fffbb65c4590cda904f1f0dfd975bb Mon Sep 17 00:00:00 2001 From: Andrei Date: Wed, 11 May 2022 13:12:48 +0300 Subject: MDEV-28461 semisync-slave server recovery fails to rollback prepared transaction that is not in binlog. Post-crash recovery of --rpl-semi-sync-slave-enabled server failed to recognize a transaction in-doubt that needed rolled back. A prepared-but-not-in-binlog transaction gets committed instead to possibly create inconsistency with a master (e.g the way it was observed in the bug report). The semisync recovery is corrected now with initializing binlog coordinates of any transaction in-doubt to the maximum offset which is unreachable. In effect when a prepared transaction that is not found in binlog it will be decided to rollback because it's guaranteed to reside in a truncated tail area of binlog. Mtr tests are reinforced to cover the described scenario. --- .../binlog/r/binlog_truncate_active_log.result | 87 ++++++++++++++++++---- .../binlog/r/binlog_truncate_multi_log.result | 8 +- .../suite/binlog/t/binlog_truncate_active_log.inc | 15 +++- .../suite/binlog/t/binlog_truncate_active_log.test | 9 ++- .../suite/binlog/t/binlog_truncate_multi_log.test | 12 ++- sql/handler.h | 13 +++- 6 files changed, 119 insertions(+), 25 deletions(-) diff --git a/mysql-test/suite/binlog/r/binlog_truncate_active_log.result b/mysql-test/suite/binlog/r/binlog_truncate_active_log.result index e76c3c3911b..85d37534c0b 100644 --- a/mysql-test/suite/binlog/r/binlog_truncate_active_log.result +++ b/mysql-test/suite/binlog/r/binlog_truncate_active_log.result @@ -4,11 +4,13 @@ RESET MASTER; SET @@global.sync_binlog=1; CREATE TABLE t (f INT) ENGINE=INNODB; CREATE TABLE t2 (f INT) ENGINE=INNODB; +CREATE TABLE t4 (f INT) ENGINE=INNODB; CREATE TABLE tm (f INT) ENGINE=Aria; # Case A. connect master1,localhost,root,,; connect master2,localhost,root,,; connect master3,localhost,root,,; +connect master4,localhost,root,,; connection default; INSERT INTO t VALUES (10); INSERT INTO tm VALUES (10); @@ -23,14 +25,23 @@ connection master3; SET DEBUG_SYNC= "now WAIT_FOR master2_ready"; SELECT @@global.gtid_binlog_pos as 'Before the crash'; Before the crash -0-1-7 +0-1-8 +connection master4; +SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL master4_ready WAIT_FOR signal_never_arrives"; +INSERT INTO t4 VALUES (13); +connection master3; +SET DEBUG_SYNC= "now WAIT_FOR master4_ready"; +SELECT @@global.gtid_binlog_pos as 'Before the crash and never logged trx'; +Before the crash and never logged trx +0-1-8 connection default; # Kill the server disconnect master1; disconnect master2; disconnect master3; -# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 -FOUND 1 /Successfully truncated.*to remove transactions starting from GTID 0-1-6/ in mysqld.1.err +disconnect master4; +# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3 +FOUND 1 /Successfully truncated.*to remove transactions starting from GTID 0-1-7/ in mysqld.1.err Pre-crash binlog file content: include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info @@ -39,6 +50,8 @@ master-bin.000001 # Query # # use `test`; CREATE TABLE t (f INT) ENGINE=INNODB master-bin.000001 # Gtid # # GTID #-#-# master-bin.000001 # Query # # use `test`; CREATE TABLE t2 (f INT) ENGINE=INNODB master-bin.000001 # Gtid # # GTID #-#-# +master-bin.000001 # Query # # use `test`; CREATE TABLE t4 (f INT) ENGINE=INNODB +master-bin.000001 # Gtid # # GTID #-#-# master-bin.000001 # Query # # use `test`; CREATE TABLE tm (f INT) ENGINE=Aria master-bin.000001 # Gtid # # BEGIN GTID #-#-# master-bin.000001 # Query # # use `test`; INSERT INTO t VALUES (10) @@ -48,16 +61,20 @@ master-bin.000001 # Query # # use `test`; INSERT INTO tm VALUES (10) master-bin.000001 # Query # # COMMIT SELECT @@global.gtid_binlog_pos as 'After the crash'; After the crash -0-1-5 +0-1-6 "One row should be present in table 't'" SELECT * FROM t; f 10 +"No row should be present in table 't4'" +SELECT * FROM t4; +f DELETE FROM t; # Case B. connect master1,localhost,root,,; connect master2,localhost,root,,; connect master3,localhost,root,,; +connect master4,localhost,root,,; connection default; INSERT INTO t VALUES (10); INSERT INTO tm VALUES (10); @@ -72,14 +89,23 @@ connection master3; SET DEBUG_SYNC= "now WAIT_FOR master2_ready"; SELECT @@global.gtid_binlog_pos as 'Before the crash'; Before the crash -0-1-10 +0-1-11 +connection master4; +SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL master4_ready WAIT_FOR signal_never_arrives"; +INSERT INTO t4 VALUES (13); +connection master3; +SET DEBUG_SYNC= "now WAIT_FOR master4_ready"; +SELECT @@global.gtid_binlog_pos as 'Before the crash and never logged trx'; +Before the crash and never logged trx +0-1-11 connection default; # Kill the server disconnect master1; disconnect master2; disconnect master3; -# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 -FOUND 1 /Successfully truncated.*to remove transactions starting from GTID 0-1-10/ in mysqld.1.err +disconnect master4; +# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3 +FOUND 1 /Successfully truncated.*to remove transactions starting from GTID 0-1-11/ in mysqld.1.err Pre-crash binlog file content: include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info @@ -97,11 +123,14 @@ master-bin.000002 # Query # # use `test`; DELETE FROM t2 WHERE f = 0 master-bin.000002 # Query # # COMMIT SELECT @@global.gtid_binlog_pos as 'After the crash'; After the crash -0-1-9 +0-1-10 "One row should be present in table 't'" SELECT * FROM t; f 10 +"No row should be present in table 't4'" +SELECT * FROM t4; +f DELETE FROM t; # Case C. CREATE PROCEDURE sp_blank_xa() @@ -114,6 +143,7 @@ END| connect master1,localhost,root,,; connect master2,localhost,root,,; connect master3,localhost,root,,; +connect master4,localhost,root,,; connection default; INSERT INTO t VALUES (10); INSERT INTO tm VALUES (10); @@ -128,14 +158,23 @@ connection master3; SET DEBUG_SYNC= "now WAIT_FOR master2_ready"; SELECT @@global.gtid_binlog_pos as 'Before the crash'; Before the crash -0-1-15 +0-1-16 +connection master4; +SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL master4_ready WAIT_FOR signal_never_arrives"; +INSERT INTO t4 VALUES (13); +connection master3; +SET DEBUG_SYNC= "now WAIT_FOR master4_ready"; +SELECT @@global.gtid_binlog_pos as 'Before the crash and never logged trx'; +Before the crash and never logged trx +0-1-16 connection default; # Kill the server disconnect master1; disconnect master2; disconnect master3; -# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 -FOUND 1 /Successfully truncated.*to remove transactions starting from GTID 0-1-14/ in mysqld.1.err +disconnect master4; +# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3 +FOUND 1 /Successfully truncated.*to remove transactions starting from GTID 0-1-15/ in mysqld.1.err Pre-crash binlog file content: include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info @@ -158,11 +197,14 @@ master-bin.000003 # Query # # use `test`; INSERT INTO tm VALUES (10) master-bin.000003 # Query # # COMMIT SELECT @@global.gtid_binlog_pos as 'After the crash'; After the crash -0-1-13 +0-1-14 "One row should be present in table 't'" SELECT * FROM t; f 10 +"No row should be present in table 't4'" +SELECT * FROM t4; +f DELETE FROM t; DROP PROCEDURE sp_blank_xa; # Case D. @@ -176,6 +218,7 @@ END| connect master1,localhost,root,,; connect master2,localhost,root,,; connect master3,localhost,root,,; +connect master4,localhost,root,,; connection default; INSERT INTO t VALUES (10); INSERT INTO tm VALUES (10); @@ -190,14 +233,23 @@ connection master3; SET DEBUG_SYNC= "now WAIT_FOR master2_ready"; SELECT @@global.gtid_binlog_pos as 'Before the crash'; Before the crash -0-1-20 +0-1-21 +connection master4; +SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL master4_ready WAIT_FOR signal_never_arrives"; +INSERT INTO t4 VALUES (13); +connection master3; +SET DEBUG_SYNC= "now WAIT_FOR master4_ready"; +SELECT @@global.gtid_binlog_pos as 'Before the crash and never logged trx'; +Before the crash and never logged trx +0-1-21 connection default; # Kill the server disconnect master1; disconnect master2; disconnect master3; -# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 -FOUND 1 /Successfully truncated.*to remove transactions starting from GTID 0-1-20/ in mysqld.1.err +disconnect master4; +# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3 +FOUND 1 /Successfully truncated.*to remove transactions starting from GTID 0-1-21/ in mysqld.1.err Pre-crash binlog file content: include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info @@ -226,11 +278,14 @@ master-bin.000004 # Query # # XA END X'786964',X'',1 master-bin.000004 # XA_prepare # # XA PREPARE X'786964',X'',1 SELECT @@global.gtid_binlog_pos as 'After the crash'; After the crash -0-1-19 +0-1-20 "One row should be present in table 't'" SELECT * FROM t; f 10 +"No row should be present in table 't4'" +SELECT * FROM t4; +f DELETE FROM t; DROP PROCEDURE sp_xa; # Cleanup diff --git a/mysql-test/suite/binlog/r/binlog_truncate_multi_log.result b/mysql-test/suite/binlog/r/binlog_truncate_multi_log.result index ae292c04c32..271e3c50b19 100644 --- a/mysql-test/suite/binlog/r/binlog_truncate_multi_log.result +++ b/mysql-test/suite/binlog/r/binlog_truncate_multi_log.result @@ -22,6 +22,11 @@ SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL master2_ready WAIT INSERT INTO ti VALUES (3, "not gonna survive"); connection default; SET DEBUG_SYNC= "now WAIT_FOR master2_ready"; +connect master3,localhost,root,,; +SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL master3_ready WAIT_FOR master3_go_never_arrives"; +INSERT INTO ti VALUES (4, "not gonna be log therefore survive"),(5, "ditto"); +connection default; +SET DEBUG_SYNC= "now WAIT_FOR master3_ready"; "List of binary logs before crash" show binary logs; Log_name File_size @@ -36,7 +41,8 @@ connection default; # Kill the server disconnect master1; disconnect master2; -# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 +disconnect master3; +# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3 FOUND 1 /truncated binlog file:.*master.*000002/ in mysqld.1.err "One record should be present in table" SELECT * FROM ti; diff --git a/mysql-test/suite/binlog/t/binlog_truncate_active_log.inc b/mysql-test/suite/binlog/t/binlog_truncate_active_log.inc index 75a8310c220..68ac752725e 100644 --- a/mysql-test/suite/binlog/t/binlog_truncate_active_log.inc +++ b/mysql-test/suite/binlog/t/binlog_truncate_active_log.inc @@ -1,6 +1,7 @@ connect(master1,localhost,root,,); connect(master2,localhost,root,,); connect(master3,localhost,root,,); +connect(master4,localhost,root,,); --connection default @@ -22,16 +23,26 @@ SET DEBUG_SYNC= "commit_before_get_LOCK_after_binlog_sync SIGNAL master2_ready"; SET DEBUG_SYNC= "now WAIT_FOR master2_ready"; SELECT @@global.gtid_binlog_pos as 'Before the crash'; +--connection master4 +# Simulate prepared & not-logged trx; it will never recover. +SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL master4_ready WAIT_FOR signal_never_arrives"; +--send INSERT INTO t4 VALUES (13) + +--connection master3 +SET DEBUG_SYNC= "now WAIT_FOR master4_ready"; +SELECT @@global.gtid_binlog_pos as 'Before the crash and never logged trx'; + --connection default --source include/kill_mysqld.inc --disconnect master1 --disconnect master2 --disconnect master3 +--disconnect master4 # # Server restart # ---let $restart_parameters= --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 +--let $restart_parameters= --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3 --source include/start_mysqld.inc # Check error log for a successful truncate message. @@ -49,6 +60,8 @@ SELECT @@global.gtid_binlog_pos as 'Before the crash'; SELECT @@global.gtid_binlog_pos as 'After the crash'; --echo "One row should be present in table 't'" SELECT * FROM t; +--echo "No row should be present in table 't4'" +SELECT * FROM t4; # prepare binlog file index for the next test --inc $binlog_file_index diff --git a/mysql-test/suite/binlog/t/binlog_truncate_active_log.test b/mysql-test/suite/binlog/t/binlog_truncate_active_log.test index 920679e99e9..7c63a853da6 100644 --- a/mysql-test/suite/binlog/t/binlog_truncate_active_log.test +++ b/mysql-test/suite/binlog/t/binlog_truncate_active_log.test @@ -25,6 +25,7 @@ RESET MASTER; SET @@global.sync_binlog=1; CREATE TABLE t (f INT) ENGINE=INNODB; CREATE TABLE t2 (f INT) ENGINE=INNODB; +CREATE TABLE t4 (f INT) ENGINE=INNODB; CREATE TABLE tm (f INT) ENGINE=Aria; # Old (pre-crash) binlog file index initial value. @@ -42,14 +43,14 @@ CREATE TABLE tm (f INT) ENGINE=Aria; # 'query1' onwards will be removed from the binary log. # Show-binlog-events is to prove that. ---let $truncate_gtid_pos = 0-1-6 +--let $truncate_gtid_pos = 0-1-7 --let $query1 = INSERT INTO t VALUES (20) --let $query2 = DELETE FROM t2 WHERE f = 0 /* no such record */ --source binlog_truncate_active_log.inc --echo # Case B. # The inverted sequence ends up to truncate starting from $query2 ---let $truncate_gtid_pos = 0-1-10 +--let $truncate_gtid_pos = 0-1-11 --let $query1 = DELETE FROM t2 WHERE f = 0 --let $query2 = INSERT INTO t VALUES (20) --source binlog_truncate_active_log.inc @@ -68,7 +69,7 @@ delimiter ;| # The same as in A with $query2 being the zero-engine XA transaction. # Both $query1 and $query2 are going to be truncated. ---let $truncate_gtid_pos = 0-1-14 +--let $truncate_gtid_pos = 0-1-15 --let $query1 = INSERT INTO t VALUES (20) --let $query2 = CALL sp_blank_xa --source binlog_truncate_active_log.inc @@ -89,7 +90,7 @@ delimiter ;| # The same as in B with $query1 being the prepared XA transaction. # Truncation must occurs at $query2. ---let $truncate_gtid_pos = 0-1-20 +--let $truncate_gtid_pos = 0-1-21 --let $query1 = CALL sp_xa --let $query2 = INSERT INTO t2 VALUES (20) --source binlog_truncate_active_log.inc diff --git a/mysql-test/suite/binlog/t/binlog_truncate_multi_log.test b/mysql-test/suite/binlog/t/binlog_truncate_multi_log.test index a7068885871..079c79b2984 100644 --- a/mysql-test/suite/binlog/t/binlog_truncate_multi_log.test +++ b/mysql-test/suite/binlog/t/binlog_truncate_multi_log.test @@ -40,6 +40,15 @@ SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL master2_ready WAIT --connection default SET DEBUG_SYNC= "now WAIT_FOR master2_ready"; + +connect(master3,localhost,root,,); +# The 3nd trx for recovery, it won't get into binlog nor therefore recover +SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL master3_ready WAIT_FOR master3_go_never_arrives"; +--send INSERT INTO ti VALUES (4, "not gonna be log therefore survive"),(5, "ditto") + +--connection default +SET DEBUG_SYNC= "now WAIT_FOR master3_ready"; + --echo "List of binary logs before crash" --source include/show_binary_logs.inc --echo # The gtid binlog state prior the crash will be truncated at the end of the test @@ -49,11 +58,12 @@ SELECT @@global.gtid_binlog_state; --source include/kill_mysqld.inc --disconnect master1 --disconnect master2 +--disconnect master3 # # Server restart # ---let $restart_parameters= --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 +--let $restart_parameters= --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3 --source include/start_mysqld.inc # Check error log for a successful truncate message. diff --git a/sql/handler.h b/sql/handler.h index 53a8c655d60..c50e1ced377 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -963,6 +963,7 @@ typedef struct xid_t XID; */ typedef uint Binlog_file_id; const Binlog_file_id MAX_binlog_id= UINT_MAX; +const my_off_t MAX_off_t = (~(my_off_t) 0); /* Compound binlog-id and byte offset of transaction's first event in a sequence (e.g the recovery sequence) of binlog files. @@ -977,14 +978,22 @@ struct xid_recovery_member my_xid xid; uint in_engine_prepare; // number of engines that have xid prepared bool decided_to_commit; - Binlog_offset binlog_coord; // semisync recovery binlog offset + /* + Semisync recovery binlog offset. It's initialized with the maximum + unreachable offset. The max value will remain for any transaction + not found in binlog to yield its rollback decision as it's guaranteed + to be within a truncated tail part of the binlog. + */ + Binlog_offset binlog_coord; XID *full_xid; // needed by wsrep or past it recovery decltype(::server_id) server_id; // server id of orginal server xid_recovery_member(my_xid xid_arg, uint prepare_arg, bool decided_arg, XID *full_xid_arg, decltype(::server_id) server_id_arg) : xid(xid_arg), in_engine_prepare(prepare_arg), - decided_to_commit(decided_arg), full_xid(full_xid_arg) , server_id(server_id_arg) {}; + decided_to_commit(decided_arg), + binlog_coord(Binlog_offset(MAX_binlog_id, MAX_off_t)), + full_xid(full_xid_arg), server_id(server_id_arg) {}; }; /* for recover() handlerton call */ -- cgit v1.2.1