diff options
author | Brandon Nesterenko <brandon.nesterenko@mariadb.com> | 2022-04-13 12:45:50 -0600 |
---|---|---|
committer | Brandon Nesterenko <brandon.nesterenko@mariadb.com> | 2022-04-25 08:07:17 -0600 |
commit | d16c3aca3c3ecd1050b2527c213f4fdf1959d9e2 (patch) | |
tree | bd938f055fcb3fbff55362d9fbec4df71076c12c | |
parent | a83c7ab1ea62954ab81cd599315e76a2f115ff92 (diff) | |
download | mariadb-git-d16c3aca3c3ecd1050b2527c213f4fdf1959d9e2.tar.gz |
MDEV-26473: mysqld got exception 0xc0000005 (rpl_slave_state/rpl_load_gtid_slave_state)
Problem:
========
During mysqld initialization, if the number of GTIDs added since
that last purge of the mysql.gtid_slave_pos tables is greater than
or equal to the –-gtid-cleanup-batch-size value, a race condition
can occur. Specifically, the binlog background thread will submit
the bg_gtid_delete_pending job to the mysql handle manager; however,
the mysql handle manager may not be initialized, leading to crashes.
Solution:
========
Force the mysql handle manager to initialize/start before the binlog
background thread is created.
Reviewed By:
============
Andrei Elkin <andrei.elkin@mariadb.com>
-rw-r--r-- | mysql-test/suite/rpl/r/rpl_mysql_manager_race_condition.result | 27 | ||||
-rw-r--r-- | mysql-test/suite/rpl/t/rpl_mysql_manager_race_condition-slave.opt | 1 | ||||
-rw-r--r-- | mysql-test/suite/rpl/t/rpl_mysql_manager_race_condition.test | 67 | ||||
-rw-r--r-- | sql/mysqld.cc | 2 | ||||
-rw-r--r-- | sql/sql_manager.cc | 1 |
5 files changed, 97 insertions, 1 deletions
diff --git a/mysql-test/suite/rpl/r/rpl_mysql_manager_race_condition.result b/mysql-test/suite/rpl/r/rpl_mysql_manager_race_condition.result new file mode 100644 index 00000000000..1172d8e39a9 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_mysql_manager_race_condition.result @@ -0,0 +1,27 @@ +include/master-slave.inc +[connection master] +connection master; +# Create a GTID event so the binlog background thread will submit a +# mysql handler job the next time mysqld is restarted. +create table t1 (a int); +include/save_master_gtid.inc +connection slave; +include/sync_with_master_gtid.inc +# Set a debug point that forces the main mysqld thread to sleep before +# anything is initialized for the mysql handle manager +# Restart the slave mysqld instance so it re-initializes with the +# binlog background thread submitting a mysql handler job and the +# mysql handler initialization suspending for a second. Without the fix +# associated with this test/patch, the following restart will error +# with a failed assertion. +include/rpl_restart_server.inc [server_number=2 parameters: --debug_dbug="+d,delay_start_handle_manager"] +include/start_slave.inc +# +# Cleanup +# +connection master; +drop table t1; +include/save_master_gtid.inc +connection slave; +include/sync_with_master_gtid.inc +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_mysql_manager_race_condition-slave.opt b/mysql-test/suite/rpl/t/rpl_mysql_manager_race_condition-slave.opt new file mode 100644 index 00000000000..d127ef62043 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_mysql_manager_race_condition-slave.opt @@ -0,0 +1 @@ +--gtid-cleanup-batch-size=1 diff --git a/mysql-test/suite/rpl/t/rpl_mysql_manager_race_condition.test b/mysql-test/suite/rpl/t/rpl_mysql_manager_race_condition.test new file mode 100644 index 00000000000..751da3158b7 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_mysql_manager_race_condition.test @@ -0,0 +1,67 @@ +# +# Purpose: +# This test ensures that, during mysqld initialization, the mysql handle +# manager starts before the binlog background thread. This is because the +# binlog background thread uses the mysql handle manager, and if the background +# thread tries to submit a job to the handle manager before it is +# initialized/started, mysqld can crash (the actual behavior is undefined). +# This race condition lead to the problem described in MDEV-26473. +# +# Methodology: +# This test ensures that the binlog background thread cannot be started +# before the mysql manager is started. Specifically, it forces a path in +# the binlog background thread to call mysql_manager_submit() by reducing +# --gtid-cleanup-batch-size to be 1 (which submits a job to delete unused rows +# from the mysql.gtid_slave_pos* tables). With this path forced, the main +# mysqld thread is suspended just before its handle manager initialization to +# allow time for the binlog thread to call mysql_manager_submit. The fix +# associated with this test should enforce that the binlog background thread is +# not created before the handle manager is initialized. +# +# References: +# MDEV-26473 mysqld got exception 0xc0000005 (rpl_slave_state/rpl_load_gtid_slave_state) +# + +--source include/have_debug.inc +--source include/master-slave.inc + +# The race condition discovered from MDEV-26473 is binlog format independent. +# We use ROW format though because it was used by the reporter. +--source include/have_binlog_format_row.inc + +--connection master + +--echo # Create a GTID event so the binlog background thread will submit a +--echo # mysql handler job the next time mysqld is restarted. +create table t1 (a int); +--source include/save_master_gtid.inc + +--connection slave +--source include/sync_with_master_gtid.inc + +--echo # Set a debug point that forces the main mysqld thread to sleep before +--echo # anything is initialized for the mysql handle manager +--let $rpl_server_parameters=--debug_dbug="+d,delay_start_handle_manager" + + +--echo # Restart the slave mysqld instance so it re-initializes with the +--echo # binlog background thread submitting a mysql handler job and the +--echo # mysql handler initialization suspending for a second. Without the fix +--echo # associated with this test/patch, the following restart will error +--echo # with a failed assertion. +--source include/rpl_restart_server.inc +--source include/start_slave.inc + + +--echo # +--echo # Cleanup +--echo # + +--connection master +drop table t1; +--source include/save_master_gtid.inc + +--connection slave +--source include/sync_with_master_gtid.inc + +--source include/rpl_end.inc diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 754f0821cd8..fb3bee9db1a 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -5405,6 +5405,7 @@ static int init_server_components() unireg_abort(1); } + start_handle_manager(); if (opt_bin_log) { int error; @@ -5864,7 +5865,6 @@ int mysqld_main(int argc, char **argv) } create_shutdown_event(); - start_handle_manager(); /* Copy default global rpl_filter to global_rpl_filter */ copy_filter_setting(global_rpl_filter, get_or_create_rpl_filter("", 0)); diff --git a/sql/sql_manager.cc b/sql/sql_manager.cc index b08e43e8af2..97f525ea3a6 100644 --- a/sql/sql_manager.cc +++ b/sql/sql_manager.cc @@ -133,6 +133,7 @@ void start_handle_manager() { pthread_t hThread; int err; + DBUG_EXECUTE_IF("delay_start_handle_manager", my_sleep(1000);); manager_thread_in_use = 1; mysql_cond_init(key_COND_manager, &COND_manager,NULL); mysql_mutex_init(key_LOCK_manager, &LOCK_manager, NULL); |