From d3837c69a2ae1493c4367d39278c05865e3d3ae5 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Mon, 6 Mar 2017 15:32:54 +0100 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table. Intermediate commit. On server start, look for and read all tables mysql.gtid_slave_pos* to restore the GTID position. Simple test case that moves the data to a new mysql.gtid_slave_pos_innodb table and verifies that the new table is read at server start. --- mysql-test/suite/rpl/r/rpl_mdev12179.result | 38 +++++++ mysql-test/suite/rpl/t/rpl_mdev12179.test | 63 +++++++++++ sql/rpl_rli.cc | 162 ++++++++++++++++++++-------- 3 files changed, 216 insertions(+), 47 deletions(-) create mode 100644 mysql-test/suite/rpl/r/rpl_mdev12179.result create mode 100644 mysql-test/suite/rpl/t/rpl_mdev12179.test diff --git a/mysql-test/suite/rpl/r/rpl_mdev12179.result b/mysql-test/suite/rpl/r/rpl_mdev12179.result new file mode 100644 index 00000000000..e32c2ba6203 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_mdev12179.result @@ -0,0 +1,38 @@ +include/rpl_init.inc [topology=1->2] +include/stop_slave.inc +CHANGE MASTER TO master_use_gtid=slave_pos; +include/start_slave.inc +CREATE TABLE t1 (a INT PRIMARY KEY); +INSERT INTO t1 VALUES (1); +SELECT * FROM t1 ORDER BY a; +a +1 +SELECT * FROM t1 ORDER BY a; +a +1 +include/stop_slave.inc +SET sql_log_bin=0; +CREATE TABLE mysql.gtid_slave_pos_innodb LIKE mysql.gtid_slave_pos; +ALTER TABLE mysql.gtid_slave_pos_innodb ENGINE=InnoDB; +INSERT INTO mysql.gtid_slave_pos_innodb SELECT * FROM mysql.gtid_slave_pos; +TRUNCATE mysql.gtid_slave_pos; +SET sql_log_bin=1; +INSERT INTO t1 VALUES (2); +INSERT INTO t1 VALUES (3); +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +include/save_master_gtid.inc +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +SET sql_log_bin=0; +DROP TABLE mysql.gtid_slave_pos_innodb; +SET sql_log_bin=1; +DROP TABLE t1; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_mdev12179.test b/mysql-test/suite/rpl/t/rpl_mdev12179.test new file mode 100644 index 00000000000..e82f3b49476 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_mdev12179.test @@ -0,0 +1,63 @@ +--let $rpl_topology=1->2 +--source include/rpl_init.inc +--source include/have_innodb.inc + +--connection server_2 +--source include/stop_slave.inc +CHANGE MASTER TO master_use_gtid=slave_pos; +--source include/start_slave.inc + +--connection server_1 +CREATE TABLE t1 (a INT PRIMARY KEY); +INSERT INTO t1 VALUES (1); +SELECT * FROM t1 ORDER BY a; +--save_master_pos + +--connection server_2 +--sync_with_master +SELECT * FROM t1 ORDER BY a; +--source include/stop_slave.inc +SET sql_log_bin=0; +CREATE TABLE mysql.gtid_slave_pos_innodb LIKE mysql.gtid_slave_pos; +ALTER TABLE mysql.gtid_slave_pos_innodb ENGINE=InnoDB; +INSERT INTO mysql.gtid_slave_pos_innodb SELECT * FROM mysql.gtid_slave_pos; +TRUNCATE mysql.gtid_slave_pos; +SET sql_log_bin=1; + +# Restart the slave mysqld server, and verify that the GTID position is +# read correctly from the new mysql.gtid_slave_pos_innodb table. + +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +wait +EOF +--shutdown_server 30 +--source include/wait_until_disconnected.inc + +--connection server_1 +INSERT INTO t1 VALUES (2); +INSERT INTO t1 VALUES (3); +SELECT * FROM t1 ORDER BY a; +--source include/save_master_gtid.inc + +# Let the slave mysqld server start again. +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +restart: --skip-slave-start=0 +EOF + +--connection server_2 +--enable_reconnect +--source include/wait_until_connected_again.inc + +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; + + +--connection server_2 +SET sql_log_bin=0; +DROP TABLE mysql.gtid_slave_pos_innodb; +SET sql_log_bin=1; + +--connection server_1 +DROP TABLE t1; + +--source include/rpl_end.inc diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index fda922c3e12..c7c72f45053 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -32,6 +32,8 @@ #include "slave.h" #include #include +#include "lock.h" +#include "sql_table.h" static int count_relay_log_space(Relay_log_info* rli); @@ -1466,41 +1468,22 @@ Relay_log_info::update_relay_log_state(rpl_gtid *gtid_list, uint32 count) #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) -int -rpl_load_gtid_slave_state(THD *thd) +struct gtid_pos_element { uint64 sub_id; rpl_gtid gtid; }; + +static int +scan_one_gtid_slave_pos_table(THD *thd, HASH *hash, DYNAMIC_ARRAY *array, + LEX_STRING *tablename) { TABLE_LIST tlist; TABLE *table; bool table_opened= false; bool table_scanned= false; - bool array_inited= false; - struct local_element { uint64 sub_id; rpl_gtid gtid; }; - struct local_element tmp_entry, *entry; - HASH hash; - DYNAMIC_ARRAY array; + struct gtid_pos_element tmp_entry, *entry; int err= 0; - uint32 i; - DBUG_ENTER("rpl_load_gtid_slave_state"); - - mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); - bool loaded= rpl_global_gtid_slave_state->loaded; - mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); - if (loaded) - DBUG_RETURN(0); - - my_hash_init(&hash, &my_charset_bin, 32, - offsetof(local_element, gtid) + offsetof(rpl_gtid, domain_id), - sizeof(uint32), NULL, my_free, HASH_UNIQUE); - if ((err= my_init_dynamic_array(&array, sizeof(local_element), 0, 0, MYF(0)))) - goto end; - array_inited= true; thd->reset_for_next_command(); - - tlist.init_one_table(STRING_WITH_LEN("mysql"), - rpl_gtid_slave_state_table_name.str, - rpl_gtid_slave_state_table_name.length, - NULL, TL_READ); + tlist.init_one_table(STRING_WITH_LEN("mysql"), tablename->str, + tablename->length, NULL, TL_READ); if ((err= open_and_lock_tables(thd, &tlist, FALSE, 0))) goto end; table_opened= true; @@ -1546,15 +1529,15 @@ rpl_load_gtid_slave_state(THD *thd) tmp_entry.gtid.domain_id= domain_id; tmp_entry.gtid.server_id= server_id; tmp_entry.gtid.seq_no= seq_no; - if ((err= insert_dynamic(&array, (uchar *)&tmp_entry))) + if ((err= insert_dynamic(array, (uchar *)&tmp_entry))) { my_error(ER_OUT_OF_RESOURCES, MYF(0)); goto end; } - if ((rec= my_hash_search(&hash, (const uchar *)&domain_id, 0))) + if ((rec= my_hash_search(hash, (const uchar *)&domain_id, 0))) { - entry= (struct local_element *)rec; + entry= (struct gtid_pos_element *)rec; if (entry->sub_id >= sub_id) continue; entry->sub_id= sub_id; @@ -1564,8 +1547,8 @@ rpl_load_gtid_slave_state(THD *thd) } else { - if (!(entry= (struct local_element *)my_malloc(sizeof(*entry), - MYF(MY_WME)))) + if (!(entry= (struct gtid_pos_element *)my_malloc(sizeof(*entry), + MYF(MY_WME)))) { my_error(ER_OUTOFMEMORY, MYF(0), (int)sizeof(*entry)); err= 1; @@ -1575,7 +1558,7 @@ rpl_load_gtid_slave_state(THD *thd) entry->gtid.domain_id= domain_id; entry->gtid.server_id= server_id; entry->gtid.seq_no= seq_no; - if ((err= my_hash_insert(&hash, (uchar *)entry))) + if ((err= my_hash_insert(hash, (uchar *)entry))) { my_free(entry); my_error(ER_OUT_OF_RESOURCES, MYF(0)); @@ -1583,6 +1566,104 @@ rpl_load_gtid_slave_state(THD *thd) } } } + err= 0; /* Clear HA_ERR_END_OF_FILE */ + +end: + if (table_scanned) + { + table->file->ha_index_or_rnd_end(); + ha_commit_trans(thd, FALSE); + ha_commit_trans(thd, TRUE); + } + if (table_opened) + { + close_thread_tables(thd); + thd->mdl_context.release_transactional_locks(); + } + return err; +} + + +/* + Look for all tables mysql.gtid_slave_pos*. Read all rows from each such + table found into ARRAY. For each domain id, put the row with highest sub_id + into HASH. +*/ +static int +scan_all_gtid_slave_pos_table(THD *thd, HASH *hash, DYNAMIC_ARRAY *array) +{ + static LEX_STRING mysql_db_name= {C_STRING_WITH_LEN("mysql")}; + char path[FN_REFLEN]; + MY_DIR *dirp; + + thd->reset_for_next_command(); + if (lock_schema_name(thd, mysql_db_name.str)) + return 1; + + build_table_filename(path, sizeof(path) - 1, mysql_db_name.str, "", "", 0); + if (!(dirp= my_dir(path, MYF(MY_DONT_SORT)))) + { + my_error(ER_FILE_NOT_FOUND, MYF(0), path, my_errno); + close_thread_tables(thd); + thd->mdl_context.release_transactional_locks(); + return 1; + } + else + { + size_t i; + Dynamic_array files(dirp->number_of_files); + Discovered_table_list tl(thd, &files); + int err; + + err= ha_discover_table_names(thd, &mysql_db_name, dirp, &tl, false); + my_dirend(dirp); + close_thread_tables(thd); + thd->mdl_context.release_transactional_locks(); + if (err) + return err; + + for (i = 0; i < files.elements(); ++i) + { + if (strncmp(files.at(i)->str, + rpl_gtid_slave_state_table_name.str, + rpl_gtid_slave_state_table_name.length) == 0) + { + if ((err= scan_one_gtid_slave_pos_table(thd, hash, array, files.at(i)))) + return err; + } + } + } + + return 0; +} + + +int +rpl_load_gtid_slave_state(THD *thd) +{ + bool array_inited= false; + struct gtid_pos_element tmp_entry, *entry; + HASH hash; + DYNAMIC_ARRAY array; + int err= 0; + uint32 i; + DBUG_ENTER("rpl_load_gtid_slave_state"); + + mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); + bool loaded= rpl_global_gtid_slave_state->loaded; + mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); + if (loaded) + DBUG_RETURN(0); + + my_hash_init(&hash, &my_charset_bin, 32, + offsetof(gtid_pos_element, gtid) + offsetof(rpl_gtid, domain_id), + sizeof(uint32), NULL, my_free, HASH_UNIQUE); + if ((err= my_init_dynamic_array(&array, sizeof(gtid_pos_element), 0, 0, MYF(0)))) + goto end; + array_inited= true; + + if ((err= scan_all_gtid_slave_pos_table(thd, &hash, &array))) + goto end; mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); if (rpl_global_gtid_slave_state->loaded) @@ -1608,7 +1689,7 @@ rpl_load_gtid_slave_state(THD *thd) for (i= 0; i < hash.records; ++i) { - entry= (struct local_element *)my_hash_element(&hash, i); + entry= (struct gtid_pos_element *)my_hash_element(&hash, i); if (opt_bin_log && mysql_bin_log.bump_seq_no_counter_if_needed(entry->gtid.domain_id, entry->gtid.seq_no)) @@ -1622,20 +1703,7 @@ rpl_load_gtid_slave_state(THD *thd) rpl_global_gtid_slave_state->loaded= true; mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); - err= 0; /* Clear HA_ERR_END_OF_FILE */ - end: - if (table_scanned) - { - table->file->ha_index_or_rnd_end(); - ha_commit_trans(thd, FALSE); - ha_commit_trans(thd, TRUE); - } - if (table_opened) - { - close_thread_tables(thd); - thd->mdl_context.release_transactional_locks(); - } if (array_inited) delete_dynamic(&array); my_hash_free(&hash); -- cgit v1.2.1 From 141a1b09e661a574ff30953f6fb34bd2fbb96607 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Wed, 8 Mar 2017 13:56:29 +0100 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Refactor scan_all_gtid_slave_pos_table() so it can do generic processing on the list of mysql.gtid_slave_pos_XXX tables. --- sql/rpl_rli.cc | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index c7c72f45053..dd85a79f0e3 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -1590,7 +1590,8 @@ end: into HASH. */ static int -scan_all_gtid_slave_pos_table(THD *thd, HASH *hash, DYNAMIC_ARRAY *array) +scan_all_gtid_slave_pos_table(THD *thd, int (*cb)(THD *, LEX_STRING *, void *), + void *cb_data) { static LEX_STRING mysql_db_name= {C_STRING_WITH_LEN("mysql")}; char path[FN_REFLEN]; @@ -1628,7 +1629,7 @@ scan_all_gtid_slave_pos_table(THD *thd, HASH *hash, DYNAMIC_ARRAY *array) rpl_gtid_slave_state_table_name.str, rpl_gtid_slave_state_table_name.length) == 0) { - if ((err= scan_one_gtid_slave_pos_table(thd, hash, array, files.at(i)))) + if ((err= (*cb)(thd, files.at(i), cb_data))) return err; } } @@ -1638,6 +1639,19 @@ scan_all_gtid_slave_pos_table(THD *thd, HASH *hash, DYNAMIC_ARRAY *array) } +struct load_gtid_state_cb_data { + HASH *hash; + DYNAMIC_ARRAY *array; +}; + +static int +load_gtid_state_cb(THD *thd, LEX_STRING *table_name, void *arg) +{ + load_gtid_state_cb_data *data= static_cast(arg); + return scan_one_gtid_slave_pos_table(thd, data->hash, data->array, table_name); +} + + int rpl_load_gtid_slave_state(THD *thd) { @@ -1647,6 +1661,7 @@ rpl_load_gtid_slave_state(THD *thd) DYNAMIC_ARRAY array; int err= 0; uint32 i; + load_gtid_state_cb_data cb_data; DBUG_ENTER("rpl_load_gtid_slave_state"); mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); @@ -1662,7 +1677,9 @@ rpl_load_gtid_slave_state(THD *thd) goto end; array_inited= true; - if ((err= scan_all_gtid_slave_pos_table(thd, &hash, &array))) + cb_data.hash = &hash; + cb_data.array = &array; + if ((err= scan_all_gtid_slave_pos_table(thd, load_gtid_state_cb, &cb_data))) goto end; mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); -- cgit v1.2.1 From 087cf0232864b60ce62550598f5903b766fe6c90 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Thu, 9 Mar 2017 12:16:15 +0100 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Keep track of which mysql.gtid_slave_posXXX tables are available for each engine, by searching for all tables in the mysql schema with names that start with "gtid_slave_pos". The list is computed at server start when the GTID position is loaded, and it is re-computed on every START SLAVE command. This way, the DBA can manually add a table for a new engine, and it will be automatically picked up on next START SLAVE, so a full server restart is not needed. The list is not yet actually used in the code. --- sql/rpl_gtid.cc | 53 ++++++++++++++++++++++++++++++- sql/rpl_gtid.h | 11 +++++++ sql/rpl_rli.cc | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- sql/rpl_rli.h | 1 + sql/sql_repl.cc | 4 +++ 5 files changed, 163 insertions(+), 3 deletions(-) diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 1d9cb39075b..4daa8b9a53b 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -243,7 +243,7 @@ rpl_slave_state_free_element(void *arg) rpl_slave_state::rpl_slave_state() - : last_sub_id(0), loaded(false) + : last_sub_id(0), gtid_pos_tables(0), loaded(false) { mysql_mutex_init(key_LOCK_slave_state, &LOCK_slave_state, MY_MUTEX_INIT_SLOW); @@ -255,6 +255,7 @@ rpl_slave_state::rpl_slave_state() rpl_slave_state::~rpl_slave_state() { + free_gtid_pos_tables(gtid_pos_tables); truncate_hash(); my_hash_free(&hash); delete_dynamic(>id_sort_array); @@ -1115,6 +1116,56 @@ rpl_slave_state::is_empty() } +void +rpl_slave_state::free_gtid_pos_tables(struct rpl_slave_state::gtid_pos_table *list) +{ + struct gtid_pos_table *cur, *next; + + cur= list; + while (cur) + { + next= cur->next; + my_free(cur); + cur= next; + } +} + + +void +rpl_slave_state::set_gtid_pos_tables_list(struct rpl_slave_state::gtid_pos_table *new_list) +{ + struct gtid_pos_table *old_list; + + mysql_mutex_assert_owner(&LOCK_slave_state); + old_list= gtid_pos_tables; + gtid_pos_tables= new_list; + free_gtid_pos_tables(old_list); +} + + +struct rpl_slave_state::gtid_pos_table * +rpl_slave_state::alloc_gtid_pos_table(LEX_STRING *table_name, handlerton *hton) +{ + struct gtid_pos_table *p; + char *allocated_str; + + if (!my_multi_malloc(MYF(MY_WME), + &p, sizeof(*p), + &allocated_str, table_name->length+1, + NULL)) + { + my_error(ER_OUTOFMEMORY, MYF(0), (int)(sizeof(*p) + table_name->length+1)); + return NULL; + } + memcpy(allocated_str, table_name->str, table_name->length+1); // Also copy '\0' + p->next = NULL; + p->table_hton= hton; + p->table_name.str= allocated_str; + p->table_name.length= table_name->length; + return p; +} + + void rpl_binlog_state::init() { my_hash_init(&hash, &my_charset_bin, 32, offsetof(element, domain_id), diff --git a/sql/rpl_gtid.h b/sql/rpl_gtid.h index f638a084e38..c8af2c4946a 100644 --- a/sql/rpl_gtid.h +++ b/sql/rpl_gtid.h @@ -155,6 +155,13 @@ struct rpl_slave_state } }; + /* Descriptor for mysql.gtid_slave_posXXX table in specific engine. */ + struct gtid_pos_table { + struct gtid_pos_table *next; + handlerton *table_hton; + LEX_STRING table_name; + }; + /* Mapping from domain_id to its element. */ HASH hash; /* Mutex protecting access to the state. */ @@ -163,6 +170,7 @@ struct rpl_slave_state DYNAMIC_ARRAY gtid_sort_array; uint64 last_sub_id; + struct gtid_pos_table *gtid_pos_tables; bool loaded; rpl_slave_state(); @@ -192,6 +200,9 @@ struct rpl_slave_state int record_and_update_gtid(THD *thd, struct rpl_group_info *rgi); int check_duplicate_gtid(rpl_gtid *gtid, rpl_group_info *rgi); void release_domain_owner(rpl_group_info *rgi); + void set_gtid_pos_tables_list(struct gtid_pos_table *new_list); + struct gtid_pos_table *alloc_gtid_pos_table(LEX_STRING *table_name, handlerton *hton); + void free_gtid_pos_tables(struct gtid_pos_table *list); }; diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index dd85a79f0e3..1be1e663d86 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -1472,7 +1472,7 @@ struct gtid_pos_element { uint64 sub_id; rpl_gtid gtid; }; static int scan_one_gtid_slave_pos_table(THD *thd, HASH *hash, DYNAMIC_ARRAY *array, - LEX_STRING *tablename) + LEX_STRING *tablename, handlerton **out_hton) { TABLE_LIST tlist; TABLE *table; @@ -1577,6 +1577,7 @@ end: } if (table_opened) { + *out_hton= table->s->db_type(); close_thread_tables(thd); thd->mdl_context.release_transactional_locks(); } @@ -1642,13 +1643,25 @@ scan_all_gtid_slave_pos_table(THD *thd, int (*cb)(THD *, LEX_STRING *, void *), struct load_gtid_state_cb_data { HASH *hash; DYNAMIC_ARRAY *array; + struct rpl_slave_state::gtid_pos_table *table_list; }; static int load_gtid_state_cb(THD *thd, LEX_STRING *table_name, void *arg) { + int err; load_gtid_state_cb_data *data= static_cast(arg); - return scan_one_gtid_slave_pos_table(thd, data->hash, data->array, table_name); + struct rpl_slave_state::gtid_pos_table *p; + handlerton *hton; + + if ((err= scan_one_gtid_slave_pos_table(thd, data->hash, data->array, + table_name, &hton))) + return err; + if (!(p= rpl_global_gtid_slave_state->alloc_gtid_pos_table(table_name, hton))) + return 1; + p->next= data->table_list; + data->table_list= p; + return 0; } @@ -1670,6 +1683,7 @@ rpl_load_gtid_slave_state(THD *thd) if (loaded) DBUG_RETURN(0); + cb_data.table_list= NULL; my_hash_init(&hash, &my_charset_bin, 32, offsetof(gtid_pos_element, gtid) + offsetof(rpl_gtid, domain_id), sizeof(uint32), NULL, my_free, HASH_UNIQUE); @@ -1717,6 +1731,8 @@ rpl_load_gtid_slave_state(THD *thd) } } + rpl_global_gtid_slave_state->set_gtid_pos_tables_list(cb_data.table_list); + cb_data.table_list= NULL; rpl_global_gtid_slave_state->loaded= true; mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); @@ -1724,10 +1740,87 @@ end: if (array_inited) delete_dynamic(&array); my_hash_free(&hash); + if (cb_data.table_list) + rpl_global_gtid_slave_state->free_gtid_pos_tables(cb_data.table_list); DBUG_RETURN(err); } +static int +find_gtid_pos_tables_cb(THD *thd, LEX_STRING *table_name, void *arg) +{ + struct rpl_slave_state::gtid_pos_table **table_list_ptr= + static_cast(arg); + TABLE_LIST tlist; + TABLE *table= NULL; + int err; + struct rpl_slave_state::gtid_pos_table *p; + + thd->reset_for_next_command(); + tlist.init_one_table(STRING_WITH_LEN("mysql"), table_name->str, + table_name->length, NULL, TL_READ); + if ((err= open_and_lock_tables(thd, &tlist, FALSE, 0))) + goto end; + table= tlist.table; + + if ((err= gtid_check_rpl_slave_state_table(table))) + goto end; + + if (!(p= rpl_global_gtid_slave_state->alloc_gtid_pos_table(table_name, table->s->db_type()))) + err= 1; + else + { + p->next= *table_list_ptr; + *table_list_ptr= p; + } + +end: + if (table) + { + ha_commit_trans(thd, FALSE); + ha_commit_trans(thd, TRUE); + close_thread_tables(thd); + thd->mdl_context.release_transactional_locks(); + } + + return err; +} + + +/* + Re-compute the list of available mysql.gtid_slave_posXXX tables. + + This is done at START SLAVE to pick up any newly created tables without + requiring server restart. +*/ +int +find_gtid_slave_pos_tables(THD *thd) +{ + int err= 0; + struct rpl_slave_state::gtid_pos_table *table_list; + + mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); + bool loaded= rpl_global_gtid_slave_state->loaded; + mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); + if (!loaded) + return 0; + + table_list= NULL; + if ((err= scan_all_gtid_slave_pos_table(thd, find_gtid_pos_tables_cb, &table_list))) + goto end; + + mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); + rpl_global_gtid_slave_state->set_gtid_pos_tables_list(table_list); + table_list= NULL; + mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); + +end: + if (table_list) + rpl_global_gtid_slave_state->free_gtid_pos_tables(table_list); + return err; +} + + void rpl_group_info::reinit(Relay_log_info *rli) { diff --git a/sql/rpl_rli.h b/sql/rpl_rli.h index 93e7b869be0..4fc264824f9 100644 --- a/sql/rpl_rli.h +++ b/sql/rpl_rli.h @@ -959,6 +959,7 @@ extern struct rpl_slave_state *rpl_global_gtid_slave_state; extern gtid_waiting rpl_global_gtid_waiting; int rpl_load_gtid_slave_state(THD *thd); +int find_gtid_slave_pos_tables(THD *thd); int event_group_new_gtid(rpl_group_info *rgi, Gtid_log_event *gev); void delete_or_keep_event_post_apply(rpl_group_info *rgi, Log_event_type typ, Log_event *ev); diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index 7c30aa38f1a..087fc293359 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -3031,6 +3031,10 @@ int start_slave(THD* thd , Master_info* mi, bool net_report) if (check_access(thd, SUPER_ACL, any_db, NULL, NULL, 0, 0)) DBUG_RETURN(-1); + /* Re-load the set of mysql.gtid_slave_posXXX tables available. */ + if (find_gtid_slave_pos_tables(thd)) + DBUG_RETURN(-1); + create_logfile_name_with_suffix(master_info_file_tmp, sizeof(master_info_file_tmp), master_info_file, 0, -- cgit v1.2.1 From c995ecbe9834bae31912e00cc98f7c872b63e1fb Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Thu, 9 Mar 2017 13:27:27 +0100 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. For each GTID recorded in mysq.gtid_slave_pos, keep track of which engine the update was made in. This will be later used to know which rows can be deleted in the table of a given engine. --- sql/log_event.cc | 18 +++++++++++------- sql/rpl_gtid.cc | 26 +++++++++++++++++--------- sql/rpl_gtid.h | 18 +++++++++++++----- sql/rpl_rli.cc | 18 +++++++++++------- 4 files changed, 52 insertions(+), 28 deletions(-) diff --git a/sql/log_event.cc b/sql/log_event.cc index 3aea3eaf2f1..d50bd5becdb 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -5027,6 +5027,7 @@ int Query_log_event::do_apply_event(rpl_group_info *rgi, int expected_error,actual_error= 0; Schema_specification_st db_options; uint64 sub_id= 0; + void *hton= NULL; rpl_gtid gtid; Relay_log_info const *rli= rgi->rli; Rpl_filter *rpl_filter= rli->mi->rpl_filter; @@ -5197,7 +5198,7 @@ int Query_log_event::do_apply_event(rpl_group_info *rgi, gtid= rgi->current_gtid; if (rpl_global_gtid_slave_state->record_gtid(thd, >id, sub_id, - true, false)) + true, false, &hton)) { int errcode= thd->get_stmt_da()->sql_errno(); if (!is_parallel_retry_error(rgi, errcode)) @@ -5417,7 +5418,7 @@ compare_errors: end: if (sub_id && !thd->is_slave_error) - rpl_global_gtid_slave_state->update_state_hash(sub_id, >id, rgi); + rpl_global_gtid_slave_state->update_state_hash(sub_id, >id, hton, rgi); /* Probably we have set thd->query, thd->db, thd->catalog to point to places @@ -7899,15 +7900,17 @@ Gtid_list_log_event::do_apply_event(rpl_group_info *rgi) int ret; if (gl_flags & FLAG_IGN_GTIDS) { + void *hton= NULL; uint32 i; + for (i= 0; i < count; ++i) { if ((ret= rpl_global_gtid_slave_state->record_gtid(thd, &list[i], - sub_id_list[i], - false, false))) + sub_id_list[i], + false, false, &hton))) return ret; rpl_global_gtid_slave_state->update_state_hash(sub_id_list[i], &list[i], - NULL); + hton, NULL); } } ret= Log_event::do_apply_event(rgi); @@ -8388,6 +8391,7 @@ int Xid_log_event::do_apply_event(rpl_group_info *rgi) rpl_gtid gtid; uint64 sub_id= 0; Relay_log_info const *rli= rgi->rli; + void *hton= NULL; /* XID_EVENT works like a COMMIT statement. And it also updates the @@ -8408,7 +8412,7 @@ int Xid_log_event::do_apply_event(rpl_group_info *rgi) gtid= rgi->current_gtid; err= rpl_global_gtid_slave_state->record_gtid(thd, >id, sub_id, true, - false); + false, &hton); if (err) { int ec= thd->get_stmt_da()->sql_errno(); @@ -8441,7 +8445,7 @@ int Xid_log_event::do_apply_event(rpl_group_info *rgi) thd->mdl_context.release_transactional_locks(); if (!res && sub_id) - rpl_global_gtid_slave_state->update_state_hash(sub_id, >id, rgi); + rpl_global_gtid_slave_state->update_state_hash(sub_id, >id, hton, rgi); /* Increment the global status commit count variable diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 4daa8b9a53b..2d12aafdd02 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -33,7 +33,7 @@ const LEX_STRING rpl_gtid_slave_state_table_name= void -rpl_slave_state::update_state_hash(uint64 sub_id, rpl_gtid *gtid, +rpl_slave_state::update_state_hash(uint64 sub_id, rpl_gtid *gtid, void *hton, rpl_group_info *rgi) { int err; @@ -45,7 +45,7 @@ rpl_slave_state::update_state_hash(uint64 sub_id, rpl_gtid *gtid, it is even committed. */ mysql_mutex_lock(&LOCK_slave_state); - err= update(gtid->domain_id, gtid->server_id, sub_id, gtid->seq_no, rgi); + err= update(gtid->domain_id, gtid->server_id, sub_id, gtid->seq_no, hton, rgi); mysql_mutex_unlock(&LOCK_slave_state); if (err) { @@ -74,12 +74,14 @@ rpl_slave_state::record_and_update_gtid(THD *thd, rpl_group_info *rgi) if (rgi->gtid_pending) { uint64 sub_id= rgi->gtid_sub_id; + void *hton= NULL; + rgi->gtid_pending= false; if (rgi->gtid_ignore_duplicate_state!=rpl_group_info::GTID_DUPLICATE_IGNORE) { - if (record_gtid(thd, &rgi->current_gtid, sub_id, false, false)) + if (record_gtid(thd, &rgi->current_gtid, sub_id, false, false, &hton)) DBUG_RETURN(1); - update_state_hash(sub_id, &rgi->current_gtid, rgi); + update_state_hash(sub_id, &rgi->current_gtid, hton, rgi); } rgi->gtid_ignore_duplicate_state= rpl_group_info::GTID_DUPLICATE_NULL; } @@ -287,11 +289,12 @@ rpl_slave_state::truncate_hash() int rpl_slave_state::update(uint32 domain_id, uint32 server_id, uint64 sub_id, - uint64 seq_no, rpl_group_info *rgi) + uint64 seq_no, void *hton, rpl_group_info *rgi) { element *elem= NULL; list_element *list_elem= NULL; + DBUG_ASSERT(hton); if (!(elem= get_element(domain_id))) return 1; @@ -336,6 +339,7 @@ rpl_slave_state::update(uint32 domain_id, uint32 server_id, uint64 sub_id, list_elem->server_id= server_id; list_elem->sub_id= sub_id; list_elem->seq_no= seq_no; + list_elem->hton= hton; elem->add(list_elem); if (last_sub_id < sub_id) @@ -482,7 +486,8 @@ gtid_check_rpl_slave_state_table(TABLE *table) */ int rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, - bool in_transaction, bool in_statement) + bool in_transaction, bool in_statement, + void **out_hton) { TABLE_LIST tlist; int err= 0; @@ -495,6 +500,7 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, wait_for_commit* suspended_wfc; DBUG_ENTER("record_gtid"); + *out_hton= NULL; if (unlikely(!loaded)) { /* @@ -582,6 +588,7 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, table->file->print_error(err, MYF(0)); goto end; } + *out_hton= table->s->db_type(); if(opt_bin_log && (err= mysql_bin_log.bump_seq_no_counter_if_needed(gtid->domain_id, @@ -1078,11 +1085,12 @@ rpl_slave_state::load(THD *thd, char *state_from_master, size_t len, { rpl_gtid gtid; uint64 sub_id; + void *hton= NULL; if (gtid_parser_helper(&state_from_master, end, >id) || !(sub_id= next_sub_id(gtid.domain_id)) || - record_gtid(thd, >id, sub_id, false, in_statement) || - update(gtid.domain_id, gtid.server_id, sub_id, gtid.seq_no, NULL)) + record_gtid(thd, >id, sub_id, false, in_statement, &hton) || + update(gtid.domain_id, gtid.server_id, sub_id, gtid.seq_no, hton, NULL)) return 1; if (state_from_master == end) break; @@ -1144,7 +1152,7 @@ rpl_slave_state::set_gtid_pos_tables_list(struct rpl_slave_state::gtid_pos_table struct rpl_slave_state::gtid_pos_table * -rpl_slave_state::alloc_gtid_pos_table(LEX_STRING *table_name, handlerton *hton) +rpl_slave_state::alloc_gtid_pos_table(LEX_STRING *table_name, void *hton) { struct gtid_pos_table *p; char *allocated_str; diff --git a/sql/rpl_gtid.h b/sql/rpl_gtid.h index c8af2c4946a..ea278427061 100644 --- a/sql/rpl_gtid.h +++ b/sql/rpl_gtid.h @@ -112,6 +112,7 @@ struct rpl_slave_state uint64 sub_id; uint64 seq_no; uint32 server_id; + void *hton; }; /* Elements in the HASH that hold the state for one domain_id. */ @@ -158,7 +159,13 @@ struct rpl_slave_state /* Descriptor for mysql.gtid_slave_posXXX table in specific engine. */ struct gtid_pos_table { struct gtid_pos_table *next; - handlerton *table_hton; + /* + Use a void * here, rather than handlerton *, to make explicit that we + are not using the value to access any functionality in the engine. It + is just used as an opaque value to identify which engine we are using + for each GTID row. + */ + void *table_hton; LEX_STRING table_name; }; @@ -179,10 +186,10 @@ struct rpl_slave_state void truncate_hash(); ulong count() const { return hash.records; } int update(uint32 domain_id, uint32 server_id, uint64 sub_id, - uint64 seq_no, rpl_group_info *rgi); + uint64 seq_no, void *hton, rpl_group_info *rgi); int truncate_state_table(THD *thd); int record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, - bool in_transaction, bool in_statement); + bool in_transaction, bool in_statement, void **out_hton); uint64 next_sub_id(uint32 domain_id); int iterate(int (*cb)(rpl_gtid *, void *), void *data, rpl_gtid *extra_gtids, uint32 num_extra, @@ -196,12 +203,13 @@ struct rpl_slave_state element *get_element(uint32 domain_id); int put_back_list(uint32 domain_id, list_element *list); - void update_state_hash(uint64 sub_id, rpl_gtid *gtid, rpl_group_info *rgi); + void update_state_hash(uint64 sub_id, rpl_gtid *gtid, void *hton, + rpl_group_info *rgi); int record_and_update_gtid(THD *thd, struct rpl_group_info *rgi); int check_duplicate_gtid(rpl_gtid *gtid, rpl_group_info *rgi); void release_domain_owner(rpl_group_info *rgi); void set_gtid_pos_tables_list(struct gtid_pos_table *new_list); - struct gtid_pos_table *alloc_gtid_pos_table(LEX_STRING *table_name, handlerton *hton); + struct gtid_pos_table *alloc_gtid_pos_table(LEX_STRING *table_name, void *hton); void free_gtid_pos_tables(struct gtid_pos_table *list); }; diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 1be1e663d86..74d831db6dc 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -1468,11 +1468,11 @@ Relay_log_info::update_relay_log_state(rpl_gtid *gtid_list, uint32 count) #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) -struct gtid_pos_element { uint64 sub_id; rpl_gtid gtid; }; +struct gtid_pos_element { uint64 sub_id; rpl_gtid gtid; void *hton; }; static int scan_one_gtid_slave_pos_table(THD *thd, HASH *hash, DYNAMIC_ARRAY *array, - LEX_STRING *tablename, handlerton **out_hton) + LEX_STRING *tablename, void **out_hton) { TABLE_LIST tlist; TABLE *table; @@ -1529,6 +1529,7 @@ scan_one_gtid_slave_pos_table(THD *thd, HASH *hash, DYNAMIC_ARRAY *array, tmp_entry.gtid.domain_id= domain_id; tmp_entry.gtid.server_id= server_id; tmp_entry.gtid.seq_no= seq_no; + tmp_entry.hton= table->s->db_type(); if ((err= insert_dynamic(array, (uchar *)&tmp_entry))) { my_error(ER_OUT_OF_RESOURCES, MYF(0)); @@ -1544,6 +1545,7 @@ scan_one_gtid_slave_pos_table(THD *thd, HASH *hash, DYNAMIC_ARRAY *array, DBUG_ASSERT(entry->gtid.domain_id == domain_id); entry->gtid.server_id= server_id; entry->gtid.seq_no= seq_no; + entry->hton= table->s->db_type(); } else { @@ -1558,6 +1560,7 @@ scan_one_gtid_slave_pos_table(THD *thd, HASH *hash, DYNAMIC_ARRAY *array, entry->gtid.domain_id= domain_id; entry->gtid.server_id= server_id; entry->gtid.seq_no= seq_no; + entry->hton= table->s->db_type(); if ((err= my_hash_insert(hash, (uchar *)entry))) { my_free(entry); @@ -1652,7 +1655,7 @@ load_gtid_state_cb(THD *thd, LEX_STRING *table_name, void *arg) int err; load_gtid_state_cb_data *data= static_cast(arg); struct rpl_slave_state::gtid_pos_table *p; - handlerton *hton; + void *hton; if ((err= scan_one_gtid_slave_pos_table(thd, data->hash, data->array, table_name, &hton))) @@ -1707,10 +1710,11 @@ rpl_load_gtid_slave_state(THD *thd) { get_dynamic(&array, (uchar *)&tmp_entry, i); if ((err= rpl_global_gtid_slave_state->update(tmp_entry.gtid.domain_id, - tmp_entry.gtid.server_id, - tmp_entry.sub_id, - tmp_entry.gtid.seq_no, - NULL))) + tmp_entry.gtid.server_id, + tmp_entry.sub_id, + tmp_entry.gtid.seq_no, + tmp_entry.hton, + NULL))) { mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); my_error(ER_OUT_OF_RESOURCES, MYF(0)); -- cgit v1.2.1 From 3501a5356e7358c0accfd3465a0c2f66f9b87b70 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Thu, 9 Mar 2017 15:30:19 +0100 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. When deleting old GTIDs, only pick those GTIDs that were recorded using the same engine. (Currently this is not used, there is only one table/engine active. This is in preparation for extending record_gtid to pick the table with the same engine as the active transaction, if any). --- sql/rpl_gtid.cc | 83 ++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 27 deletions(-) diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 2d12aafdd02..9e6f6ec7009 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -493,11 +493,13 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, int err= 0; bool table_opened= false; TABLE *table; - list_element *elist= 0, *next; + list_element *delete_list= 0, *next, *cur, **next_ptr_ptr, **best_ptr_ptr; + uint64_t best_sub_id; element *elem; ulonglong thd_saved_option= thd->variables.option_bits; Query_tables_list lex_backup; wait_for_commit* suspended_wfc; + void *hton= NULL; DBUG_ENTER("record_gtid"); *out_hton= NULL; @@ -553,6 +555,7 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, goto end; table_opened= true; table= tlist.table; + hton= table->s->db_type(); if ((err= gtid_check_rpl_slave_state_table(table))) goto end; @@ -588,7 +591,7 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, table->file->print_error(err, MYF(0)); goto end; } - *out_hton= table->s->db_type(); + *out_hton= hton; if(opt_bin_log && (err= mysql_bin_log.bump_seq_no_counter_if_needed(gtid->domain_id, @@ -606,36 +609,62 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, err= 1; goto end; } - if ((elist= elem->grab_list()) != NULL) + + /* Now pull out all GTIDs that were recorded in this engine. */ + delete_list = NULL; + next_ptr_ptr= &elem->list; + cur= elem->list; + best_sub_id= 0; + best_ptr_ptr= NULL; + while (cur) { - /* Delete any old stuff, but keep around the most recent one. */ - list_element *cur= elist; - uint64 best_sub_id= cur->sub_id; - list_element **best_ptr_ptr= &elist; - while ((next= cur->next)) + list_element *next= cur->next; + if (cur->hton == hton) { - if (next->sub_id > best_sub_id) + /* Belongs to same engine, so move it to the delete list. */ + cur->next= delete_list; + delete_list= cur; + if (cur->sub_id > best_sub_id) { - best_sub_id= next->sub_id; + best_sub_id= cur->sub_id; + best_ptr_ptr= &delete_list; + } + else if (best_ptr_ptr == &delete_list) best_ptr_ptr= &cur->next; + } + else + { + /* Another engine, leave it in the list. */ + if (cur->sub_id > best_sub_id) + { + best_sub_id= cur->sub_id; + /* Current best is not on the delete list. */ + best_ptr_ptr= NULL; } - cur= next; + *next_ptr_ptr= cur; + next_ptr_ptr= &cur->next; } - /* - Delete the highest sub_id element from the old list, and put it back as - the single-element new list. - */ + cur= next; + } + *next_ptr_ptr= NULL; + /* + If the highest sub_id element is on the delete list, put it back on the + original list, to preserve the highest sub_id element in the table for + GTID position recovery. + */ + if (best_ptr_ptr) + { cur= *best_ptr_ptr; *best_ptr_ptr= cur->next; - cur->next= NULL; + cur->next= elem->list; elem->list= cur; } mysql_mutex_unlock(&LOCK_slave_state); - if (!elist) + if (!delete_list) goto end; - /* Now delete any already committed rows. */ + /* Now delete any already committed GTIDs. */ bitmap_set_bit(table->read_set, table->field[0]->field_index); bitmap_set_bit(table->read_set, table->field[1]->field_index); @@ -644,7 +673,7 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, table->file->print_error(err, MYF(0)); goto end; } - while (elist) + while (delete_list) { uchar key_buffer[4+8]; @@ -654,9 +683,9 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, /* `break' does not work inside DBUG_EXECUTE_IF */ goto dbug_break; }); - next= elist->next; + next= delete_list->next; - table->field[1]->store(elist->sub_id, true); + table->field[1]->store(delete_list->sub_id, true); /* domain_id is already set in table->record[0] from write_row() above. */ key_copy(key_buffer, table->record[0], &table->key_info[0], 0, false); if (table->file->ha_index_read_map(table->record[1], key_buffer, @@ -670,8 +699,8 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, not want to endlessly error on the same element in case of table corruption or such. */ - my_free(elist); - elist= next; + my_free(delete_list); + delete_list= next; if (err) break; } @@ -689,13 +718,13 @@ end: if (err || (err= ha_commit_trans(thd, FALSE))) { /* - If error, we need to put any remaining elist back into the HASH so we - can do another delete attempt later. + If error, we need to put any remaining delete_list back into the HASH + so we can do another delete attempt later. */ - if (elist) + if (delete_list) { mysql_mutex_lock(&LOCK_slave_state); - put_back_list(gtid->domain_id, elist); + put_back_list(gtid->domain_id, delete_list); mysql_mutex_unlock(&LOCK_slave_state); } -- cgit v1.2.1 From 6a84473c28af10e072267bc811f280a49bdc8ca8 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Tue, 14 Mar 2017 12:54:10 +0100 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. This commit implements that record_gtid() selects a gtid_slave_posXXX table with a storage engine already in use by current transaction, if any. The default table mysql.gtid_slave_pos is used if no match can be found on storage engine, or for GTID position updates with no specific storage engine. Table discovery of mysql.gtid_slave_pos* happens on initial GTID state load as well as on every START SLAVE. Some effort is made to make this possible without additional locking. New tables are added using lock-free atomics. Removing tables requires stopping all slaves first. A warning is given in the error log when a table is removed but a non-stopped slave still has a reference to it. If multiple mysql.gtid_slave_posXXX tables with same storage engine exist, one is chosen arbitrarily to be used, with a warning in the error log. GTID data from all tables is still read, but only one among redundant tables with same storage engine will be updated. --- .../suite/multi_source/gtid_slave_pos.result | 141 ++++++++++++++++ mysql-test/suite/multi_source/gtid_slave_pos.test | 156 +++++++++++++++++ mysql-test/suite/rpl/t/rpl_mdev12179.test | 2 +- sql/rpl_gtid.cc | 73 +++++++- sql/rpl_gtid.h | 25 ++- sql/rpl_rli.cc | 184 ++++++++++++++++++--- .../mysql-test/tokudb_rpl/r/mdev12179.result | 60 +++++++ .../tokudb/mysql-test/tokudb_rpl/t/mdev12179.test | 56 +++++++ 8 files changed, 663 insertions(+), 34 deletions(-) create mode 100644 mysql-test/suite/multi_source/gtid_slave_pos.result create mode 100644 mysql-test/suite/multi_source/gtid_slave_pos.test create mode 100644 storage/tokudb/mysql-test/tokudb_rpl/r/mdev12179.result create mode 100644 storage/tokudb/mysql-test/tokudb_rpl/t/mdev12179.test diff --git a/mysql-test/suite/multi_source/gtid_slave_pos.result b/mysql-test/suite/multi_source/gtid_slave_pos.result new file mode 100644 index 00000000000..ccccc664110 --- /dev/null +++ b/mysql-test/suite/multi_source/gtid_slave_pos.result @@ -0,0 +1,141 @@ +connect slave1,127.0.0.1,root,,,$SERVER_MYPORT_3; +connect master1,127.0.0.1,root,,,$SERVER_MYPORT_1; +connect master2,127.0.0.1,root,,,$SERVER_MYPORT_2; +connection slave1; +CHANGE MASTER 'slave1' TO master_port=MYPORT_1, master_host='127.0.0.1', master_user='root', master_use_gtid=slave_pos; +CHANGE MASTER 'slave2' TO master_port=MYPORT_2, master_host='127.0.0.1', master_user='root', master_use_gtid=slave_pos; +set default_master_connection = 'slave1'; +START SLAVE; +include/wait_for_slave_to_start.inc +set default_master_connection = 'slave2'; +START SLAVE; +include/wait_for_slave_to_start.inc +set default_master_connection = ''; +connection master1; +SET GLOBAL gtid_domain_id= 1; +SET SESSION gtid_domain_id= 1; +CREATE TABLE t3 (a INT PRIMARY KEY, b VARCHAR(10)) ENGINE=InnoDB; +CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(10)); +INSERT INTO t1 VALUES (1, "initial"); +INSERT INTO t3 VALUES (101, "initial 1"); +include/save_master_gtid.inc +connection master2; +SET GLOBAL gtid_domain_id= 2; +SET SESSION gtid_domain_id= 2; +CREATE TABLE t2 (a INT PRIMARY KEY, b VARCHAR(10)) ENGINE=InnoDB; +INSERT INTO t2 VALUES (1, "initial"); +connection slave1; +include/sync_with_master_gtid.inc +connection master2; +include/save_master_gtid.inc +connection slave1; +include/sync_with_master_gtid.inc +*** Add an innodb gtid_slave_pos table. It is not used yet as slaves are already running *** +SET sql_log_bin=0; +CREATE TABLE mysql.gtid_slave_pos_innodb LIKE mysql.gtid_slave_pos; +ALTER TABLE mysql.gtid_slave_pos_innodb ENGINE=InnoDB; +SET sql_log_bin=0; +connection master1; +INSERT INTO t3 VALUES (102, "secondary"); +include/save_master_gtid.inc +connection slave1; +include/sync_with_master_gtid.inc +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; +domain_id max(seq_no) +1 5 +2 2 +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos_innodb GROUP BY domain_id; +domain_id max(seq_no) +*** Restart one slave thread, the other keeps running. Now the new table is used *** +connection slave1; +set default_master_connection = 'slave1'; +STOP SLAVE; +include/wait_for_slave_to_stop.inc +START SLAVE; +include/wait_for_slave_to_start.inc +connection master2; +INSERT INTO t2 VALUES (2, "secondary2"); +include/save_master_gtid.inc +connection slave1; +include/sync_with_master_gtid.inc +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; +domain_id max(seq_no) +1 5 +2 2 +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos_innodb GROUP BY domain_id; +domain_id max(seq_no) +2 3 +*** Remove a gtid_slave_posXXX table, restart one slave *** +*** Get a warning that the change is not yet picked up *** +*** See that updates fail due to trying to use the missing table *** +connection slave1; +SET sql_log_bin=0; +DROP TABLE mysql.gtid_slave_pos_innodb; +SET sql_log_bin=1; +set default_master_connection = 'slave2'; +STOP SLAVE; +include/wait_for_slave_to_stop.inc +START SLAVE; +include/wait_for_slave_to_start.inc +CALL mtr.add_suppression("The table mysql.gtid_slave_pos_innodb was removed."); +connection master2; +INSERT INTO t2 VALUES (3, "tertiary 2"); +include/save_master_gtid.inc +connection slave1; +include/wait_for_slave_sql_error.inc [errno=1942] +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; +domain_id max(seq_no) +1 5 +2 2 +*** Stop both slaves, see that the drop of mysql.gtid_slave_pos_innodb is now picked up *** +connection slave1; +set default_master_connection = 'slave1'; +STOP SLAVE; +include/wait_for_slave_to_stop.inc +set default_master_connection = 'slave2'; +STOP SLAVE; +include/wait_for_slave_to_stop.inc +set default_master_connection = 'slave1'; +START SLAVE; +include/wait_for_slave_to_start.inc +set default_master_connection = 'slave2'; +START SLAVE; +include/wait_for_slave_to_start.inc +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a b +1 initial +SELECT * FROM t2 ORDER BY a; +a b +1 initial +2 secondary2 +3 tertiary 2 +SELECT * FROM t3 ORDER BY a; +a b +101 initial 1 +102 secondary +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; +domain_id max(seq_no) +1 5 +2 4 +connection master1; +DROP TABLE t1; +DROP TABLE t3; +connection master2; +DROP TABLE t2; +connection slave1; +SET GLOBAL gtid_domain_id=0; +STOP ALL SLAVES; +Warnings: +Note 1938 SLAVE 'slave1' stopped +Note 1938 SLAVE 'slave2' stopped +include/reset_master_slave.inc +disconnect slave1; +connection master1; +SET GLOBAL gtid_domain_id=0; +include/reset_master_slave.inc +disconnect master1; +connection master2; +SET GLOBAL gtid_domain_id=0; +include/reset_master_slave.inc +disconnect master2; diff --git a/mysql-test/suite/multi_source/gtid_slave_pos.test b/mysql-test/suite/multi_source/gtid_slave_pos.test new file mode 100644 index 00000000000..b8701ff078c --- /dev/null +++ b/mysql-test/suite/multi_source/gtid_slave_pos.test @@ -0,0 +1,156 @@ +--source include/not_embedded.inc +--source include/have_innodb.inc + +# +# Test multiple mysql.gtid_slave_posXXX tables with multiple master connections +# + +--connect (slave1,127.0.0.1,root,,,$SERVER_MYPORT_3) +--connect (master1,127.0.0.1,root,,,$SERVER_MYPORT_1) +--connect (master2,127.0.0.1,root,,,$SERVER_MYPORT_2) + +--connection slave1 +--replace_result $SERVER_MYPORT_1 MYPORT_1 +eval CHANGE MASTER 'slave1' TO master_port=$SERVER_MYPORT_1, master_host='127.0.0.1', master_user='root', master_use_gtid=slave_pos; +--replace_result $SERVER_MYPORT_2 MYPORT_2 +eval CHANGE MASTER 'slave2' TO master_port=$SERVER_MYPORT_2, master_host='127.0.0.1', master_user='root', master_use_gtid=slave_pos; +set default_master_connection = 'slave1'; +START SLAVE; +--source include/wait_for_slave_to_start.inc +set default_master_connection = 'slave2'; +START SLAVE; +--source include/wait_for_slave_to_start.inc +set default_master_connection = ''; + + +--connection master1 +SET GLOBAL gtid_domain_id= 1; +SET SESSION gtid_domain_id= 1; +CREATE TABLE t3 (a INT PRIMARY KEY, b VARCHAR(10)) ENGINE=InnoDB; +CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(10)); +INSERT INTO t1 VALUES (1, "initial"); +INSERT INTO t3 VALUES (101, "initial 1"); +--source include/save_master_gtid.inc + +--connection master2 +SET GLOBAL gtid_domain_id= 2; +SET SESSION gtid_domain_id= 2; +CREATE TABLE t2 (a INT PRIMARY KEY, b VARCHAR(10)) ENGINE=InnoDB; +INSERT INTO t2 VALUES (1, "initial"); + + +--connection slave1 +--source include/sync_with_master_gtid.inc + +--connection master2 +--source include/save_master_gtid.inc + +--connection slave1 +--source include/sync_with_master_gtid.inc + + +--echo *** Add an innodb gtid_slave_pos table. It is not used yet as slaves are already running *** + +SET sql_log_bin=0; +CREATE TABLE mysql.gtid_slave_pos_innodb LIKE mysql.gtid_slave_pos; +ALTER TABLE mysql.gtid_slave_pos_innodb ENGINE=InnoDB; +SET sql_log_bin=0; + +--connection master1 +INSERT INTO t3 VALUES (102, "secondary"); +--source include/save_master_gtid.inc + +--connection slave1 +--source include/sync_with_master_gtid.inc +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos_innodb GROUP BY domain_id; + +--echo *** Restart one slave thread, the other keeps running. Now the new table is used *** +--connection slave1 +set default_master_connection = 'slave1'; +STOP SLAVE; +--source include/wait_for_slave_to_stop.inc +START SLAVE; +--source include/wait_for_slave_to_start.inc + +--connection master2 +INSERT INTO t2 VALUES (2, "secondary2"); +--source include/save_master_gtid.inc + +--connection slave1 +--source include/sync_with_master_gtid.inc +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos_innodb GROUP BY domain_id; + +--echo *** Remove a gtid_slave_posXXX table, restart one slave *** +--echo *** Get a warning that the change is not yet picked up *** +--echo *** See that updates fail due to trying to use the missing table *** +--connection slave1 +SET sql_log_bin=0; +DROP TABLE mysql.gtid_slave_pos_innodb; +SET sql_log_bin=1; +set default_master_connection = 'slave2'; +STOP SLAVE; +--source include/wait_for_slave_to_stop.inc +START SLAVE; +--source include/wait_for_slave_to_start.inc +CALL mtr.add_suppression("The table mysql.gtid_slave_pos_innodb was removed."); + +--connection master2 +INSERT INTO t2 VALUES (3, "tertiary 2"); +--source include/save_master_gtid.inc + +--connection slave1 +--let $slave_sql_errno= 1942 +--source include/wait_for_slave_sql_error.inc +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; + +--echo *** Stop both slaves, see that the drop of mysql.gtid_slave_pos_innodb is now picked up *** +--connection slave1 +set default_master_connection = 'slave1'; +STOP SLAVE; +--source include/wait_for_slave_to_stop.inc +set default_master_connection = 'slave2'; +STOP SLAVE; +--source include/wait_for_slave_to_stop.inc +set default_master_connection = 'slave1'; +START SLAVE; +--source include/wait_for_slave_to_start.inc +set default_master_connection = 'slave2'; +START SLAVE; +--source include/wait_for_slave_to_start.inc + +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; +SELECT * FROM t3 ORDER BY a; +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; + + +# Cleanup. +--connection master1 +DROP TABLE t1; +DROP TABLE t3; + +--connection master2 +DROP TABLE t2; + +--connection slave1 +SET GLOBAL gtid_domain_id=0; +--let $wait_condition= SELECT COUNT(*)=0 FROM information_schema.tables WHERE table_name IN ("t1", "t2", "t3") AND table_schema = "test" +--source include/wait_condition.inc +--sorted_result +STOP ALL SLAVES; +--source include/reset_master_slave.inc +--disconnect slave1 + + +--connection master1 +SET GLOBAL gtid_domain_id=0; +--source include/reset_master_slave.inc +--disconnect master1 + +--connection master2 +SET GLOBAL gtid_domain_id=0; +--source include/reset_master_slave.inc +--disconnect master2 diff --git a/mysql-test/suite/rpl/t/rpl_mdev12179.test b/mysql-test/suite/rpl/t/rpl_mdev12179.test index e82f3b49476..026d5628381 100644 --- a/mysql-test/suite/rpl/t/rpl_mdev12179.test +++ b/mysql-test/suite/rpl/t/rpl_mdev12179.test @@ -1,6 +1,6 @@ +--source include/have_innodb.inc --let $rpl_topology=1->2 --source include/rpl_init.inc ---source include/have_innodb.inc --connection server_2 --source include/stop_slave.inc diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 9e6f6ec7009..ac3c621a5c1 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -470,6 +470,48 @@ gtid_check_rpl_slave_state_table(TABLE *table) } +/* + Attempt to find a mysql.gtid_slave_posXXX table that has a storage engine + that is already in use by the current transaction, if any. +*/ +void +rpl_slave_state::select_gtid_pos_table(THD *thd, LEX_STRING *out_tablename) +{ + struct gtid_pos_table *list, *table_entry, *default_entry; + + /* + See comments on rpl_slave_state::gtid_pos_tables for rules around proper + access to the list. + */ + list= my_atomic_loadptr_explicit(>id_pos_tables, MY_MEMORY_ORDER_ACQUIRE); + + Ha_trx_info *ha_info= thd->transaction.all.ha_list; + while (ha_info) + { + void *trx_hton= ha_info->ht(); + table_entry= list; + while (table_entry) + { + if (table_entry->table_hton == trx_hton) + { + *out_tablename= table_entry->table_name; + return; + } + table_entry= table_entry->next; + } + ha_info= ha_info->next(); + } + /* + If we cannot find any table whose engine matches an engine that is + already active in the transaction, or if there is no current transaction + engines available, we return the default gtid_slave_pos table. + */ + default_entry= my_atomic_loadptr_explicit(&default_gtid_pos_table, + MY_MEMORY_ORDER_ACQUIRE); + *out_tablename= default_entry->table_name; +} + + /* Write a gtid to the replication slave state table. @@ -500,6 +542,7 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, Query_tables_list lex_backup; wait_for_commit* suspended_wfc; void *hton= NULL; + LEX_STRING gtid_pos_table_name; DBUG_ENTER("record_gtid"); *out_hton= NULL; @@ -517,6 +560,7 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, if (!in_statement) thd->reset_for_next_command(); + select_gtid_pos_table(thd, >id_pos_table_name); DBUG_EXECUTE_IF("gtid_inject_record_gtid", { @@ -547,10 +591,8 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, */ suspended_wfc= thd->suspend_subsequent_commits(); thd->lex->reset_n_backup_query_tables_list(&lex_backup); - tlist.init_one_table(STRING_WITH_LEN("mysql"), - rpl_gtid_slave_state_table_name.str, - rpl_gtid_slave_state_table_name.length, - NULL, TL_WRITE); + tlist.init_one_table(STRING_WITH_LEN("mysql"), gtid_pos_table_name.str, + gtid_pos_table_name.length, NULL, TL_WRITE); if ((err= open_and_lock_tables(thd, &tlist, FALSE, 0))) goto end; table_opened= true; @@ -1168,18 +1210,35 @@ rpl_slave_state::free_gtid_pos_tables(struct rpl_slave_state::gtid_pos_table *li } +/* + Replace the list of available mysql.gtid_slave_posXXX tables with a new list. + The caller must be holding LOCK_slave_state. Additionally, this function + must only be called while all SQL threads are stopped. +*/ void -rpl_slave_state::set_gtid_pos_tables_list(struct rpl_slave_state::gtid_pos_table *new_list) +rpl_slave_state::set_gtid_pos_tables_list(rpl_slave_state::gtid_pos_table *new_list, + rpl_slave_state::gtid_pos_table *default_entry) { - struct gtid_pos_table *old_list; + gtid_pos_table *old_list; mysql_mutex_assert_owner(&LOCK_slave_state); old_list= gtid_pos_tables; - gtid_pos_tables= new_list; + my_atomic_storeptr_explicit(>id_pos_tables, new_list, MY_MEMORY_ORDER_RELEASE); + my_atomic_storeptr_explicit(&default_gtid_pos_table, default_entry, + MY_MEMORY_ORDER_RELEASE); free_gtid_pos_tables(old_list); } +void +rpl_slave_state::add_gtid_pos_table(rpl_slave_state::gtid_pos_table *entry) +{ + mysql_mutex_assert_owner(&LOCK_slave_state); + entry->next= gtid_pos_tables; + my_atomic_storeptr_explicit(>id_pos_tables, entry, MY_MEMORY_ORDER_RELEASE); +} + + struct rpl_slave_state::gtid_pos_table * rpl_slave_state::alloc_gtid_pos_table(LEX_STRING *table_name, void *hton) { diff --git a/sql/rpl_gtid.h b/sql/rpl_gtid.h index ea278427061..b4f80fe3ddf 100644 --- a/sql/rpl_gtid.h +++ b/sql/rpl_gtid.h @@ -177,7 +177,27 @@ struct rpl_slave_state DYNAMIC_ARRAY gtid_sort_array; uint64 last_sub_id; + /* + List of tables available for durably storing the slave GTID position. + + Accesses to this table is protected by LOCK_slave_state. However for + efficiency, there is also a provision for read access to it from a running + slave without lock. + + An element can be added at the head of a list by storing the new + gtid_pos_tables pointer atomically with release semantics, to ensure that + the next pointer of the new element is visible to readers of the new list. + Other changes (like deleting or replacing elements) must happen only while + all SQL driver threads are stopped. LOCK_slave_state must be held in any + case. + + The list can be read without lock by an SQL driver thread or worker thread + by reading the gtid_pos_tables pointer atomically with acquire semantics, + to ensure that it will see the correct next pointer of a new head element. + */ struct gtid_pos_table *gtid_pos_tables; + /* The default entry in gtid_pos_tables, mysql.gtid_slave_pos. */ + struct gtid_pos_table *default_gtid_pos_table; bool loaded; rpl_slave_state(); @@ -188,6 +208,7 @@ struct rpl_slave_state int update(uint32 domain_id, uint32 server_id, uint64 sub_id, uint64 seq_no, void *hton, rpl_group_info *rgi); int truncate_state_table(THD *thd); + void select_gtid_pos_table(THD *thd, LEX_STRING *out_tablename); int record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, bool in_transaction, bool in_statement, void **out_hton); uint64 next_sub_id(uint32 domain_id); @@ -208,7 +229,9 @@ struct rpl_slave_state int record_and_update_gtid(THD *thd, struct rpl_group_info *rgi); int check_duplicate_gtid(rpl_gtid *gtid, rpl_group_info *rgi); void release_domain_owner(rpl_group_info *rgi); - void set_gtid_pos_tables_list(struct gtid_pos_table *new_list); + void set_gtid_pos_tables_list(gtid_pos_table *new_list, + gtid_pos_table *default_entry); + void add_gtid_pos_table(gtid_pos_table *entry); struct gtid_pos_table *alloc_gtid_pos_table(LEX_STRING *table_name, void *hton); void free_gtid_pos_tables(struct gtid_pos_table *list); }; diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 74d831db6dc..74e4be524b9 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -1647,24 +1647,70 @@ struct load_gtid_state_cb_data { HASH *hash; DYNAMIC_ARRAY *array; struct rpl_slave_state::gtid_pos_table *table_list; + struct rpl_slave_state::gtid_pos_table *default_entry; }; +static int +process_gtid_pos_table(THD *thd, LEX_STRING *table_name, void *hton, + struct load_gtid_state_cb_data *data) +{ + struct rpl_slave_state::gtid_pos_table *p, *entry, **next_ptr; + bool is_default= + (strcmp(table_name->str, rpl_gtid_slave_state_table_name.str) == 0); + + /* + Ignore tables with duplicate storage engine, with a warning. + Prefer the default mysql.gtid_slave_pos over another table + mysql.gtid_slave_posXXX with the same storage engine. + */ + next_ptr= &data->table_list; + entry= data->table_list; + while (entry) + { + if (entry->table_hton == hton) + { + static const char *warning_msg= "Ignoring redundant table mysql.%s " + "since mysql.%s has the same storage engine"; + if (!is_default) + { + /* Ignore the redundant table. */ + sql_print_warning(warning_msg, table_name->str, entry->table_name); + return 0; + } + else + { + sql_print_warning(warning_msg, entry->table_name, table_name->str); + /* Delete the redundant table, and proceed to add this one instead. */ + *next_ptr= entry->next; + my_free(entry); + break; + } + } + next_ptr= &entry->next; + entry= entry->next; + } + + if (!(p= rpl_global_gtid_slave_state->alloc_gtid_pos_table(table_name, hton))) + return 1; + p->next= data->table_list; + data->table_list= p; + if (is_default) + data->default_entry= p; + return 0; +} + + static int load_gtid_state_cb(THD *thd, LEX_STRING *table_name, void *arg) { int err; load_gtid_state_cb_data *data= static_cast(arg); - struct rpl_slave_state::gtid_pos_table *p; void *hton; if ((err= scan_one_gtid_slave_pos_table(thd, data->hash, data->array, table_name, &hton))) return err; - if (!(p= rpl_global_gtid_slave_state->alloc_gtid_pos_table(table_name, hton))) - return 1; - p->next= data->table_list; - data->table_list= p; - return 0; + return process_gtid_pos_table(thd, table_name, hton, data); } @@ -1687,6 +1733,7 @@ rpl_load_gtid_slave_state(THD *thd) DBUG_RETURN(0); cb_data.table_list= NULL; + cb_data.default_entry= NULL; my_hash_init(&hash, &my_charset_bin, 32, offsetof(gtid_pos_element, gtid) + offsetof(rpl_gtid, domain_id), sizeof(uint32), NULL, my_free, HASH_UNIQUE); @@ -1706,6 +1753,23 @@ rpl_load_gtid_slave_state(THD *thd) goto end; } + if (!cb_data.table_list) + { + my_error(ER_NO_SUCH_TABLE, MYF(0), "mysql", + rpl_gtid_slave_state_table_name.str); + err= 1; + goto end; + } + else if (!cb_data.default_entry) + { + /* + If the mysql.gtid_slave_pos table does not exist, but at least one other + table is available, arbitrarily pick the first in the list to use as + default. + */ + cb_data.default_entry= cb_data.table_list; + } + for (i= 0; i < array.elements; ++i) { get_dynamic(&array, (uchar *)&tmp_entry, i); @@ -1735,7 +1799,8 @@ rpl_load_gtid_slave_state(THD *thd) } } - rpl_global_gtid_slave_state->set_gtid_pos_tables_list(cb_data.table_list); + rpl_global_gtid_slave_state->set_gtid_pos_tables_list(cb_data.table_list, + cb_data.default_entry); cb_data.table_list= NULL; rpl_global_gtid_slave_state->loaded= true; mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); @@ -1753,12 +1818,10 @@ end: static int find_gtid_pos_tables_cb(THD *thd, LEX_STRING *table_name, void *arg) { - struct rpl_slave_state::gtid_pos_table **table_list_ptr= - static_cast(arg); + load_gtid_state_cb_data *data= static_cast(arg); TABLE_LIST tlist; TABLE *table= NULL; int err; - struct rpl_slave_state::gtid_pos_table *p; thd->reset_for_next_command(); tlist.init_one_table(STRING_WITH_LEN("mysql"), table_name->str, @@ -1769,14 +1832,7 @@ find_gtid_pos_tables_cb(THD *thd, LEX_STRING *table_name, void *arg) if ((err= gtid_check_rpl_slave_state_table(table))) goto end; - - if (!(p= rpl_global_gtid_slave_state->alloc_gtid_pos_table(table_name, table->s->db_type()))) - err= 1; - else - { - p->next= *table_list_ptr; - *table_list_ptr= p; - } + err= process_gtid_pos_table(thd, table_name, table->s->db_type(), data); end: if (table) @@ -1801,7 +1857,8 @@ int find_gtid_slave_pos_tables(THD *thd) { int err= 0; - struct rpl_slave_state::gtid_pos_table *table_list; + load_gtid_state_cb_data cb_data; + bool any_running; mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); bool loaded= rpl_global_gtid_slave_state->loaded; @@ -1809,18 +1866,95 @@ find_gtid_slave_pos_tables(THD *thd) if (!loaded) return 0; - table_list= NULL; - if ((err= scan_all_gtid_slave_pos_table(thd, find_gtid_pos_tables_cb, &table_list))) + cb_data.table_list= NULL; + cb_data.default_entry= NULL; + if ((err= scan_all_gtid_slave_pos_table(thd, find_gtid_pos_tables_cb, &cb_data))) + goto end; + + if (!cb_data.table_list) + { + my_error(ER_NO_SUCH_TABLE, MYF(0), "mysql", + rpl_gtid_slave_state_table_name.str); + err= 1; goto end; + } + else if (!cb_data.default_entry) + { + /* + If the mysql.gtid_slave_pos table does not exist, but at least one other + table is available, arbitrarily pick the first in the list to use as + default. + */ + cb_data.default_entry= cb_data.table_list; + } + any_running= any_slave_sql_running(); mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); - rpl_global_gtid_slave_state->set_gtid_pos_tables_list(table_list); - table_list= NULL; + if (!any_running) + { + rpl_global_gtid_slave_state->set_gtid_pos_tables_list(cb_data.table_list, + cb_data.default_entry); + cb_data.table_list= NULL; + } + else + { + /* + If there are SQL threads running, we cannot safely remove the old list. + However we can add new entries, and warn about any tables that + disappeared, but may still be visible to running SQL threads. + */ + rpl_slave_state::gtid_pos_table *old_entry, *new_entry, **next_ptr_ptr; + + old_entry= rpl_global_gtid_slave_state->gtid_pos_tables; + while (old_entry) + { + new_entry= cb_data.table_list; + while (new_entry) + { + if (new_entry->table_hton == old_entry->table_hton) + break; + new_entry= new_entry->next; + } + if (!new_entry) + sql_print_warning("The table mysql.%s was removed. " + "This change will not take full effect " + "until all SQL threads have been restarted", + old_entry->table_name.str); + old_entry= old_entry->next; + } + next_ptr_ptr= &cb_data.table_list; + new_entry= cb_data.table_list; + while (new_entry) + { + /* Check if we already have a table with this storage engine. */ + old_entry= rpl_global_gtid_slave_state->gtid_pos_tables; + while (old_entry) + { + if (new_entry->table_hton == old_entry->table_hton) + break; + old_entry= old_entry->next; + } + if (old_entry) + { + /* This new_entry is already available in the list. */ + next_ptr_ptr= &new_entry->next; + new_entry= new_entry->next; + } + else + { + /* Move this new_entry to the list. */ + rpl_slave_state::gtid_pos_table *next= new_entry->next; + rpl_global_gtid_slave_state->add_gtid_pos_table(new_entry); + *next_ptr_ptr= next; + new_entry= next; + } + } + } mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); end: - if (table_list) - rpl_global_gtid_slave_state->free_gtid_pos_tables(table_list); + if (cb_data.table_list) + rpl_global_gtid_slave_state->free_gtid_pos_tables(cb_data.table_list); return err; } diff --git a/storage/tokudb/mysql-test/tokudb_rpl/r/mdev12179.result b/storage/tokudb/mysql-test/tokudb_rpl/r/mdev12179.result new file mode 100644 index 00000000000..ce1f5bb3a8e --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_rpl/r/mdev12179.result @@ -0,0 +1,60 @@ +include/master-slave.inc +[connection master] +include/stop_slave.inc +CHANGE MASTER TO master_use_gtid=slave_pos; +SET sql_log_bin=0; +CREATE TABLE mysql.gtid_slave_pos_innodb LIKE mysql.gtid_slave_pos; +ALTER TABLE mysql.gtid_slave_pos_innodb ENGINE=InnoDB; +CREATE TABLE mysql.gtid_slave_pos_tokudb LIKE mysql.gtid_slave_pos; +ALTER TABLE mysql.gtid_slave_pos_tokudb ENGINE=TokuDB; +CREATE TABLE mysql.gtid_slave_pos_myisam_redundant LIKE mysql.gtid_slave_pos; +CREATE TABLE mysql.gtid_slave_pos_innodb_redundant LIKE mysql.gtid_slave_pos; +ALTER TABLE mysql.gtid_slave_pos_innodb_redundant ENGINE=InnoDB; +call mtr.add_suppression("Ignoring redundant table.*since.*has the same storage engine"); +include/start_slave.inc +CREATE TABLE t1 (a INT PRIMARY KEY); +CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB; +CREATE TABLE t3 (a INT PRIMARY KEY) ENGINE=TokuDB; +INSERT INTO t1 VALUES (1); +INSERT INTO t2 VALUES (1); +INSERT INTO t3 VALUES (1); +SELECT * FROM t1 ORDER BY a; +a +1 +SELECT * FROM t2 ORDER BY a; +a +1 +SELECT * FROM t3 ORDER BY a; +a +1 +SELECT * FROM t1 ORDER BY a; +a +1 +SELECT * FROM t2 ORDER BY a; +a +1 +SELECT * FROM t3 ORDER BY a; +a +1 +SELECT * FROM mysql.gtid_slave_pos ORDER BY sub_id; +domain_id sub_id server_id seq_no +0 3 1 3 +0 4 1 4 +SELECT * FROM ( SELECT * FROM mysql.gtid_slave_pos_innodb +UNION ALL SELECT * FROM mysql.gtid_slave_pos_innodb_redundant) inner_select +ORDER BY sub_id; +domain_id sub_id server_id seq_no +0 5 1 5 +SELECT * FROM mysql.gtid_slave_pos_tokudb ORDER BY sub_id; +domain_id sub_id server_id seq_no +0 6 1 6 +SET sql_log_bin=0; +DROP TABLE mysql.gtid_slave_pos_innodb; +DROP TABLE mysql.gtid_slave_pos_tokudb; +DROP TABLE mysql.gtid_slave_pos_myisam_redundant; +DROP TABLE mysql.gtid_slave_pos_innodb_redundant; +SET sql_log_bin=1; +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +include/rpl_end.inc diff --git a/storage/tokudb/mysql-test/tokudb_rpl/t/mdev12179.test b/storage/tokudb/mysql-test/tokudb_rpl/t/mdev12179.test new file mode 100644 index 00000000000..822fc6f9e33 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_rpl/t/mdev12179.test @@ -0,0 +1,56 @@ +--source include/have_tokudb.inc +--source include/have_innodb.inc +--source include/master-slave.inc + +--connection server_2 +--source include/stop_slave.inc +CHANGE MASTER TO master_use_gtid=slave_pos; +SET sql_log_bin=0; +CREATE TABLE mysql.gtid_slave_pos_innodb LIKE mysql.gtid_slave_pos; +ALTER TABLE mysql.gtid_slave_pos_innodb ENGINE=InnoDB; +CREATE TABLE mysql.gtid_slave_pos_tokudb LIKE mysql.gtid_slave_pos; +ALTER TABLE mysql.gtid_slave_pos_tokudb ENGINE=TokuDB; +CREATE TABLE mysql.gtid_slave_pos_myisam_redundant LIKE mysql.gtid_slave_pos; +CREATE TABLE mysql.gtid_slave_pos_innodb_redundant LIKE mysql.gtid_slave_pos; +ALTER TABLE mysql.gtid_slave_pos_innodb_redundant ENGINE=InnoDB; +call mtr.add_suppression("Ignoring redundant table.*since.*has the same storage engine"); +--source include/start_slave.inc + +--connection server_1 +CREATE TABLE t1 (a INT PRIMARY KEY); +CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB; +CREATE TABLE t3 (a INT PRIMARY KEY) ENGINE=TokuDB; +INSERT INTO t1 VALUES (1); +INSERT INTO t2 VALUES (1); +INSERT INTO t3 VALUES (1); +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; +SELECT * FROM t3 ORDER BY a; +--save_master_pos + +--connection server_2 +--sync_with_master +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; +SELECT * FROM t3 ORDER BY a; +SELECT * FROM mysql.gtid_slave_pos ORDER BY sub_id; +SELECT * FROM ( SELECT * FROM mysql.gtid_slave_pos_innodb + UNION ALL SELECT * FROM mysql.gtid_slave_pos_innodb_redundant) inner_select + ORDER BY sub_id; +SELECT * FROM mysql.gtid_slave_pos_tokudb ORDER BY sub_id; + + +--connection server_2 +SET sql_log_bin=0; +DROP TABLE mysql.gtid_slave_pos_innodb; +DROP TABLE mysql.gtid_slave_pos_tokudb; +DROP TABLE mysql.gtid_slave_pos_myisam_redundant; +DROP TABLE mysql.gtid_slave_pos_innodb_redundant; +SET sql_log_bin=1; + +--connection server_1 +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; + +--source include/rpl_end.inc -- cgit v1.2.1 From 363d6a16ae3469496e9f17eab168c5b14fad7203 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Mon, 20 Mar 2017 11:55:24 +0100 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Implement a --gtid-pos-auto-engines system variable. The variable is a list of engines for which mysql.gtid_slave_pos_ENGINE should be auto-created if needed. This commit only implements the option variable. It is not yet used to actually auto-create any tables, nor is there a corresponding command-line version of the option yet. --- mysql-test/suite/rpl/r/rpl_mdev12179.result | 33 ++++++ mysql-test/suite/rpl/t/rpl_mdev12179.test | 22 ++++ sql/mysqld.cc | 3 + sql/mysqld.h | 3 + sql/set_var.cc | 174 ++++++++++++++++++++++++++++ sql/set_var.h | 5 + sql/sql_plugin.cc | 6 + sql/sys_vars.cc | 40 +++++++ sql/sys_vars.ic | 98 ++++++++++++++++ 9 files changed, 384 insertions(+) diff --git a/mysql-test/suite/rpl/r/rpl_mdev12179.result b/mysql-test/suite/rpl/r/rpl_mdev12179.result index e32c2ba6203..17ae0074046 100644 --- a/mysql-test/suite/rpl/r/rpl_mdev12179.result +++ b/mysql-test/suite/rpl/r/rpl_mdev12179.result @@ -1,6 +1,39 @@ include/rpl_init.inc [topology=1->2] +SET GLOBAL gtid_pos_auto_engines="innodb"; +ERROR HY000: This operation cannot be performed as you have a running slave ''; run STOP SLAVE '' first include/stop_slave.inc CHANGE MASTER TO master_use_gtid=slave_pos; +SELECT @@gtid_pos_auto_engines; +@@gtid_pos_auto_engines + +SELECT @@SESSION.gtid_pos_auto_engines; +ERROR HY000: Variable 'gtid_pos_auto_engines' is a GLOBAL variable +SET GLOBAL gtid_pos_auto_engines= NULL; +ERROR 42000: Variable 'gtid_pos_auto_engines' can't be set to the value of 'NULL' +SET GLOBAL gtid_pos_auto_engines="innodb"; +SELECT @@gtid_pos_auto_engines; +@@gtid_pos_auto_engines +InnoDB +SET GLOBAL gtid_pos_auto_engines="myisam,innodb"; +SELECT @@gtid_pos_auto_engines; +@@gtid_pos_auto_engines +MyISAM,InnoDB +SET GLOBAL gtid_pos_auto_engines="innodb,myisam"; +SELECT @@gtid_pos_auto_engines; +@@gtid_pos_auto_engines +InnoDB,MyISAM +SET GLOBAL gtid_pos_auto_engines="innodb,innodb,myisam,innodb,myisam,myisam,innodb"; +SELECT @@gtid_pos_auto_engines; +@@gtid_pos_auto_engines +InnoDB,MyISAM +SET GLOBAL gtid_pos_auto_engines=DEFAULT; +SELECT @@gtid_pos_auto_engines; +@@gtid_pos_auto_engines + +SET GLOBAL gtid_pos_auto_engines=""; +SELECT @@gtid_pos_auto_engines; +@@gtid_pos_auto_engines + include/start_slave.inc CREATE TABLE t1 (a INT PRIMARY KEY); INSERT INTO t1 VALUES (1); diff --git a/mysql-test/suite/rpl/t/rpl_mdev12179.test b/mysql-test/suite/rpl/t/rpl_mdev12179.test index 026d5628381..a85195eb6b4 100644 --- a/mysql-test/suite/rpl/t/rpl_mdev12179.test +++ b/mysql-test/suite/rpl/t/rpl_mdev12179.test @@ -3,8 +3,30 @@ --source include/rpl_init.inc --connection server_2 +--error ER_SLAVE_MUST_STOP +SET GLOBAL gtid_pos_auto_engines="innodb"; --source include/stop_slave.inc CHANGE MASTER TO master_use_gtid=slave_pos; + +# Test the @@gtid_pos_auto_engines sysvar. +SELECT @@gtid_pos_auto_engines; +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT @@SESSION.gtid_pos_auto_engines; +--error ER_WRONG_VALUE_FOR_VAR +SET GLOBAL gtid_pos_auto_engines= NULL; +SET GLOBAL gtid_pos_auto_engines="innodb"; +SELECT @@gtid_pos_auto_engines; +SET GLOBAL gtid_pos_auto_engines="myisam,innodb"; +SELECT @@gtid_pos_auto_engines; +SET GLOBAL gtid_pos_auto_engines="innodb,myisam"; +SELECT @@gtid_pos_auto_engines; +SET GLOBAL gtid_pos_auto_engines="innodb,innodb,myisam,innodb,myisam,myisam,innodb"; +SELECT @@gtid_pos_auto_engines; +SET GLOBAL gtid_pos_auto_engines=DEFAULT; +SELECT @@gtid_pos_auto_engines; +SET GLOBAL gtid_pos_auto_engines=""; +SELECT @@gtid_pos_auto_engines; + --source include/start_slave.inc --connection server_1 diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 6108ed6ffd4..3c3acdf6030 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -374,6 +374,8 @@ char *my_bind_addr_str; static char *default_collation_name; char *default_storage_engine, *default_tmp_storage_engine; char *enforced_storage_engine=NULL; +char *gtid_pos_auto_engines; +plugin_ref *opt_gtid_pos_auto_plugins; static char compiled_default_collation_name[]= MYSQL_DEFAULT_COLLATION_NAME; static I_List thread_cache; static bool binlog_format_used= false; @@ -4234,6 +4236,7 @@ static int init_common_variables() default_storage_engine= const_cast("MyISAM"); #endif default_tmp_storage_engine= NULL; + gtid_pos_auto_engines= const_cast(""); /* Add server status variables to the dynamic list of diff --git a/sql/mysqld.h b/sql/mysqld.h index 613b57b133d..be8b97c80c4 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -19,6 +19,7 @@ #include /* MYSQL_PLUGIN_IMPORT, FN_REFLEN, FN_EXTLEN */ #include "sql_basic_types.h" /* query_id_t */ +#include "sql_plugin.h" #include "sql_bitmap.h" /* Bitmap */ #include "my_decimal.h" /* my_decimal */ #include "mysql_com.h" /* SERVER_VERSION_LENGTH */ @@ -153,6 +154,8 @@ extern char *default_tz_name; extern Time_zone *default_tz; extern char *default_storage_engine, *default_tmp_storage_engine; extern char *enforced_storage_engine; +extern char *gtid_pos_auto_engines; +extern plugin_ref *opt_gtid_pos_auto_plugins; extern bool opt_endinfo, using_udf_functions; extern my_bool locked_in_memory; extern bool opt_using_transactions; diff --git a/sql/set_var.cc b/sql/set_var.cc index 07395e3e708..7b190f6244e 100644 --- a/sql/set_var.cc +++ b/sql/set_var.cc @@ -1278,3 +1278,177 @@ enum sys_var::where get_sys_var_value_origin(void *ptr) return sys_var::CONFIG; } + +/* + Find the next item in string of comma-separated items. + END_POS points at the end of the string. + ITEM_START and ITEM_END return the limits of the next item. + Returns true while items are available, false at the end. +*/ +static bool +engine_list_next_item(const char **pos, const char *end_pos, + const char **item_start, const char **item_end) +{ + if (*pos >= end_pos) + return false; + *item_start= *pos; + while (*pos < end_pos && **pos != ',') + ++*pos; + *item_end= *pos; + ++*pos; + return true; +} + + +static bool +resolve_engine_list_item(plugin_ref *list, uint32 *idx, + const char *pos, const char *pos_end) +{ + LEX_STRING item_str; + plugin_ref ref; + uint32_t i; + + item_str.str= const_cast(pos); + item_str.length= pos_end-pos; + ref= ha_resolve_by_name(NULL, &item_str, false); + if (!ref) + { + ErrConvString err(pos, pos_end-pos, system_charset_info); + my_error(ER_UNKNOWN_STORAGE_ENGINE, MYF(0), err.ptr()); + return true; + } + /* Ignore duplicates, like --plugin-load does. */ + for (i= 0; i < *idx; ++i) + { + if (plugin_hton(list[i]) == plugin_hton(ref)) + { + plugin_unlock(NULL, ref); + return false; + } + } + list[*idx]= ref; + ++*idx; + return false; +} + + +/* + Helper for class Sys_var_pluginlist. + Resolve a comma-separated list of storage engine names to a null-terminated + array of plugin_ref. +*/ +plugin_ref * +resolve_engine_list(const char *str_arg, size_t str_arg_len) +{ + uint32 count, idx; + const char *pos, *item_start, *item_end; + const char *str_arg_end= str_arg + str_arg_len; + plugin_ref *res; + + count= 0; + pos= str_arg; + for (;;) + { + if (!engine_list_next_item(&pos, str_arg_end, &item_start, &item_end)) + break; + ++count; + } + + res= (plugin_ref *)my_malloc((count+1)*sizeof(*res), MYF(MY_ZEROFILL|MY_WME)); + if (!res) + { + my_error(ER_OUTOFMEMORY, MYF(0), (int)((count+1)*sizeof(*res))); + goto err; + } + + idx= 0; + pos= str_arg; + for (;;) + { + if (!engine_list_next_item(&pos, str_arg_end, &item_start, &item_end)) + break; + DBUG_ASSERT(idx < count); + if (idx >= count) + break; + if (resolve_engine_list_item(res, &idx, item_start, item_end)) + goto err; + } + + return res; + +err: + free_engine_list(res); + return NULL; +} + + +void +free_engine_list(plugin_ref *list) +{ + plugin_ref *p; + + if (!list) + return; + for (p= list; *p; ++p) + plugin_unlock(NULL, *p); + my_free(list); +} + + +plugin_ref * +copy_engine_list(plugin_ref *list) +{ + plugin_ref *p; + uint32 count, i; + + for (p= list, count= 0; *p; ++p, ++count) + ; + p= (plugin_ref *)my_malloc((count+1)*sizeof(*p), MYF(0)); + if (!p) + { + my_error(ER_OUTOFMEMORY, MYF(0), (int)((count+1)*sizeof(*p))); + return NULL; + } + for (i= 0; i < count; ++i) + p[i]= my_plugin_lock(NULL, list[i]); + return p; +} + + +char * +pretty_print_engine_list(THD *thd, plugin_ref *list) +{ + plugin_ref *p; + size_t size; + char *buf, *pos; + + if (!list) + return thd->strmake("", 0); + + size= 0; + for (p= list; *p; ++p) + size+= plugin_name(*p)->length + 1; + buf= static_cast(thd->alloc(size)); + if (!buf) + return NULL; + pos= buf; + for (p= list; *p; ++p) + { + LEX_STRING *name; + size_t remain; + + remain= buf + size - pos; + DBUG_ASSERT(remain > 0); + if (remain <= 1) + break; + if (pos != buf) + { + pos= strmake(pos, ",", remain-1); + --remain; + } + name= plugin_name(*p); + pos= strmake(pos, name->str, MY_MIN(name->length, remain-1)); + } + *pos= '\0'; + return buf; +} diff --git a/sql/set_var.h b/sql/set_var.h index 97dc3b5ba51..a6f0cb02fe8 100644 --- a/sql/set_var.h +++ b/sql/set_var.h @@ -286,6 +286,7 @@ public: longlong longlong_value; ///< for signed integer double double_value; ///< for Sys_var_double plugin_ref plugin; ///< for Sys_var_plugin + plugin_ref *plugins; ///< for Sys_var_pluginlist Time_zone *time_zone; ///< for Sys_var_tz LEX_STRING string_value; ///< for Sys_var_charptr and others const void *ptr; ///< for Sys_var_struct @@ -422,6 +423,10 @@ int sys_var_init(); uint sys_var_elements(); int sys_var_add_options(DYNAMIC_ARRAY *long_options, int parse_flags); void sys_var_end(void); +plugin_ref *resolve_engine_list(const char *str_arg, size_t str_arg_len); +void free_engine_list(plugin_ref *list); +plugin_ref *copy_engine_list(plugin_ref *list); +char *pretty_print_engine_list(THD *thd, plugin_ref *list); #endif diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index 101ea3fd3c7..45d78c525fe 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -1922,6 +1922,12 @@ void plugin_shutdown(void) if (initialized) { + if (opt_gtid_pos_auto_plugins) + { + free_engine_list(opt_gtid_pos_auto_plugins); + opt_gtid_pos_auto_plugins= NULL; + } + mysql_mutex_lock(&LOCK_plugin); reap_needed= true; diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index a3bc3f6d9cf..8f0a081acc7 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -3521,6 +3521,46 @@ static Sys_var_plugin Sys_enforce_storage_engine( NO_CMD_LINE, MYSQL_STORAGE_ENGINE_PLUGIN, DEFAULT(&enforced_storage_engine), NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(check_has_super)); +/* + Check + 1. Value for gtid_pos_auto_engines is not NULL. + 2. No slave SQL thread is running. +*/ +static bool +check_gtid_pos_auto_engines(sys_var *self, THD *thd, set_var *var) +{ + bool running; + bool err= false; + + DBUG_ASSERT(var->type == OPT_GLOBAL); + if (var->value && var->value->is_null()) + err= true; + else + { + running= give_error_if_slave_running(false); + if (running) + err= true; + } + if (err && var->save_result.plugins) + { + free_engine_list(var->save_result.plugins); + var->save_result.plugins= NULL; + } + return err; +} + + +static Sys_var_pluginlist Sys_gtid_pos_auto_engines( + "gtid_pos_auto_engines", + "List of engines for which to automatically create a " + "mysql.gtid_slave_pos_ENGINE table, if a transaction using that engine " + "is replicated. This can be used to avoid introducing cross-engine " + "transactions, if engines are used different from that used by table " + "mysql.gtid_slave_pos", + GLOBAL_VAR(opt_gtid_pos_auto_plugins), NO_CMD_LINE, + DEFAULT(>id_pos_auto_engines), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(check_gtid_pos_auto_engines)); + #if defined(ENABLED_DEBUG_SYNC) /* Variable can be set for the session only. diff --git a/sql/sys_vars.ic b/sql/sys_vars.ic index 780450b484b..e76f60b56b0 100644 --- a/sql/sys_vars.ic +++ b/sql/sys_vars.ic @@ -1533,6 +1533,104 @@ public: { return valptr(thd, get_default(thd)); } }; +/** + Class for variables that containg a list of plugins. + Currently this is used only for @@gtid_pos_auto_create_engines + + Backing store: plugin_ref + + @note + Currently this is only used for storage engine type plugins, and thus only + storage engine type plugin is implemented. It could be extended to other + plugin types later if needed, similar to Sys_var_plugin. + + These variables don't support command-line equivalents, any such + command-line options should be added manually to my_long_options in mysqld.cc +*/ +class Sys_var_pluginlist: public sys_var +{ + int plugin_type; +public: + Sys_var_pluginlist(const char *name_arg, + const char *comment, int flag_args, ptrdiff_t off, size_t size, + CMD_LINE getopt, + char **def_val, PolyLock *lock=0, + enum binlog_status_enum binlog_status_arg=VARIABLE_NOT_IN_BINLOG, + on_check_function on_check_func=0, + on_update_function on_update_func=0, + const char *substitute=0) + : sys_var(&all_sys_vars, name_arg, comment, flag_args, off, getopt.id, + getopt.arg_type, SHOW_CHAR, (intptr)def_val, + lock, binlog_status_arg, on_check_func, on_update_func, + substitute) + { + option.var_type|= GET_STR; + SYSVAR_ASSERT(size == sizeof(plugin_ref)); + SYSVAR_ASSERT(getopt.id < 0); // force NO_CMD_LINE + } + bool do_check(THD *thd, set_var *var) + { + char buff[STRING_BUFFER_USUAL_SIZE]; + String str(buff,sizeof(buff), system_charset_info), *res; + plugin_ref *plugins; + + if (!(res=var->value->val_str(&str))) + plugins= resolve_engine_list("", 0); + else + plugins= resolve_engine_list(res->ptr(), res->length()); + if (!plugins) + return true; + var->save_result.plugins= plugins; + return false; + } + void do_update(plugin_ref **valptr, plugin_ref* newval) + { + plugin_ref *oldval= *valptr; + *valptr= newval; + free_engine_list(oldval); + } + bool session_update(THD *thd, set_var *var) + { + do_update((plugin_ref**)session_var_ptr(thd), + var->save_result.plugins); + return false; + } + bool global_update(THD *thd, set_var *var) + { + do_update((plugin_ref**)global_var_ptr(), + var->save_result.plugins); + return false; + } + void session_save_default(THD *thd, set_var *var) + { + plugin_ref* plugins= global_var(plugin_ref *); + var->save_result.plugins= plugins ? copy_engine_list(plugins) : 0; + } + plugin_ref *get_default(THD *thd) + { + char *default_value= *reinterpret_cast(option.def_value); + if (!default_value) + return 0; + return resolve_engine_list(default_value, strlen(default_value)); + } + + void global_save_default(THD *thd, set_var *var) + { + var->save_result.plugins= get_default(thd); + } + + uchar *valptr(THD *thd, plugin_ref *plugins) + { + return (uchar*)pretty_print_engine_list(thd, plugins); + } + uchar *session_value_ptr(THD *thd, const LEX_STRING *base) + { return valptr(thd, session_var(thd, plugin_ref*)); } + uchar *global_value_ptr(THD *thd, const LEX_STRING *base) + { return valptr(thd, global_var(plugin_ref*)); } + uchar *default_value_ptr(THD *thd) + { return valptr(thd, get_default(thd)); } +}; + #if defined(ENABLED_DEBUG_SYNC) #include "debug_sync.h" -- cgit v1.2.1 From 8bc1632ea54624b6bdb4aac692147603e734e9c5 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Mon, 20 Mar 2017 13:29:37 +0100 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Implement the command-line part of --gtid-pos-auto-engines. (The option is still not actually used for anything, this will be for a later commit). --- mysql-test/suite/rpl/r/rpl_mdev12179.result | 6 ++++++ mysql-test/suite/rpl/t/rpl_mdev12179.test | 7 ++++++- sql/mysqld.cc | 31 +++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/rpl/r/rpl_mdev12179.result b/mysql-test/suite/rpl/r/rpl_mdev12179.result index 17ae0074046..c7a791faeb6 100644 --- a/mysql-test/suite/rpl/r/rpl_mdev12179.result +++ b/mysql-test/suite/rpl/r/rpl_mdev12179.result @@ -64,6 +64,12 @@ a 1 2 3 +SELECT @@gtid_pos_auto_engines; +@@gtid_pos_auto_engines +InnoDB,MyISAM +include/stop_slave.inc +SET GLOBAL gtid_pos_auto_engines=""; +include/start_slave.inc SET sql_log_bin=0; DROP TABLE mysql.gtid_slave_pos_innodb; SET sql_log_bin=1; diff --git a/mysql-test/suite/rpl/t/rpl_mdev12179.test b/mysql-test/suite/rpl/t/rpl_mdev12179.test index a85195eb6b4..a4b344de3f1 100644 --- a/mysql-test/suite/rpl/t/rpl_mdev12179.test +++ b/mysql-test/suite/rpl/t/rpl_mdev12179.test @@ -62,8 +62,9 @@ SELECT * FROM t1 ORDER BY a; --source include/save_master_gtid.inc # Let the slave mysqld server start again. +# As we are restarting, also take the opportunity to test --gtid-pos-auto-engines --append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect -restart: --skip-slave-start=0 +restart: --skip-slave-start=0 --gtid-pos-auto-engines=innodb,myisam EOF --connection server_2 @@ -73,6 +74,10 @@ EOF --source include/sync_with_master_gtid.inc SELECT * FROM t1 ORDER BY a; +SELECT @@gtid_pos_auto_engines; +--source include/stop_slave.inc +SET GLOBAL gtid_pos_auto_engines=""; +--source include/start_slave.inc --connection server_2 SET sql_log_bin=0; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 3c3acdf6030..e43b074a018 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4917,6 +4917,26 @@ static int init_default_storage_engine_impl(const char *opt_name, return 0; } + +static int +init_gtid_pos_auto_engines(void) +{ + plugin_ref *plugins; + + if (gtid_pos_auto_engines) + plugins= resolve_engine_list(gtid_pos_auto_engines, + strlen(gtid_pos_auto_engines)); + else + plugins= resolve_engine_list("", 0); + if (!plugins) + return 1; + mysql_mutex_lock(&LOCK_global_system_variables); + opt_gtid_pos_auto_plugins= plugins; + mysql_mutex_unlock(&LOCK_global_system_variables); + return 0; +} + + static int init_server_components() { DBUG_ENTER("init_server_components"); @@ -5354,6 +5374,9 @@ static int init_server_components() if (init_default_storage_engine(enforced_storage_engine, enforced_table_plugin)) unireg_abort(1); + if (init_gtid_pos_auto_engines()) + unireg_abort(1); + #ifdef USE_ARIA_FOR_TMP_TABLES if (!ha_storage_engine_is_enabled(maria_hton) && !opt_bootstrap) { @@ -7354,6 +7377,14 @@ struct my_option my_long_options[]= "Set up signals usable for debugging. Deprecated, use --debug-gdb instead.", &opt_debugging, &opt_debugging, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"gtid-pos-auto-engines", 0, + "List of engines for which to automatically create a " + "mysql.gtid_slave_pos_ENGINE table, if a transaction using that engine " + "is replicated. This can be used to avoid introducing cross-engine " + "transactions, if engines are used different from that used by table " + "mysql.gtid_slave_pos", + >id_pos_auto_engines, 0, 0, GET_STR, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0 }, #ifdef HAVE_LARGE_PAGE_OPTION {"super-large-pages", 0, "Enable support for super large pages.", &opt_super_large_pages, &opt_super_large_pages, 0, -- cgit v1.2.1 From 4eebf431b07b93cc05e1b15f1e77b6cfcdcf737f Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Fri, 24 Mar 2017 11:51:23 +0100 Subject: Fix that MTR leaks restart: options to following test This was found during MDEV-12179 development, but unrelated so put in its own commit. The "restart: --extra-option" facility in expect file for server restart sets extra options for the restarted mysqld. These options were incorrectly not cleared when starting a new test case, so the server was restarted with whatever extra options were left by the previous testcase, causing random failures depending on order of running tests. --- mysql-test/mysql-test-run.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 296a646d9ba..15ac72a838c 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -5215,6 +5215,7 @@ sub server_need_restart { if (!My::Options::same($started_opts, $extra_opts) || exists $server->{'restart_opts'}) { + delete $server->{'restart_opts'}; my $use_dynamic_option_switch= 0; if (!$use_dynamic_option_switch) { -- cgit v1.2.1 From 3cc89b3e85605ecb09b4b2222c8b0b8222a29fde Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Fri, 24 Mar 2017 12:06:29 +0100 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Ignore unknown engines in --gtid-pos-auto-engines command-line options (but not SET GLOBAL). This seems useful, to allow a default that auto-creates the gtid pos table for engines like TokuDB and MyRocks (which greatly benefit from such), but does not prevent server startup when those engines are not available. --- sql/mysqld.cc | 12 ++++++++++-- sql/set_var.cc | 19 +++++++++++++------ sql/set_var.h | 3 ++- sql/sys_vars.ic | 6 +++--- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index e43b074a018..09c9c5c681e 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4923,11 +4923,19 @@ init_gtid_pos_auto_engines(void) { plugin_ref *plugins; + /* + For the command-line option --gtid_pos_auto_engines, we allow (and ignore) + engines that are unknown. This is convenient, since it allows to set + default auto-create engines that might not be used by particular users. + The option sets a list of storage engines that will have gtid position + table auto-created for them if needed. And if the engine is not available, + then it will certainly not be needed. + */ if (gtid_pos_auto_engines) plugins= resolve_engine_list(gtid_pos_auto_engines, - strlen(gtid_pos_auto_engines)); + strlen(gtid_pos_auto_engines), false); else - plugins= resolve_engine_list("", 0); + plugins= resolve_engine_list("", 0, false); if (!plugins) return 1; mysql_mutex_lock(&LOCK_global_system_variables); diff --git a/sql/set_var.cc b/sql/set_var.cc index 7b190f6244e..78904f75661 100644 --- a/sql/set_var.cc +++ b/sql/set_var.cc @@ -1302,7 +1302,8 @@ engine_list_next_item(const char **pos, const char *end_pos, static bool resolve_engine_list_item(plugin_ref *list, uint32 *idx, - const char *pos, const char *pos_end) + const char *pos, const char *pos_end, + bool error_on_unknown_engine) { LEX_STRING item_str; plugin_ref ref; @@ -1313,9 +1314,13 @@ resolve_engine_list_item(plugin_ref *list, uint32 *idx, ref= ha_resolve_by_name(NULL, &item_str, false); if (!ref) { - ErrConvString err(pos, pos_end-pos, system_charset_info); - my_error(ER_UNKNOWN_STORAGE_ENGINE, MYF(0), err.ptr()); - return true; + if (error_on_unknown_engine) + { + ErrConvString err(pos, pos_end-pos, system_charset_info); + my_error(ER_UNKNOWN_STORAGE_ENGINE, MYF(0), err.ptr()); + return true; + } + return false; } /* Ignore duplicates, like --plugin-load does. */ for (i= 0; i < *idx; ++i) @@ -1338,7 +1343,8 @@ resolve_engine_list_item(plugin_ref *list, uint32 *idx, array of plugin_ref. */ plugin_ref * -resolve_engine_list(const char *str_arg, size_t str_arg_len) +resolve_engine_list(const char *str_arg, size_t str_arg_len, + bool error_on_unknown_engine) { uint32 count, idx; const char *pos, *item_start, *item_end; @@ -1370,7 +1376,8 @@ resolve_engine_list(const char *str_arg, size_t str_arg_len) DBUG_ASSERT(idx < count); if (idx >= count) break; - if (resolve_engine_list_item(res, &idx, item_start, item_end)) + if (resolve_engine_list_item(res, &idx, item_start, item_end, + error_on_unknown_engine)) goto err; } diff --git a/sql/set_var.h b/sql/set_var.h index a6f0cb02fe8..cdd0f7da1ba 100644 --- a/sql/set_var.h +++ b/sql/set_var.h @@ -423,7 +423,8 @@ int sys_var_init(); uint sys_var_elements(); int sys_var_add_options(DYNAMIC_ARRAY *long_options, int parse_flags); void sys_var_end(void); -plugin_ref *resolve_engine_list(const char *str_arg, size_t str_arg_len); +plugin_ref *resolve_engine_list(const char *str_arg, size_t str_arg_len, + bool error_on_unknown_engine); void free_engine_list(plugin_ref *list); plugin_ref *copy_engine_list(plugin_ref *list); char *pretty_print_engine_list(THD *thd, plugin_ref *list); diff --git a/sql/sys_vars.ic b/sql/sys_vars.ic index e76f60b56b0..c5fe8c0af2c 100644 --- a/sql/sys_vars.ic +++ b/sql/sys_vars.ic @@ -1575,9 +1575,9 @@ public: plugin_ref *plugins; if (!(res=var->value->val_str(&str))) - plugins= resolve_engine_list("", 0); + plugins= resolve_engine_list("", 0, true); else - plugins= resolve_engine_list(res->ptr(), res->length()); + plugins= resolve_engine_list(res->ptr(), res->length(), true); if (!plugins) return true; var->save_result.plugins= plugins; @@ -1611,7 +1611,7 @@ public: char *default_value= *reinterpret_cast(option.def_value); if (!default_value) return 0; - return resolve_engine_list(default_value, strlen(default_value)); + return resolve_engine_list(default_value, strlen(default_value), false); } void global_save_default(THD *thd, set_var *var) -- cgit v1.2.1 From fdf2d407707faf05b8b7d67662a70cc5537d15aa Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Fri, 24 Mar 2017 12:07:07 +0100 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Implement auto-creation of mysql.gtid_slave_pos* tables with needed engines, if listed in --gtid-pos-auto-engines. Uses an asynchronous approach to minimise locking overhead. The list of available tables is extended with a flag. Extra entries are added for --gtid-pos-auto-engines tables that do not exist yet, marked as not existing but ready for auto-creation. If record_gtid() needs a table marked for auto-creation, it sends a request to the slave background thread to create the table, and continues to use an existing table for the current and immediately coming transactions. As soon as the slave background thread has made the new table available, it will be used for all subsequent relevant transactions in record_gtid(). This asynchronous approach also avoids a lot of complex issues around trying to do DDL in the middle of an on-going transaction. --- mysql-test/suite/rpl/r/rpl_mdev12179.result | 193 ++++++++++++++++++++++++- mysql-test/suite/rpl/t/rpl_mdev12179.test | 193 ++++++++++++++++++++++++- scripts/mysql_system_tables.sql | 2 + sql/rpl_gtid.cc | 21 ++- sql/rpl_gtid.h | 10 +- sql/rpl_rli.cc | 83 +++++++++-- sql/slave.cc | 216 +++++++++++++++++++++++++++- sql/slave.h | 3 + 8 files changed, 698 insertions(+), 23 deletions(-) diff --git a/mysql-test/suite/rpl/r/rpl_mdev12179.result b/mysql-test/suite/rpl/r/rpl_mdev12179.result index c7a791faeb6..8bf8f183181 100644 --- a/mysql-test/suite/rpl/r/rpl_mdev12179.result +++ b/mysql-test/suite/rpl/r/rpl_mdev12179.result @@ -1,4 +1,5 @@ include/rpl_init.inc [topology=1->2] +connection server_2; SET GLOBAL gtid_pos_auto_engines="innodb"; ERROR HY000: This operation cannot be performed as you have a running slave ''; run STOP SLAVE '' first include/stop_slave.inc @@ -35,11 +36,13 @@ SELECT @@gtid_pos_auto_engines; @@gtid_pos_auto_engines include/start_slave.inc +connection server_1; CREATE TABLE t1 (a INT PRIMARY KEY); INSERT INTO t1 VALUES (1); SELECT * FROM t1 ORDER BY a; a 1 +connection server_2; SELECT * FROM t1 ORDER BY a; a 1 @@ -50,6 +53,7 @@ ALTER TABLE mysql.gtid_slave_pos_innodb ENGINE=InnoDB; INSERT INTO mysql.gtid_slave_pos_innodb SELECT * FROM mysql.gtid_slave_pos; TRUNCATE mysql.gtid_slave_pos; SET sql_log_bin=1; +connection server_1; INSERT INTO t1 VALUES (2); INSERT INTO t1 VALUES (3); SELECT * FROM t1 ORDER BY a; @@ -58,20 +62,203 @@ a 2 3 include/save_master_gtid.inc +*** Restart server with --gtid-pos-auto-engines=innodb,myisam *** +connection server_2; include/sync_with_master_gtid.inc SELECT * FROM t1 ORDER BY a; a 1 2 3 +*** Verify no new gtid_slave_pos* tables are created *** +SELECT table_name, engine FROM information_schema.tables +WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; +table_name engine +gtid_slave_pos MyISAM +gtid_slave_pos_innodb InnoDB SELECT @@gtid_pos_auto_engines; @@gtid_pos_auto_engines InnoDB,MyISAM include/stop_slave.inc -SET GLOBAL gtid_pos_auto_engines=""; +SET sql_log_bin=0; +INSERT INTO mysql.gtid_slave_pos_innodb SELECT * FROM mysql.gtid_slave_pos; +DROP TABLE mysql.gtid_slave_pos; +RENAME TABLE mysql.gtid_slave_pos_innodb TO mysql.gtid_slave_pos; +SET sql_log_bin=1; +connection server_1; +CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (4); +INSERT INTO t2 VALUES (1); +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +SELECT * FROM t2 ORDER BY a; +a +1 +include/save_master_gtid.inc +*** Restart server with --gtid-pos-auto-engines=myisam,innodb *** +connection server_2; +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +SELECT * FROM t2 ORDER BY a; +a +1 +*** Verify that no new gtid_slave_pos* tables are auto-created *** +SELECT table_name, engine FROM information_schema.tables +WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; +table_name engine +gtid_slave_pos InnoDB +include/stop_slave.inc +SET sql_log_bin=0; +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; +SET sql_log_bin=1; +connection server_1; +INSERT INTO t1 VALUES (5); +INSERT INTO t2 VALUES (2); +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +5 +SELECT * FROM t2 ORDER BY a; +a +1 +2 +include/save_master_gtid.inc +connection server_2; +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +5 +SELECT * FROM t2 ORDER BY a; +a +1 +2 +*** Verify that mysql.gtid_slave_pos_InnoDB is auto-created *** +SELECT table_name, engine FROM information_schema.tables +WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; +table_name engine +gtid_slave_pos MyISAM +gtid_slave_pos_InnoDB InnoDB +include/stop_slave.inc +SET sql_log_bin=0; +INSERT INTO mysql.gtid_slave_pos SELECT * FROM mysql.gtid_slave_pos_InnoDB; +DROP TABLE mysql.gtid_slave_pos_InnoDB; +SET sql_log_bin=1; +connection server_1; +INSERT INTO t1 VALUES (6); +INSERT INTO t2 VALUES (3); +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +5 +6 +SELECT * FROM t2 ORDER BY a; +a +1 +2 +3 +include/save_master_gtid.inc +*** Restart server without --gtid-pos-auto-engines *** +connection server_2; +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +5 +6 +SELECT * FROM t2 ORDER BY a; +a +1 +2 +3 +*** Verify that no mysql.gtid_slave_pos* table is auto-created *** +SELECT table_name, engine FROM information_schema.tables +WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; +table_name engine +gtid_slave_pos MyISAM +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; +domain_id max(seq_no) +0 11 +include/stop_slave.inc +SET GLOBAL gtid_pos_auto_engines="innodb"; include/start_slave.inc +connection server_1; +INSERT INTO t1 VALUES (7); +INSERT INTO t2 VALUES (4); +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +5 +6 +7 +SELECT * FROM t2 ORDER BY a; +a +1 +2 +3 +4 +include/save_master_gtid.inc +connection server_2; +include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +a +1 +2 +3 +4 +5 +6 +7 +SELECT * FROM t2 ORDER BY a; +a +1 +2 +3 +4 +*** Verify that mysql.gtid_slave_pos_InnoDB is auto-created *** +SELECT table_name, engine FROM information_schema.tables +WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; +table_name engine +gtid_slave_pos MyISAM +gtid_slave_pos_InnoDB InnoDB +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; +domain_id max(seq_no) +0 13 +include/stop_slave.inc +SET GLOBAL gtid_pos_auto_engines=""; SET sql_log_bin=0; -DROP TABLE mysql.gtid_slave_pos_innodb; +DROP TABLE mysql.gtid_slave_pos_InnoDB; SET sql_log_bin=1; -DROP TABLE t1; +include/start_slave.inc +connection server_1; +DROP TABLE t1, t2; include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_mdev12179.test b/mysql-test/suite/rpl/t/rpl_mdev12179.test index a4b344de3f1..b4609c2ed7a 100644 --- a/mysql-test/suite/rpl/t/rpl_mdev12179.test +++ b/mysql-test/suite/rpl/t/rpl_mdev12179.test @@ -63,6 +63,7 @@ SELECT * FROM t1 ORDER BY a; # Let the slave mysqld server start again. # As we are restarting, also take the opportunity to test --gtid-pos-auto-engines +--echo *** Restart server with --gtid-pos-auto-engines=innodb,myisam *** --append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect restart: --skip-slave-start=0 --gtid-pos-auto-engines=innodb,myisam EOF @@ -74,17 +75,203 @@ EOF --source include/sync_with_master_gtid.inc SELECT * FROM t1 ORDER BY a; +--echo *** Verify no new gtid_slave_pos* tables are created *** +SELECT table_name, engine FROM information_schema.tables + WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; + SELECT @@gtid_pos_auto_engines; --source include/stop_slave.inc -SET GLOBAL gtid_pos_auto_engines=""; +SET sql_log_bin=0; +INSERT INTO mysql.gtid_slave_pos_innodb SELECT * FROM mysql.gtid_slave_pos; +DROP TABLE mysql.gtid_slave_pos; +RENAME TABLE mysql.gtid_slave_pos_innodb TO mysql.gtid_slave_pos; +SET sql_log_bin=1; + +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +wait +EOF +--shutdown_server 30 +--source include/wait_until_disconnected.inc + +--connection server_1 +CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (4); +INSERT INTO t2 VALUES (1); +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; +--source include/save_master_gtid.inc + +--echo *** Restart server with --gtid-pos-auto-engines=myisam,innodb *** +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +restart: --skip-slave-start=0 --gtid-pos-auto-engines=myisam,innodb +EOF + +--connection server_2 +--enable_reconnect +--source include/wait_until_connected_again.inc + +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; + +--echo *** Verify that no new gtid_slave_pos* tables are auto-created *** +SELECT table_name, engine FROM information_schema.tables + WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; + + +--source include/stop_slave.inc +SET sql_log_bin=0; +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; +SET sql_log_bin=1; + +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +wait +EOF +--shutdown_server 30 +--source include/wait_until_disconnected.inc + +--connection server_1 +INSERT INTO t1 VALUES (5); +INSERT INTO t2 VALUES (2); +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; +--source include/save_master_gtid.inc + +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +--echo *** Restart server with --gtid-pos-auto-engines=innodb *** +restart: --skip-slave-start=0 --gtid-pos-auto-engines=innodb +EOF + +--connection server_2 +--enable_reconnect +--source include/wait_until_connected_again.inc + +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; + +--echo *** Verify that mysql.gtid_slave_pos_InnoDB is auto-created *** +# Note, the create happens asynchronously, so wait for it. +let $wait_condition= + SELECT EXISTS (SELECT * FROM information_schema.tables + WHERE table_schema='mysql' AND table_name='gtid_slave_pos_InnoDB'); +--source include/wait_condition.inc +SELECT table_name, engine FROM information_schema.tables + WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; + + +--source include/stop_slave.inc +SET sql_log_bin=0; +INSERT INTO mysql.gtid_slave_pos SELECT * FROM mysql.gtid_slave_pos_InnoDB; +DROP TABLE mysql.gtid_slave_pos_InnoDB; +SET sql_log_bin=1; + +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +wait +EOF +--shutdown_server 30 +--source include/wait_until_disconnected.inc + +--connection server_1 +INSERT INTO t1 VALUES (6); +INSERT INTO t2 VALUES (3); +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; +--source include/save_master_gtid.inc + +--echo *** Restart server without --gtid-pos-auto-engines *** +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +restart: --skip-slave-start=0 +EOF + +--connection server_2 +--enable_reconnect +--source include/wait_until_connected_again.inc + +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; + +--echo *** Verify that no mysql.gtid_slave_pos* table is auto-created *** +SELECT table_name, engine FROM information_schema.tables + WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; + +--source include/stop_slave.inc +SET GLOBAL gtid_pos_auto_engines="innodb"; --source include/start_slave.inc +--connection server_1 +INSERT INTO t1 VALUES (7); +INSERT INTO t2 VALUES (4); +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; +--source include/save_master_gtid.inc + --connection server_2 +--source include/sync_with_master_gtid.inc +SELECT * FROM t1 ORDER BY a; +SELECT * FROM t2 ORDER BY a; + +--echo *** Verify that mysql.gtid_slave_pos_InnoDB is auto-created *** +let $wait_condition= + SELECT EXISTS (SELECT * FROM information_schema.tables + WHERE table_schema='mysql' AND table_name='gtid_slave_pos_InnoDB'); +--source include/wait_condition.inc +SELECT table_name, engine FROM information_schema.tables + WHERE table_schema='mysql' AND table_name LIKE 'gtid_slave_pos%' + ORDER BY table_name; +SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; + +# Check that the auto-created InnoDB table starts being used without +# needing slave restart. The auto-create happens asynchronously, so it +# is non-deterministic when it will start being used. But we can wait +# for it to happen. + +--let $count=300 +--let $done=0 +--let $old_silent= $keep_include_silent +--let $keep_include_silent= 1 +--disable_query_log +while (!$done) +{ + --connection server_1 + INSERT INTO t2(a) SELECT 1+MAX(a) FROM t2; + --source include/save_master_gtid.inc + + --connection server_2 + --source include/sync_with_master_gtid.inc + --let $done=`SELECT COUNT(*) > 0 FROM mysql.gtid_slave_pos_InnoDB` + if (!$done) + { + dec $count; + if (!$count) + { + SELECT * FROM mysql.gtid_slave_pos_InnoDB; + --die Timeout waiting for mysql.gtid_slave_pos_InnoDB to be used + } + real_sleep 0.1; + } +} +--enable_query_log +--let $keep_include_silent=$old_silent +# Note that at this point, the contents of table t2, as well as the GTID +# position, is non-deterministic. + + +#--connection server_2 +--source include/stop_slave.inc +SET GLOBAL gtid_pos_auto_engines=""; SET sql_log_bin=0; -DROP TABLE mysql.gtid_slave_pos_innodb; +DROP TABLE mysql.gtid_slave_pos_InnoDB; SET sql_log_bin=1; +--source include/start_slave.inc --connection server_1 -DROP TABLE t1; +DROP TABLE t1, t2; --source include/rpl_end.inc diff --git a/scripts/mysql_system_tables.sql b/scripts/mysql_system_tables.sql index 7b614163f46..60c1ac97955 100644 --- a/scripts/mysql_system_tables.sql +++ b/scripts/mysql_system_tables.sql @@ -224,6 +224,8 @@ CREATE TABLE IF NOT EXISTS column_stats (db_name varchar(64) NOT NULL, table_nam CREATE TABLE IF NOT EXISTS index_stats (db_name varchar(64) NOT NULL, table_name varchar(64) NOT NULL, index_name varchar(64) NOT NULL, prefix_arity int(11) unsigned NOT NULL, avg_frequency decimal(12,4) DEFAULT NULL, PRIMARY KEY (db_name,table_name,index_name,prefix_arity) ) ENGINE=MyISAM CHARACTER SET utf8 COLLATE utf8_bin comment='Statistics on Indexes'; +-- Note: This definition must be kept in sync with the one used in +-- build_gtid_pos_create_query() in sql/slave.cc SET @cmd= "CREATE TABLE IF NOT EXISTS gtid_slave_pos ( domain_id INT UNSIGNED NOT NULL, sub_id BIGINT UNSIGNED NOT NULL, diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index ac3c621a5c1..af70f281d8c 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -26,6 +26,7 @@ #include "key.h" #include "rpl_gtid.h" #include "rpl_rli.h" +#include "slave.h" const LEX_STRING rpl_gtid_slave_state_table_name= @@ -494,8 +495,20 @@ rpl_slave_state::select_gtid_pos_table(THD *thd, LEX_STRING *out_tablename) { if (table_entry->table_hton == trx_hton) { - *out_tablename= table_entry->table_name; - return; + if (likely(table_entry->state == GTID_POS_AVAILABLE)) + { + *out_tablename= table_entry->table_name; + return; + } + /* + This engine is marked to automatically create the table. + We cannot easily do this here (possibly in the middle of a + transaction). But we can request the slave background thread + to create it, and in a short while it should become available + for following transactions. + */ + slave_background_gtid_pos_create_request(table_entry); + break; } table_entry= table_entry->next; } @@ -1240,7 +1253,8 @@ rpl_slave_state::add_gtid_pos_table(rpl_slave_state::gtid_pos_table *entry) struct rpl_slave_state::gtid_pos_table * -rpl_slave_state::alloc_gtid_pos_table(LEX_STRING *table_name, void *hton) +rpl_slave_state::alloc_gtid_pos_table(LEX_STRING *table_name, void *hton, + rpl_slave_state::gtid_pos_table_state state) { struct gtid_pos_table *p; char *allocated_str; @@ -1258,6 +1272,7 @@ rpl_slave_state::alloc_gtid_pos_table(LEX_STRING *table_name, void *hton) p->table_hton= hton; p->table_name.str= allocated_str; p->table_name.length= table_name->length; + p->state= state; return p; } diff --git a/sql/rpl_gtid.h b/sql/rpl_gtid.h index b4f80fe3ddf..e6a3ea1a667 100644 --- a/sql/rpl_gtid.h +++ b/sql/rpl_gtid.h @@ -157,6 +157,12 @@ struct rpl_slave_state }; /* Descriptor for mysql.gtid_slave_posXXX table in specific engine. */ + enum gtid_pos_table_state { + GTID_POS_AUTO_CREATE, + GTID_POS_CREATE_REQUESTED, + GTID_POS_CREATE_IN_PROGRESS, + GTID_POS_AVAILABLE + }; struct gtid_pos_table { struct gtid_pos_table *next; /* @@ -167,6 +173,7 @@ struct rpl_slave_state */ void *table_hton; LEX_STRING table_name; + uint8 state; }; /* Mapping from domain_id to its element. */ @@ -232,7 +239,8 @@ struct rpl_slave_state void set_gtid_pos_tables_list(gtid_pos_table *new_list, gtid_pos_table *default_entry); void add_gtid_pos_table(gtid_pos_table *entry); - struct gtid_pos_table *alloc_gtid_pos_table(LEX_STRING *table_name, void *hton); + struct gtid_pos_table *alloc_gtid_pos_table(LEX_STRING *table_name, + void *hton, rpl_slave_state::gtid_pos_table_state state); void free_gtid_pos_tables(struct gtid_pos_table *list); }; diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 74e4be524b9..2cda87925d1 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -1690,7 +1690,9 @@ process_gtid_pos_table(THD *thd, LEX_STRING *table_name, void *hton, entry= entry->next; } - if (!(p= rpl_global_gtid_slave_state->alloc_gtid_pos_table(table_name, hton))) + p= rpl_global_gtid_slave_state->alloc_gtid_pos_table(table_name, + hton, rpl_slave_state::GTID_POS_AVAILABLE); + if (!p) return 1; p->next= data->table_list; data->table_list= p; @@ -1700,6 +1702,59 @@ process_gtid_pos_table(THD *thd, LEX_STRING *table_name, void *hton, } +/* + Put tables corresponding to @@gtid_pos_auto_engines at the end of the list, + marked to be auto-created if needed. +*/ +static int +gtid_pos_auto_create_tables(rpl_slave_state::gtid_pos_table **list_ptr) +{ + plugin_ref *auto_engines; + int err= 0; + mysql_mutex_lock(&LOCK_global_system_variables); + for (auto_engines= opt_gtid_pos_auto_plugins; + !err && auto_engines && *auto_engines; + ++auto_engines) + { + void *hton= plugin_hton(*auto_engines); + char buf[FN_REFLEN+1]; + LEX_STRING table_name; + char *p; + rpl_slave_state::gtid_pos_table *entry, **next_ptr; + + /* See if this engine is already in the list. */ + next_ptr= list_ptr; + entry= *list_ptr; + while (entry) + { + if (entry->table_hton == hton) + break; + next_ptr= &entry->next; + entry= entry->next; + } + if (entry) + continue; + + /* Add an auto-create entry for this engine at end of list. */ + p= strmake(buf, rpl_gtid_slave_state_table_name.str, FN_REFLEN); + p= strmake(p, "_", FN_REFLEN - (p - buf)); + p= strmake(p, plugin_name(*auto_engines)->str, FN_REFLEN - (p - buf)); + table_name.str= buf; + table_name.length= p - buf; + entry= rpl_global_gtid_slave_state->alloc_gtid_pos_table + (&table_name, hton, rpl_slave_state::GTID_POS_AUTO_CREATE); + if (!entry) + { + err= 1; + break; + } + *next_ptr= entry; + } + mysql_mutex_unlock(&LOCK_global_system_variables); + return err; +} + + static int load_gtid_state_cb(THD *thd, LEX_STRING *table_name, void *arg) { @@ -1746,6 +1801,18 @@ rpl_load_gtid_slave_state(THD *thd) if ((err= scan_all_gtid_slave_pos_table(thd, load_gtid_state_cb, &cb_data))) goto end; + if (!cb_data.default_entry) + { + /* + If the mysql.gtid_slave_pos table does not exist, but at least one other + table is available, arbitrarily pick the first in the list to use as + default. + */ + cb_data.default_entry= cb_data.table_list; + } + if ((err= gtid_pos_auto_create_tables(&cb_data.table_list))) + goto end; + mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); if (rpl_global_gtid_slave_state->loaded) { @@ -1757,18 +1824,10 @@ rpl_load_gtid_slave_state(THD *thd) { my_error(ER_NO_SUCH_TABLE, MYF(0), "mysql", rpl_gtid_slave_state_table_name.str); + mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); err= 1; goto end; } - else if (!cb_data.default_entry) - { - /* - If the mysql.gtid_slave_pos table does not exist, but at least one other - table is available, arbitrarily pick the first in the list to use as - default. - */ - cb_data.default_entry= cb_data.table_list; - } for (i= 0; i < array.elements; ++i) { @@ -1878,7 +1937,7 @@ find_gtid_slave_pos_tables(THD *thd) err= 1; goto end; } - else if (!cb_data.default_entry) + if (!cb_data.default_entry) { /* If the mysql.gtid_slave_pos table does not exist, but at least one other @@ -1887,6 +1946,8 @@ find_gtid_slave_pos_tables(THD *thd) */ cb_data.default_entry= cb_data.table_list; } + if ((err= gtid_pos_auto_create_tables(&cb_data.table_list))) + goto end; any_running= any_slave_sql_running(); mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); diff --git a/sql/slave.cc b/sql/slave.cc index fbdb78b5c5d..443e5b7a5f1 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -59,6 +59,7 @@ #include "rpl_tblmap.h" #include "debug_sync.h" #include "rpl_parallel.h" +#include "sql_show.h" #define FLAGSTR(V,F) ((V)&(F)?#F" ":"") @@ -279,15 +280,178 @@ static void init_slave_psi_keys(void) #endif /* HAVE_PSI_INTERFACE */ +/* + Note: This definition needs to be kept in sync with the one in + mysql_system_tables.sql which is used by mysql_create_db. +*/ +static const char gtid_pos_table_definition1[]= + "CREATE TABLE "; +static const char gtid_pos_table_definition2[]= + " (domain_id INT UNSIGNED NOT NULL, " + "sub_id BIGINT UNSIGNED NOT NULL, " + "server_id INT UNSIGNED NOT NULL, " + "seq_no BIGINT UNSIGNED NOT NULL, " + "PRIMARY KEY (domain_id, sub_id)) CHARSET=latin1 " + "COMMENT='Replication slave GTID position' " + "ENGINE="; + +/* + Build a query string + CREATE TABLE mysql.gtid_slave_pos_ ... ENGINE= +*/ +static bool +build_gtid_pos_create_query(THD *thd, String *query, + LEX_STRING *table_name, + LEX_STRING *engine_name) +{ + bool err= false; + err|= query->append(gtid_pos_table_definition1); + err|= append_identifier(thd, query, table_name->str, table_name->length); + err|= query->append(gtid_pos_table_definition2); + err|= append_identifier(thd, query, engine_name->str, engine_name->length); + return err; +} + + +static int +gtid_pos_table_creation(THD *thd, plugin_ref engine, LEX_STRING *table_name) +{ + int err; + StringBuffer query; + + if (build_gtid_pos_create_query(thd, &query, table_name, plugin_name(engine))) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return 1; + } + + thd->set_db("mysql", 5); + thd->clear_error(); + ulonglong thd_saved_option= thd->variables.option_bits; + /* This query shuold not be binlogged. */ + thd->variables.option_bits&= ~(ulonglong)OPTION_BIN_LOG; + thd->set_query_and_id(query.c_ptr(), query.length(), thd->charset(), + next_query_id()); + Parser_state parser_state; + err= parser_state.init(thd, thd->query(), thd->query_length()); + if (err) + goto end; + mysql_parse(thd, thd->query(), thd->query_length(), &parser_state, + FALSE, FALSE); + if (thd->is_error()) + err= 1; +end: + thd->variables.option_bits= thd_saved_option; + thd->reset_query(); + return err; +} + + +static void +handle_gtid_pos_auto_create_request(THD *thd, void *hton) +{ + int err; + plugin_ref engine= NULL, *auto_engines; + rpl_slave_state::gtid_pos_table *entry; + StringBuffer loc_table_name; + LEX_STRING table_name; + + /* + Check that the plugin is still in @@gtid_pos_auto_engines, and lock + it. + */ + mysql_mutex_lock(&LOCK_global_system_variables); + engine= NULL; + for (auto_engines= opt_gtid_pos_auto_plugins; + auto_engines && *auto_engines; + ++auto_engines) + { + if (plugin_hton(*auto_engines) == hton) + { + engine= my_plugin_lock(NULL, *auto_engines); + break; + } + } + mysql_mutex_unlock(&LOCK_global_system_variables); + if (!engine) + { + /* The engine is gone from @@gtid_pos_auto_engines, so no action. */ + goto end; + } + + /* Find the entry for the table to auto-create. */ + mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); + entry= rpl_global_gtid_slave_state->gtid_pos_tables; + while (entry) + { + if (entry->table_hton == hton && + entry->state == rpl_slave_state::GTID_POS_CREATE_REQUESTED) + break; + entry= entry->next; + } + if (entry) + { + entry->state = rpl_slave_state::GTID_POS_CREATE_IN_PROGRESS; + err= loc_table_name.append(entry->table_name.str, entry->table_name.length); + } + mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); + if (!entry) + goto end; + if (err) + { + sql_print_error("Out of memory while trying to auto-create GTID position table"); + goto end; + } + table_name.str= loc_table_name.c_ptr_safe(); + table_name.length= loc_table_name.length(); + + err= gtid_pos_table_creation(thd, engine, &table_name); + if (err) + { + sql_print_error("Error auto-creating GTID position table `mysql.%s`: %s Error_code: %d", + table_name.str, thd->get_stmt_da()->message(), + thd->get_stmt_da()->sql_errno()); + thd->clear_error(); + goto end; + } + + /* Now enable the entry for the auto-created table. */ + mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); + entry= rpl_global_gtid_slave_state->gtid_pos_tables; + while (entry) + { + if (entry->table_hton == hton && + entry->state == rpl_slave_state::GTID_POS_CREATE_IN_PROGRESS) + { + entry->state= rpl_slave_state::GTID_POS_AVAILABLE; + break; + } + entry= entry->next; + } + mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); + +end: + if (engine) + plugin_unlock(NULL, engine); +} + + static bool slave_background_thread_running; static bool slave_background_thread_stop; static bool slave_background_thread_gtid_loaded; -struct slave_background_kill_t { +static struct slave_background_kill_t { slave_background_kill_t *next; THD *to_kill; } *slave_background_kill_list; +static struct slave_background_gtid_pos_create_t { + slave_background_gtid_pos_create_t *next; + void *hton; +} *slave_background_gtid_pos_create_list; + pthread_handler_t handle_slave_background(void *arg __attribute__((unused))) @@ -321,6 +485,7 @@ handle_slave_background(void *arg __attribute__((unused))) do { slave_background_kill_t *kill_list; + slave_background_gtid_pos_create_t *create_list; thd->ENTER_COND(&COND_slave_background, &LOCK_slave_background, &stage_slave_background_wait_request, @@ -329,12 +494,14 @@ handle_slave_background(void *arg __attribute__((unused))) { stop= abort_loop || thd->killed || slave_background_thread_stop; kill_list= slave_background_kill_list; - if (stop || kill_list) + create_list= slave_background_gtid_pos_create_list; + if (stop || kill_list || create_list) break; mysql_cond_wait(&COND_slave_background, &LOCK_slave_background); } slave_background_kill_list= NULL; + slave_background_gtid_pos_create_list= NULL; thd->EXIT_COND(&old_stage); while (kill_list) @@ -353,6 +520,16 @@ handle_slave_background(void *arg __attribute__((unused))) mysql_mutex_unlock(&to_kill->LOCK_wakeup_ready); my_free(p); } + + while (create_list) + { + slave_background_gtid_pos_create_t *next= create_list->next; + void *hton= create_list->hton; + handle_gtid_pos_auto_create_request(thd, hton); + my_free(create_list); + create_list= next; + } + mysql_mutex_lock(&LOCK_slave_background); } while (!stop); @@ -391,6 +568,41 @@ slave_background_kill_request(THD *to_kill) } +/* + This function must only be called from a slave SQL thread (or worker thread), + to ensure that the table_entry will not go away before we can lock the + LOCK_slave_state. +*/ +void +slave_background_gtid_pos_create_request( + rpl_slave_state::gtid_pos_table *table_entry) +{ + slave_background_gtid_pos_create_t *p; + + if (table_entry->state != rpl_slave_state::GTID_POS_AUTO_CREATE) + return; + p= (slave_background_gtid_pos_create_t *)my_malloc(sizeof(*p), MYF(MY_WME)); + if (!p) + return; + mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); + if (table_entry->state != rpl_slave_state::GTID_POS_AUTO_CREATE) + { + my_free(p); + mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); + return; + } + table_entry->state= rpl_slave_state::GTID_POS_CREATE_REQUESTED; + mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); + + p->hton= table_entry->table_hton; + mysql_mutex_lock(&LOCK_slave_background); + p->next= slave_background_gtid_pos_create_list; + slave_background_gtid_pos_create_list= p; + mysql_cond_signal(&COND_slave_background); + mysql_mutex_unlock(&LOCK_slave_background); +} + + /* Start the slave background thread. diff --git a/sql/slave.h b/sql/slave.h index ded9d76e49d..2afd5277e25 100644 --- a/sql/slave.h +++ b/sql/slave.h @@ -48,6 +48,7 @@ #include "my_list.h" #include "rpl_filter.h" #include "rpl_tblmap.h" +#include "rpl_gtid.h" #define SLAVE_NET_TIMEOUT 60 @@ -268,6 +269,8 @@ void slave_output_error_info(rpl_group_info *rgi, THD *thd); pthread_handler_t handle_slave_sql(void *arg); bool net_request_file(NET* net, const char* fname); void slave_background_kill_request(THD *to_kill); +void slave_background_gtid_pos_create_request + (rpl_slave_state::gtid_pos_table *table_entry); extern bool volatile abort_loop; extern Master_info *active_mi; /* active_mi for multi-master */ -- cgit v1.2.1 From da9decdccf102f639bae43c8cdc7594b6dcbcb41 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Thu, 20 Apr 2017 16:07:27 +0200 Subject: Use separate connection for START SLAVE in rpl_deadlock.test START SLAVE is documented to implicitly commit any active transaction. So do this in a separate connection, to not break the lock on which the test case depends. Found during MDEV-12179 development, but independent of this, so done in a separate commit. --- mysql-test/extra/rpl_tests/rpl_deadlock.test | 8 ++++++++ mysql-test/suite/rpl/r/rpl_deadlock_innodb.result | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/mysql-test/extra/rpl_tests/rpl_deadlock.test b/mysql-test/extra/rpl_tests/rpl_deadlock.test index f7a1e71d5d9..e9191d5fcd8 100644 --- a/mysql-test/extra/rpl_tests/rpl_deadlock.test +++ b/mysql-test/extra/rpl_tests/rpl_deadlock.test @@ -46,6 +46,9 @@ BEGIN; SELECT * FROM t1 FOR UPDATE; # Save variable 'Slave_retried_transactions' before deadlock let $slave_retried_transactions= query_get_value(SHOW GLOBAL STATUS LIKE 'Slave_retried_transactions', Value, 1); +# Run the START SLAVE in a separate connection. Otherwise it terminates +# the SELECT FOR UPDATE transaction (START SLAVE does implicit COMMIT!). +connection slave1; START SLAVE; # Wait until SQL thread blocked: variable 'Slave_retried_transactions' will incremented let $status_var= Slave_retried_transactions; @@ -53,6 +56,7 @@ let $status_var_value= $slave_retried_transactions; let $status_type= GLOBAL; let $status_var_comparsion= >; --source include/wait_for_status_var.inc +connection slave; SELECT COUNT(*) FROM t2; COMMIT; sync_with_master; @@ -78,9 +82,11 @@ BEGIN; # Hold lock SELECT * FROM t1 FOR UPDATE; # Wait until slave stopped with error 'Lock wait timeout exceeded' +connection slave1; START SLAVE; let $slave_sql_errno= 1205; --source include/wait_for_slave_sql_error.inc +connection slave; SELECT COUNT(*) FROM t2; COMMIT; --source include/start_slave.inc @@ -109,9 +115,11 @@ BEGIN; # Hold lock SELECT * FROM t1 FOR UPDATE; # Wait until slave stopped with error 'Lock wait timeout exceeded' +connection slave1; START SLAVE; let $slave_sql_errno= 1205; --source include/wait_for_slave_sql_error.inc +connection slave; SELECT COUNT(*) FROM t2; COMMIT; --source include/start_slave.inc diff --git a/mysql-test/suite/rpl/r/rpl_deadlock_innodb.result b/mysql-test/suite/rpl/r/rpl_deadlock_innodb.result index 1c9611ba8f0..bb8c45ae4eb 100644 --- a/mysql-test/suite/rpl/r/rpl_deadlock_innodb.result +++ b/mysql-test/suite/rpl/r/rpl_deadlock_innodb.result @@ -39,7 +39,9 @@ connection slave; BEGIN; SELECT * FROM t1 FOR UPDATE; a +connection slave1; START SLAVE; +connection slave; SELECT COUNT(*) FROM t2; COUNT(*) 0 @@ -61,8 +63,10 @@ BEGIN; SELECT * FROM t1 FOR UPDATE; a 1 +connection slave1; START SLAVE; include/wait_for_slave_sql_error.inc [errno=1205] +connection slave; SELECT COUNT(*) FROM t2; COUNT(*) 0 @@ -92,8 +96,10 @@ SELECT * FROM t1 FOR UPDATE; a 1 1 +connection slave1; START SLAVE; include/wait_for_slave_sql_error.inc [errno=1205] +connection slave; SELECT COUNT(*) FROM t2; COUNT(*) 0 -- cgit v1.2.1 From 094e4b264cf54b0cdb0f0fc2d7b978e93e83bfce Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Thu, 20 Apr 2017 16:11:14 +0200 Subject: Clarify plugin_ref lifetimes in function comments The mechanism in plugin_lock() et.al. which automatically unlocks plugins unless called with NULL THD is quite subtle, and deserves proper documentation. (This was found during MDEV-12179 development, but is independent of it, and so done in a separate commit). --- sql/sql_plugin.cc | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index 45d78c525fe..b7f14b24b66 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -938,6 +938,10 @@ SHOW_COMP_OPTION plugin_status(const char *name, size_t len, int type) } +/* + If LEX is passed non-NULL, an automatic unlock of the plugin will happen + in the LEX destructor. +*/ static plugin_ref intern_plugin_lock(LEX *lex, plugin_ref rc) { st_plugin_int *pi= plugin_ref_to_int(rc); @@ -981,6 +985,16 @@ static plugin_ref intern_plugin_lock(LEX *lex, plugin_ref rc) } +/* + Notes on lifetime: + + If THD is passed as non-NULL (and with a non-NULL thd->lex), an entry is made + in the thd->lex which will cause an automatic unlock of the plugin in the LEX + destructor. In this case, no manual unlock must be done. + + Otherwise, when passing a NULL THD, the caller must arrange that plugin + unlock happens later. +*/ plugin_ref plugin_lock(THD *thd, plugin_ref ptr) { LEX *lex= thd ? thd->lex : 0; @@ -1017,6 +1031,16 @@ plugin_ref plugin_lock(THD *thd, plugin_ref ptr) } +/* + Notes on lifetime: + + If THD is passed as non-NULL (and with a non-NULL thd->lex), an entry is made + in the thd->lex which will cause an automatic unlock of the plugin in the LEX + destructor. In this case, no manual unlock must be done. + + Otherwise, when passing a NULL THD, the caller must arrange that plugin + unlock happens later. +*/ plugin_ref plugin_lock_by_name(THD *thd, const LEX_STRING *name, int type) { LEX *lex= thd ? thd->lex : 0; -- cgit v1.2.1 From 638d4e90e823756922f162af7783ad122d03f6dc Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Thu, 20 Apr 2017 16:14:37 +0200 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. .result file updates due to added --gtid-pos-auto-engines sysvar. --- mysql-test/r/mysqld--help.result | 8 ++++++++ .../suite/sys_vars/r/sysvars_server_notembedded.result | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index cfd2ede6ef9..c5f8be932e0 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -257,6 +257,13 @@ The following options may be given as the first argument: applied; this means it is the responsibility of the user to ensure that GTID sequence numbers are strictly increasing. + --gtid-pos-auto-engines=name + List of engines for which to automatically create a + mysql.gtid_slave_pos_ENGINE table, if a transaction using + that engine is replicated. This can be used to avoid + introducing cross-engine transactions, if engines are + used different from that used by table + mysql.gtid_slave_pos --gtid-strict-mode Enforce strict seq_no ordering of events in the binary log. Slave stops with an error if it encounters an event that would cause it to generate an out-of-order binlog if @@ -1254,6 +1261,7 @@ getopt-prefix-matching TRUE group-concat-max-len 1048576 gtid-domain-id 0 gtid-ignore-duplicates FALSE +gtid-pos-auto-engines gtid-strict-mode FALSE help TRUE histogram-size 0 diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result index 052d30563b2..a6709bcb863 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result @@ -1115,6 +1115,20 @@ NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST OFF,ON READ_ONLY NO COMMAND_LINE_ARGUMENT OPTIONAL +VARIABLE_NAME GTID_POS_AUTO_ENGINES +SESSION_VALUE NULL +GLOBAL_VALUE +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE VARCHAR +VARIABLE_COMMENT List of engines for which to automatically create a mysql.gtid_slave_pos_ENGINE table, if a transaction using that engine is replicated. This can be used to avoid introducing cross-engine transactions, if engines are used different from that used by table mysql.gtid_slave_pos +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST NULL +READ_ONLY NO +COMMAND_LINE_ARGUMENT NULL VARIABLE_NAME GTID_SEQ_NO SESSION_VALUE 0 GLOBAL_VALUE NULL -- cgit v1.2.1 From 00eebb22435c871bbe9938582d96e6a3d1c00861 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Thu, 20 Apr 2017 16:16:26 +0200 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Fix incorrect assertion. The hton in the list of pending GTIDs can be NULL, in the special case where we failed to load the mysql.gtid_slave_pos table at server startup, but nevertheless allow non-GTID replication to proceed. --- sql/rpl_gtid.cc | 2 +- sql/rpl_gtid.h | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index af70f281d8c..d0f5694fb5a 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -295,7 +295,7 @@ rpl_slave_state::update(uint32 domain_id, uint32 server_id, uint64 sub_id, element *elem= NULL; list_element *list_elem= NULL; - DBUG_ASSERT(hton); + DBUG_ASSERT(hton || !loaded); if (!(elem= get_element(domain_id))) return 1; diff --git a/sql/rpl_gtid.h b/sql/rpl_gtid.h index e6a3ea1a667..4ff1ca563b8 100644 --- a/sql/rpl_gtid.h +++ b/sql/rpl_gtid.h @@ -112,6 +112,11 @@ struct rpl_slave_state uint64 sub_id; uint64 seq_no; uint32 server_id; + /* + hton of mysql.gtid_slave_pos* table used to record this GTID. + Can be NULL if the gtid table failed to load (eg. missing + mysql.gtid_slave_pos table following an upgrade). + */ void *hton; }; -- cgit v1.2.1 From 8953c7e48426671f8fb3a68cae22eb7a00cfee61 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Thu, 20 Apr 2017 16:19:01 +0200 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Fix engine list lifetime for sys_var_pluginlist. The Sys_var class assumes that some operations can be done without explicitly freeing resources, for example default_value_ptr(). Thus, methods (like Sys_var_pluginlist::do_check) need to generally work with temporary lists, which are registered in the THD to be freed/unlocked automatically. And do_update() needs to make a permanent copy to store in the global variable. --- sql/mysqld.cc | 6 +++--- sql/set_var.cc | 58 +++++++++++++++++++++++++++++++++++++++++++++++---------- sql/set_var.h | 5 +++-- sql/sys_vars.cc | 5 ----- sql/sys_vars.ic | 22 +++++++++++++++++----- 5 files changed, 71 insertions(+), 25 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 09c9c5c681e..0b8aaf1de78 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4932,10 +4932,10 @@ init_gtid_pos_auto_engines(void) then it will certainly not be needed. */ if (gtid_pos_auto_engines) - plugins= resolve_engine_list(gtid_pos_auto_engines, - strlen(gtid_pos_auto_engines), false); + plugins= resolve_engine_list(NULL, gtid_pos_auto_engines, + strlen(gtid_pos_auto_engines), false, false); else - plugins= resolve_engine_list("", 0, false); + plugins= resolve_engine_list(NULL, "", 0, false, false); if (!plugins) return 1; mysql_mutex_lock(&LOCK_global_system_variables); diff --git a/sql/set_var.cc b/sql/set_var.cc index 78904f75661..acc744808cb 100644 --- a/sql/set_var.cc +++ b/sql/set_var.cc @@ -1301,17 +1301,18 @@ engine_list_next_item(const char **pos, const char *end_pos, static bool -resolve_engine_list_item(plugin_ref *list, uint32 *idx, +resolve_engine_list_item(THD *thd, plugin_ref *list, uint32 *idx, const char *pos, const char *pos_end, - bool error_on_unknown_engine) + bool error_on_unknown_engine, bool temp_copy) { LEX_STRING item_str; plugin_ref ref; uint32_t i; + THD *thd_or_null = (temp_copy ? thd : NULL); item_str.str= const_cast(pos); item_str.length= pos_end-pos; - ref= ha_resolve_by_name(NULL, &item_str, false); + ref= ha_resolve_by_name(thd_or_null, &item_str, false); if (!ref) { if (error_on_unknown_engine) @@ -1327,7 +1328,8 @@ resolve_engine_list_item(plugin_ref *list, uint32 *idx, { if (plugin_hton(list[i]) == plugin_hton(ref)) { - plugin_unlock(NULL, ref); + if (!temp_copy) + plugin_unlock(NULL, ref); return false; } } @@ -1341,10 +1343,16 @@ resolve_engine_list_item(plugin_ref *list, uint32 *idx, Helper for class Sys_var_pluginlist. Resolve a comma-separated list of storage engine names to a null-terminated array of plugin_ref. + + If TEMP_COPY is true, a THD must be given as well. In this case, the + allocated memory and locked plugins are registered in the THD and will + be freed / unlocked automatically. If TEMP_COPY is true, THD can be + passed as NULL, and resources must be freed explicitly later with + free_engine_list(). */ plugin_ref * -resolve_engine_list(const char *str_arg, size_t str_arg_len, - bool error_on_unknown_engine) +resolve_engine_list(THD *thd, const char *str_arg, size_t str_arg_len, + bool error_on_unknown_engine, bool temp_copy) { uint32 count, idx; const char *pos, *item_start, *item_end; @@ -1360,7 +1368,10 @@ resolve_engine_list(const char *str_arg, size_t str_arg_len, ++count; } - res= (plugin_ref *)my_malloc((count+1)*sizeof(*res), MYF(MY_ZEROFILL|MY_WME)); + if (temp_copy) + res= (plugin_ref *)thd->calloc((count+1)*sizeof(*res)); + else + res= (plugin_ref *)my_malloc((count+1)*sizeof(*res), MYF(MY_ZEROFILL|MY_WME)); if (!res) { my_error(ER_OUTOFMEMORY, MYF(0), (int)((count+1)*sizeof(*res))); @@ -1376,15 +1387,16 @@ resolve_engine_list(const char *str_arg, size_t str_arg_len, DBUG_ASSERT(idx < count); if (idx >= count) break; - if (resolve_engine_list_item(res, &idx, item_start, item_end, - error_on_unknown_engine)) + if (resolve_engine_list_item(thd, res, &idx, item_start, item_end, + error_on_unknown_engine, temp_copy)) goto err; } return res; err: - free_engine_list(res); + if (!temp_copy) + free_engine_list(res); return NULL; } @@ -1418,6 +1430,32 @@ copy_engine_list(plugin_ref *list) } for (i= 0; i < count; ++i) p[i]= my_plugin_lock(NULL, list[i]); + p[i] = NULL; + return p; +} + + +/* + Create a temporary copy of an engine list. The memory will be freed + (and the plugins unlocked) automatically, on the passed THD. +*/ +plugin_ref * +temp_copy_engine_list(THD *thd, plugin_ref *list) +{ + plugin_ref *p; + uint32 count, i; + + for (p= list, count= 0; *p; ++p, ++count) + ; + p= (plugin_ref *)thd->alloc((count+1)*sizeof(*p)); + if (!p) + { + my_error(ER_OUTOFMEMORY, MYF(0), (int)((count+1)*sizeof(*p))); + return NULL; + } + for (i= 0; i < count; ++i) + p[i]= my_plugin_lock(thd, list[i]); + p[i] = NULL; return p; } diff --git a/sql/set_var.h b/sql/set_var.h index cdd0f7da1ba..66a3eeae42d 100644 --- a/sql/set_var.h +++ b/sql/set_var.h @@ -423,10 +423,11 @@ int sys_var_init(); uint sys_var_elements(); int sys_var_add_options(DYNAMIC_ARRAY *long_options, int parse_flags); void sys_var_end(void); -plugin_ref *resolve_engine_list(const char *str_arg, size_t str_arg_len, - bool error_on_unknown_engine); +plugin_ref *resolve_engine_list(THD *thd, const char *str_arg, size_t str_arg_len, + bool error_on_unknown_engine, bool temp_copy); void free_engine_list(plugin_ref *list); plugin_ref *copy_engine_list(plugin_ref *list); +plugin_ref *temp_copy_engine_list(THD *thd, plugin_ref *list); char *pretty_print_engine_list(THD *thd, plugin_ref *list); #endif diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 8f0a081acc7..84b88f84bbf 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -3541,11 +3541,6 @@ check_gtid_pos_auto_engines(sys_var *self, THD *thd, set_var *var) if (running) err= true; } - if (err && var->save_result.plugins) - { - free_engine_list(var->save_result.plugins); - var->save_result.plugins= NULL; - } return err; } diff --git a/sql/sys_vars.ic b/sql/sys_vars.ic index c5fe8c0af2c..5227384f203 100644 --- a/sql/sys_vars.ic +++ b/sql/sys_vars.ic @@ -1546,6 +1546,17 @@ public: These variables don't support command-line equivalents, any such command-line options should be added manually to my_long_options in mysqld.cc + + Note on lifetimes of resources allocated: We allocate a zero-terminated array + of plugin_ref*, and lock the contained plugins. The list in the global + variable must be freed (with free_engine_list()). However, the way Sys_var + works, there is no place to explicitly free other lists, like the one + returned from get_default(). + + Therefore, the code needs to work with temporary lists, which are + registered in the THD to be automatically freed (and plugins similarly + automatically unlocked). This is why do_check() allocates a temporary + list, from which do_update() then makes a permanent copy. */ class Sys_var_pluginlist: public sys_var { @@ -1575,9 +1586,9 @@ public: plugin_ref *plugins; if (!(res=var->value->val_str(&str))) - plugins= resolve_engine_list("", 0, true); + plugins= resolve_engine_list(thd, "", 0, true, true); else - plugins= resolve_engine_list(res->ptr(), res->length(), true); + plugins= resolve_engine_list(thd, res->ptr(), res->length(), true, true); if (!plugins) return true; var->save_result.plugins= plugins; @@ -1586,7 +1597,7 @@ public: void do_update(plugin_ref **valptr, plugin_ref* newval) { plugin_ref *oldval= *valptr; - *valptr= newval; + *valptr= copy_engine_list(newval); free_engine_list(oldval); } bool session_update(THD *thd, set_var *var) @@ -1604,14 +1615,15 @@ public: void session_save_default(THD *thd, set_var *var) { plugin_ref* plugins= global_var(plugin_ref *); - var->save_result.plugins= plugins ? copy_engine_list(plugins) : 0; + var->save_result.plugins= plugins ? temp_copy_engine_list(thd, plugins) : 0; } plugin_ref *get_default(THD *thd) { char *default_value= *reinterpret_cast(option.def_value); if (!default_value) return 0; - return resolve_engine_list(default_value, strlen(default_value), false); + return resolve_engine_list(thd, default_value, strlen(default_value), + false, true); } void global_save_default(THD *thd, set_var *var) -- cgit v1.2.1 From 1b54cb3b7741548a02ff520d730dcab140d60777 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Thu, 20 Apr 2017 16:22:21 +0200 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Update some existing test cases to work with the new handling of mysql.gtid_slave_pos* tables: - The tables are now checked during START SLAVE, which causes some errors or error injections to trigger differently. - Some test cases that play games with renaming or altering the mysql.gtid_slave_pos table need adjustments. --- mysql-test/suite/rpl/r/rpl_gtid_crash.result | 4 ++-- mysql-test/suite/rpl/r/rpl_gtid_errorhandling.result | 14 +++++--------- mysql-test/suite/rpl/r/rpl_gtid_stop_start.result | 4 ++-- mysql-test/suite/rpl/r/rpl_gtid_until.result | 2 ++ mysql-test/suite/rpl/t/rpl_gtid_crash.test | 4 ++-- mysql-test/suite/rpl/t/rpl_gtid_errorhandling.test | 19 +++++-------------- mysql-test/suite/rpl/t/rpl_gtid_stop_start.test | 18 ++++++++++++++++-- mysql-test/suite/rpl/t/rpl_gtid_until.test | 3 +++ 8 files changed, 37 insertions(+), 31 deletions(-) diff --git a/mysql-test/suite/rpl/r/rpl_gtid_crash.result b/mysql-test/suite/rpl/r/rpl_gtid_crash.result index 7b6e95bf718..ed2a2b287e9 100644 --- a/mysql-test/suite/rpl/r/rpl_gtid_crash.result +++ b/mysql-test/suite/rpl/r/rpl_gtid_crash.result @@ -88,16 +88,16 @@ include/save_master_gtid.inc connection server_2; include/sync_with_master_gtid.inc include/stop_slave.inc -SET GLOBAL debug_dbug="+d,crash_commit_before"; START SLAVE; +SET GLOBAL debug_dbug="+d,crash_commit_before"; connection server_1; INSERT INTO t1 VALUES (5); include/save_master_gtid.inc connection server_2; include/sync_with_master_gtid.inc include/stop_slave.inc -SET GLOBAL debug_dbug="+d,crash_commit_after"; START SLAVE; +SET GLOBAL debug_dbug="+d,crash_commit_after"; connection server_1; INSERT INTO t1 VALUES (6); include/save_master_gtid.inc diff --git a/mysql-test/suite/rpl/r/rpl_gtid_errorhandling.result b/mysql-test/suite/rpl/r/rpl_gtid_errorhandling.result index 7c2471ba37c..04c3fa7966f 100644 --- a/mysql-test/suite/rpl/r/rpl_gtid_errorhandling.result +++ b/mysql-test/suite/rpl/r/rpl_gtid_errorhandling.result @@ -8,26 +8,22 @@ connection slave; include/stop_slave.inc ALTER TABLE mysql.gtid_slave_pos CHANGE seq_no seq_no VARCHAR(20); START SLAVE; +ERROR HY000: Failed to open mysql.gtid_slave_pos connection master; INSERT INTO t1 VALUES (1); connection slave; -CALL mtr.add_suppression("Slave: Failed to open mysql.gtid_slave_pos"); -include/wait_for_slave_sql_error.inc [errno=1942] -include/stop_slave.inc +CALL mtr.add_suppression("Incorrect definition of table mysql.gtid_slave_pos"); ALTER TABLE mysql.gtid_slave_pos CHANGE seq_no seq_no BIGINT UNSIGNED NOT NULL; ALTER TABLE mysql.gtid_slave_pos DROP PRIMARY KEY; ALTER TABLE mysql.gtid_slave_pos ADD PRIMARY KEY (sub_id, domain_id); START SLAVE; -include/wait_for_slave_sql_error.inc [errno=1942] -include/stop_slave.inc +ERROR HY000: Failed to open mysql.gtid_slave_pos ALTER TABLE mysql.gtid_slave_pos DROP PRIMARY KEY; START SLAVE; -include/wait_for_slave_sql_error.inc [errno=1942] -include/stop_slave.inc +ERROR HY000: Failed to open mysql.gtid_slave_pos ALTER TABLE mysql.gtid_slave_pos ADD PRIMARY KEY (sub_id); START SLAVE; -include/wait_for_slave_sql_error.inc [errno=1942] -include/stop_slave.inc +ERROR HY000: Failed to open mysql.gtid_slave_pos ALTER TABLE mysql.gtid_slave_pos DROP PRIMARY KEY; ALTER TABLE mysql.gtid_slave_pos ADD PRIMARY KEY (domain_id, sub_id); include/start_slave.inc diff --git a/mysql-test/suite/rpl/r/rpl_gtid_stop_start.result b/mysql-test/suite/rpl/r/rpl_gtid_stop_start.result index 3f3b5e4344a..ff845794c22 100644 --- a/mysql-test/suite/rpl/r/rpl_gtid_stop_start.result +++ b/mysql-test/suite/rpl/r/rpl_gtid_stop_start.result @@ -194,7 +194,7 @@ domain_id COUNT(*) *** MDEV-4650: show variables; ERROR 1946 (HY000): Failed to load replication slave GTID position *** connection server_2; SET sql_log_bin=0; -RENAME TABLE mysql.gtid_slave_pos TO mysql.gtid_slave_pos_old; +RENAME TABLE mysql.gtid_slave_pos TO mysql.old_gtid_slave_pos; SET sql_log_bin=1; SHOW VARIABLES; SHOW VARIABLES LIKE 'gtid_slave_pos'; @@ -207,7 +207,7 @@ Level Code Message Error 1146 Table 'mysql.gtid_slave_pos' doesn't exist Error 1946 Failed to load replication slave GTID position from table mysql.gtid_slave_pos SET sql_log_bin=0; -RENAME TABLE mysql.gtid_slave_pos_old TO mysql.gtid_slave_pos; +RENAME TABLE mysql.old_gtid_slave_pos TO mysql.gtid_slave_pos; CALL mtr.add_suppression("Failed to load slave replication state from table mysql.gtid_slave_pos"); SET sql_log_bin=1; SHOW VARIABLES LIKE 'gtid_slave_pos'; diff --git a/mysql-test/suite/rpl/r/rpl_gtid_until.result b/mysql-test/suite/rpl/r/rpl_gtid_until.result index 886f6cfd2cb..2295aad34ac 100644 --- a/mysql-test/suite/rpl/r/rpl_gtid_until.result +++ b/mysql-test/suite/rpl/r/rpl_gtid_until.result @@ -10,6 +10,8 @@ SET s= SUBSTR(s FROM 1 FOR LOCATE(",", s) - 1); RETURN s; END| connection server_2; +include/stop_slave.inc +include/start_slave.inc START SLAVE UNTIL master_gtid_pos = ""; ERROR HY000: Slave is already running include/stop_slave_io.inc diff --git a/mysql-test/suite/rpl/t/rpl_gtid_crash.test b/mysql-test/suite/rpl/t/rpl_gtid_crash.test index b81cbd38cd3..5cf28b6e49a 100644 --- a/mysql-test/suite/rpl/t/rpl_gtid_crash.test +++ b/mysql-test/suite/rpl/t/rpl_gtid_crash.test @@ -161,8 +161,8 @@ EOF --write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect wait EOF -SET GLOBAL debug_dbug="+d,crash_commit_before"; START SLAVE; +SET GLOBAL debug_dbug="+d,crash_commit_before"; --connection server_1 INSERT INTO t1 VALUES (5); @@ -185,8 +185,8 @@ EOF --write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect wait EOF -SET GLOBAL debug_dbug="+d,crash_commit_after"; START SLAVE; +SET GLOBAL debug_dbug="+d,crash_commit_after"; --connection server_1 INSERT INTO t1 VALUES (6); diff --git a/mysql-test/suite/rpl/t/rpl_gtid_errorhandling.test b/mysql-test/suite/rpl/t/rpl_gtid_errorhandling.test index 05da466597e..a5946411144 100644 --- a/mysql-test/suite/rpl/t/rpl_gtid_errorhandling.test +++ b/mysql-test/suite/rpl/t/rpl_gtid_errorhandling.test @@ -10,37 +10,28 @@ CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB; --connection slave --source include/stop_slave.inc ALTER TABLE mysql.gtid_slave_pos CHANGE seq_no seq_no VARCHAR(20); +--error ER_GTID_OPEN_TABLE_FAILED START SLAVE; --connection master INSERT INTO t1 VALUES (1); --connection slave -CALL mtr.add_suppression("Slave: Failed to open mysql.gtid_slave_pos"); ---let $slave_sql_errno=1942 ---source include/wait_for_slave_sql_error.inc - ---source include/stop_slave.inc +CALL mtr.add_suppression("Incorrect definition of table mysql.gtid_slave_pos"); ALTER TABLE mysql.gtid_slave_pos CHANGE seq_no seq_no BIGINT UNSIGNED NOT NULL; ALTER TABLE mysql.gtid_slave_pos DROP PRIMARY KEY; ALTER TABLE mysql.gtid_slave_pos ADD PRIMARY KEY (sub_id, domain_id); +--error ER_GTID_OPEN_TABLE_FAILED START SLAVE; ---let $slave_sql_errno=1942 ---source include/wait_for_slave_sql_error.inc ---source include/stop_slave.inc ALTER TABLE mysql.gtid_slave_pos DROP PRIMARY KEY; +--error ER_GTID_OPEN_TABLE_FAILED START SLAVE; ---let $slave_sql_errno=1942 ---source include/wait_for_slave_sql_error.inc ---source include/stop_slave.inc ALTER TABLE mysql.gtid_slave_pos ADD PRIMARY KEY (sub_id); +--error ER_GTID_OPEN_TABLE_FAILED START SLAVE; ---let $slave_sql_errno=1942 ---source include/wait_for_slave_sql_error.inc ---source include/stop_slave.inc ALTER TABLE mysql.gtid_slave_pos DROP PRIMARY KEY; ALTER TABLE mysql.gtid_slave_pos ADD PRIMARY KEY (domain_id, sub_id); --source include/start_slave.inc diff --git a/mysql-test/suite/rpl/t/rpl_gtid_stop_start.test b/mysql-test/suite/rpl/t/rpl_gtid_stop_start.test index 09b35011f1f..309debd87c5 100644 --- a/mysql-test/suite/rpl/t/rpl_gtid_stop_start.test +++ b/mysql-test/suite/rpl/t/rpl_gtid_stop_start.test @@ -232,6 +232,20 @@ EOF SET sql_log_bin= 0; ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; SET sql_log_bin= 1; +# Do a second restart to get the mysql.gtid_slave_pos table loaded with +# the right engine. +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +wait +EOF +--shutdown_server 30 +--source include/wait_until_disconnected.inc + +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +restart: +EOF +--enable_reconnect +--source include/wait_until_connected_again.inc + --source include/start_slave.inc --connection server_1 @@ -285,7 +299,7 @@ SELECT domain_id, COUNT(*) FROM mysql.gtid_slave_pos GROUP BY domain_id; --connection server_2 SET sql_log_bin=0; --let $old_pos= `SELECT @@GLOBAL.gtid_slave_pos` -RENAME TABLE mysql.gtid_slave_pos TO mysql.gtid_slave_pos_old; +RENAME TABLE mysql.gtid_slave_pos TO mysql.old_gtid_slave_pos; SET sql_log_bin=1; --write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect @@ -313,7 +327,7 @@ SHOW WARNINGS; # Restore things. SET sql_log_bin=0; -RENAME TABLE mysql.gtid_slave_pos_old TO mysql.gtid_slave_pos; +RENAME TABLE mysql.old_gtid_slave_pos TO mysql.gtid_slave_pos; CALL mtr.add_suppression("Failed to load slave replication state from table mysql.gtid_slave_pos"); SET sql_log_bin=1; diff --git a/mysql-test/suite/rpl/t/rpl_gtid_until.test b/mysql-test/suite/rpl/t/rpl_gtid_until.test index 20d4510ccc8..aa05ecf79ab 100644 --- a/mysql-test/suite/rpl/t/rpl_gtid_until.test +++ b/mysql-test/suite/rpl/t/rpl_gtid_until.test @@ -19,6 +19,9 @@ delimiter ;| --connection server_2 --sync_with_master +# Restart SQL thread to pick up ALTER TABLE of mysql.gtid_slave_pos. +--source include/stop_slave.inc +--source include/start_slave.inc # Both replication threads must be stopped for UNTIL master_gtid_pos. --error ER_SLAVE_WAS_RUNNING -- cgit v1.2.1 From df2f01c14a3b10c5c084ee89dcd7c6aa2db5409e Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Sun, 23 Apr 2017 10:44:45 +0200 Subject: Add cleanup of mysql.gtid_slave_pos table. (Found during MDEV-12179 development, but independent of it, so put into a separate commit). --- mysql-test/suite/rpl/r/rpl_mdev10863.result | 1 + mysql-test/suite/rpl/t/rpl_mdev10863.test | 1 + 2 files changed, 2 insertions(+) diff --git a/mysql-test/suite/rpl/r/rpl_mdev10863.result b/mysql-test/suite/rpl/r/rpl_mdev10863.result index 158d4a921b7..6accd1ee830 100644 --- a/mysql-test/suite/rpl/r/rpl_mdev10863.result +++ b/mysql-test/suite/rpl/r/rpl_mdev10863.result @@ -46,5 +46,6 @@ SET GLOBAL slave_parallel_threads=@old_parallel_threads; SET GLOBAL max_relay_log_size= @old_max_relay; include/start_slave.inc connection server_1; +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; DROP TABLE t1; include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_mdev10863.test b/mysql-test/suite/rpl/t/rpl_mdev10863.test index 796e770672d..81cdfd84dbe 100644 --- a/mysql-test/suite/rpl/t/rpl_mdev10863.test +++ b/mysql-test/suite/rpl/t/rpl_mdev10863.test @@ -99,6 +99,7 @@ SET GLOBAL max_relay_log_size= @old_max_relay; --source include/start_slave.inc --connection server_1 +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; DROP TABLE t1; --source include/rpl_end.inc -- cgit v1.2.1 From 4a8381ad34f66cadb4322d617df66fb3b8dcf28d Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Sun, 23 Apr 2017 10:46:07 +0200 Subject: Fix test to wait for slave to fully stop. (Found during MDEV-12179 development, but independent of it, so put into a separate commit). --- mysql-test/suite/rpl/r/show_status_stop_slave_race-7126.result | 1 + mysql-test/suite/rpl/t/show_status_stop_slave_race-7126.test | 1 + 2 files changed, 2 insertions(+) diff --git a/mysql-test/suite/rpl/r/show_status_stop_slave_race-7126.result b/mysql-test/suite/rpl/r/show_status_stop_slave_race-7126.result index 64219e3908d..999d9417b3f 100644 --- a/mysql-test/suite/rpl/r/show_status_stop_slave_race-7126.result +++ b/mysql-test/suite/rpl/r/show_status_stop_slave_race-7126.result @@ -3,6 +3,7 @@ include/master-slave.inc call mtr.add_suppression("Master is configured to log replication events"); connection slave; connection slave; +include/wait_for_slave_to_stop.inc start slave; connection master; include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/show_status_stop_slave_race-7126.test b/mysql-test/suite/rpl/t/show_status_stop_slave_race-7126.test index 88d7076b1e4..06a9e8ad75b 100644 --- a/mysql-test/suite/rpl/t/show_status_stop_slave_race-7126.test +++ b/mysql-test/suite/rpl/t/show_status_stop_slave_race-7126.test @@ -15,6 +15,7 @@ call mtr.add_suppression("Master is configured to log replication events"); # All done. --connection slave +--source include/wait_for_slave_to_stop.inc start slave; --connection master -- cgit v1.2.1 From 1af3165f98b7f8122af1649babc790e844a660df Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Sun, 23 Apr 2017 10:47:22 +0200 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Make test case tolerant to an earlier test case changing the storage engine for mysql.gtid_slave_pos. --- mysql-test/suite/rpl/r/rpl_mdev12179.result | 1 + mysql-test/suite/rpl/t/rpl_mdev12179.test | 3 +++ 2 files changed, 4 insertions(+) diff --git a/mysql-test/suite/rpl/r/rpl_mdev12179.result b/mysql-test/suite/rpl/r/rpl_mdev12179.result index 8bf8f183181..40059375356 100644 --- a/mysql-test/suite/rpl/r/rpl_mdev12179.result +++ b/mysql-test/suite/rpl/r/rpl_mdev12179.result @@ -48,6 +48,7 @@ a 1 include/stop_slave.inc SET sql_log_bin=0; +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; CREATE TABLE mysql.gtid_slave_pos_innodb LIKE mysql.gtid_slave_pos; ALTER TABLE mysql.gtid_slave_pos_innodb ENGINE=InnoDB; INSERT INTO mysql.gtid_slave_pos_innodb SELECT * FROM mysql.gtid_slave_pos; diff --git a/mysql-test/suite/rpl/t/rpl_mdev12179.test b/mysql-test/suite/rpl/t/rpl_mdev12179.test index b4609c2ed7a..a9113c91797 100644 --- a/mysql-test/suite/rpl/t/rpl_mdev12179.test +++ b/mysql-test/suite/rpl/t/rpl_mdev12179.test @@ -40,6 +40,9 @@ SELECT * FROM t1 ORDER BY a; SELECT * FROM t1 ORDER BY a; --source include/stop_slave.inc SET sql_log_bin=0; +# Reset storage engine for mysql.gtid_slave_pos in case an earlier test +# might have changed it to InnoDB. +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; CREATE TABLE mysql.gtid_slave_pos_innodb LIKE mysql.gtid_slave_pos; ALTER TABLE mysql.gtid_slave_pos_innodb ENGINE=InnoDB; INSERT INTO mysql.gtid_slave_pos_innodb SELECT * FROM mysql.gtid_slave_pos; -- cgit v1.2.1 From 89aad233de9371476d9424c7df72d66c37c23e8a Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Sun, 23 Apr 2017 10:49:58 +0200 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Move the discovery of mysql.gtid_slave_pos* tables into the SQL thread. This avoids doing things like opening tables and scanning the mysql schema for tables inside of the START SLAVE statement, which might interact badly with existing transaction or table locks. (Even though START SLAVE is documented to implicitly commit any active transactions, this appears not to be the case in current code). Table discovery fits naturally in the SQL thread init code, next to the loading of mysql.gtid_slave_pos state. --- .../suite/rpl/r/rpl_gtid_errorhandling.result | 14 +++++++---- mysql-test/suite/rpl/t/rpl_gtid_errorhandling.test | 19 ++++++++++---- sql/mysqld.cc | 2 +- sql/rpl_mi.cc | 29 +++++++++++++--------- sql/rpl_mi.h | 2 +- sql/rpl_parallel.cc | 4 +-- sql/rpl_rli.cc | 14 ++++++++--- sql/slave.cc | 8 ++++++ sql/sql_repl.cc | 4 --- 9 files changed, 63 insertions(+), 33 deletions(-) diff --git a/mysql-test/suite/rpl/r/rpl_gtid_errorhandling.result b/mysql-test/suite/rpl/r/rpl_gtid_errorhandling.result index 04c3fa7966f..62a5b9c3531 100644 --- a/mysql-test/suite/rpl/r/rpl_gtid_errorhandling.result +++ b/mysql-test/suite/rpl/r/rpl_gtid_errorhandling.result @@ -8,22 +8,26 @@ connection slave; include/stop_slave.inc ALTER TABLE mysql.gtid_slave_pos CHANGE seq_no seq_no VARCHAR(20); START SLAVE; -ERROR HY000: Failed to open mysql.gtid_slave_pos connection master; INSERT INTO t1 VALUES (1); connection slave; -CALL mtr.add_suppression("Incorrect definition of table mysql.gtid_slave_pos"); +CALL mtr.add_suppression("Slave: Failed to open mysql.gtid_slave_pos"); +include/wait_for_slave_sql_error.inc [errno=1944] +include/stop_slave.inc ALTER TABLE mysql.gtid_slave_pos CHANGE seq_no seq_no BIGINT UNSIGNED NOT NULL; ALTER TABLE mysql.gtid_slave_pos DROP PRIMARY KEY; ALTER TABLE mysql.gtid_slave_pos ADD PRIMARY KEY (sub_id, domain_id); START SLAVE; -ERROR HY000: Failed to open mysql.gtid_slave_pos +include/wait_for_slave_sql_error.inc [errno=1944] +include/stop_slave.inc ALTER TABLE mysql.gtid_slave_pos DROP PRIMARY KEY; START SLAVE; -ERROR HY000: Failed to open mysql.gtid_slave_pos +include/wait_for_slave_sql_error.inc [errno=1944] +include/stop_slave.inc ALTER TABLE mysql.gtid_slave_pos ADD PRIMARY KEY (sub_id); START SLAVE; -ERROR HY000: Failed to open mysql.gtid_slave_pos +include/wait_for_slave_sql_error.inc [errno=1944] +include/stop_slave.inc ALTER TABLE mysql.gtid_slave_pos DROP PRIMARY KEY; ALTER TABLE mysql.gtid_slave_pos ADD PRIMARY KEY (domain_id, sub_id); include/start_slave.inc diff --git a/mysql-test/suite/rpl/t/rpl_gtid_errorhandling.test b/mysql-test/suite/rpl/t/rpl_gtid_errorhandling.test index a5946411144..796f6894f19 100644 --- a/mysql-test/suite/rpl/t/rpl_gtid_errorhandling.test +++ b/mysql-test/suite/rpl/t/rpl_gtid_errorhandling.test @@ -10,28 +10,37 @@ CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB; --connection slave --source include/stop_slave.inc ALTER TABLE mysql.gtid_slave_pos CHANGE seq_no seq_no VARCHAR(20); ---error ER_GTID_OPEN_TABLE_FAILED START SLAVE; --connection master INSERT INTO t1 VALUES (1); --connection slave -CALL mtr.add_suppression("Incorrect definition of table mysql.gtid_slave_pos"); +CALL mtr.add_suppression("Slave: Failed to open mysql.gtid_slave_pos"); +--let $slave_sql_errno=1944 +--source include/wait_for_slave_sql_error.inc + +--source include/stop_slave.inc ALTER TABLE mysql.gtid_slave_pos CHANGE seq_no seq_no BIGINT UNSIGNED NOT NULL; ALTER TABLE mysql.gtid_slave_pos DROP PRIMARY KEY; ALTER TABLE mysql.gtid_slave_pos ADD PRIMARY KEY (sub_id, domain_id); ---error ER_GTID_OPEN_TABLE_FAILED START SLAVE; +--let $slave_sql_errno=1944 +--source include/wait_for_slave_sql_error.inc +--source include/stop_slave.inc ALTER TABLE mysql.gtid_slave_pos DROP PRIMARY KEY; ---error ER_GTID_OPEN_TABLE_FAILED START SLAVE; +--let $slave_sql_errno=1944 +--source include/wait_for_slave_sql_error.inc +--source include/stop_slave.inc ALTER TABLE mysql.gtid_slave_pos ADD PRIMARY KEY (sub_id); ---error ER_GTID_OPEN_TABLE_FAILED START SLAVE; +--let $slave_sql_errno=1944 +--source include/wait_for_slave_sql_error.inc +--source include/stop_slave.inc ALTER TABLE mysql.gtid_slave_pos DROP PRIMARY KEY; ALTER TABLE mysql.gtid_slave_pos ADD PRIMARY KEY (domain_id, sub_id); --source include/start_slave.inc diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 0b8aaf1de78..a3c8686fee4 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -7790,7 +7790,7 @@ static int show_slaves_running(THD *thd, SHOW_VAR *var, char *buff) var->type= SHOW_LONGLONG; var->value= buff; - *((longlong *)buff)= any_slave_sql_running(); + *((longlong *)buff)= any_slave_sql_running(false); return 0; } diff --git a/sql/rpl_mi.cc b/sql/rpl_mi.cc index f30b7e161f2..4cf09d0bb76 100644 --- a/sql/rpl_mi.cc +++ b/sql/rpl_mi.cc @@ -1558,6 +1558,9 @@ bool give_error_if_slave_running(bool already_locked) /** any_slave_sql_running() + @param + already_locked 0 if we need to lock, 1 if we have LOCK_active_mi_locked + @return 0 No Slave SQL thread is running # Number of slave SQL thread running @@ -1568,26 +1571,28 @@ bool give_error_if_slave_running(bool already_locked) hash entries can't be accessed. */ -uint any_slave_sql_running() +uint any_slave_sql_running(bool already_locked) { uint count= 0; HASH *hash; DBUG_ENTER("any_slave_sql_running"); - mysql_mutex_lock(&LOCK_active_mi); + if (!already_locked) + mysql_mutex_lock(&LOCK_active_mi); if (unlikely(shutdown_in_progress || !master_info_index)) + count= 1; + else { - mysql_mutex_unlock(&LOCK_active_mi); - DBUG_RETURN(1); - } - hash= &master_info_index->master_info_hash; - for (uint i= 0; i< hash->records; ++i) - { - Master_info *mi= (Master_info *)my_hash_element(hash, i); - if (mi->rli.slave_running != MYSQL_SLAVE_NOT_RUN) - count++; + hash= &master_info_index->master_info_hash; + for (uint i= 0; i< hash->records; ++i) + { + Master_info *mi= (Master_info *)my_hash_element(hash, i); + if (mi->rli.slave_running != MYSQL_SLAVE_NOT_RUN) + count++; + } } - mysql_mutex_unlock(&LOCK_active_mi); + if (!already_locked) + mysql_mutex_unlock(&LOCK_active_mi); DBUG_RETURN(count); } diff --git a/sql/rpl_mi.h b/sql/rpl_mi.h index 31c0f280ac1..104a7ce3da6 100644 --- a/sql/rpl_mi.h +++ b/sql/rpl_mi.h @@ -377,7 +377,7 @@ void create_logfile_name_with_suffix(char *res_file_name, size_t length, uchar *get_key_master_info(Master_info *mi, size_t *length, my_bool not_used __attribute__((unused))); void free_key_master_info(Master_info *mi); -uint any_slave_sql_running(); +uint any_slave_sql_running(bool already_locked); bool give_error_if_slave_running(bool already_lock); #endif /* HAVE_REPLICATION */ diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 6788f422cbd..5215faaa17b 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -1465,7 +1465,7 @@ rpl_parallel_change_thread_count(rpl_parallel_thread_pool *pool, */ if (!new_count && !force) { - if (any_slave_sql_running()) + if (any_slave_sql_running(false)) { DBUG_PRINT("warning", ("SQL threads running while trying to reset parallel pool")); @@ -1620,7 +1620,7 @@ err: int rpl_parallel_resize_pool_if_no_slaves(void) { /* master_info_index is set to NULL on shutdown */ - if (opt_slave_parallel_threads > 0 && !any_slave_sql_running()) + if (opt_slave_parallel_threads > 0 && !any_slave_sql_running(false)) return rpl_parallel_inactivate_pool(&global_rpl_thread_pool); return 0; } diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 2cda87925d1..59138a23daf 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -1917,7 +1917,7 @@ find_gtid_slave_pos_tables(THD *thd) { int err= 0; load_gtid_state_cb_data cb_data; - bool any_running; + uint num_running; mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); bool loaded= rpl_global_gtid_slave_state->loaded; @@ -1949,10 +1949,17 @@ find_gtid_slave_pos_tables(THD *thd) if ((err= gtid_pos_auto_create_tables(&cb_data.table_list))) goto end; - any_running= any_slave_sql_running(); + mysql_mutex_lock(&LOCK_active_mi); + num_running= any_slave_sql_running(true); mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); - if (!any_running) + if (num_running <= 1) { + /* + If no slave is running now, the count will be 1, since this SQL thread + which is starting is included in the count. In this case, we can safely + replace the list, no-one can be trying to read it without lock. + */ + DBUG_ASSERT(num_running == 1); rpl_global_gtid_slave_state->set_gtid_pos_tables_list(cb_data.table_list, cb_data.default_entry); cb_data.table_list= NULL; @@ -2012,6 +2019,7 @@ find_gtid_slave_pos_tables(THD *thd) } } mysql_mutex_unlock(&rpl_global_gtid_slave_state->LOCK_slave_state); + mysql_mutex_unlock(&LOCK_active_mi); end: if (cb_data.table_list) diff --git a/sql/slave.cc b/sql/slave.cc index 443e5b7a5f1..a892bba82c9 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -5279,6 +5279,14 @@ pthread_handler_t handle_slave_sql(void *arg) if (mi->using_gtid != Master_info::USE_GTID_NO || opt_gtid_strict_mode) goto err; } + /* Re-load the set of mysql.gtid_slave_posXXX tables available. */ + if (find_gtid_slave_pos_tables(thd)) + { + rli->report(ERROR_LEVEL, thd->get_stmt_da()->sql_errno(), NULL, + "Error processing replication GTID position tables: %s", + thd->get_stmt_da()->message()); + goto err; + } /* execute init_slave variable */ if (opt_init_slave.length) diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index 087fc293359..7c30aa38f1a 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -3031,10 +3031,6 @@ int start_slave(THD* thd , Master_info* mi, bool net_report) if (check_access(thd, SUPER_ACL, any_db, NULL, NULL, 0, 0)) DBUG_RETURN(-1); - /* Re-load the set of mysql.gtid_slave_posXXX tables available. */ - if (find_gtid_slave_pos_tables(thd)) - DBUG_RETURN(-1); - create_logfile_name_with_suffix(master_info_file_tmp, sizeof(master_info_file_tmp), master_info_file, 0, -- cgit v1.2.1 From 59bab556a04e5945d540ec618a97fa1f818b5d1f Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Tue, 25 Apr 2017 10:01:33 +0200 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Fix record_gtid()'s choice of mysql.gtid_slave_pos* table to only consider engines that participated in the transaction with write operations (ie. ignore read-only participation). --- sql/rpl_gtid.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index d0f5694fb5a..83b1b402272 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -489,8 +489,12 @@ rpl_slave_state::select_gtid_pos_table(THD *thd, LEX_STRING *out_tablename) Ha_trx_info *ha_info= thd->transaction.all.ha_list; while (ha_info) { - void *trx_hton= ha_info->ht(); + void *trx_hton; table_entry= list; + + if (!ha_info->is_trx_read_write()) + continue; + trx_hton= ha_info->ht(); while (table_entry) { if (table_entry->table_hton == trx_hton) -- cgit v1.2.1 From 86fa6f9b3d7b85f1e629ec4a428dd37670bc3c3e Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Tue, 25 Apr 2017 12:00:43 +0200 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. .result file update following merge (now mysqltest logs --connection commands to .result files). --- storage/tokudb/mysql-test/tokudb_rpl/r/mdev12179.result | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/storage/tokudb/mysql-test/tokudb_rpl/r/mdev12179.result b/storage/tokudb/mysql-test/tokudb_rpl/r/mdev12179.result index ce1f5bb3a8e..da9204eabca 100644 --- a/storage/tokudb/mysql-test/tokudb_rpl/r/mdev12179.result +++ b/storage/tokudb/mysql-test/tokudb_rpl/r/mdev12179.result @@ -1,5 +1,6 @@ include/master-slave.inc [connection master] +connection server_2; include/stop_slave.inc CHANGE MASTER TO master_use_gtid=slave_pos; SET sql_log_bin=0; @@ -12,6 +13,7 @@ CREATE TABLE mysql.gtid_slave_pos_innodb_redundant LIKE mysql.gtid_slave_pos; ALTER TABLE mysql.gtid_slave_pos_innodb_redundant ENGINE=InnoDB; call mtr.add_suppression("Ignoring redundant table.*since.*has the same storage engine"); include/start_slave.inc +connection server_1; CREATE TABLE t1 (a INT PRIMARY KEY); CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB; CREATE TABLE t3 (a INT PRIMARY KEY) ENGINE=TokuDB; @@ -27,6 +29,7 @@ a SELECT * FROM t3 ORDER BY a; a 1 +connection server_2; SELECT * FROM t1 ORDER BY a; a 1 @@ -48,12 +51,14 @@ domain_id sub_id server_id seq_no SELECT * FROM mysql.gtid_slave_pos_tokudb ORDER BY sub_id; domain_id sub_id server_id seq_no 0 6 1 6 +connection server_2; SET sql_log_bin=0; DROP TABLE mysql.gtid_slave_pos_innodb; DROP TABLE mysql.gtid_slave_pos_tokudb; DROP TABLE mysql.gtid_slave_pos_myisam_redundant; DROP TABLE mysql.gtid_slave_pos_innodb_redundant; SET sql_log_bin=1; +connection server_1; DROP TABLE t1; DROP TABLE t2; DROP TABLE t3; -- cgit v1.2.1 From c174718aed075a3e9102e000710f047c6582b3f1 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Tue, 25 Apr 2017 19:08:45 +0200 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Implement status variables to aid the DBA in determining the need and/or effectiveness of the per-engine mylsq.gtid_slave_pos feature: transactions_multi_engine Number of transactions that changed data in multiple (transactional) storage engines. rpl_transactions_multi_engine Number of replicated transactions that involved changes in multiple (transactional) storage engines, before considering the update of the mysql.gtid_slave_posXXX table. transactions_gtid_foreign_engine Number of replicated transactions where the update of the mysql.gtid_slave_posXXX table had to choose a storage engine that did not otherwise participate in the transaction. --- sql/handler.cc | 5 + sql/log.h | 1 + sql/mysqld.cc | 9 + sql/mysqld.h | 3 + sql/rpl_gtid.cc | 39 +++- .../mysql-test/tokudb_rpl/r/mdev12179.result | 202 ++++++++++++++++++++- .../tokudb/mysql-test/tokudb_rpl/t/mdev12179.test | 178 +++++++++++++++++- 7 files changed, 429 insertions(+), 8 deletions(-) diff --git a/sql/handler.cc b/sql/handler.cc index a6016646d3c..72e94847763 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -1542,6 +1542,7 @@ static int commit_one_phase_2(THD *thd, bool all, THD_TRANS *trans, bool is_real_trans) { int error= 0; + uint count= 0; Ha_trx_info *ha_info= trans->ha_list, *ha_info_next; DBUG_ENTER("commit_one_phase_2"); if (is_real_trans) @@ -1559,6 +1560,8 @@ commit_one_phase_2(THD *thd, bool all, THD_TRANS *trans, bool is_real_trans) } /* Should this be done only if is_real_trans is set ? */ status_var_increment(thd->status_var.ha_commit_count); + if (is_real_trans && ht != binlog_hton && ha_info->is_trx_read_write()) + ++count; ha_info_next= ha_info->next(); ha_info->reset(); /* keep it conveniently zero-filled */ } @@ -1577,6 +1580,8 @@ commit_one_phase_2(THD *thd, bool all, THD_TRANS *trans, bool is_real_trans) { thd->has_waiter= false; thd->transaction.cleanup(); + if (count >= 2) + statistic_increment(transactions_multi_engine, LOCK_status); } DBUG_RETURN(error); diff --git a/sql/log.h b/sql/log.h index f1a025edfb9..c0d7f338cf5 100644 --- a/sql/log.h +++ b/sql/log.h @@ -1095,6 +1095,7 @@ void make_default_log_name(char **out, const char* log_ext, bool once); void binlog_reset_cache(THD *thd); extern MYSQL_PLUGIN_IMPORT MYSQL_BIN_LOG mysql_bin_log; +extern handlerton *binlog_hton; extern LOGGER logger; extern const char *log_bin_index; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index a3c8686fee4..cf4e7a322f8 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -527,6 +527,9 @@ ulong max_connections, max_connect_errors; ulong extra_max_connections; uint max_digest_length= 0; ulong slave_retried_transactions; +ulong transactions_multi_engine; +ulong rpl_transactions_multi_engine; +ulong transactions_gtid_foreign_engine; ulonglong slave_skipped_errors; ulong feature_files_opened_with_delayed_keys= 0, feature_check_constraint= 0; ulonglong denied_connections; @@ -8564,6 +8567,9 @@ SHOW_VAR status_vars[]= { {"Threads_connected", (char*) &connection_count, SHOW_INT}, {"Threads_created", (char*) &thread_created, SHOW_LONG_NOFLUSH}, {"Threads_running", (char*) &thread_running, SHOW_INT}, + {"Transactions_multi_engine", (char*) &transactions_multi_engine, SHOW_LONG}, + {"Rpl_transactions_multi_engine", (char*) &rpl_transactions_multi_engine, SHOW_LONG}, + {"Transactions_gtid_foreign_engine", (char*) &transactions_gtid_foreign_engine, SHOW_LONG}, {"Update_scan", (char*) offsetof(STATUS_VAR, update_scan_count), SHOW_LONG_STATUS}, {"Uptime", (char*) &show_starttime, SHOW_SIMPLE_FUNC}, #ifdef ENABLED_PROFILING @@ -8807,6 +8813,9 @@ static int mysql_init_variables(void) report_user= report_password = report_host= 0; /* TO BE DELETED */ opt_relay_logname= opt_relaylog_index_name= 0; slave_retried_transactions= 0; + transactions_multi_engine= 0; + rpl_transactions_multi_engine= 0; + transactions_gtid_foreign_engine= 0; log_bin_basename= NULL; log_bin_index= NULL; diff --git a/sql/mysqld.h b/sql/mysqld.h index be8b97c80c4..5de7a3209f0 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -131,6 +131,9 @@ extern my_bool opt_safe_show_db, opt_local_infile, opt_myisam_use_mmap; extern my_bool opt_slave_compressed_protocol, use_temp_pool; extern ulong slave_exec_mode_options, slave_ddl_exec_mode_options; extern ulong slave_retried_transactions; +extern ulong transactions_multi_engine; +extern ulong rpl_transactions_multi_engine; +extern ulong transactions_gtid_foreign_engine; extern ulong slave_run_triggers_for_rbr; extern ulonglong slave_type_conversions_options; extern my_bool read_only, opt_readonly; diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 83b1b402272..925115af88a 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -486,15 +486,15 @@ rpl_slave_state::select_gtid_pos_table(THD *thd, LEX_STRING *out_tablename) */ list= my_atomic_loadptr_explicit(>id_pos_tables, MY_MEMORY_ORDER_ACQUIRE); - Ha_trx_info *ha_info= thd->transaction.all.ha_list; - while (ha_info) + Ha_trx_info *ha_info; + uint count = 0; + for (ha_info= thd->transaction.all.ha_list; ha_info; ha_info= ha_info->next()) { - void *trx_hton; + void *trx_hton= ha_info->ht(); table_entry= list; - if (!ha_info->is_trx_read_write()) + if (!ha_info->is_trx_read_write() || trx_hton == binlog_hton) continue; - trx_hton= ha_info->ht(); while (table_entry) { if (table_entry->table_hton == trx_hton) @@ -502,6 +502,26 @@ rpl_slave_state::select_gtid_pos_table(THD *thd, LEX_STRING *out_tablename) if (likely(table_entry->state == GTID_POS_AVAILABLE)) { *out_tablename= table_entry->table_name; + /* + Check if this is a cross-engine transaction, so we can correctly + maintain the rpl_transactions_multi_engine status variable. + */ + if (count >= 1) + statistic_increment(rpl_transactions_multi_engine, LOCK_status); + else + { + for (;;) + { + ha_info= ha_info->next(); + if (!ha_info) + break; + if (ha_info->is_trx_read_write() && ha_info->ht() != binlog_hton) + { + statistic_increment(rpl_transactions_multi_engine, LOCK_status); + break; + } + } + } return; } /* @@ -516,7 +536,7 @@ rpl_slave_state::select_gtid_pos_table(THD *thd, LEX_STRING *out_tablename) } table_entry= table_entry->next; } - ha_info= ha_info->next(); + ++count; } /* If we cannot find any table whose engine matches an engine that is @@ -526,6 +546,13 @@ rpl_slave_state::select_gtid_pos_table(THD *thd, LEX_STRING *out_tablename) default_entry= my_atomic_loadptr_explicit(&default_gtid_pos_table, MY_MEMORY_ORDER_ACQUIRE); *out_tablename= default_entry->table_name; + /* Record in status that we failed to find a suitable gtid_pos table. */ + if (count > 0) + { + statistic_increment(transactions_gtid_foreign_engine, LOCK_status); + if (count > 1) + statistic_increment(rpl_transactions_multi_engine, LOCK_status); + } } diff --git a/storage/tokudb/mysql-test/tokudb_rpl/r/mdev12179.result b/storage/tokudb/mysql-test/tokudb_rpl/r/mdev12179.result index da9204eabca..d4532eec4e2 100644 --- a/storage/tokudb/mysql-test/tokudb_rpl/r/mdev12179.result +++ b/storage/tokudb/mysql-test/tokudb_rpl/r/mdev12179.result @@ -52,12 +52,212 @@ SELECT * FROM mysql.gtid_slave_pos_tokudb ORDER BY sub_id; domain_id sub_id server_id seq_no 0 6 1 6 connection server_2; +FLUSH NO_WRITE_TO_BINLOG STATUS; +SET sql_log_bin=0; +SHOW STATUS LIKE "Transactions_multi_engine"; +Variable_name Value +Transactions_multi_engine 0 +INSERT INTO t1 VALUES (100); +SHOW STATUS LIKE "Transactions_multi_engine"; +Variable_name Value +Transactions_multi_engine 0 +INSERT INTO t2 VALUES (101); +SHOW STATUS LIKE "Transactions_multi_engine"; +Variable_name Value +Transactions_multi_engine 0 +INSERT INTO t3 VALUES (101); +SHOW STATUS LIKE "Transactions_multi_engine"; +Variable_name Value +Transactions_multi_engine 0 +BEGIN; +INSERT INTO t3 VALUES (102); +INSERT INTO t2 VALUES (103); +COMMIT; +SHOW STATUS LIKE "Transactions_multi_engine"; +Variable_name Value +Transactions_multi_engine 1 +BEGIN; +INSERT INTO t2 VALUES (104); +INSERT INTO t3 VALUES (105); +COMMIT; +SHOW STATUS LIKE "Transactions_multi_engine"; +Variable_name Value +Transactions_multi_engine 2 +UPDATE t2, t3 SET t2.a=106, t3.a=107 WHERE t2.a=104 AND t3.a=105; +SHOW STATUS LIKE "Transactions_multi_engine"; +Variable_name Value +Transactions_multi_engine 3 +SET sql_log_bin=1; +INSERT INTO t1 VALUES (200); +SHOW STATUS LIKE "Transactions_multi_engine"; +Variable_name Value +Transactions_multi_engine 3 +INSERT INTO t2 VALUES (201); +SHOW STATUS LIKE "Transactions_multi_engine"; +Variable_name Value +Transactions_multi_engine 3 +INSERT INTO t3 VALUES (201); +SHOW STATUS LIKE "Transactions_multi_engine"; +Variable_name Value +Transactions_multi_engine 3 +BEGIN; +INSERT INTO t3 VALUES (202); +INSERT INTO t2 VALUES (203); +COMMIT; +SHOW STATUS LIKE "Transactions_multi_engine"; +Variable_name Value +Transactions_multi_engine 4 +BEGIN; +INSERT INTO t2 VALUES (204); +INSERT INTO t3 VALUES (205); +COMMIT; +SHOW STATUS LIKE "Transactions_multi_engine"; +Variable_name Value +Transactions_multi_engine 5 +UPDATE t2, t3 SET t2.a=206, t3.a=207 WHERE t2.a=204 AND t3.a=205; +SHOW STATUS LIKE "Transactions_multi_engine"; +Variable_name Value +Transactions_multi_engine 6 +DELETE FROM t1 WHERE a >= 100; +DELETE FROM t2 WHERE a >= 100; +DELETE FROM t3 WHERE a >= 100; +connection server_2; +include/stop_slave.inc SET sql_log_bin=0; -DROP TABLE mysql.gtid_slave_pos_innodb; DROP TABLE mysql.gtid_slave_pos_tokudb; DROP TABLE mysql.gtid_slave_pos_myisam_redundant; DROP TABLE mysql.gtid_slave_pos_innodb_redundant; SET sql_log_bin=1; +FLUSH NO_WRITE_TO_BINLOG STATUS; +include/start_slave.inc +SHOW STATUS LIKE "%transactions%engine"; +Variable_name Value +Rpl_transactions_multi_engine 0 +Transactions_gtid_foreign_engine 0 +Transactions_multi_engine 0 +connection server_1; +INSERT INTO t1 VALUES (100); +connection server_2; +SHOW STATUS LIKE "%transactions%engine"; +Variable_name Value +Rpl_transactions_multi_engine 0 +Transactions_gtid_foreign_engine 0 +Transactions_multi_engine 0 +connection server_1; +INSERT INTO t2 VALUES (101); +connection server_2; +SHOW STATUS LIKE "%transactions%engine"; +Variable_name Value +Rpl_transactions_multi_engine 0 +Transactions_gtid_foreign_engine 0 +Transactions_multi_engine 0 +connection server_1; +INSERT INTO t3 VALUES (101); +connection server_2; +SHOW STATUS LIKE "%transactions%engine"; +Variable_name Value +Rpl_transactions_multi_engine 0 +Transactions_gtid_foreign_engine 1 +Transactions_multi_engine 0 +connection server_1; +BEGIN; +INSERT INTO t3 VALUES (102); +INSERT INTO t2 VALUES (103); +COMMIT; +connection server_2; +SHOW STATUS LIKE "%transactions%engine"; +Variable_name Value +Rpl_transactions_multi_engine 1 +Transactions_gtid_foreign_engine 1 +Transactions_multi_engine 1 +connection server_1; +BEGIN; +INSERT INTO t2 VALUES (104); +INSERT INTO t3 VALUES (105); +COMMIT; +connection server_2; +SHOW STATUS LIKE "%transactions%engine"; +Variable_name Value +Rpl_transactions_multi_engine 2 +Transactions_gtid_foreign_engine 1 +Transactions_multi_engine 2 +connection server_1; +UPDATE t2, t3 SET t2.a=106, t3.a=107 WHERE t2.a=104 AND t3.a=105; +connection server_2; +SHOW STATUS LIKE "%transactions%engine"; +Variable_name Value +Rpl_transactions_multi_engine 3 +Transactions_gtid_foreign_engine 1 +Transactions_multi_engine 3 +connection server_2; +connection server_2; +SHOW VARIABLES LIKE 'log_bin'; +Variable_name Value +log_bin OFF +include/start_slave.inc +SHOW STATUS LIKE "%transactions%engine"; +Variable_name Value +Rpl_transactions_multi_engine 0 +Transactions_gtid_foreign_engine 0 +Transactions_multi_engine 0 +connection server_1; +INSERT INTO t1 VALUES (200); +connection server_2; +SHOW STATUS LIKE "%transactions%engine"; +Variable_name Value +Rpl_transactions_multi_engine 0 +Transactions_gtid_foreign_engine 0 +Transactions_multi_engine 0 +connection server_1; +INSERT INTO t2 VALUES (201); +connection server_2; +SHOW STATUS LIKE "%transactions%engine"; +Variable_name Value +Rpl_transactions_multi_engine 0 +Transactions_gtid_foreign_engine 0 +Transactions_multi_engine 0 +connection server_1; +INSERT INTO t3 VALUES (201); +connection server_2; +SHOW STATUS LIKE "%transactions%engine"; +Variable_name Value +Rpl_transactions_multi_engine 0 +Transactions_gtid_foreign_engine 1 +Transactions_multi_engine 0 +connection server_1; +BEGIN; +INSERT INTO t3 VALUES (202); +INSERT INTO t2 VALUES (203); +COMMIT; +connection server_2; +SHOW STATUS LIKE "%transactions%engine"; +Variable_name Value +Rpl_transactions_multi_engine 1 +Transactions_gtid_foreign_engine 1 +Transactions_multi_engine 1 +connection server_1; +BEGIN; +INSERT INTO t2 VALUES (204); +INSERT INTO t3 VALUES (205); +COMMIT; +connection server_2; +SHOW STATUS LIKE "%transactions%engine"; +Variable_name Value +Rpl_transactions_multi_engine 2 +Transactions_gtid_foreign_engine 1 +Transactions_multi_engine 2 +connection server_1; +UPDATE t2, t3 SET t2.a=206, t3.a=207 WHERE t2.a=204 AND t3.a=205; +connection server_2; +SHOW STATUS LIKE "%transactions%engine"; +Variable_name Value +Rpl_transactions_multi_engine 3 +Transactions_gtid_foreign_engine 1 +Transactions_multi_engine 3 +connection server_2; +SET sql_log_bin=0; +DROP TABLE mysql.gtid_slave_pos_innodb; +SET sql_log_bin=1; connection server_1; DROP TABLE t1; DROP TABLE t2; diff --git a/storage/tokudb/mysql-test/tokudb_rpl/t/mdev12179.test b/storage/tokudb/mysql-test/tokudb_rpl/t/mdev12179.test index 822fc6f9e33..ceb119cd0dc 100644 --- a/storage/tokudb/mysql-test/tokudb_rpl/t/mdev12179.test +++ b/storage/tokudb/mysql-test/tokudb_rpl/t/mdev12179.test @@ -40,13 +40,189 @@ SELECT * FROM ( SELECT * FROM mysql.gtid_slave_pos_innodb SELECT * FROM mysql.gtid_slave_pos_tokudb ORDER BY sub_id; +# Test status variable Transactions_multi_engine. --connection server_2 +FLUSH NO_WRITE_TO_BINLOG STATUS; +SET sql_log_bin=0; +SHOW STATUS LIKE "Transactions_multi_engine"; +INSERT INTO t1 VALUES (100); +SHOW STATUS LIKE "Transactions_multi_engine"; +INSERT INTO t2 VALUES (101); +SHOW STATUS LIKE "Transactions_multi_engine"; +INSERT INTO t3 VALUES (101); +SHOW STATUS LIKE "Transactions_multi_engine"; +BEGIN; +INSERT INTO t3 VALUES (102); +INSERT INTO t2 VALUES (103); +COMMIT; +SHOW STATUS LIKE "Transactions_multi_engine"; +BEGIN; +INSERT INTO t2 VALUES (104); +INSERT INTO t3 VALUES (105); +COMMIT; +SHOW STATUS LIKE "Transactions_multi_engine"; +UPDATE t2, t3 SET t2.a=106, t3.a=107 WHERE t2.a=104 AND t3.a=105; +SHOW STATUS LIKE "Transactions_multi_engine"; +# Try again with binlog enabled. +SET sql_log_bin=1; +INSERT INTO t1 VALUES (200); +SHOW STATUS LIKE "Transactions_multi_engine"; +INSERT INTO t2 VALUES (201); +SHOW STATUS LIKE "Transactions_multi_engine"; +INSERT INTO t3 VALUES (201); +SHOW STATUS LIKE "Transactions_multi_engine"; +BEGIN; +INSERT INTO t3 VALUES (202); +INSERT INTO t2 VALUES (203); +COMMIT; +SHOW STATUS LIKE "Transactions_multi_engine"; +BEGIN; +INSERT INTO t2 VALUES (204); +INSERT INTO t3 VALUES (205); +COMMIT; +SHOW STATUS LIKE "Transactions_multi_engine"; +UPDATE t2, t3 SET t2.a=206, t3.a=207 WHERE t2.a=204 AND t3.a=205; +SHOW STATUS LIKE "Transactions_multi_engine"; + +DELETE FROM t1 WHERE a >= 100; +DELETE FROM t2 WHERE a >= 100; +DELETE FROM t3 WHERE a >= 100; + + +# Test status variables Rpl_transactions_multi_engine and Transactions_gtid_foreign_engine. +# Have mysql.gtid_slave_pos* for myisam and innodb but not tokudb. +--connection server_2 +--source include/stop_slave.inc SET sql_log_bin=0; -DROP TABLE mysql.gtid_slave_pos_innodb; DROP TABLE mysql.gtid_slave_pos_tokudb; DROP TABLE mysql.gtid_slave_pos_myisam_redundant; DROP TABLE mysql.gtid_slave_pos_innodb_redundant; SET sql_log_bin=1; +FLUSH NO_WRITE_TO_BINLOG STATUS; +--source include/start_slave.inc +SHOW STATUS LIKE "%transactions%engine"; + +--connection server_1 +INSERT INTO t1 VALUES (100); +--save_master_pos +--connection server_2 +--sync_with_master +SHOW STATUS LIKE "%transactions%engine"; + +--connection server_1 +INSERT INTO t2 VALUES (101); +--save_master_pos +--connection server_2 +--sync_with_master +SHOW STATUS LIKE "%transactions%engine"; + +--connection server_1 +INSERT INTO t3 VALUES (101); +--save_master_pos +--connection server_2 +--sync_with_master +SHOW STATUS LIKE "%transactions%engine"; + +--connection server_1 +BEGIN; +INSERT INTO t3 VALUES (102); +INSERT INTO t2 VALUES (103); +COMMIT; +--save_master_pos +--connection server_2 +--sync_with_master +SHOW STATUS LIKE "%transactions%engine"; + +--connection server_1 +BEGIN; +INSERT INTO t2 VALUES (104); +INSERT INTO t3 VALUES (105); +COMMIT; +--save_master_pos +--connection server_2 +--sync_with_master +SHOW STATUS LIKE "%transactions%engine"; + +--connection server_1 +UPDATE t2, t3 SET t2.a=106, t3.a=107 WHERE t2.a=104 AND t3.a=105; +--save_master_pos +--connection server_2 +--sync_with_master +SHOW STATUS LIKE "%transactions%engine"; + +# Now the same thing, but without binlogging on the slave. +--connection server_2 +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +wait +EOF +--shutdown_server 30 +--source include/wait_until_disconnected.inc + +# Restart without binary log. +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +restart: --skip-log-bin +EOF + +--connection server_2 +--enable_reconnect +--source include/wait_until_connected_again.inc +SHOW VARIABLES LIKE 'log_bin'; +--source include/start_slave.inc +SHOW STATUS LIKE "%transactions%engine"; + +--connection server_1 +INSERT INTO t1 VALUES (200); +--save_master_pos +--connection server_2 +--sync_with_master +SHOW STATUS LIKE "%transactions%engine"; + +--connection server_1 +INSERT INTO t2 VALUES (201); +--save_master_pos +--connection server_2 +--sync_with_master +SHOW STATUS LIKE "%transactions%engine"; + +--connection server_1 +INSERT INTO t3 VALUES (201); +--save_master_pos +--connection server_2 +--sync_with_master +SHOW STATUS LIKE "%transactions%engine"; + +--connection server_1 +BEGIN; +INSERT INTO t3 VALUES (202); +INSERT INTO t2 VALUES (203); +COMMIT; +--save_master_pos +--connection server_2 +--sync_with_master +SHOW STATUS LIKE "%transactions%engine"; + +--connection server_1 +BEGIN; +INSERT INTO t2 VALUES (204); +INSERT INTO t3 VALUES (205); +COMMIT; +--save_master_pos +--connection server_2 +--sync_with_master +SHOW STATUS LIKE "%transactions%engine"; + +--connection server_1 +UPDATE t2, t3 SET t2.a=206, t3.a=207 WHERE t2.a=204 AND t3.a=205; +--save_master_pos +--connection server_2 +--sync_with_master +SHOW STATUS LIKE "%transactions%engine"; + + +--connection server_2 +SET sql_log_bin=0; +DROP TABLE mysql.gtid_slave_pos_innodb; +SET sql_log_bin=1; --connection server_1 DROP TABLE t1; -- cgit v1.2.1 From 8683052389e8f59caeb4218dc300d98627be1750 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Sat, 29 Apr 2017 11:19:09 +0200 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Fix compilation for embedded server. --- sql/rpl_gtid.cc | 2 ++ sql/sys_vars.cc | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 925115af88a..a4a774a0200 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -531,7 +531,9 @@ rpl_slave_state::select_gtid_pos_table(THD *thd, LEX_STRING *out_tablename) to create it, and in a short while it should become available for following transactions. */ +#ifdef HAVE_REPLICATION slave_background_gtid_pos_create_request(table_entry); +#endif break; } table_entry= table_entry->next; diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 84b88f84bbf..6aec02a8291 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -3521,6 +3521,8 @@ static Sys_var_plugin Sys_enforce_storage_engine( NO_CMD_LINE, MYSQL_STORAGE_ENGINE_PLUGIN, DEFAULT(&enforced_storage_engine), NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(check_has_super)); + +#ifdef HAVE_REPLICATION /* Check 1. Value for gtid_pos_auto_engines is not NULL. @@ -3555,6 +3557,8 @@ static Sys_var_pluginlist Sys_gtid_pos_auto_engines( GLOBAL_VAR(opt_gtid_pos_auto_plugins), NO_CMD_LINE, DEFAULT(>id_pos_auto_engines), NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(check_gtid_pos_auto_engines)); +#endif + #if defined(ENABLED_DEBUG_SYNC) /* -- cgit v1.2.1 From 0db2cd7c7600c8cd03a1005d299cd0112d80b4ad Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Wed, 10 May 2017 09:56:31 +0200 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Fix compilation failure with different my_atomic implementation. The my_atomic_loadptr* takes void ** as first argument, so variables updated with it needs to be void * (it is not legal C to cast some_type ** to void **). --- sql/rpl_gtid.cc | 13 +++++++------ sql/rpl_gtid.h | 7 +++++-- sql/rpl_rli.cc | 6 ++++-- sql/slave.cc | 6 ++++-- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index a4a774a0200..b1b65119d0b 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -258,7 +258,7 @@ rpl_slave_state::rpl_slave_state() rpl_slave_state::~rpl_slave_state() { - free_gtid_pos_tables(gtid_pos_tables); + free_gtid_pos_tables((struct gtid_pos_table *)gtid_pos_tables); truncate_hash(); my_hash_free(&hash); delete_dynamic(>id_sort_array); @@ -484,7 +484,8 @@ rpl_slave_state::select_gtid_pos_table(THD *thd, LEX_STRING *out_tablename) See comments on rpl_slave_state::gtid_pos_tables for rules around proper access to the list. */ - list= my_atomic_loadptr_explicit(>id_pos_tables, MY_MEMORY_ORDER_ACQUIRE); + list= (struct gtid_pos_table *) + my_atomic_loadptr_explicit(>id_pos_tables, MY_MEMORY_ORDER_ACQUIRE); Ha_trx_info *ha_info; uint count = 0; @@ -545,8 +546,8 @@ rpl_slave_state::select_gtid_pos_table(THD *thd, LEX_STRING *out_tablename) already active in the transaction, or if there is no current transaction engines available, we return the default gtid_slave_pos table. */ - default_entry= my_atomic_loadptr_explicit(&default_gtid_pos_table, - MY_MEMORY_ORDER_ACQUIRE); + default_entry= (struct gtid_pos_table *) + my_atomic_loadptr_explicit(&default_gtid_pos_table, MY_MEMORY_ORDER_ACQUIRE); *out_tablename= default_entry->table_name; /* Record in status that we failed to find a suitable gtid_pos table. */ if (count > 0) @@ -1268,7 +1269,7 @@ rpl_slave_state::set_gtid_pos_tables_list(rpl_slave_state::gtid_pos_table *new_l gtid_pos_table *old_list; mysql_mutex_assert_owner(&LOCK_slave_state); - old_list= gtid_pos_tables; + old_list= (struct gtid_pos_table *)gtid_pos_tables; my_atomic_storeptr_explicit(>id_pos_tables, new_list, MY_MEMORY_ORDER_RELEASE); my_atomic_storeptr_explicit(&default_gtid_pos_table, default_entry, MY_MEMORY_ORDER_RELEASE); @@ -1280,7 +1281,7 @@ void rpl_slave_state::add_gtid_pos_table(rpl_slave_state::gtid_pos_table *entry) { mysql_mutex_assert_owner(&LOCK_slave_state); - entry->next= gtid_pos_tables; + entry->next= (struct gtid_pos_table *)gtid_pos_tables; my_atomic_storeptr_explicit(>id_pos_tables, entry, MY_MEMORY_ORDER_RELEASE); } diff --git a/sql/rpl_gtid.h b/sql/rpl_gtid.h index 4ff1ca563b8..452f3676704 100644 --- a/sql/rpl_gtid.h +++ b/sql/rpl_gtid.h @@ -206,10 +206,13 @@ struct rpl_slave_state The list can be read without lock by an SQL driver thread or worker thread by reading the gtid_pos_tables pointer atomically with acquire semantics, to ensure that it will see the correct next pointer of a new head element. + + The type is struct gtid_pos_table *, but needs to be void * to allow using + my_atomic operations without violating C strict aliasing semantics. */ - struct gtid_pos_table *gtid_pos_tables; + void * volatile gtid_pos_tables; /* The default entry in gtid_pos_tables, mysql.gtid_slave_pos. */ - struct gtid_pos_table *default_gtid_pos_table; + void * volatile default_gtid_pos_table; bool loaded; rpl_slave_state(); diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 59138a23daf..63179f5d8e9 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -1973,7 +1973,8 @@ find_gtid_slave_pos_tables(THD *thd) */ rpl_slave_state::gtid_pos_table *old_entry, *new_entry, **next_ptr_ptr; - old_entry= rpl_global_gtid_slave_state->gtid_pos_tables; + old_entry= (rpl_slave_state::gtid_pos_table *) + rpl_global_gtid_slave_state->gtid_pos_tables; while (old_entry) { new_entry= cb_data.table_list; @@ -1995,7 +1996,8 @@ find_gtid_slave_pos_tables(THD *thd) while (new_entry) { /* Check if we already have a table with this storage engine. */ - old_entry= rpl_global_gtid_slave_state->gtid_pos_tables; + old_entry= (rpl_slave_state::gtid_pos_table *) + rpl_global_gtid_slave_state->gtid_pos_tables; while (old_entry) { if (new_entry->table_hton == old_entry->table_hton) diff --git a/sql/slave.cc b/sql/slave.cc index a892bba82c9..ffce356c2ec 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -383,7 +383,8 @@ handle_gtid_pos_auto_create_request(THD *thd, void *hton) /* Find the entry for the table to auto-create. */ mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); - entry= rpl_global_gtid_slave_state->gtid_pos_tables; + entry= (rpl_slave_state::gtid_pos_table *) + rpl_global_gtid_slave_state->gtid_pos_tables; while (entry) { if (entry->table_hton == hton && @@ -419,7 +420,8 @@ handle_gtid_pos_auto_create_request(THD *thd, void *hton) /* Now enable the entry for the auto-created table. */ mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); - entry= rpl_global_gtid_slave_state->gtid_pos_tables; + entry= (rpl_slave_state::gtid_pos_table *) + rpl_global_gtid_slave_state->gtid_pos_tables; while (entry) { if (entry->table_hton == hton && -- cgit v1.2.1 From 01b86a0ad99263570ebd6d87c95268e130409a3e Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Wed, 10 May 2017 15:48:32 +0200 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Fix a couple races in test multi_source.gtid_slave_pos which caused occasional failures. --- .../suite/multi_source/gtid_slave_pos.result | 22 ++++++++++++++++++---- mysql-test/suite/multi_source/gtid_slave_pos.test | 19 ++++++++++++++++++- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/multi_source/gtid_slave_pos.result b/mysql-test/suite/multi_source/gtid_slave_pos.result index ccccc664110..d57cfc17959 100644 --- a/mysql-test/suite/multi_source/gtid_slave_pos.result +++ b/mysql-test/suite/multi_source/gtid_slave_pos.result @@ -53,6 +53,11 @@ STOP SLAVE; include/wait_for_slave_to_stop.inc START SLAVE; include/wait_for_slave_to_start.inc +connection master1; +INSERT INTO t1 VALUES (2, "followup"); +include/save_master_gtid.inc +connection slave1; +include/sync_with_master_gtid.inc connection master2; INSERT INTO t2 VALUES (2, "secondary2"); include/save_master_gtid.inc @@ -60,7 +65,7 @@ connection slave1; include/sync_with_master_gtid.inc SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; domain_id max(seq_no) -1 5 +1 6 2 2 SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos_innodb GROUP BY domain_id; domain_id max(seq_no) @@ -80,12 +85,11 @@ include/wait_for_slave_to_start.inc CALL mtr.add_suppression("The table mysql.gtid_slave_pos_innodb was removed."); connection master2; INSERT INTO t2 VALUES (3, "tertiary 2"); -include/save_master_gtid.inc connection slave1; include/wait_for_slave_sql_error.inc [errno=1942] SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; domain_id max(seq_no) -1 5 +1 6 2 2 *** Stop both slaves, see that the drop of mysql.gtid_slave_pos_innodb is now picked up *** connection slave1; @@ -98,13 +102,23 @@ include/wait_for_slave_to_stop.inc set default_master_connection = 'slave1'; START SLAVE; include/wait_for_slave_to_start.inc +connection master1; +INSERT INTO t1 VALUES (3, "more stuff"); +include/save_master_gtid.inc +connection slave1; +include/sync_with_master_gtid.inc set default_master_connection = 'slave2'; START SLAVE; include/wait_for_slave_to_start.inc +connection master2; +include/save_master_gtid.inc +connection slave1; include/sync_with_master_gtid.inc SELECT * FROM t1 ORDER BY a; a b 1 initial +2 followup +3 more stuff SELECT * FROM t2 ORDER BY a; a b 1 initial @@ -116,7 +130,7 @@ a b 102 secondary SELECT domain_id, max(seq_no) FROM mysql.gtid_slave_pos GROUP BY domain_id; domain_id max(seq_no) -1 5 +1 7 2 4 connection master1; DROP TABLE t1; diff --git a/mysql-test/suite/multi_source/gtid_slave_pos.test b/mysql-test/suite/multi_source/gtid_slave_pos.test index b8701ff078c..c01130f8cd5 100644 --- a/mysql-test/suite/multi_source/gtid_slave_pos.test +++ b/mysql-test/suite/multi_source/gtid_slave_pos.test @@ -73,6 +73,14 @@ STOP SLAVE; START SLAVE; --source include/wait_for_slave_to_start.inc +# Send through a transaction on the slave1 connection, to be sure that it has +# had time to update the state with the new table. +--connection master1 +INSERT INTO t1 VALUES (2, "followup"); +--source include/save_master_gtid.inc +--connection slave1 +--source include/sync_with_master_gtid.inc + --connection master2 INSERT INTO t2 VALUES (2, "secondary2"); --source include/save_master_gtid.inc @@ -98,7 +106,6 @@ CALL mtr.add_suppression("The table mysql.gtid_slave_pos_innodb was removed."); --connection master2 INSERT INTO t2 VALUES (3, "tertiary 2"); ---source include/save_master_gtid.inc --connection slave1 --let $slave_sql_errno= 1942 @@ -116,10 +123,20 @@ STOP SLAVE; set default_master_connection = 'slave1'; START SLAVE; --source include/wait_for_slave_to_start.inc +# Send through a transaction on the slave1 connection, to be sure that it has +# had time to update the state with the new table. +--connection master1 +INSERT INTO t1 VALUES (3, "more stuff"); +--source include/save_master_gtid.inc +--connection slave1 +--source include/sync_with_master_gtid.inc set default_master_connection = 'slave2'; START SLAVE; --source include/wait_for_slave_to_start.inc +--connection master2 +--source include/save_master_gtid.inc +--connection slave1 --source include/sync_with_master_gtid.inc SELECT * FROM t1 ORDER BY a; SELECT * FROM t2 ORDER BY a; -- cgit v1.2.1 From 105ea511dc9eed25936838408e12147159659087 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Wed, 10 May 2017 16:54:28 +0200 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Fix test failure in rpl.rpl_gtid_mdev4484 when an earlier test changed the mysql.gtid_slave_pos table to innodb. With MDEV-12179, the delete of the old GTID is skipped, because it is recorded as belonging to another engine, due to ALTER TABLE in the test case. Fix by restart the SQL thread after the ALTER to have the correct engine registered. Also fix the test case rpl.rpl_gtid_ignored to not leave mysql.innodb_slave_pos as a different engine. --- mysql-test/suite/rpl/r/rpl_gtid_ignored.result | 1 + mysql-test/suite/rpl/r/rpl_gtid_mdev4484.result | 8 +++++++- mysql-test/suite/rpl/t/rpl_gtid_ignored.test | 1 + mysql-test/suite/rpl/t/rpl_gtid_mdev4484.test | 16 ++++++++++++---- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/rpl/r/rpl_gtid_ignored.result b/mysql-test/suite/rpl/r/rpl_gtid_ignored.result index 8d5b9be2ca0..ac608c3c2a3 100644 --- a/mysql-test/suite/rpl/r/rpl_gtid_ignored.result +++ b/mysql-test/suite/rpl/r/rpl_gtid_ignored.result @@ -79,6 +79,7 @@ a 9 connection server_1; DROP TABLE t1; +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; SET GLOBAL gtid_strict_mode= @old_gtid_strict_mode; SET debug_sync = "reset"; connection server_2; diff --git a/mysql-test/suite/rpl/r/rpl_gtid_mdev4484.result b/mysql-test/suite/rpl/r/rpl_gtid_mdev4484.result index c49207b99fa..aaeb0c8f119 100644 --- a/mysql-test/suite/rpl/r/rpl_gtid_mdev4484.result +++ b/mysql-test/suite/rpl/r/rpl_gtid_mdev4484.result @@ -1,5 +1,12 @@ include/master-slave.inc [connection master] +connection slave; +include/stop_slave.inc +SET sql_log_bin=0; +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; +SET sql_log_bin=1; +include/start_slave.inc +connection master; CREATE TABLE t1 (i int) ENGINE=InnoDB; connection slave; *** MDEV-4484, incorrect error handling when entries in gtid_slave_pos not found. *** @@ -13,7 +20,6 @@ SET @old_dbug= @@GLOBAL.debug_dbug; SET GLOBAL debug_dbug="+d,gtid_slave_pos_simulate_failed_delete"; SET sql_log_bin= 0; CALL mtr.add_suppression("Can't find file"); -ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; SET sql_log_bin= 1; include/start_slave.inc connection master; diff --git a/mysql-test/suite/rpl/t/rpl_gtid_ignored.test b/mysql-test/suite/rpl/t/rpl_gtid_ignored.test index cb98be3c838..6e927bd5a77 100644 --- a/mysql-test/suite/rpl/t/rpl_gtid_ignored.test +++ b/mysql-test/suite/rpl/t/rpl_gtid_ignored.test @@ -129,6 +129,7 @@ SELECT * FROM t1 ORDER BY a; # Clean up. --connection server_1 DROP TABLE t1; +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; SET GLOBAL gtid_strict_mode= @old_gtid_strict_mode; SET debug_sync = "reset"; diff --git a/mysql-test/suite/rpl/t/rpl_gtid_mdev4484.test b/mysql-test/suite/rpl/t/rpl_gtid_mdev4484.test index 43634ec1528..57f78530b01 100644 --- a/mysql-test/suite/rpl/t/rpl_gtid_mdev4484.test +++ b/mysql-test/suite/rpl/t/rpl_gtid_mdev4484.test @@ -2,6 +2,18 @@ --source include/have_innodb.inc --source include/have_debug.inc +--connection slave +--source include/stop_slave.inc +# Since we inject an error updating mysql.gtid_slave_pos, we will get different +# output depending on whether it is InnoDB or MyISAM (roll back or no roll +# back). So fix it to make sure we are consistent, in case an earlier test case +# left it as InnoDB. +SET sql_log_bin=0; +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; +SET sql_log_bin=1; +--source include/start_slave.inc + +--connection master CREATE TABLE t1 (i int) ENGINE=InnoDB; --sync_slave_with_master @@ -20,10 +32,6 @@ SET @old_dbug= @@GLOBAL.debug_dbug; SET GLOBAL debug_dbug="+d,gtid_slave_pos_simulate_failed_delete"; SET sql_log_bin= 0; CALL mtr.add_suppression("Can't find file"); -# Since we inject an error updating mysql.gtid_slave_pos, we will get different -# output depending on whether it is InnoDB or MyISAM (roll back or no roll -# back). So fix it to make sure we are consistent. -ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; SET sql_log_bin= 1; --source include/start_slave.inc -- cgit v1.2.1 From b19f055772b0735f7c1757691a42aa3974344b4c Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Thu, 11 May 2017 08:03:27 +0200 Subject: TokuDB .result file update following earlier commit. --- storage/tokudb/mysql-test/rpl/r/rpl_deadlock_tokudb.result | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/storage/tokudb/mysql-test/rpl/r/rpl_deadlock_tokudb.result b/storage/tokudb/mysql-test/rpl/r/rpl_deadlock_tokudb.result index 2348fd0d9d4..1698dc6df85 100644 --- a/storage/tokudb/mysql-test/rpl/r/rpl_deadlock_tokudb.result +++ b/storage/tokudb/mysql-test/rpl/r/rpl_deadlock_tokudb.result @@ -39,7 +39,9 @@ connection slave; BEGIN; SELECT * FROM t1 FOR UPDATE; a +connection slave1; START SLAVE; +connection slave; SELECT COUNT(*) FROM t2; COUNT(*) 0 @@ -61,8 +63,10 @@ BEGIN; SELECT * FROM t1 FOR UPDATE; a 1 +connection slave1; START SLAVE; include/wait_for_slave_sql_error.inc [errno=1205] +connection slave; SELECT COUNT(*) FROM t2; COUNT(*) 0 @@ -92,8 +96,10 @@ SELECT * FROM t1 FOR UPDATE; a 1 1 +connection slave1; START SLAVE; include/wait_for_slave_sql_error.inc [errno=1205] +connection slave; SELECT COUNT(*) FROM t2; COUNT(*) 0 -- cgit v1.2.1 From 95e09f0766f037530d2dcfbb6c530137a4ee0db4 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Thu, 11 May 2017 08:45:29 +0200 Subject: MDEV-12179: Per-engine mysql.gtid_slave_pos table Intermediate commit. Update multi_source.gtid_ignore_duplicates test to avoid a sporadic failure following MDEV-12179 functionality. The test case manually messes with the mysql.gtid_slave_pos table. Make sure to clean up that at the end of the test, and suppress the messages from the server about these changes. --- mysql-test/suite/multi_source/gtid_ignore_duplicates.result | 5 +++++ mysql-test/suite/multi_source/gtid_ignore_duplicates.test | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/mysql-test/suite/multi_source/gtid_ignore_duplicates.result b/mysql-test/suite/multi_source/gtid_ignore_duplicates.result index 92d096245c7..96627b42c97 100644 --- a/mysql-test/suite/multi_source/gtid_ignore_duplicates.result +++ b/mysql-test/suite/multi_source/gtid_ignore_duplicates.result @@ -65,6 +65,7 @@ include/wait_for_slave_to_start.inc set default_master_connection = ''; connection server_1; ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +CALL mtr.add_suppression("This change will not take full effect until all SQL threads have been restarted"); CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB; INSERT INTO t1 VALUES (1); BEGIN; @@ -491,17 +492,21 @@ SET GLOBAL slave_parallel_threads= @old_parallel; SET GLOBAL gtid_ignore_duplicates= @old_ignore_duplicates; connection server_1; DROP TABLE t1; +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; include/reset_master_slave.inc disconnect server_1; connection server_2; DROP TABLE t1; +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; include/reset_master_slave.inc disconnect server_2; connection server_3; DROP TABLE t1; +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; include/reset_master_slave.inc disconnect server_3; connection server_4; DROP TABLE t1; +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; include/reset_master_slave.inc disconnect server_4; diff --git a/mysql-test/suite/multi_source/gtid_ignore_duplicates.test b/mysql-test/suite/multi_source/gtid_ignore_duplicates.test index 218d91aa7fb..b61da0f0f33 100644 --- a/mysql-test/suite/multi_source/gtid_ignore_duplicates.test +++ b/mysql-test/suite/multi_source/gtid_ignore_duplicates.test @@ -86,6 +86,7 @@ set default_master_connection = ''; --connection server_1 ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +CALL mtr.add_suppression("This change will not take full effect until all SQL threads have been restarted"); CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB; INSERT INTO t1 VALUES (1); BEGIN; @@ -431,20 +432,24 @@ SET GLOBAL gtid_ignore_duplicates= @old_ignore_duplicates; --connection server_1 DROP TABLE t1; +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; --source include/reset_master_slave.inc --disconnect server_1 --connection server_2 DROP TABLE t1; +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; --source include/reset_master_slave.inc --disconnect server_2 --connection server_3 DROP TABLE t1; +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; --source include/reset_master_slave.inc --disconnect server_3 --connection server_4 DROP TABLE t1; +ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM; --source include/reset_master_slave.inc --disconnect server_4 -- cgit v1.2.1