Implementation of Multi-source replication (MDEV:253)

Documentation of the feature can be found at: http://kb.askmonty.org/en/multi-source-replication/
This code is based on code from Taobao, developed by Plinux

Other things:
- Added new commands: START ALL SLAVES, STOP ALL SLAVES and SHOW FULL SLAVE STATUS
- Almost all usage of 'active_mi' is deleted.
- Added parameter to reset_logs() so that one can specify if new logs should be created.
- Check wildcard match as early as possible for SHOW STATUS. This makes SHOW STATUS like 'xxx' a lot faster and use less mutex
- Made max_relay_log_size depending on master connection.
- Added sys_vars.default_master_connection_basic to fix a failure in sys_vars.all_vars, modified sql_slave_skip_counter_basic to allow session-level settings
- Added commands to mysqladmin: start-all-slaves & stop-all-slaves
- Removed logging of "next log '%s' is currently active | not active"
- Fixed bug in my_vsnprintf() when using positional parameters with length
- Added fn_ext2(), which returns pointer to last '.' in file name
- max_relay_log_size now acts as a normal slave specific variable
- Don't store replication position if innobase_overwrite_relay_log_info is not set
- max_relay_log_size copies it's values from max_binlog_size at startup



BUILD/SETUP.sh:
  Added -Wno-invalid-offsetof to get rid of warning of offsetof() on C++ class (safe in the contex we use it)
client/mysqladmin.cc:
  Added commands start-all-slaves & stop-all-slaves
client/mysqltest.cc:
  Added support for error names starting with 'W'
  Added connection_name support to --sync_with_master
cmake/maintainer.cmake:
  Added -Wno-invalid-offsetof to get rid of warning of offsetof() on C++ class (safe in the contex we use it)
include/my_sys.h:
  Added fn_ext2(), which returns pointer to last '.' in file name
include/mysql/plugin.h:
  Added SHOW_SIMPLE_FUNC
include/mysql/plugin_audit.h.pp:
  Updated signature file
include/mysql/plugin_auth.h.pp:
  Updated signature file
include/mysql/plugin_ftparser.h.pp:
  Updated signature file
mysql-test/extra/rpl_tests/rpl_max_relay_size.test:
  Updated test
mysql-test/include/setup_fake_relay_log.inc:
  There is no orphan relay log files anymore
mysql-test/mysql-test-run.pl:
  Added multi_source to test suite
mysql-test/r/flush.result:
  Added test for new syntax of flush relay logs (can't repeat relay logs or slave)
mysql-test/r/mysqld--help.result:
  Updated result
mysql-test/r/mysqltest.result:
  Updated result
mysql-test/r/parser.result:
  Updated result
mysql-test/r/signal_code.result:
  Updated result after introducing new commands
mysql-test/r/sp-code.result:
  Updated result after introducing new commands
mysql-test/suite/multi_source:
  Tests for multi-source
mysql-test/suite/multi_source/info_logs-master.opt:
  Test with strange file names
mysql-test/suite/multi_source/info_logs.result:
  Test of logs
mysql-test/suite/multi_source/info_logs.test:
  Test of logs
mysql-test/suite/multi_source/my.cnf:
  Setup of multi-master tests
  Added log-warnings to get more information to the log files
mysql-test/suite/multi_source/relaylog_events.result:
  Test relay log handling
mysql-test/suite/multi_source/relaylog_events.test:
  Test relay log handling
mysql-test/suite/multi_source/reset_slave.result:
  Test RESET SLAVE
mysql-test/suite/multi_source/reset_slave.test:
  Test RESET SLAVE
mysql-test/suite/multi_source/simple.result:
  Simple basic test of multi-source functionality
mysql-test/suite/multi_source/simple.test:
  Simple basic test of multi-source functionality
mysql-test/suite/multi_source/skip_counter.result:
  Testing skip_counter and max_relay_log_size
mysql-test/suite/multi_source/skip_counter.test:
  Testing skip_counter and max_relay_log_size
mysql-test/suite/multi_source/syntax.result:
  Test of multi-source syntax
mysql-test/suite/multi_source/syntax.test:
  Test of multi-source syntax
mysql-test/suite/rpl/r/rpl_deadlock_innodb.result:
  New warnings
mysql-test/suite/rpl/r/rpl_filter_dbs_dynamic.result:
  Improved error texts
mysql-test/suite/rpl/r/rpl_filter_tables_dynamic.result:
  Improved error texts
mysql-test/suite/rpl/r/rpl_filter_wild_tables_dynamic.result:
  Improved error texts
mysql-test/suite/rpl/r/rpl_flush_logs.result:
  Update results
mysql-test/suite/rpl/r/rpl_heartbeat.result:
  Warning was removed
mysql-test/suite/rpl/r/rpl_heartbeat_basic.result:
  Warning was removed
mysql-test/suite/rpl/r/rpl_rotate_logs.result:
  Updated results because of new error messages
mysql-test/suite/rpl/r/rpl_row_max_relay_size.result:
  Updated results
mysql-test/suite/rpl/r/rpl_row_show_relaylog_events.result:
  Updated results after improved RESET SLAVE (we now use less relay log files)
mysql-test/suite/rpl/r/rpl_skip_replication.result:
  New error message
mysql-test/suite/rpl/r/rpl_start_slave_deadlock_sys_vars.result:
  Test removed as the old DBUG_SYNC entries doesn't exist anymore
mysql-test/suite/rpl/r/rpl_stm_max_relay_size.result:
  Updated results
mysql-test/suite/rpl/r/rpl_stm_mix_show_relaylog_events.result:
  Updated results after improved RESET SLAVE (we now use less relay log files)
mysql-test/suite/rpl/t/rpl_flush_logs.test:
  Updated tests as relay log files is created a bit differently than before
mysql-test/suite/rpl/t/rpl_start_slave_deadlock_sys_vars.test:
  Test removed as the old DBUG_SYNC entries doesn't exist anymore
mysql-test/suite/sys_vars/r/default_master_connection_basic.result:
  New test to test usage of default_master_connection
mysql-test/suite/sys_vars/r/max_relay_log_size_basic.result:
  Updated results
mysql-test/suite/sys_vars/r/sql_slave_skip_counter_basic.result:
  Updated usage of sql_slave_skip_counter
mysql-test/suite/sys_vars/t/default_master_connection_basic.test:
  New test to test usage of default_master_connection
mysql-test/suite/sys_vars/t/max_relay_log_size_basic.test:
  Updated results
mysql-test/suite/sys_vars/t/sql_slave_skip_counter_basic.test:
  Updated test for multi-source
mysql-test/t/flush.test:
  Added test for new syntax of flush relay logs (can't repeat relay logs or slave)
mysql-test/t/parser.test:
  Updated test as master_pos_wait() now takes more arguments than before
mysys/mf_fn_ext.c:
  Added fn_ext2(), which returns pointer to last '.' in file name
plugin/semisync/semisync_master_plugin.cc:
  Use SHOW_SIMPLE_FUNC to optimize SHOW STATUS
sql/event_scheduler.cc:
  No reason to initialize slave_thread (it's guaranteed to be zero here)
sql/item_create.cc:
  Added connection_name argument to master_pos_wait()
  Simplified code
sql/item_func.cc:
  Added connection_name argument to master_pos_wait()
sql/item_func.h:
  Added connection_name argument to master_pos_wait()
sql/lex.h:
  Added SLAVES keyword
sql/log.cc:
  Added tag "Master 'connection_name'" to slave errors that has a connection name.
  Added parameter to reset_logs() so that one can specify if new logs should be created.
  Removed some wrong casts
  Changed some constants to defines
sql/log.h:
  Added parameter to reset_logs()
  Updated comment to reflect new code
sql/log_event.cc:
  Updated DBUG_PRINT
sql/mysqld.cc:
  Added variable mysqld_server_initialized so that other functions can test if server is fully initialized.
  Free all slave data in one place (fewer ifdef's)
  Removed not needed call to close_active_mi()
  Initialize slaves() later in startup to ensure that everthing is really initialized when slaves start.
  Made status variable slave_running multi-source safe
  max_relay_log_size copies it's values from max_binlog_size at startup
  SHOW_FUNC -> SHOW_SIMPLE_FUNC
sql/mysqld.h:
  Added mysqld_server_initialized
  Removed max_relay_log_size
sql/rpl_mi.cc:
  Store connection name and cmp_connection_name (only used for show full slave status) in Master_info
  Added code for Master_info_index, which handles storage of multi-master information
  Don't write the empty "" connection_name to multi-master.info file. This is handled by the original code.
  Create Master_info_index::index_file_names once at init
  More DBUG_PRINT
  Give error if Master_info_index::check_duplicate_master_info fails
  Added start|stop all slaves
sql/rpl_mi.h:
  Added connection_name and Master_info_index
  Updated prototypes
sql/rpl_rli.cc:
  Added connection_name to relay log files.
  If we do a full reset, don't create any new relay log files.
  Updated comment to reflect new code
  Made max_relay_log_size depending on master connection.
sql/rpl_rli.h:
  Fixed type of slave_skip_counter as we now access it directly in sys_vars.cc, so it must be ulong
  Made executed_entries and max_relay_log_size depending on master connection.
sql/set_var.cc:
  Fixed that one can get variable name also when one uses DEFAULT
sql/set_var.h:
  Made option global so that one can check and change min & max values (sorry Sergei)
sql/share/errmsg-utf8.txt:
  Added new error messages needed for multi-source
  Added multi-source name to error ER_MASTER_INFO and WARN_NO_MASTER_INFO
  Improved error message if connection exists
sql/slave.cc:
  Moved things a bit around to make it easier to handle error conditions.
  Create a global master_info_index and add the "" connection to it
  Ensure that new Master_info doesn't fail.
  Don't call terminate_slave_threads(active_mi..) on end_slave() as this is now done automaticly when deleting master_info_index.
  Delete not needed function close_active_mi(). One can achive same thing by calling end_slave().
  Added support for SHOW FULL SLAVE STATUS (show status for all master connections with connection_name as first column)
  Removed logging of "next log '%s' is currently active | not active"
  Added Slave_received_heartbeats and Slave_heartbeat_period
sql/slave.h:
  Added new defines and prototypes
sql/sql_base.cc:
  More DBUG_PRINT
sql/sql_class.cc:
  Reset thd->connection_name and thd-->default_master_connection
sql/sql_class.h:
  Added thd->connection_name and thd-->default_master_connection
  Made slave_skip_count and max_relay_log_size depending on master connection. These variables are in THD to make changing them thread safe.
sql/sql_const.h:
  Added MAX_CONNECTION_NAME
sql/sql_insert.cc:
  thd->slave_thread -> thd->rli_slave
  Removed usage of active_mi
sql/sql_lex.cc:
  Reset 'lex->verbose' (to simplify some sql_yacc.yy code)
sql/sql_lex.h:
  Added connection_name, relay_log_connection_name, SQLCOM_SLAVE_ALL_START and SQLCOM_SLAVE_ALL_STOP
sql/sql_load.cc:
  thd->slave_thread -> thd->rli_slave
  Removed usage of active_mi
sql/sql_parse.cc:
  Added support for connection_name to all SLAVE commands.
  - Instead of using active_mi, we now get the current Master_info from master_info_index.
  - Create new replication threads with CHANGE MASTER
  - Added support for show_all_master_info()-
sql/sql_prepare.cc:
  Added SQLCOM_SLAVE_ALL_START and SQLCOM_SLAVE_ALL_STOP
sql/sql_reload.cc:
  Made reset/full slave use master_info_index->get_master_info() instead of active_mi.
  If one uses 'RESET SLAVE "connection_name" all' the connection is removed from master_info_index.
  Fixed issues with FLUSH RELAY LOGS
sql/sql_repl.cc:
  sql_slave_skip_counter is moved to thd->variables to make it thread safe and fix some bugs with it
  Add connection name to relay log files.
  Added connection name to errors.
  Added some logging for multi-master if log_warnings > 1
  stop_slave():
  - Don't check if thd is set. It's guaranteed to always be set.
  change_master():
  - Check for duplicate connection names in change_master()
  - Check for wrong arguments first in file (to simplify error handling)
  - Register new connections in master_info_index
  
  
  
  
  
  
  ******
  Added multi-source support to show relaylog events
  ******
  check_duplicate_master_info() now generates an error
  Added parameter to reset_logs()
  ******
  Updated calls to create_signed_file_name()
sql/sql_show.cc:
  Check wildcard match as early as possible for SHOW STATUS. This makes SHOW STATUS like 'xxx' a lot faster and use less mutex
sql/sql_yacc.yy:
  Added optional connection_name to a all relevant master/slave commands
  Added multi-source support to show relaylog events
  Added new commands START ALL SLAVES, STOP ALL SLAVES and SHOW FULL SLAVE STATUS
sql/strfunc.cc:
  my_global.h shoud always be included first.
sql/sys_vars.cc:
  Added variable default_master_connection
  Made variable sql_slave_skip_counter multi-source safe
  Made max_relay_log_size depending on master connection.
  Made old code more reusable
sql/sys_vars.h:
  Added Sys_var_session_lexstring (needed for default_master_connection)
  Added Sys_var_multi_source_uint (needed for sql_slave_skip_counter).
  Changed Sys_var_multi_source_uint to ulong to be able to handle max_relay_log_size
  Made old code more reusable
storage/example/ha_example.cc:
  Use SHOW_SIMPLE_FUNC to optimize SHOW STATUS
storage/sphinx/ha_sphinx.cc:
  Use SHOW_SIMPLE_FUNC to optimize SHOW STATUS
storage/xtradb/handler/ha_innodb.cc:
  Don't store replication position if innobase_overwrite_relay_log_info is not set
strings/my_vsnprintf.c:
  Fixed bug when using positional parameters with length

1 files changed, 167 insertions, 7 deletions

diff --git a/sql/sys_vars.h b/sql/sys_vars.h
index 21bebcd762c..b438295c1b0 100644
--- a/sql/sys_vars.h
+++ b/sql/sys_vars.h
@@ -566,11 +566,13 @@ public:
     option.var_type= GET_STR;
   }
 
+  bool do_check(THD *thd, set_var *var)
+  {
+    return Sys_var_charptr::do_string_check(thd, var, charset(thd));
+  }
   bool check_update_type(Item_result type)
   { return type != STRING_RESULT; }
 
-  bool do_check(THD *thd, set_var *var);
-
   void session_save_default(THD *thd, set_var *var)
   { DBUG_ASSERT(FALSE); }
 
@@ -588,8 +590,6 @@ public:
 protected:
   uchar *global_value_ptr(THD *thd, LEX_STRING *base);
   bool set_filter_value(const char *value);
-  void lock(void);
-  void unlock(void);
 };
 
 /**
@@ -633,6 +633,93 @@ public:
   }
 };
 
+
+/*
+  A LEX_STRING stored only in thd->variables
+  Only to be used for small buffers
+*/
+
+class Sys_var_session_lexstring: public sys_var
+{
+  size_t max_length;
+public:
+  Sys_var_session_lexstring(const char *name_arg,
+                            const char *comment, int flag_args,
+                            ptrdiff_t off, size_t size, CMD_LINE getopt,
+                            enum charset_enum is_os_charset_arg,
+                            const char *def_val, size_t max_length_arg,
+                            on_check_function on_check_func=0,
+                            on_update_function on_update_func=0)
+    : sys_var(&all_sys_vars, name_arg, comment, flag_args, off, getopt.id,
+              getopt.arg_type, SHOW_CHAR, (intptr)def_val,
+              0, VARIABLE_NOT_IN_BINLOG, on_check_func, on_update_func,
+              0, 0),max_length(max_length_arg)
+  {
+    option.var_type= GET_NO_ARG;
+    SYSVAR_ASSERT(scope() == ONLY_SESSION)
+    *const_cast<SHOW_TYPE*>(&show_val_type)= SHOW_LEX_STRING;
+  }
+  bool do_check(THD *thd, set_var *var)
+  {
+    char buff[STRING_BUFFER_USUAL_SIZE];
+    String str(buff, sizeof(buff), system_charset_info), *res;
+
+    if (!(res=var->value->val_str(&str)))
+    {
+      var->save_result.string_value.str= 0;     /* NULL */
+      var->save_result.string_value.length= 0;
+    }
+    else
+    {
+      if (res->length() > max_length)
+      {
+        my_error(ER_WRONG_STRING_LENGTH, MYF(0),
+                 res->ptr(), name.str, (int) max_length);
+        return true;
+      }
+      var->save_result.string_value.str= thd->strmake(res->ptr(),
+                                                      res->length());
+      var->save_result.string_value.length= res->length();
+    }
+    return false;
+  }
+  bool session_update(THD *thd, set_var *var)
+  {
+    LEX_STRING *tmp= &session_var(thd, LEX_STRING);
+    tmp->length= var->save_result.string_value.length;
+    /* Store as \0 terminated string (just to be safe) */
+    strmake(tmp->str, var->save_result.string_value.str, tmp->length);
+    return false;
+  }
+  bool global_update(THD *thd, set_var *var)
+  {
+    DBUG_ASSERT(FALSE);
+    return false;
+  }
+  void session_save_default(THD *thd, set_var *var)
+  {
+    char *ptr= (char*)(intptr)option.def_value;
+    var->save_result.string_value.str= ptr;
+    var->save_result.string_value.length= strlen(ptr);
+  }
+  void global_save_default(THD *thd, set_var *var)
+  {
+    DBUG_ASSERT(FALSE);
+  }
+  uchar *session_value_ptr(THD *thd, LEX_STRING *base)
+  {
+    return (uchar*) &session_var(thd, LEX_STRING);
+  }
+  uchar *global_value_ptr(THD *thd, LEX_STRING *base)
+  {
+    DBUG_ASSERT(FALSE);
+    return NULL;
+  }
+  bool check_update_type(Item_result type)
+  { return type != STRING_RESULT; }
+};
+
+
 #ifndef DBUG_OFF
 /**
   @@session.dbug and @@global.dbug variables.
@@ -1389,6 +1476,7 @@ public:
 };
 #endif /* defined(ENABLED_DEBUG_SYNC) */
 
+
 /**
   The class for bit variables - a variant of boolean that stores the value
   in a bit.
@@ -1854,14 +1942,86 @@ public:
           const char *comment, int flag_args, ptrdiff_t off, size_t size,
           CMD_LINE getopt,
           const char *values[], uint def_val, PolyLock *lock,
-          enum binlog_status_enum binlog_status_arg,
-          on_check_function on_check_func)
+          enum binlog_status_enum binlog_status_arg)
     :Sys_var_enum(name_arg, comment, flag_args, off, size, getopt,
-                  values, def_val, lock, binlog_status_arg, on_check_func)
+                  values, def_val, lock, binlog_status_arg)
   {}
   bool global_update(THD *thd, set_var *var);
 };
 
+/*
+  Class for handing multi-source replication variables
+  Variable values are store in Master_info, but to make it possible to
+  access variable without locks we also store it thd->variables.
+  These can be used as GLOBAL or SESSION, but both points to the same
+  variable.  This is to make things compatible with MySQL 5.5 where variables
+  like sql_slave_skip_counter are GLOBAL.
+*/
+
+class Sys_var_multi_source_ulong;
+class Master_info;
+
+typedef bool (*on_multi_source_update_function)(sys_var *self, THD *thd,
+                                                Master_info *mi);
+bool update_multi_source_variable(sys_var *self,
+                                  THD *thd, enum_var_type type);
+
+
+class Sys_var_multi_source_ulong :public Sys_var_ulong
+{ 
+  ptrdiff_t master_info_offset;
+  on_multi_source_update_function update_multi_source_variable_func;
+public:
+  Sys_var_multi_source_ulong(const char *name_arg,
+                             const char *comment, int flag_args,
+                             ptrdiff_t off, size_t size,
+                             CMD_LINE getopt,
+                             ptrdiff_t master_info_offset_arg,
+                             uint min_val, uint max_val, uint def_val,
+                             uint block_size,
+                             on_multi_source_update_function on_update_func)
+    :Sys_var_ulong(name_arg, comment, flag_args, off, size,
+                   getopt, min_val, max_val, def_val, block_size,
+                   0, VARIABLE_NOT_IN_BINLOG, 0, update_multi_source_variable),
+    master_info_offset(master_info_offset_arg),
+    update_multi_source_variable_func(on_update_func)
+  {
+  }
+  bool session_update(THD *thd, set_var *var)
+  {
+    session_var(thd, uint)= (uint) (var->save_result.ulonglong_value);
+    /* Value should be moved to multi_master in on_update_func */
+    return false;
+  }
+  bool global_update(THD *thd, set_var *var)
+  {
+    return session_update(thd, var);
+  }
+  void session_save_default(THD *thd, set_var *var)
+  {
+    /* Use value given in variable declaration */
+    global_save_default(thd, var);
+  }
+  uchar *session_value_ptr(THD *thd,LEX_STRING *base)
+  {
+    uint *tmp, res;
+    tmp= (uint*) (((uchar*)&(thd->variables)) + offset);
+    res= get_master_info_uint_value(thd, master_info_offset);
+    *tmp= res;
+    return (uchar*) tmp;
+  }
+  uchar *global_value_ptr(THD *thd, LEX_STRING *base)
+  {
+    return session_value_ptr(thd, base);
+  }
+  uint get_master_info_uint_value(THD *thd, ptrdiff_t offset);
+  bool update_variable(THD *thd, Master_info *mi)
+  {
+    return update_multi_source_variable_func(this, thd, mi);
+  }
+};
+
+
 /****************************************************************************
   Used templates
 ****************************************************************************/