BUG#46364 MyISAM transbuffer problems (NTM problem)

It is well-known that due to concurrency issues, a slave can become
inconsistent when a transaction contains updates to both transaction and
non-transactional tables in statement and mixed modes.

In a nutshell, the current code-base tries to preserve causality among the
statements by writing non-transactional statements to the txn-cache which
is flushed upon commit. However, modifications done to non-transactional
tables on behalf of a transaction become immediately visible to other
connections but may not immediately get into the binary log and therefore
consistency may be broken.

In general, it is impossible to automatically detect causality/dependency
among statements by just analyzing the statements sent to the server. This
happen because dependency may be hidden in the application code and it is
necessary to know a priori all the statements processed in the context of
a transaction such as in a procedure. Moreover, even for the few cases that
we could automatically address in the server, the computation effort
required could make the approach infeasible.

So, in this patch we introduce the option
    - "--binlog-direct-non-transactional-updates" that can be used to bypass
    the current behavior in order to write directly to binary log statements
    that change non-transactional tables.

mysql-test/extra/rpl_tests/rpl_mixing_engines.inc:
  Backported this from Celosia to improve the test cases related to the NTM issue.
sql/log.cc:
  Checks the --binlog-direct-non-transactional-updates before choosing
  to either use the trxn-cache or not.
sql/mysqld.cc:
  Introduces the option --binlog-direct-non-transactional-updates.
sql/set_var.cc:
  Introduces the option --binlog-direct-non-transactional-updates.
sql/sql_class.h:
  Introduces the option --binlog-direct-non-transactional-updates.

4 files changed, 25 insertions, 8 deletions

diff --git a/sql/log.cc b/sql/log.cc
index 36d57271a7d..bc9efd75ea2 100644
--- a/sql/log.cc
+++ b/sql/log.cc
@@ -4284,12 +4284,20 @@ bool MYSQL_BIN_LOG::write(Log_event *event_info)
 #if defined(USING_TRANSACTIONS) 
     /*
       Should we write to the binlog cache or to the binlog on disk?
+
       Write to the binlog cache if:
-      - it is already not empty (meaning we're in a transaction; note that the
-     present event could be about a non-transactional table, but still we need
-     to write to the binlog cache in that case to handle updates to mixed
-     trans/non-trans table types the best possible in binlogging)
-      - or if the event asks for it (cache_stmt == TRUE).
+      1 - a transactional engine/table is updated (stmt_has_updated_trans_table == TRUE);
+      2 - or the event asks for it (cache_stmt == TRUE);
+      3 - or the cache is already not empty (meaning we're in a transaction;
+      note that the present event could be about a non-transactional table, but
+      still we need to write to the binlog cache in that case to handle updates
+      to mixed trans/non-trans table types).
+      
+      Write to the binlog on disk if only a non-transactional engine is
+      updated and:
+      1 - the binlog cache is empty or;
+      2 - --binlog-direct-non-transactional-updates is set and we are about to
+      use the statement format. When using the row format (cache_stmt == TRUE).
     */
     if (opt_using_transactions && thd)
     {
@@ -4300,8 +4308,9 @@ bool MYSQL_BIN_LOG::write(Log_event *event_info)
         (binlog_trx_data*) thd_get_ha_data(thd, binlog_hton);
       IO_CACHE *trans_log= &trx_data->trans_log;
       my_off_t trans_log_pos= my_b_tell(trans_log);
-      if (event_info->get_cache_stmt() || trans_log_pos != 0 ||
-          stmt_has_updated_trans_table(thd))
+      if (event_info->get_cache_stmt() || stmt_has_updated_trans_table(thd) ||
+          (!thd->variables.binlog_direct_non_trans_update &&
+            trans_log_pos != 0))
       {
         DBUG_PRINT("info", ("Using trans_log: cache: %d, trans_log_pos: %lu",
                             event_info->get_cache_stmt(),
diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index 7bb357827c8..242acd28ecc 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -5727,7 +5727,8 @@ enum options_mysqld
   OPT_SLAVE_EXEC_MODE,
   OPT_GENERAL_LOG_FILE,
   OPT_SLOW_QUERY_LOG_FILE,
-  OPT_IGNORE_BUILTIN_INNODB
+  OPT_IGNORE_BUILTIN_INNODB,
+  OPT_BINLOG_DIRECT_NON_TRANS_UPDATE
 };
 
 
@@ -7071,6 +7072,10 @@ The minimum value for this variable is 4096.",
    (uchar**) &max_system_variables.net_wait_timeout, 0, GET_ULONG,
    REQUIRED_ARG, NET_WAIT_TIMEOUT, 1, IF_WIN(INT_MAX32/1000, LONG_TIMEOUT),
    0, 1, 0},
+  {"binlog-direct-non-transactional-updates", OPT_BINLOG_DIRECT_NON_TRANS_UPDATE,
+   "Causes updates to non-transactional engines using statement format to be written directly to binary log. Before using this option make sure that there are no dependencies between transactional and non-transactional tables such as in the statement INSERT INTO t_myisam SELECT * FROM t_innodb; otherwise, slaves may diverge from the master.",
+   (uchar**) &global_system_variables.binlog_direct_non_trans_update, (uchar**) &max_system_variables.binlog_direct_non_trans_update, 0, GET_BOOL, NO_ARG, 0,
+    0, 0, 0, 0, 0},
   {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
 };
 
diff --git a/sql/set_var.cc b/sql/set_var.cc
index 3c6b259045d..e06ad9dcc59 100644
--- a/sql/set_var.cc
+++ b/sql/set_var.cc
@@ -181,6 +181,8 @@ static sys_var_long_ptr	sys_binlog_cache_size(&vars, "binlog_cache_size",
 					      &binlog_cache_size);
 static sys_var_thd_binlog_format sys_binlog_format(&vars, "binlog_format",
                                             &SV::binlog_format);
+static sys_var_thd_bool sys_binlog_direct_non_trans_update(&vars, "binlog_direct_non_transactional_updates",
+                                                           &SV::binlog_direct_non_trans_update);
 static sys_var_thd_ulong	sys_bulk_insert_buff_size(&vars, "bulk_insert_buffer_size",
 						  &SV::bulk_insert_buff_size);
 static sys_var_const_os         sys_character_sets_dir(&vars,
diff --git a/sql/sql_class.h b/sql/sql_class.h
index 8acc03f929d..56d31be79c7 100644
--- a/sql/sql_class.h
+++ b/sql/sql_class.h
@@ -353,6 +353,7 @@ struct system_variables
   ulong ndb_index_stat_cache_entries;
   ulong ndb_index_stat_update_freq;
   ulong binlog_format; // binlog format for this thd (see enum_binlog_format)
+  my_bool binlog_direct_non_trans_update;
   /*
     In slave thread we need to know in behalf of which
     thread the query is being run to replicate temp tables properly