MDEV-6082 Assertion `0' fails in TC_LOG_DUMMY::log_and_order on DML after installing TokuDB at runtime on server with disabled InnoDB

We don't support changing tc_log implementation at run time.
If the first XA-capable engine is loaded with INSTALL PLUGIN - disable its
XA capabilities with a warning

3 files changed, 23 insertions, 5 deletions

diff --git a/sql/handler.cc b/sql/handler.cc
index 2e20944a96b..efecd478019 100644
--- a/sql/handler.cc
+++ b/sql/handler.cc
@@ -543,7 +543,19 @@ int ha_initialize_handlerton(st_plugin_int *plugin)
       savepoint_alloc_size+= tmp;
       hton2plugin[hton->slot]=plugin;
       if (hton->prepare)
+      {
         total_ha_2pc++;
+        if (tc_log && tc_log != get_tc_log_implementation())
+        {
+          total_ha_2pc--;
+          hton->prepare= 0;
+          push_warning_printf(current_thd, MYSQL_ERROR::WARN_LEVEL_WARN,
+                              ER_UNKNOWN_ERROR,
+                              "Cannot enable tc-log at run-time. "
+                              "XA features of %s are disabled",
+                              plugin->name.str);
+        }
+      }
       break;
     }
     /* fall through */
diff --git a/sql/log.h b/sql/log.h
index e388df61b38..4eb23d4878f 100644
--- a/sql/log.h
+++ b/sql/log.h
@@ -954,6 +954,13 @@ end:
   DBUG_RETURN(error);
 }
 
-
+static inline TC_LOG *get_tc_log_implementation()
+{
+  if (total_ha_2pc <= 1)
+    return &tc_log_dummy;
+  if (opt_bin_log)
+    return &mysql_bin_log;
+  return &tc_log_mmap;
+}
 
 #endif /* LOG_H */
diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index 193c1af0f99..0559afa7616 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -4327,6 +4327,8 @@ a file name for --log-bin-index option", opt_binlog_index_name);
   if (ha_init_errors())
     DBUG_RETURN(1);
 
+  tc_log= 0; // ha_initialize_handlerton() needs that
+
   if (plugin_init(&remaining_argc, remaining_argv,
                   (opt_noacl ? PLUGIN_INIT_SKIP_PLUGIN_TABLE : 0) |
                   (opt_abort ? PLUGIN_INIT_SKIP_INITIALIZATION : 0)))
@@ -4469,10 +4471,7 @@ a file name for --log-bin-index option", opt_binlog_index_name);
   internal_tmp_table_max_key_segments= myisam_max_key_segments();
 #endif
 
-  tc_log= (total_ha_2pc > 1 ? (opt_bin_log  ?
-                               (TC_LOG *) &mysql_bin_log :
-                               (TC_LOG *) &tc_log_mmap) :
-           (TC_LOG *) &tc_log_dummy);
+  tc_log= get_tc_log_implementation();
 
   if (tc_log->open(opt_bin_log ? opt_bin_logname : opt_tc_log_file))
   {