MDEV-4605: Failing to load GTID slave position from rpl.gtid_slave_pos

There were several cases where the slave GTID position was not loaded
correctly before being used. This caused various failures such as
corrupting the position at slave start and empty values of
@@gtid_slave_pos and @@gtid_current_pos.

Fixed by adding more checks for loaded position, and by always loading
the position at server startup.

6 files changed, 115 insertions, 5 deletions

diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index 1102a527546..eb102696f64 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -931,7 +931,8 @@ static PSI_cond_info all_server_conds[]=
 
 PSI_thread_key key_thread_bootstrap, key_thread_delayed_insert,
   key_thread_handle_manager, key_thread_main,
-  key_thread_one_connection, key_thread_signal_hand;
+  key_thread_one_connection, key_thread_signal_hand,
+  key_thread_slave_init;
 
 static PSI_thread_info all_server_threads[]=
 {
@@ -956,7 +957,8 @@ static PSI_thread_info all_server_threads[]=
   { &key_thread_handle_manager, "manager", PSI_FLAG_GLOBAL},
   { &key_thread_main, "main", PSI_FLAG_GLOBAL},
   { &key_thread_one_connection, "one_connection", 0},
-  { &key_thread_signal_hand, "signal_handler", PSI_FLAG_GLOBAL}
+  { &key_thread_signal_hand, "signal_handler", PSI_FLAG_GLOBAL},
+  { &key_thread_slave_init, "slave_init", PSI_FLAG_GLOBAL}
 };
 
 PSI_file_key key_file_binlog, key_file_binlog_index, key_file_casetest,
diff --git a/sql/mysqld.h b/sql/mysqld.h
index 9c9bf71fec8..02d6b41cf69 100644
--- a/sql/mysqld.h
+++ b/sql/mysqld.h
@@ -283,7 +283,7 @@ extern PSI_cond_key key_TC_LOG_MMAP_COND_queue_busy;
 
 extern PSI_thread_key key_thread_bootstrap, key_thread_delayed_insert,
   key_thread_handle_manager, key_thread_kill_server, key_thread_main,
-  key_thread_one_connection, key_thread_signal_hand;
+  key_thread_one_connection, key_thread_signal_hand, key_thread_slave_init;
 
 extern PSI_file_key key_file_binlog, key_file_binlog_index, key_file_casetest,
   key_file_dbopt, key_file_des_key_file, key_file_ERRMSG, key_select_to_file,
diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc
index 03ec77e1433..9cb210f398f 100644
--- a/sql/rpl_rli.cc
+++ b/sql/rpl_rli.cc
@@ -1425,7 +1425,10 @@ rpl_load_gtid_slave_state(THD *thd)
 
   bitmap_set_all(table->read_set);
   if ((err= table->file->ha_rnd_init_with_error(1)))
+  {
+    table->file->print_error(err, MYF(0));
     goto end;
+  }
   table_scanned= true;
   for (;;)
   {
@@ -1440,7 +1443,10 @@ rpl_load_gtid_slave_state(THD *thd)
       else if (err == HA_ERR_END_OF_FILE)
         break;
       else
+      {
+        table->file->print_error(err, MYF(0));
         goto end;
+      }
     }
     domain_id= (ulonglong)table->field[0]->val_int();
     sub_id= (ulonglong)table->field[1]->val_int();
@@ -1465,6 +1471,7 @@ rpl_load_gtid_slave_state(THD *thd)
       if (!(entry= (struct local_element *)my_malloc(sizeof(*entry),
                                                      MYF(MY_WME))))
       {
+        my_error(ER_OUTOFMEMORY, MYF(0), (int)sizeof(*entry));
         err= 1;
         goto end;
       }
@@ -1475,12 +1482,18 @@ rpl_load_gtid_slave_state(THD *thd)
       if ((err= my_hash_insert(&hash, (uchar *)entry)))
       {
         my_free(entry);
+        my_error(ER_OUT_OF_RESOURCES, MYF(0));
         goto end;
       }
     }
   }
 
   rpl_global_gtid_slave_state.lock();
+  if (rpl_global_gtid_slave_state.loaded)
+  {
+    rpl_global_gtid_slave_state.unlock();
+    goto end;
+  }
   for (i= 0; i < hash.records; ++i)
   {
     entry= (struct local_element *)my_hash_element(&hash, i);
@@ -1490,14 +1503,15 @@ rpl_load_gtid_slave_state(THD *thd)
                                                  entry->gtid.seq_no)))
     {
       rpl_global_gtid_slave_state.unlock();
+      my_error(ER_OUT_OF_RESOURCES, MYF(0));
       goto end;
     }
     if (opt_bin_log &&
         mysql_bin_log.bump_seq_no_counter_if_needed(entry->gtid.domain_id,
                                                     entry->gtid.seq_no))
     {
-      my_error(ER_OUT_OF_RESOURCES, MYF(0));
       rpl_global_gtid_slave_state.unlock();
+      my_error(ER_OUT_OF_RESOURCES, MYF(0));
       goto end;
     }
   }
diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt
index f08ba26fa4d..a6a331eeb18 100644
--- a/sql/share/errmsg-utf8.txt
+++ b/sql/share/errmsg-utf8.txt
@@ -6538,7 +6538,7 @@ ER_GTID_OPEN_TABLE_FAILED
 ER_GTID_POSITION_NOT_FOUND_IN_BINLOG
 	eng "Connecting slave requested to start from GTID %u-%u-%llu, which is not in the master's binlog"
 ER_CANNOT_LOAD_SLAVE_GTID_STATE
-	eng "Failed to load replication slave GTID state from table %s.%s"
+	eng "Failed to load replication slave GTID position from table %s.%s"
 ER_MASTER_GTID_POS_CONFLICTS_WITH_BINLOG
 	eng "Specified GTID %u-%u-%llu conflicts with the binary log which contains a more recent GTID %u-%u-%llu. If MASTER_GTID_POS=CURRENT_POS is used, the binlog position will override the new value of @@gtid_slave_pos."
 ER_MASTER_GTID_POS_MISSING_DOMAIN
diff --git a/sql/slave.cc b/sql/slave.cc
index 10411f9a5a5..6b876c5e863 100644
--- a/sql/slave.cc
+++ b/sql/slave.cc
@@ -253,6 +253,66 @@ static void init_slave_psi_keys(void)
 }
 #endif /* HAVE_PSI_INTERFACE */
 
+
+static bool slave_init_thread_running;
+
+
+pthread_handler_t
+handle_slave_init(void *arg __attribute__((unused)))
+{
+  THD *thd;
+
+  my_thread_init();
+  thd= new THD;
+  thd->thread_stack= (char*) &thd;           /* Set approximate stack start */
+  mysql_mutex_lock(&LOCK_thread_count);
+  thd->thread_id= thread_id++;
+  mysql_mutex_unlock(&LOCK_thread_count);
+  thd->store_globals();
+
+  thd_proc_info(thd, "Loading slave GTID position from table");
+  if (rpl_load_gtid_slave_state(thd))
+    sql_print_warning("Failed to load slave replication state from table "
+                      "%s.%s: %u: %s", "mysql",
+                      rpl_gtid_slave_state_table_name.str,
+                      thd->stmt_da->sql_errno(), thd->stmt_da->message());
+
+  mysql_mutex_lock(&LOCK_thread_count);
+  delete thd;
+  mysql_mutex_unlock(&LOCK_thread_count);
+  my_thread_end();
+
+  mysql_mutex_lock(&LOCK_thread_count);
+  slave_init_thread_running= false;
+  mysql_cond_signal(&COND_thread_count);
+  mysql_mutex_unlock(&LOCK_thread_count);
+
+  return 0;
+}
+
+
+static int
+run_slave_init_thread()
+{
+  pthread_t th;
+
+  slave_init_thread_running= true;
+  if (mysql_thread_create(key_thread_slave_init, &th, NULL,
+                          handle_slave_init, NULL))
+  {
+    sql_print_error("Failed to create thread while initialising slave");
+    return 1;
+  }
+
+  mysql_mutex_lock(&LOCK_thread_count);
+  while (slave_init_thread_running)
+    mysql_cond_wait(&COND_thread_count, &LOCK_thread_count);
+  mysql_mutex_unlock(&LOCK_thread_count);
+
+  return 0;
+}
+
+
 /* Initialize slave structures */
 
 int init_slave()
@@ -264,6 +324,9 @@ int init_slave()
   init_slave_psi_keys();
 #endif
 
+  if (run_slave_init_thread())
+    return 1;
+
   /*
     This is called when mysqld starts. Before client connections are
     accepted. However bootstrap may conflict with us if it does START SLAVE.
@@ -3454,6 +3517,16 @@ pthread_handler_t handle_slave_io(void *arg)
   /* This must be called before run any binlog_relay_io hooks */
   my_pthread_setspecific_ptr(RPL_MASTER_INFO, mi);
 
+  /* Load the set of seen GTIDs, if we did not already. */
+  if (rpl_load_gtid_slave_state(thd))
+  {
+    mi->report(ERROR_LEVEL, thd->stmt_da->sql_errno(), 
+                "Unable to load replication GTID slave state from mysql.%s: %s",
+                rpl_gtid_slave_state_table_name.str, thd->stmt_da->message());
+    goto err;
+  }
+
+
   if (RUN_HOOK(binlog_relay_io, thread_start, (thd, mi)))
   {
     mi->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR,
diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc
index 75d11589be6..b25f979cf79 100644
--- a/sql/sys_vars.cc
+++ b/sql/sys_vars.cc
@@ -1262,6 +1262,12 @@ Sys_var_gtid_binlog_pos::global_value_ptr(THD *thd, LEX_STRING *base)
   String str(buf, sizeof(buf), system_charset_info);
   char *p;
 
+  if (!rpl_global_gtid_slave_state.loaded)
+  {
+    my_error(ER_CANNOT_LOAD_SLAVE_GTID_STATE, MYF(0), "mysql",
+             rpl_gtid_slave_state_table_name.str);
+    return NULL;
+  }
   str.length(0);
   if ((opt_bin_log && mysql_bin_log.append_state_pos(&str)) ||
       !(p= thd->strmake(str.ptr(), str.length())))
@@ -1308,6 +1314,14 @@ Sys_var_gtid_slave_pos::do_check(THD *thd, set_var *var)
   bool running;
 
   DBUG_ASSERT(var->type == OPT_GLOBAL);
+
+  if (!rpl_global_gtid_slave_state.loaded)
+  {
+    my_error(ER_CANNOT_LOAD_SLAVE_GTID_STATE, MYF(0), "mysql",
+             rpl_gtid_slave_state_table_name.str);
+    return true;
+  }
+
   mysql_mutex_lock(&LOCK_active_mi);
   running= master_info_index->give_error_if_slave_running();
   mysql_mutex_unlock(&LOCK_active_mi);
@@ -1366,6 +1380,13 @@ Sys_var_gtid_slave_pos::global_value_ptr(THD *thd, LEX_STRING *base)
   String str;
   char *p;
 
+  if (!rpl_global_gtid_slave_state.loaded)
+  {
+    my_error(ER_CANNOT_LOAD_SLAVE_GTID_STATE, MYF(0), "mysql",
+             rpl_gtid_slave_state_table_name.str);
+    return NULL;
+  }
+
   str.length(0);
   if (rpl_append_gtid_state(&str, false) ||
       !(p= thd->strmake(str.ptr(), str.length())))