MDEV-6391: GTID binlog state not recovered if mariadb-bin.state is removed

When the server starts up, check if the master-bin.state file was lost.
If it was, recover its contents by scanning the last binlog file, thus
avoiding running with a corrupt binlog state.

3 files changed, 174 insertions, 8 deletions

diff --git a/mysql-test/suite/rpl/r/rpl_gtid_crash.result b/mysql-test/suite/rpl/r/rpl_gtid_crash.result
index 75bd9d0cbb1..0c2249f2dce 100644
--- a/mysql-test/suite/rpl/r/rpl_gtid_crash.result
+++ b/mysql-test/suite/rpl/r/rpl_gtid_crash.result
@@ -267,5 +267,55 @@ a
 24
 26
 27
+*** MDEV-6391: GTID binlog state not recovered if mariadb-bin.state is removed ***
+include/stop_slave.inc
+INSERT INTO t1 VALUES (30);
+SET @old_server_id= @@server_id;
+SET @old_domain_id= @@gtid_domain_id;
+SET SESSION server_id= 10;
+INSERT INTO t1 VALUES (31);
+INSERT INTO t1 VALUES (32);
+SET SESSION gtid_domain_id= 1;
+SET SESSION server_id=11;
+INSERT INTO t1 VALUES (33);
+SET SESSION gtid_domain_id= 2;
+INSERT INTO t1 VALUES (34);
+SET SESSION server_id= 10;
+INSERT INTO t1 VALUES (35);
+INSERT INTO t1 VALUES (36);
+SET SESSION gtid_domain_id= 0;
+SET SESSION server_id= 12;
+INSERT INTO t1 VALUES (37);
+SET SESSION gtid_domain_id= @old_domain_id;
+SET SESSION server_id= @old_server_id;
+INSERT INTO t1 VALUES (38);
+INSERT INTO t1 VALUES (39);
+SELECT * FROM t1 WHERE a >= 30 ORDER BY a;
+a
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+include/save_master_gtid.inc
+include/start_slave.inc
+include/sync_with_master_gtid.inc
+SELECT * FROM t1 WHERE a >= 30 ORDER BY a;
+a
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
 DROP TABLE t1;
 include/rpl_end.inc
diff --git a/mysql-test/suite/rpl/t/rpl_gtid_crash.test b/mysql-test/suite/rpl/t/rpl_gtid_crash.test
index 90b4e454a74..df3ba9a3420 100644
--- a/mysql-test/suite/rpl/t/rpl_gtid_crash.test
+++ b/mysql-test/suite/rpl/t/rpl_gtid_crash.test
@@ -587,6 +587,77 @@ eval SELECT IF(INSTR(@@gtid_current_pos, '$saved_gtid'), "Current pos ok", CONCA
 SELECT * from t1 WHERE a > 10 ORDER BY a;
 
 
+--echo *** MDEV-6391: GTID binlog state not recovered if mariadb-bin.state is removed ***
+
+--connection server_2
+--source include/stop_slave.inc
+
+# Do some misc. transactions, stop the master, drop the master-bin.state file.
+# Start the master back up, check that binlog state is correct.
+
+--connection server_1
+
+INSERT INTO t1 VALUES (30);
+SET @old_server_id= @@server_id;
+SET @old_domain_id= @@gtid_domain_id;
+
+SET SESSION server_id= 10;
+INSERT INTO t1 VALUES (31);
+INSERT INTO t1 VALUES (32);
+SET SESSION gtid_domain_id= 1;
+SET SESSION server_id=11;
+INSERT INTO t1 VALUES (33);
+SET SESSION gtid_domain_id= 2;
+INSERT INTO t1 VALUES (34);
+SET SESSION server_id= 10;
+INSERT INTO t1 VALUES (35);
+INSERT INTO t1 VALUES (36);
+SET SESSION gtid_domain_id= 0;
+SET SESSION server_id= 12;
+INSERT INTO t1 VALUES (37);
+SET SESSION gtid_domain_id= @old_domain_id;
+SET SESSION server_id= @old_server_id;
+INSERT INTO t1 VALUES (38);
+INSERT INTO t1 VALUES (39);
+SELECT * FROM t1 WHERE a >= 30 ORDER BY a;
+--source include/save_master_gtid.inc
+
+--let OLD_STATE= `SELECT @@gtid_binlog_state`
+
+--let $datadir= `SELECT @@datadir`
+
+--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+wait
+EOF
+shutdown_server 10;
+--source include/wait_until_disconnected.inc
+
+--remove_file $datadir/master-bin.state
+
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+restart
+EOF
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+--let NEW_STATE= `SELECT @@gtid_binlog_state`
+
+--perl
+my $old= $ENV{'OLD_STATE'};
+my $new= $ENV{'NEW_STATE'};
+# Make them order-independent, for easy comparison.
+$old= join(",", sort(split(",", $old)));
+$new= join(",", sort(split(",", $new)));
+die "ERROR: new binlog state '$new' differs from old '$old'\n"
+  unless $old eq $new;
+EOF
+
+--connection server_2
+--source include/start_slave.inc
+--source include/sync_with_master_gtid.inc
+SELECT * FROM t1 WHERE a >= 30 ORDER BY a;
+
+
 --connection server_1
 DROP TABLE t1;
 
diff --git a/sql/log.cc b/sql/log.cc
index 38fe1066896..a2b072b1a35 100644
--- a/sql/log.cc
+++ b/sql/log.cc
@@ -5653,6 +5653,14 @@ end:
 }
 
 
+/*
+  Initialize the binlog state from the master-bin.state file, at server startup.
+
+  Returns:
+    0 for success.
+    2 for when .state file did not exist.
+    1 for other error.
+*/
 int
 MYSQL_BIN_LOG::read_state_from_file()
 {
@@ -5680,7 +5688,7 @@ MYSQL_BIN_LOG::read_state_from_file()
         with GTID enabled. So initialize to empty state.
       */
       rpl_global_gtid_binlog_state.reset();
-      err= 0;
+      err= 2;
       goto end;
     }
   }
@@ -9444,7 +9452,17 @@ MYSQL_BIN_LOG::do_binlog_recovery(const char *opt_name, bool do_xa_recovery)
     if (error != LOG_INFO_EOF)
       sql_print_error("find_log_pos() failed (error: %d)", error);
     else
+    {
       error= read_state_from_file();
+      if (error == 2)
+      {
+        /*
+          No binlog files and no binlog state is not an error (eg. just initial
+          server start after fresh installation).
+        */
+        error= 0;
+      }
+    }
     return error;
   }
 
@@ -9470,15 +9488,42 @@ MYSQL_BIN_LOG::do_binlog_recovery(const char *opt_name, bool do_xa_recovery)
 
   if ((ev= Log_event::read_log_event(&log, 0, &fdle,
                                      opt_master_verify_checksum)) &&
-      ev->get_type_code() == FORMAT_DESCRIPTION_EVENT &&
-      ev->flags & LOG_EVENT_BINLOG_IN_USE_F)
+      ev->get_type_code() == FORMAT_DESCRIPTION_EVENT)
   {
-    sql_print_information("Recovering after a crash using %s", opt_name);
-    error= recover(&log_info, log_name, &log,
-                   (Format_description_log_event *)ev, do_xa_recovery);
+    if (ev->flags & LOG_EVENT_BINLOG_IN_USE_F)
+    {
+      sql_print_information("Recovering after a crash using %s", opt_name);
+      error= recover(&log_info, log_name, &log,
+                     (Format_description_log_event *)ev, do_xa_recovery);
+    }
+    else
+    {
+      error= read_state_from_file();
+      if (error == 2)
+      {
+        /*
+          The binlog exists, but the .state file is missing. This is normal if
+          this is the first master start after a major upgrade to 10.0 (with
+          GTID support).
+
+          However, it could also be that the .state file was lost somehow, and
+          in this case it could be a serious issue, as we would set the wrong
+          binlog state in the next binlog file to be created, and GTID
+          processing would be corrupted. A common way would be copying files
+          from an old server to a new one and forgetting the .state file.
+
+          So in this case, we want to try to recover the binlog state by
+          scanning the last binlog file (but we do not need any XA recovery).
+
+          ToDo: We could avoid one scan at first start after major upgrade, by
+          detecting that there is no GTID_LIST event at the start of the
+          binlog file, and stopping the scan in that case.
+        */
+        error= recover(&log_info, log_name, &log,
+                       (Format_description_log_event *)ev, false);
+      }
+    }
   }
-  else
-    error= read_state_from_file();
 
   delete ev;
   end_io_cache(&log);