Mysql GTID replication fails when data inserted

If you have a master and a slave configured and you insert
new data into the master, it will cause subsequent replica
create to fail.

The problem is that we weren't setting the gtid_purged variable
using the metadata in the xtrabackup_binlog_info file. The
Mysql GTID replication strategy was adjusted to account for this.

A release note has been added.

The replication scenario tests were enhanced to validate this
issue.

Note: this issue doesn't occur with MariaDB GTID replication
because it mechanism is different.

Scenario tests were run succesfully on Mysql, Percona and
MariaDB with this change in place.

Change-Id: I66c8b6278afa50ba14e4bb7888e3a25dc657a9e4
Closes-bug: 1563574
(cherry picked from commit 09a312ae3af1958df6dadcfd68dfae7261971fa3)

4 files changed, 56 insertions, 0 deletions

diff --git a/releasenotes/notes/fix-mysql-replication-bf2b131994a5a772.yaml b/releasenotes/notes/fix-mysql-replication-bf2b131994a5a772.yaml
new file mode 100644
index 00000000..0c8494ab
--- /dev/null
+++ b/releasenotes/notes/fix-mysql-replication-bf2b131994a5a772.yaml
@@ -0,0 +1,5 @@
+---
+fixes:
+  - Fixes an issue with a failure to establish a new replica for MySQL
+    in some cases where a replica already exists and some data has
+    been inserted into the master. Bug 1563574
\ No newline at end of file
diff --git a/trove/guestagent/strategies/replication/mysql_gtid.py b/trove/guestagent/strategies/replication/mysql_gtid.py
index f76aa04f..f25606b9 100644
--- a/trove/guestagent/strategies/replication/mysql_gtid.py
+++ b/trove/guestagent/strategies/replication/mysql_gtid.py
@@ -13,11 +13,17 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 #
+import csv
 
 from oslo_log import log as logging
 
 from trove.common import cfg
+from trove.common import exception
+from trove.common.i18n import _
 from trove.guestagent.backup.backupagent import BackupAgent
+from trove.guestagent.common import operating_system
+from trove.guestagent.common.operating_system import FileMode
+from trove.guestagent.datastore.mysql.service import MySqlApp
 from trove.guestagent.strategies.replication import mysql_base
 
 AGENT = BackupAgent()
@@ -29,7 +35,21 @@ LOG = logging.getLogger(__name__)
 class MysqlGTIDReplication(mysql_base.MysqlReplicationBase):
     """MySql Replication coordinated by GTIDs."""
 
+    class UnableToDetermineLastMasterGTID(exception.TroveError):
+        message = _("Unable to determine last GTID executed on master "
+                    "(from file %(binlog_file)s).")
+
     def connect_to_master(self, service, snapshot):
+        if 'dataset' in snapshot:
+            # pull the last executed GTID from the master via
+            # the xtrabackup metadata file. If that value is
+            # provided we need to set the gtid_purged variable
+            # before executing the CHANGE MASTER TO command
+            last_gtid = self._read_last_master_gtid()
+            if last_gtid:
+                set_gtid_cmd = "SET GLOBAL gtid_purged='%s'" % last_gtid
+                service.execute_on_client(set_gtid_cmd)
+
         logging_config = snapshot['log_position']
         LOG.debug("connect_to_master %s" % logging_config['replication_user'])
         change_master_cmd = (
@@ -47,3 +67,18 @@ class MysqlGTIDReplication(mysql_base.MysqlReplicationBase):
             })
         service.execute_on_client(change_master_cmd)
         service.start_slave()
+
+    def _read_last_master_gtid(self):
+        INFO_FILE = ('%s/xtrabackup_binlog_info' % MySqlApp.get_data_dir())
+        LOG.info(_("Setting read permissions on %s") % INFO_FILE)
+        operating_system.chmod(INFO_FILE, FileMode.ADD_READ_ALL, as_root=True)
+        LOG.info(_("Reading last master GTID from %s") % INFO_FILE)
+        try:
+            with open(INFO_FILE, 'rb') as f:
+                row = csv.reader(f, delimiter='\t',
+                                 skipinitialspace=True).next()
+                return row[2]
+        except (IOError, IndexError) as ex:
+            LOG.exception(ex)
+            raise self.UnableToDetermineLastMasterGTID(
+                {'binlog_file': INFO_FILE})
diff --git a/trove/tests/scenario/groups/replication_group.py b/trove/tests/scenario/groups/replication_group.py
index cea7a7b8..2f153b24 100644
--- a/trove/tests/scenario/groups/replication_group.py
+++ b/trove/tests/scenario/groups/replication_group.py
@@ -46,6 +46,16 @@ class ReplicationGroup(TestGroup):
         self.test_runner.run_create_single_replica()
 
     @test(runs_after=[create_single_replica])
+    def add_data_after_replica(self):
+        """Add data to master after initial replica is setup"""
+        self.test_runner.run_add_data_after_replica()
+
+    @test(runs_after=[add_data_after_replica])
+    def verify_replica_data_after_single(self):
+        """Verify data exists on single replica"""
+        self.test_runner.run_verify_replica_data_after_single()
+
+    @test(runs_after=[verify_replica_data_after_single])
     def create_multiple_replicas(self):
         """Test creating multiple replicas."""
         self.test_runner.run_create_multiple_replicas()
diff --git a/trove/tests/scenario/runners/replication_runners.py b/trove/tests/scenario/runners/replication_runners.py
index 7f763803..1ebbc0fb 100644
--- a/trove/tests/scenario/runners/replication_runners.py
+++ b/trove/tests/scenario/runners/replication_runners.py
@@ -43,6 +43,9 @@ class ReplicationRunner(TestRunner):
         self.test_helper.add_data(data_type, host)
         self.used_data_sets.add(data_type)
 
+    def run_add_data_after_replica(self, data_type=DataType.micro):
+        self.assert_add_replication_data(data_type, self.master_host)
+
     def run_verify_data_for_replication(self, data_type=DataType.small):
         self.assert_verify_replication_data(data_type, self.master_host)
 
@@ -124,6 +127,9 @@ class ReplicationRunner(TestRunner):
             self.report.log("Checking data on host %s" % host)
             self.assert_verify_replication_data(data_type, host)
 
+    def run_verify_replica_data_after_single(self):
+        self.assert_verify_replica_data(self.instance_info.id, DataType.micro)
+
     def run_verify_replica_data_new(self):
         self.assert_verify_replica_data(self.instance_info.id, DataType.tiny)