MDEV-18426: Most of the mtr tests in the galera_3nodes suite fail

Most of the mtr tests in the galera_3nodes suite fail
for a variety of reasons with a variety of errors.
Some tests simply need to add the missing "connection"
lines to the result files, but many of them fail due
to substantial errors that require reworking test files.

This patch adds the missing "connection" lines to
the result files and fixes several substantial flaws
in the galera_3nodes suite tests and in the mtr framework
service files, adapting the tests from galera_3nodes
for the current version of MariaDB.

https://jira.mariadb.org/browse/MDEV-18426

12 files changed, 100 insertions, 27 deletions

diff --git a/mysql-test/suite/galera_3nodes/include/galera_suspend.inc b/mysql-test/suite/galera_3nodes/include/galera_suspend.inc
deleted file mode 100644
index 3495ad2342b..00000000000
--- a/mysql-test/suite/galera_3nodes/include/galera_suspend.inc
+++ /dev/null
@@ -1,14 +0,0 @@
-#
-# This macro suspends the current node
-#
-
---let _SUSPEND_NODE_PIDFILE = `SELECT @@pid_file`
---echo Suspending node ...
-
---perl
-        my $pid_filename = $ENV{'_SUSPEND_NODE_PIDFILE'};
-        my $mysqld_pid = `cat $pid_filename`;
-        chomp($mysqld_pid);
-        system("kill -19 $mysqld_pid");
-        exit(0);
-EOF
diff --git a/mysql-test/suite/galera_3nodes/r/galera_evs_suspect_timeout.result b/mysql-test/suite/galera_3nodes/r/galera_evs_suspect_timeout.result
index 7e0d282ec7f..6a7eea94077 100644
--- a/mysql-test/suite/galera_3nodes/r/galera_evs_suspect_timeout.result
+++ b/mysql-test/suite/galera_3nodes/r/galera_evs_suspect_timeout.result
@@ -1,12 +1,17 @@
+connection node_1;
 SET GLOBAL wsrep_provider_options = 'evs.inactive_timeout=PT100M; evs.suspect_timeout=PT1S';
+connection node_2;
 SET GLOBAL wsrep_provider_options = 'evs.inactive_timeout=PT100M; evs.suspect_timeout=PT1S';
+connection node_3;
 Suspending node ...
+connection node_1;
 SET SESSION wsrep_sync_wait = 0;
 SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
 VARIABLE_VALUE = 2
 1
 CREATE TABLE t1 (f1 INTEGER);
 INSERT INTO t1 VALUES (1);
+connection node_2;
 SET SESSION wsrep_sync_wait = 0;
 SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
 VARIABLE_VALUE = 2
@@ -17,4 +22,5 @@ COUNT(*) = 1
 1
 DROP TABLE t1;
 Resuming node ...
+connection node_3;
 CALL mtr.add_suppression("WSREP: gcs_caused() returned -1 \\(Operation not permitted\\)");
diff --git a/mysql-test/suite/galera_3nodes/r/galera_garbd.result b/mysql-test/suite/galera_3nodes/r/galera_garbd.result
index 180aade029c..fb7e729dc77 100644
--- a/mysql-test/suite/galera_3nodes/r/galera_garbd.result
+++ b/mysql-test/suite/galera_3nodes/r/galera_garbd.result
@@ -1,17 +1,29 @@
+connection node_1;
+connection node_2;
+connection node_3;
 Killing node #3 to free ports for garbd ...
+connection node_3;
+connection node_1;
 Starting garbd ...
 CREATE TABLE t1 (f1 INTEGER);
 INSERT INTO t1 VALUES (1);
+connection node_2;
 SELECT COUNT(*) = 1 FROM t1;
 COUNT(*) = 1
 1
 Killing garbd ...
+connection node_1;
 INSERT INTO t1 VALUES (2);
+connection node_2;
 SELECT COUNT(*) = 2 FROM t1;
 COUNT(*) = 2
 1
 DROP TABLE t1;
 Restarting node #3 to satisfy MTR's end-of-test checks
+connection node_3;
+connection node_1;
 CALL mtr.add_suppression("WSREP: Protocol violation\. JOIN message sender 1\.0 \(.*\) is not in state transfer \(SYNCED\)");
+connection node_2;
 CALL mtr.add_suppression("WSREP: Protocol violation\. JOIN message sender 1\.0 \(.*\) is not in state transfer \(SYNCED\)");
+connection node_3;
 CALL mtr.add_suppression("WSREP: Protocol violation\. JOIN message sender 1\.0 \(.*\) is not in state transfer \(SYNCED\)");
diff --git a/mysql-test/suite/galera_3nodes/r/galera_ist_gcache_rollover.result b/mysql-test/suite/galera_3nodes/r/galera_ist_gcache_rollover.result
index 3d4dbcc00b0..aa3e349eda7 100644
--- a/mysql-test/suite/galera_3nodes/r/galera_ist_gcache_rollover.result
+++ b/mysql-test/suite/galera_3nodes/r/galera_ist_gcache_rollover.result
@@ -1,6 +1,7 @@
 connection node_1;
 connection node_2;
 connection node_3;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY);
 INSERT INTO t1 VALUES (01), (02), (03), (04), (05);
 connection node_2;
diff --git a/mysql-test/suite/galera_3nodes/r/galera_safe_to_bootstrap.result b/mysql-test/suite/galera_3nodes/r/galera_safe_to_bootstrap.result
index 1676ef9b07b..45b4d63fb4f 100644
--- a/mysql-test/suite/galera_3nodes/r/galera_safe_to_bootstrap.result
+++ b/mysql-test/suite/galera_3nodes/r/galera_safe_to_bootstrap.result
@@ -1,3 +1,7 @@
+connection node_1;
+connection node_2;
+connection node_3;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
@@ -7,7 +11,6 @@ connection node_1;
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
-connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
 connection node_3;
 connection node_1;
 include/assert_grep.inc [grastate.dat does not have 'safe_to_bootstrap: 0']
@@ -29,8 +32,18 @@ connection node_1;
 connection node_2;
 connection node_3;
 connection node_2;
+CALL mtr.add_suppression("WSREP: no nodes coming from prim view, prim not possible");
+CALL mtr.add_suppression("WSREP: It may not be safe to bootstrap the cluster from this node");
+CALL mtr.add_suppression("WSREP: wsrep::connect(.*) failed: 7");
+CALL mtr.add_suppression("Aborting");
+CALL mtr.add_suppression("WSREP: moving position backwards: [0-9]+ -> 0");
 CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
 connection node_3;
+CALL mtr.add_suppression("WSREP: no nodes coming from prim view, prim not possible");
+CALL mtr.add_suppression("WSREP: It may not be safe to bootstrap the cluster from this node");
+CALL mtr.add_suppression("WSREP: wsrep::connect(.*) failed: 7");
+CALL mtr.add_suppression("Aborting");
+CALL mtr.add_suppression("WSREP: moving position backwards: [0-9]+ -> 0");
 CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
 SHOW CREATE TABLE t1;
 Table	Create Table
diff --git a/mysql-test/suite/galera_3nodes/r/galera_var_dirty_reads2.result b/mysql-test/suite/galera_3nodes/r/galera_var_dirty_reads2.result
index c6756bce210..77991a6d468 100644
--- a/mysql-test/suite/galera_3nodes/r/galera_var_dirty_reads2.result
+++ b/mysql-test/suite/galera_3nodes/r/galera_var_dirty_reads2.result
@@ -1,3 +1,7 @@
+connection node_1;
+connection node_2;
+connection node_3;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER);
 INSERT INTO t1 VALUES (1);
 connection node_2;
diff --git a/mysql-test/suite/galera_3nodes/t/galera_evs_suspect_timeout.test b/mysql-test/suite/galera_3nodes/t/galera_evs_suspect_timeout.test
index 03236a3cb93..4dab936c343 100644
--- a/mysql-test/suite/galera_3nodes/t/galera_evs_suspect_timeout.test
+++ b/mysql-test/suite/galera_3nodes/t/galera_evs_suspect_timeout.test
@@ -56,8 +56,8 @@ DROP TABLE t1;
 
 # Reconnect node #3 so that MTR's end-of-test checks can run
 
---connection node_3
 --source include/galera_resume.inc
+--connection node_3
 --source include/wait_until_connected_again.inc
 
 CALL mtr.add_suppression("WSREP: gcs_caused() returned -1 \\(Operation not permitted\\)");
diff --git a/mysql-test/suite/galera_3nodes/t/galera_garbd.test b/mysql-test/suite/galera_3nodes/t/galera_garbd.test
index a68ba8ce15b..2d03e8897b9 100644
--- a/mysql-test/suite/galera_3nodes/t/galera_garbd.test
+++ b/mysql-test/suite/galera_3nodes/t/galera_garbd.test
@@ -7,10 +7,20 @@
 --source include/have_innodb.inc
 --source include/big_test.inc
 
---echo Killing node #3 to free ports for garbd ...
 --let $galera_connection_name = node_3
 --let $galera_server_number = 3
 --source include/galera_connect.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--let $node_3=node_3
+--source ../galera/include/auto_increment_offset_save.inc
+
+--echo Killing node #3 to free ports for garbd ...
+--connection node_3
+--let $gp3 = `SELECT SUBSTR(@@wsrep_provider_options, LOCATE('base_port =', @@wsrep_provider_options) + LENGTH('base_port = '))`
+--let $galera_port_3 = `SELECT SUBSTR('$gp3', 1, LOCATE(';', '$gp3') - 1)`
 --source include/shutdown_mysqld.inc
 
 --connection node_1
@@ -18,7 +28,9 @@
 --source include/wait_condition.inc
 
 --echo Starting garbd ...
---exec `dirname $WSREP_PROVIDER`/garb/garbd --address "gcomm://127.0.0.1:$NODE_GALERAPORT_1" --group my_wsrep_cluster --options 'base_port=$NODE_GALERAPORT_3' > $MYSQL_TMP_DIR/garbd.log 2>&1 &
+--let $gp1 = `SELECT SUBSTR(@@wsrep_provider_options, LOCATE('base_port =', @@wsrep_provider_options) + LENGTH('base_port = '))`
+--let $galera_port_1 = `SELECT SUBSTR('$gp1', 1, LOCATE(';', '$gp1') - 1)`
+--exec `dirname $WSREP_PROVIDER`/../../bin/garb/garbd --address "gcomm://127.0.0.1:$galera_port_1" --group my_wsrep_cluster --options 'base_port=$galera_port_3' > $MYSQL_TMP_DIR/garbd.log 2>&1 &
 
 --sleep 5
 
@@ -32,7 +44,7 @@ INSERT INTO t1 VALUES (1);
 SELECT COUNT(*) = 1 FROM t1;
 
 --echo Killing garbd ...
---exec pkill --oldest --full garbd.*$NODE_GALERAPORT_3
+--exec pkill --oldest --full garbd.*$galera_port_3
 
 --sleep 5
 
@@ -51,6 +63,8 @@ DROP TABLE t1;
 --connection node_3
 --source include/start_mysqld.inc
 
+# Restore original auto_increment_offset values.
+--source ../galera/include/auto_increment_offset_restore.inc
 
 # Workaround for galera#101
 
diff --git a/mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.cnf b/mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.cnf
index 821175220ac..303087dffbb 100644
--- a/mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.cnf
+++ b/mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.cnf
@@ -1,11 +1,11 @@
 !include ../galera_3nodes.cnf
 
 [mysqld.1]
-wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true;gcache.size=1M'
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.1.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.ignore_sb=true;gcache.size=1M'
 
 [mysqld.2]
-wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true;gcache.size=1M'
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.2.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.ignore_sb=true;gcache.size=1M'
 
 [mysqld.3]
-wsrep_provider_options='base_port=@mysqld.3.#galera_port;pc.ignore_sb=true;gcache.size=1M'
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.3.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.ignore_sb=true;gcache.size=1M'
 
diff --git a/mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.test b/mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.test
index a67b30e3fa1..ebc756d60b1 100644
--- a/mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.test
+++ b/mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.test
@@ -23,6 +23,7 @@
 --let $node_3=node_3
 --source ../galera/include/auto_increment_offset_save.inc
 
+--connection node_1
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY);
 INSERT INTO t1 VALUES (01), (02), (03), (04), (05);
 
diff --git a/mysql-test/suite/galera_3nodes/t/galera_safe_to_bootstrap.test b/mysql-test/suite/galera_3nodes/t/galera_safe_to_bootstrap.test
index 88d0cfba4f4..b7b6c66e5ad 100644
--- a/mysql-test/suite/galera_3nodes/t/galera_safe_to_bootstrap.test
+++ b/mysql-test/suite/galera_3nodes/t/galera_safe_to_bootstrap.test
@@ -1,8 +1,19 @@
 #
 # Test the safe_to_bootstrap in grastate.dat 
 #
-
 --source include/galera_cluster.inc
+
+--let $galera_connection_name = node_3
+--let $galera_server_number = 3
+--source include/galera_connect.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--let $node_3=node_3
+--source ../galera/include/auto_increment_offset_save.inc
+
+--connection node_1
 CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 
 #
@@ -47,8 +58,6 @@ CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 #
 # Shut down one more node
 #
-
---connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3
 --connection node_3
 --source include/shutdown_mysqld.inc
 
@@ -129,9 +138,9 @@ SET SESSION wsrep_on = OFF;
 #
 
 --error 1
---exec $MYSQLD --defaults-group-suffix=.2 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --wsrep-new-cluster | grep 'This node is not safe to bootstrap the cluster'
+--exec $MYSQLD --defaults-group-suffix=.2 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --wsrep-new-cluster --wsrep-cluster-address='gcomm://' | grep 'This node is not safe to bootstrap the cluster'
 --error 1
---exec $MYSQLD --defaults-group-suffix=.3 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --wsrep-new-cluster | grep 'This node is not safe to bootstrap the cluster'
+--exec $MYSQLD --defaults-group-suffix=.3 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --wsrep-new-cluster --wsrep-cluster-address='gcomm://' | grep 'This node is not safe to bootstrap the cluster'
 
 #
 # Attempt to bootstrap starting from node #1, should succeed
@@ -154,10 +163,23 @@ SET SESSION wsrep_on = OFF;
 --source include/wait_condition.inc
 
 --connection node_2
+CALL mtr.add_suppression("WSREP: no nodes coming from prim view, prim not possible");
+CALL mtr.add_suppression("WSREP: It may not be safe to bootstrap the cluster from this node");
+CALL mtr.add_suppression("WSREP: wsrep::connect(.*) failed: 7");
+CALL mtr.add_suppression("Aborting");
+CALL mtr.add_suppression("WSREP: moving position backwards: [0-9]+ -> 0");
 CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
 
 --connection node_3
+CALL mtr.add_suppression("WSREP: no nodes coming from prim view, prim not possible");
+CALL mtr.add_suppression("WSREP: It may not be safe to bootstrap the cluster from this node");
+CALL mtr.add_suppression("WSREP: wsrep::connect(.*) failed: 7");
+CALL mtr.add_suppression("Aborting");
+CALL mtr.add_suppression("WSREP: moving position backwards: [0-9]+ -> 0");
 CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
 SHOW CREATE TABLE t1;
 
 DROP TABLE t1;
+
+# Restore original auto_increment_offset values.
+--source ../galera/include/auto_increment_offset_restore.inc
diff --git a/mysql-test/suite/galera_3nodes/t/galera_var_dirty_reads2.test b/mysql-test/suite/galera_3nodes/t/galera_var_dirty_reads2.test
index 129ba2e1f38..e3f94a012b8 100644
--- a/mysql-test/suite/galera_3nodes/t/galera_var_dirty_reads2.test
+++ b/mysql-test/suite/galera_3nodes/t/galera_var_dirty_reads2.test
@@ -5,6 +5,17 @@
 --source include/galera_cluster.inc
 --source include/have_innodb.inc
 
+--let $galera_connection_name = node_3
+--let $galera_server_number = 3
+--source include/galera_connect.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--let $node_3=node_3
+--source ../galera/include/auto_increment_offset_save.inc
+
+--connection node_1
 CREATE TABLE t1 (f1 INTEGER);
 INSERT INTO t1 VALUES (1);
 
@@ -110,3 +121,6 @@ SET GLOBAL wsrep_provider_options='gmcast.isolate=0';
 --source include/wait_condition.inc
 
 DROP TABLE t1;
+
+# Restore original auto_increment_offset values.
+--source ../galera/include/auto_increment_offset_restore.inc