Speed up `mtr --parallel=<lots>` by scheduling some slow tests earlier.

The patch also fixes a race in rpl_stop_slave.test.

On machines with lots of CPU and memory, something like `mtr --parallel=10`
can speed up the test suite enormously. However, we have a few test cases
that run for long (several minutes), and if we are unlucky and happen to
schedule those towards the end of the test suite, we end up with most
workers idle while waiting for the last slow test to end, significantly
delaying the finish of the entire suite.

Improve this by marking the offending tests as taking "long", and trying
to schedule those tests early. This reduces the time towards the end of
the test suite run where some workers are waiting with nothing to do for
the remaining workers each to finish their last test.

Also, the rpl_stop_slave test had a race which could cause it to take
a 300 seconds debug_sync timeout; this is fixed.

Testing on a 4-core 8GB machine, this patch speeds up the test suite with
around 30% for --parallel=10 (debug build), allowing to run the entire
suite in 5 minutes.

20 files changed, 60 insertions, 9 deletions

diff --git a/mysql-test/extra/rpl_tests/rpl_stop_slave.test b/mysql-test/extra/rpl_tests/rpl_stop_slave.test
index 7c88afe3532..5adfc99d59e 100644
--- a/mysql-test/extra/rpl_tests/rpl_stop_slave.test
+++ b/mysql-test/extra/rpl_tests/rpl_stop_slave.test
@@ -42,6 +42,7 @@ send STOP SLAVE SQL_THREAD;
 connection slave1;
 --echo # To resume slave SQL thread
 SET DEBUG_SYNC= 'now SIGNAL signal.continue';
+SET DEBUG_SYNC= 'now WAIT_FOR signal.continued';
 SET DEBUG_SYNC= 'RESET';
 
 --echo
diff --git a/mysql-test/include/long_test.inc b/mysql-test/include/long_test.inc
new file mode 100644
index 00000000000..d9a3b338229
--- /dev/null
+++ b/mysql-test/include/long_test.inc
@@ -0,0 +1,4 @@
+# We use this --source include to mark a test as taking long to run.
+# We can use this to schedule such test early (to not be left with
+# only one or two long tests running, and rests of works idle), or to
+# run a quick test skipping long-running test cases.
diff --git a/mysql-test/lib/mtr_cases.pm b/mysql-test/lib/mtr_cases.pm
index 3ed2f072e9e..39a4e28823b 100644
--- a/mysql-test/lib/mtr_cases.pm
+++ b/mysql-test/lib/mtr_cases.pm
@@ -89,6 +89,20 @@ sub init_pattern {
 }
 
 
+sub testcase_sort_order {
+  my ($a, $b, $sort_criteria)= @_;
+  my $a_sort_criteria= $sort_criteria->{$a->fullname()};
+  my $b_sort_criteria= $sort_criteria->{$b->fullname()};
+  my $res= $a_sort_criteria cmp $b_sort_criteria;
+  return $res if $res;
+  # Run slow tests first, trying to avoid getting stuck at the end
+  # with a slow test in one worker and the other workers idle.
+  return -1 if $a->{'long_test'} && !$b->{'long_test'};
+  return 1 if !$a->{'long_test'} && $b->{'long_test'};
+
+  return $a->fullname() cmp $b->fullname();
+}
+
 ##############################################################################
 #
 #  Collect information about test cases to be run
@@ -177,9 +191,7 @@ sub collect_test_cases ($$$) {
       $sort_criteria{$tinfo->fullname()} = join(" ", @criteria);
     }
 
-    @$cases = sort {
-      $sort_criteria{$a->fullname()} . $a->fullname() cmp
-	$sort_criteria{$b->fullname()} . $b->fullname() } @$cases;
+    @$cases = sort { testcase_sort_order($a, $b, \%sort_criteria) } @$cases;
 
     # For debugging the sort-order
     # foreach my $tinfo (@$cases)
@@ -1054,6 +1066,7 @@ my @tags=
  ["include/not_valgrind.inc", "not_valgrind", 1],
  ["include/have_example_plugin.inc", "example_plugin_test", 1],
  ["include/have_ssl.inc", "need_ssl", 1],
+ ["include/long_test.inc", "long_test", 1],
 );
 
 
diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl
index 1f420d2acde..4a652580d97 100755
--- a/mysql-test/mysql-test-run.pl
+++ b/mysql-test/mysql-test-run.pl
@@ -729,9 +729,11 @@ sub run_test_server ($$$) {
 	    last;
 	  }
 
-	  # Second best choice is the first that does not fulfill
-	  # any of the above conditions
-	  if (!defined $second_best){
+	  # From secondary choices, we prefer to pick a 'long-running' test if
+          # possible; this helps avoid getting stuck with a few of those at the
+          # end of high --parallel runs, with most workers being idle.
+	  if (!defined $second_best ||
+              ($t->{'long_test'} && !($tests->[$second_best]{'long_test'}))){
 	    #mtr_report("Setting second_best to $i");
 	    $second_best= $i;
 	  }
diff --git a/mysql-test/suite/federated/federated_debug.test b/mysql-test/suite/federated/federated_debug.test
index 4152d2975b3..d63c1007088 100644
--- a/mysql-test/suite/federated/federated_debug.test
+++ b/mysql-test/suite/federated/federated_debug.test
@@ -1,4 +1,5 @@
 --source include/have_debug.inc
+--source include/long_test.inc
 --source federated.inc
 
 --echo #
diff --git a/mysql-test/suite/maria/t/maria-recovery-rtree-ft.test b/mysql-test/suite/maria/t/maria-recovery-rtree-ft.test
index 3ede5002b72..ac71be376f1 100644
--- a/mysql-test/suite/maria/t/maria-recovery-rtree-ft.test
+++ b/mysql-test/suite/maria/t/maria-recovery-rtree-ft.test
@@ -6,6 +6,7 @@
 # Binary must be compiled with debug for crash to occur
 --source include/have_debug.inc
 --source include/have_maria.inc
+--source include/long_test.inc
 
 set global maria_log_file_size=4294967295;
 let $MARIA_LOG=.;
diff --git a/mysql-test/suite/parts/t/partition_alter2_1_myisam.test b/mysql-test/suite/parts/t/partition_alter2_1_myisam.test
index 1c89a82b960..11ec9b51f7c 100644
--- a/mysql-test/suite/parts/t/partition_alter2_1_myisam.test
+++ b/mysql-test/suite/parts/t/partition_alter2_1_myisam.test
@@ -22,6 +22,8 @@
 # any of the variables.
 #
 
+--source include/long_test.inc
+
 #------------------------------------------------------------------------------#
 # General not engine specific settings and requirements
 
diff --git a/mysql-test/suite/parts/t/partition_alter2_2_myisam.test b/mysql-test/suite/parts/t/partition_alter2_2_myisam.test
index c9b22ed8595..8fbb943a48d 100644
--- a/mysql-test/suite/parts/t/partition_alter2_2_myisam.test
+++ b/mysql-test/suite/parts/t/partition_alter2_2_myisam.test
@@ -22,6 +22,8 @@
 # any of the variables.
 #
 
+--source include/long_test.inc
+
 #------------------------------------------------------------------------------#
 # General not engine specific settings and requirements
 
diff --git a/mysql-test/suite/parts/t/partition_basic_innodb.test b/mysql-test/suite/parts/t/partition_basic_innodb.test
index 2fa94cbde21..8240257f087 100644
--- a/mysql-test/suite/parts/t/partition_basic_innodb.test
+++ b/mysql-test/suite/parts/t/partition_basic_innodb.test
@@ -22,6 +22,8 @@
 # any of the variables.
 #
 
+--source include/long_test.inc
+
 #------------------------------------------------------------------------------#
 # General not engine specific settings and requirements
 
diff --git a/mysql-test/suite/parts/t/partition_decimal_myisam.test b/mysql-test/suite/parts/t/partition_decimal_myisam.test
index 49fc64cbd37..9808dc878f8 100644
--- a/mysql-test/suite/parts/t/partition_decimal_myisam.test
+++ b/mysql-test/suite/parts/t/partition_decimal_myisam.test
@@ -22,6 +22,8 @@
 # any of the variables.
 #
 
+--source include/long_test.inc
+
 #------------------------------------------------------------------------------#
 # General not engine specific settings and requirements
 
diff --git a/mysql-test/suite/parts/t/partition_float_myisam.test b/mysql-test/suite/parts/t/partition_float_myisam.test
index 51e0f1f5a21..f15e6ad3636 100644
--- a/mysql-test/suite/parts/t/partition_float_myisam.test
+++ b/mysql-test/suite/parts/t/partition_float_myisam.test
@@ -22,6 +22,8 @@
 # any of the variables.
 #
 
+--source include/long_test.inc
+
 #------------------------------------------------------------------------------#
 # General not engine specific settings and requirements
 
diff --git a/mysql-test/suite/parts/t/partition_int_myisam.test b/mysql-test/suite/parts/t/partition_int_myisam.test
index b0ede4995e8..5f29b575244 100644
--- a/mysql-test/suite/parts/t/partition_int_myisam.test
+++ b/mysql-test/suite/parts/t/partition_int_myisam.test
@@ -22,6 +22,8 @@
 # any of the variables.
 #
 
+--source include/long_test.inc
+
 #------------------------------------------------------------------------------#
 # General not engine specific settings and requirements
 
diff --git a/mysql-test/suite/rpl/r/rpl_stop_slave.result b/mysql-test/suite/rpl/r/rpl_stop_slave.result
index 49146417ab7..4e16e8264f6 100644
--- a/mysql-test/suite/rpl/r/rpl_stop_slave.result
+++ b/mysql-test/suite/rpl/r/rpl_stop_slave.result
@@ -38,6 +38,7 @@ STOP SLAVE SQL_THREAD;
 [ On Slave1 ]
 # To resume slave SQL thread
 SET DEBUG_SYNC= 'now SIGNAL signal.continue';
+SET DEBUG_SYNC= 'now WAIT_FOR signal.continued';
 SET DEBUG_SYNC= 'RESET';
 
 [ On Slave ]
@@ -63,6 +64,7 @@ STOP SLAVE SQL_THREAD;
 [ On Slave1 ]
 # To resume slave SQL thread
 SET DEBUG_SYNC= 'now SIGNAL signal.continue';
+SET DEBUG_SYNC= 'now WAIT_FOR signal.continued';
 SET DEBUG_SYNC= 'RESET';
 
 [ On Slave ]
@@ -89,6 +91,7 @@ STOP SLAVE SQL_THREAD;
 [ On Slave1 ]
 # To resume slave SQL thread
 SET DEBUG_SYNC= 'now SIGNAL signal.continue';
+SET DEBUG_SYNC= 'now WAIT_FOR signal.continued';
 SET DEBUG_SYNC= 'RESET';
 
 [ On Slave ]
@@ -115,6 +118,7 @@ STOP SLAVE SQL_THREAD;
 [ On Slave1 ]
 # To resume slave SQL thread
 SET DEBUG_SYNC= 'now SIGNAL signal.continue';
+SET DEBUG_SYNC= 'now WAIT_FOR signal.continued';
 SET DEBUG_SYNC= 'RESET';
 
 [ On Slave ]
diff --git a/mysql-test/suite/rpl/t/rpl_deadlock_innodb.test b/mysql-test/suite/rpl/t/rpl_deadlock_innodb.test
index ee907f81b22..169ccbf7f18 100644
--- a/mysql-test/suite/rpl/t/rpl_deadlock_innodb.test
+++ b/mysql-test/suite/rpl/t/rpl_deadlock_innodb.test
@@ -7,5 +7,6 @@
 ########################################################
 -- source include/not_ndb_default.inc
 -- source include/have_innodb.inc
+-- source include/long_test.inc
 let $engine_type=innodb;
 -- source extra/rpl_tests/rpl_deadlock.test
diff --git a/mysql-test/suite/rpl/t/rpl_row_sp003.test b/mysql-test/suite/rpl/t/rpl_row_sp003.test
index ab49174ddfa..8ed47232ba9 100644
--- a/mysql-test/suite/rpl/t/rpl_row_sp003.test
+++ b/mysql-test/suite/rpl/t/rpl_row_sp003.test
@@ -10,6 +10,7 @@
 -- source include/have_binlog_format_row.inc
 # Slow test, don't run during staging part
 -- source include/not_staging.inc
+--source include/long_test.inc
 -- source include/master-slave.inc
 
 let $engine_type=INNODB;
diff --git a/mysql-test/t/join_outer.test b/mysql-test/t/join_outer.test
index a957c715bda..c8fa2a35538 100644
--- a/mysql-test/t/join_outer.test
+++ b/mysql-test/t/join_outer.test
@@ -1,3 +1,5 @@
+--source include/long_test.inc
+
 #
 # test of left outer join
 #
diff --git a/mysql-test/t/multi_update2.test b/mysql-test/t/multi_update2.test
index 9c5078efb6f..a0f17fabec4 100644
--- a/mysql-test/t/multi_update2.test
+++ b/mysql-test/t/multi_update2.test
@@ -1,3 +1,5 @@
+--source include/long_test.inc
+
 #
 # Test of update statement that uses many tables.
 #
diff --git a/mysql-test/t/pool_of_threads.test b/mysql-test/t/pool_of_threads.test
index e71b16e1f89..530038cee91 100644
--- a/mysql-test/t/pool_of_threads.test
+++ b/mysql-test/t/pool_of_threads.test
@@ -4,6 +4,7 @@
 -- source include/have_pool_of_threads.inc
 # Slow test, don't run during staging part
 -- source include/not_staging.inc
+-- source include/long_test.inc
 -- source include/common-tests.inc
 
 
diff --git a/mysql-test/t/query_cache.test b/mysql-test/t/query_cache.test
index de8467b509d..24487db826a 100644
--- a/mysql-test/t/query_cache.test
+++ b/mysql-test/t/query_cache.test
@@ -1,4 +1,5 @@
 -- source include/have_query_cache.inc
+-- source include/long_test.inc
 
 #
 # Tests with query cache
diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc
index 57ee5686bed..49879cb7edb 100644
--- a/sql/sql_parse.cc
+++ b/sql/sql_parse.cc
@@ -3286,12 +3286,17 @@ end_with_restore_list:
 
     DBUG_EXECUTE_IF("after_mysql_insert",
                     {
-                      const char act[]=
+                      const char act1[]=
                         "now "
                         "wait_for signal.continue";
+                      const char act2[]=
+                        "now "
+                        "signal signal.continued";
                       DBUG_ASSERT(opt_debug_sync_timeout > 0);
-                      DBUG_ASSERT(!debug_sync_set_action(current_thd,
-                                                         STRING_WITH_LEN(act)));
+                      DBUG_ASSERT(!debug_sync_set_action(thd,
+                                                         STRING_WITH_LEN(act1)));
+                      DBUG_ASSERT(!debug_sync_set_action(thd,
+                                                         STRING_WITH_LEN(act2)));
                     };);
     break;
   }