Merge parallel mtr

10 files changed, 1353 insertions, 480 deletions

diff --git a/client/mysqltest.c b/client/mysqltest.c
index 31d85c36ed1..aea64c88247 100644
--- a/client/mysqltest.c
+++ b/client/mysqltest.c
@@ -1345,23 +1345,21 @@ void show_diff(DYNAMIC_STRING* ds,
   if (init_dynamic_string(&ds_tmp, "", 256, 256))
     die("Out of memory");
 
-  /* First try with diff --help to see if the command exists at all */
+  /* First try with unified diff */
   if (run_tool("diff",
                &ds_tmp, /* Get output from diff in ds_tmp */
-               "--help",
+               "-u",
+               filename1,
+               filename2,
                "2>&1",
-               NULL) != 0) /* Most "diff --help" tools return 0 */
-  {
-    diff_failed= "You don't appear to have diff installed";
-  }
-  else
+               NULL) > 1) /* Most "diff" tools return >1 if error */
   {
     dynstr_set(&ds_tmp, "");
-    /* First try with unified diff */
 
+    /* Fallback to context diff with "diff -c" */
     if (run_tool("diff",
                  &ds_tmp, /* Get output from diff in ds_tmp */
-                 "-u",
+                 "-c",
                  filename1,
                  filename2,
                  "2>&1",
@@ -1369,17 +1367,17 @@ void show_diff(DYNAMIC_STRING* ds,
     {
       dynstr_set(&ds_tmp, "");
 
-      /* Fallback to context diff with "diff -c" */
+      /* Fallback to plain "diff" */
       if (run_tool("diff",
                    &ds_tmp, /* Get output from diff in ds_tmp */
-                   "-c",
                    filename1,
                    filename2,
                    "2>&1",
                    NULL) > 1) /* Most "diff" tools return >1 if error */
       {
         dynstr_set(&ds_tmp, "");
-        diff_failed= "Could not execute 'diff -u' or 'diff -c'";
+
+        diff_failed= "Could not execute 'diff -u', 'diff -c' or 'diff'";
       }
     }
   }
diff --git a/mysql-test/Makefile.am b/mysql-test/Makefile.am
index 6e706ba022e..3348b258115 100644
--- a/mysql-test/Makefile.am
+++ b/mysql-test/Makefile.am
@@ -38,9 +38,11 @@ nobase_test_DATA = lib/mtr_cases.pm \
 	lib/My/Config.pm \
 	lib/My/Find.pm \
 	lib/My/Options.pm \
+	lib/My/Test.pm \
 	lib/My/Platform.pm \
 	lib/My/SafeProcess.pm \
 	lib/My/File/Path.pm \
+	lib/My/SysInfo.pm \
 	lib/My/SafeProcess/Base.pm \
 	lib/My/SafeProcess/safe_process.pl
 
diff --git a/mysql-test/lib/My/ConfigFactory.pm b/mysql-test/lib/My/ConfigFactory.pm
index 69bb969d7b8..31927f5e8d5 100644
--- a/mysql-test/lib/My/ConfigFactory.pm
+++ b/mysql-test/lib/My/ConfigFactory.pm
@@ -104,10 +104,16 @@ sub fix_server_id {
 sub fix_socket {
   my ($self, $config, $group_name, $group)= @_;
   # Put socket file in tmpdir
-  my $dir= $group->value('tmpdir');
+  my $dir= $self->{ARGS}->{tmpdir};
   return "$dir/$group_name.sock";
 }
 
+sub fix_tmpdir {
+  my ($self, $config, $group_name, $group)= @_;
+  my $dir= $self->{ARGS}->{tmpdir};
+  return "$dir/$group_name";
+}
+
 sub fix_log_error {
   my ($self, $config, $group_name, $group)= @_;
   my $dir= dirname($group->value('datadir'));
@@ -182,7 +188,7 @@ sub fix_ssl_client_key {
 my @mysqld_rules=
   (
  { 'basedir' => sub { return shift->{ARGS}->{basedir}; } },
- { 'tmpdir' => sub { return shift->{ARGS}->{tmpdir}; } },
+ { 'tmpdir' => \&fix_tmpdir },
  { 'character-sets-dir' => \&fix_charset_dir },
  { 'language' => \&fix_language },
  { 'datadir' => \&fix_datadir },
@@ -295,6 +301,16 @@ my @mysqlbinlog_rules=
 
 
 #
+# Rules to run for [mysql_upgrade] section
+#  - will be run in order listed here
+#
+my @mysql_upgrade_rules=
+(
+ { 'tmpdir' => sub { return shift->{ARGS}->{tmpdir}; } },
+);
+
+
+#
 # Generate a [client.<suffix>] group to be
 # used for connecting to [mysqld.<suffix>]
 #
@@ -600,6 +616,11 @@ sub new_config {
 			     $config->insert('mysqlbinlog'),
 			     @mysqlbinlog_rules);
 
+  # [mysql_upgrade] need additional settings
+  $self->run_rules_for_group($config,
+			     $config->insert('mysql_upgrade'),
+			     @mysql_upgrade_rules);
+
   # Additional rules required for [client]
   $self->run_rules_for_group($config,
 			     $config->insert('client'),
diff --git a/mysql-test/lib/My/Options.pm b/mysql-test/lib/My/Options.pm
index f0fc4ba75b5..40f05c41d1c 100644
--- a/mysql-test/lib/My/Options.pm
+++ b/mysql-test/lib/My/Options.pm
@@ -61,6 +61,9 @@ sub _split_option {
   elsif ($option=~ /^--(.*)$/){
     return ($1, undef)
   }
+  elsif ($option=~ /^\$(.*)$/){ # $VAR
+    return ($1, undef)
+  }
   elsif ($option=~ /^(.*)=(.*)$/){
     return ($1, $2)
   }
diff --git a/mysql-test/lib/My/SafeProcess.pm b/mysql-test/lib/My/SafeProcess.pm
index 4d703d73b9a..7762441496c 100644
--- a/mysql-test/lib/My/SafeProcess.pm
+++ b/mysql-test/lib/My/SafeProcess.pm
@@ -65,7 +65,7 @@ END {
   # Kill any children still running
   for my $proc (values %running){
     if ( $proc->is_child($$) ){
-      print "Killing: $proc\n";
+      #print "Killing: $proc\n";
       $proc->kill();
     }
   }
@@ -128,6 +128,7 @@ sub new {
   my $verbose  = delete($opts{'verbose'});
   my $host     = delete($opts{'host'});
   my $shutdown = delete($opts{'shutdown'});
+  my $user_data= delete($opts{'user_data'});
 
 #  if (defined $host) {
 #    $safe_script=  "lib/My/SafeProcess/safe_process_cpcd.pl";
@@ -173,6 +174,7 @@ sub new {
       SAFE_NAME => $name,
       SAFE_SHUTDOWN => $shutdown,
       PARENT => $$,
+      SAFE_USER_DATA => $user_data,
      }, $class);
 
   # Put the new process in list of running
@@ -461,8 +463,8 @@ sub wait_one {
     return 1;
   }
 
-  warn "wait_one: expected pid $pid but got $retpid"
-    unless( $retpid == $pid );
+  #warn "wait_one: expected pid $pid but got $retpid"
+  #  unless( $retpid == $pid );
 
   $self->_collect();
   return 0;
@@ -546,4 +548,16 @@ sub _verbose {
   print STDERR " ## ", @_, "\n";
 }
 
+
+sub pid {
+  my ($self)= @_;
+  return $self->{SAFE_PID};
+}
+
+sub user_data {
+  my ($self)= @_;
+  return $self->{SAFE_USER_DATA};
+}
+
+
 1;
diff --git a/mysql-test/lib/My/SysInfo.pm b/mysql-test/lib/My/SysInfo.pm
new file mode 100644
index 00000000000..87f768f4866
--- /dev/null
+++ b/mysql-test/lib/My/SysInfo.pm
@@ -0,0 +1,197 @@
+# -*- cperl -*-
+# Copyright (C) 2004-2006 MySQL AB
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+
+package My::SysInfo;
+
+use strict;
+use Carp;
+use My::Platform;
+
+use constant DEFAULT_BOGO_MIPS => 2000;
+
+sub _cpuinfo {
+  my ($self)= @_;
+  print "_cpuinfo\n";
+
+  my $info_file= "/proc/cpuinfo";
+  if ( !(  -e $info_file and -f $info_file) ) {
+    return undef;
+  }
+
+  my $F= IO::File->new($info_file) or return undef;
+
+  # Set input separator to blank line
+  local $/ = '';
+
+  while ( my $cpu_chunk= <$F>) {
+    chomp($cpu_chunk);
+
+    my $cpuinfo = {};
+
+    foreach my $cpuline ( split(/\n/, $cpu_chunk) ) {
+      my ( $attribute, $value ) = split(/\s*:\s*/, $cpuline);
+
+      $attribute =~ s/\s+/_/;
+      $attribute = lc($attribute);
+
+      if ( $value =~ /^(no|not available|yes)$/ ) {
+	$value = $value eq 'yes' ? 1 : 0;
+      }
+
+      if ( $attribute eq 'flags' ) {
+	@{ $cpuinfo->{flags} } = split / /, $value;
+      } else {
+	$cpuinfo->{$attribute} = $value;
+      }
+    }
+
+    # Make sure bogomips is set to some value
+    $cpuinfo->{bogomips} |= DEFAULT_BOGO_MIPS;
+
+    # Cpus reported once, but with 'cpu_count' set to the actual number
+    my $cpu_count= $cpuinfo->{cpu_count} || 1;
+    for(1..$cpu_count){
+      push(@{$self->{cpus}}, $cpuinfo);
+    }
+  }
+  $F= undef; # Close file
+  return $self;
+}
+
+
+sub _kstat {
+  my ($self)= @_;
+  while (1){
+    my $instance_num= $self->{cpus} ? @{$self->{cpus}} : 0;
+    my $list= `kstat -p -m cpu_info -i $instance_num`;
+    my @lines= split('\n', $list) or last; # Break loop
+
+    my $cpuinfo= {};
+    foreach my $line (@lines)
+    {
+      my ($module, $instance, $name, $statistic, $value)=
+	$line=~ /(\w*):(\w*):(\w*):(\w*)\t(.*)/;
+
+      $cpuinfo->{$statistic}= $value;
+    }
+
+    # Default value, the actual cpu values can be used to decrease this
+    # on slower cpus
+    $cpuinfo->{bogomips}= DEFAULT_BOGO_MIPS;
+
+    push(@{$self->{cpus}}, $cpuinfo);
+  }
+
+  # At least one cpu should have been found
+  # if this method worked
+  if ( $self->{cpus} ) {
+    return $self;
+  }
+  return undef;
+}
+
+
+sub _unamex {
+  my ($self)= @_;
+  # TODO
+  return undef;
+}
+
+
+sub new {
+  my ($class)= @_;
+
+
+  my $self= bless {
+		   cpus => (),
+		  }, $class;
+
+  my @info_methods =
+    (
+     \&_cpuinfo,
+     \&_kstat,
+     \&_unamex,
+   );
+
+  foreach my $method (@info_methods){
+    if ($method->($self)){
+      return $self;
+    }
+  }
+
+  # Push a dummy cpu
+  push(@{$self->{cpus}},
+     {
+      bogomips => DEFAULT_BOGO_MIPS,
+      model_name => "unknown",
+     });
+
+  return $self;
+}
+
+
+# Return the list of cpus found
+sub cpus {
+  my ($self)= @_;
+  return @{$self->{cpus}} or
+    confess "INTERNAL ERROR: No cpus in list";
+}
+
+
+# Return the number of cpus found
+sub num_cpus {
+  my ($self)= @_;
+  return int(@{$self->{cpus}}) or
+    confess "INTERNAL ERROR: No cpus in list";
+}
+
+
+# Return the smallest bogomips value amongst the processors
+sub min_bogomips {
+  my ($self)= @_;
+
+  my $bogomips;
+
+  foreach my $cpu (@{$self->{cpus}}) {
+    if (!defined $bogomips or $bogomips > $cpu->{bogomips}) {
+      $bogomips= $cpu->{bogomips};
+    }
+  }
+
+  return $bogomips;
+}
+
+
+# Prit the cpuinfo
+sub print_info {
+  my ($self)= @_;
+
+  foreach my $cpu (@{$self->{cpus}}) {
+    while ((my ($key, $value)) = each(%$cpu)) {
+      print " ", $key, "= ";
+      if (ref $value eq "ARRAY") {
+	print "[", join(", ", @$value), "]";
+      } else {
+	print $value;
+      }
+      print "\n";
+    }
+    print "\n";
+  }
+}
+
+1;
diff --git a/mysql-test/lib/My/Test.pm b/mysql-test/lib/My/Test.pm
new file mode 100644
index 00000000000..890ae76b282
--- /dev/null
+++ b/mysql-test/lib/My/Test.pm
@@ -0,0 +1,122 @@
+# -*- cperl -*-
+
+
+#
+# One test
+#
+package My::Test;
+
+use strict;
+use warnings;
+use Carp;
+
+
+sub new {
+  my $class= shift;
+  my $self= bless {
+		   @_,
+		  }, $class;
+  return $self;
+}
+
+
+#
+# Return a unique key that can be used to
+# identify this test in a hash
+#
+sub key {
+  my ($self)= @_;
+  my $key= $self->{name};
+  $key.= "+".$self->{combination} if $self->{combination};
+  return $key;
+}
+
+
+sub _encode {
+  my ($value)= @_;
+  $value =~ s/([|\\\x{0a}\x{0d}])/sprintf('\%02X', ord($1))/eg;
+  return $value;
+}
+
+sub _decode {
+  my ($value)= @_;
+  $value =~ s/\\([0-9a-fA-F]{2})/chr(hex($1))/ge;
+  return $value;
+}
+
+sub is_failed {
+  my ($self)= @_;
+  my $result= $self->{result};
+  croak "'is_failed' can't be called until test has been run!"
+    unless defined $result;
+
+  return ($result eq 'MTR_RES_FAILED');
+}
+
+
+sub write_test {
+  my ($test, $sock, $header)= @_;
+
+  print $sock $header, "\n";
+  while ((my ($key, $value)) = each(%$test)) {
+    print $sock  $key, "= ";
+    if (ref $value eq "ARRAY") {
+      print $sock "[", _encode(join(", ", @$value)), "]";
+    } else {
+      print $sock _encode($value);
+    }
+    print $sock "\n";
+  }
+  print $sock "\n";
+}
+
+
+sub read_test {
+  my ($sock)= @_;
+  my $test= My::Test->new();
+  # Read the : separated key value pairs until a
+  # single newline on it's own line
+  my $line;
+  while (defined($line= <$sock>)) {
+    # List is terminated by newline on it's own
+    if ($line eq "\n") {
+      # Correctly terminated reply
+      # print "Got newline\n";
+      last;
+    }
+    chomp($line);
+
+    # Split key/value on the first "="
+    my ($key, $value)= split("= ", $line, 2);
+
+    if ($value =~ /^\[(.*)\]/){
+      my @values= split(", ", _decode($1));
+      push(@{$test->{$key}}, @values);
+    }
+    else
+    {
+      $test->{$key}= _decode($value);
+    }
+  }
+  return $test;
+}
+
+
+sub print_test {
+  my ($self)= @_;
+
+  print "[", $self->{name}, "]", "\n";
+  while ((my ($key, $value)) = each(%$self)) {
+    print " ", $key, "= ";
+    if (ref $value eq "ARRAY") {
+      print "[", join(", ", @$value), "]";
+    } else {
+      print $value;
+    }
+    print "\n";
+  }
+  print "\n";
+}
+
+
+1;
diff --git a/mysql-test/lib/mtr_cases.pm b/mysql-test/lib/mtr_cases.pm
index 65061efa50a..8c061dbc691 100644
--- a/mysql-test/lib/mtr_cases.pm
+++ b/mysql-test/lib/mtr_cases.pm
@@ -40,7 +40,7 @@ our $default_storage_engine;
 our $opt_with_ndbcluster_only;
 our $defaults_file;
 our $defaults_extra_file;
-our $reorder;
+our $reorder= 1;
 
 sub collect_option {
   my ($opt, $value)= @_;
@@ -56,6 +56,7 @@ use File::Spec::Functions qw / splitdir /;
 use IO::File();
 use My::Config;
 use My::Platform;
+use My::Test;
 use My::Find;
 
 require "mtr_misc.pl";
@@ -136,52 +137,16 @@ sub collect_test_cases ($$) {
     {
       my @criteria = ();
 
-      # Look for tests that must be run in a defined order - that is
-      # defined by test having the same name except for the ending digit
+      #
+      # Append the criteria for sorting, in order of importance.
+      #
+      push(@criteria, "ndb=" . ($tinfo->{'ndb_test'} ? "A" : "B"));
+      # Group test with equal options together.
+      # Ending with "~" makes empty sort later than filled
+      my $opts= $tinfo->{'master_opt'} ? $tinfo->{'master_opt'} : [];
+      push(@criteria, join("!", sort @{$opts}) . "~");
 
-      # Put variables into hash
-      my $test_name= $tinfo->{'name'};
-      my $depend_on_test_name;
-      if ( $test_name =~ /^([\D]+)([0-9]{1})$/ )
-      {
-	my $base_name= $1;
-	my $idx= $2;
-	mtr_verbose("$test_name =>  $base_name idx=$idx");
-	if ( $idx > 1 )
-	{
-	  $idx-= 1;
-	  $base_name= "$base_name$idx";
-	  mtr_verbose("New basename $base_name");
-	}
-
-	foreach my $tinfo2 (@$cases)
-	{
-	  if ( $tinfo2->{'name'} eq $base_name )
-	  {
-	    mtr_verbose("found dependent test $tinfo2->{'name'}");
-	    $depend_on_test_name=$base_name;
-	  }
-	}
-      }
-
-      if ( defined $depend_on_test_name )
-      {
-	mtr_verbose("Giving $test_name same critera as $depend_on_test_name");
-	$sort_criteria{$test_name} = $sort_criteria{$depend_on_test_name};
-      }
-      else
-      {
-	#
-	# Append the criteria for sorting, in order of importance.
-	#
-	push(@criteria, "ndb=" . ($tinfo->{'ndb_test'} ? "1" : "0"));
-	# Group test with equal options together.
-	# Ending with "~" makes empty sort later than filled
-	my $opts= $tinfo->{'master_opt'} ? $tinfo->{'master_opt'} : [];
-	push(@criteria, join("!", sort @{$opts}) . "~");
-
-	$sort_criteria{$test_name} = join(" ", @criteria);
-      }
+      $sort_criteria{$tinfo->{name}} = join(" ", @criteria);
     }
 
     @$cases = sort {
@@ -459,7 +424,7 @@ sub collect_one_suite($)
 	  }
 
 	  # Copy test options
-	  my $new_test= {};
+	  my $new_test= My::Test->new();
 	  while (my ($key, $value) = each(%$test)) {
 	    if (ref $value eq "ARRAY") {
 	      push(@{$new_test->{$key}}, @$value);
@@ -687,13 +652,16 @@ sub collect_one_test_case {
   # ----------------------------------------------------------------------
   # Set defaults
   # ----------------------------------------------------------------------
-  my $tinfo= {};
-  $tinfo->{'name'}= $suitename . ".$tname";
-  $tinfo->{'path'}= "$testdir/$filename";
+  my $tinfo= My::Test->new
+    (
+     name          => "$suitename.$tname",
+     path          => "$testdir/$filename",
 
-  # TODO allow nonexistsing result file
-  # in that case .test must issue "exit" otherwise test should fail by default
-  $tinfo->{'result_file'}= "$resdir/$tname.result";
+     # TODO allow nonexistsing result file
+     # in that case .test must issue "exit" otherwise test
+     # should fail by default
+     result_file   => "$resdir/$tname.result",
+    );
 
   # ----------------------------------------------------------------------
   # Skip some tests but include in list, just mark them as skipped
@@ -1065,19 +1033,6 @@ sub unspace {
 }
 
 
-
-sub envsubst {
-  my $string= shift;
-
-  if ( ! defined $ENV{$string} )
-  {
-    mtr_error(".opt file references '$string' which is not set");
-  }
-
-  return $ENV{$string};
-}
-
-
 sub opts_from_file ($) {
   my $file=  shift;
 
@@ -1114,10 +1069,6 @@ sub opts_from_file ($) {
         or $arg =~ s/^([^\'\"]*)\"(.*)\"([^\'\"]*)$/$1$2$3/;
       $arg =~ s/\\\\/\\/g;
 
-      # Expand environment variables
-      $arg =~ s/\$\{(\w+)\}/envsubst($1)/ge;
-      $arg =~ s/\$(\w+)/envsubst($1)/ge;
-
       # Do not pass empty string since my_getopt is not capable to handle it.
       if (length($arg)) {
 	push(@args, $arg);
@@ -1133,17 +1084,7 @@ sub print_testcases {
 
   print "=" x 60, "\n";
   foreach my $test (@cases){
-    print "[", $test->{name}, "]", "\n";
-    while ((my ($key, $value)) = each(%$test)) {
-      print " ", $key, "= ";
-      if (ref $value eq "ARRAY") {
-	print "[", join(", ", @$value), "]";
-      } else {
-	print $value;
-      }
-      print "\n";
-    }
-    print "\n";
+    $test->print_test();
   }
   print "=" x 60, "\n";
 }
diff --git a/mysql-test/lib/mtr_report.pm b/mysql-test/lib/mtr_report.pm
index 84c3c0dc226..786e73f0680 100644
--- a/mysql-test/lib/mtr_report.pm
+++ b/mysql-test/lib/mtr_report.pm
@@ -26,15 +26,20 @@ our @EXPORT= qw(report_option mtr_print_line mtr_print_thick_line
 		mtr_print_header mtr_report mtr_report_stats
 		mtr_warning mtr_error mtr_debug mtr_verbose
 		mtr_verbose_restart mtr_report_test_passed
-		mtr_report_test_failed mtr_report_test_skipped);
+		mtr_report_test_skipped mtr_print
+		mtr_report_test);
 
 use mtr_match;
 require "mtr_io.pl";
 
 my $tot_real_time= 0;
 
-our $timestamp= 0;
-our $timediff= 1;
+our $timestamp= 1;
+our $timediff= 0;
+our $name;
+our $verbose;
+our $verbose_restart= 0;
+our $timer= 1;
 
 sub report_option {
   my ($opt, $value)= @_;
@@ -43,115 +48,134 @@ sub report_option {
   $opt =~ s/-/_/g;
   no strict 'refs';
   ${$opt}= $value;
+
+  #print $name, " setting $opt to ", (defined $value? $value : "undef") ,"\n";
 }
 
-sub SHOW_SUITE_NAME() { return  1; };
+sub _name {
+  return $name ? $name." " : undef;
+}
 
 sub _mtr_report_test_name ($) {
   my $tinfo= shift;
   my $tname= $tinfo->{name};
 
-  # Remove suite part of name
-  $tname =~ s/.*\.// unless SHOW_SUITE_NAME;
+  return unless defined $verbose;
 
   # Add combination name if any
   $tname.= " '$tinfo->{combination}'"
     if defined $tinfo->{combination};
 
-  print _timestamp();
-  printf "%-30s ", $tname;
+  print _name(), _timestamp();
+  printf "%-40s ", $tname;
 }
 
 
 sub mtr_report_test_skipped ($) {
-  my $tinfo= shift;
-  _mtr_report_test_name($tinfo);
-
+  my ($tinfo)= @_;
   $tinfo->{'result'}= 'MTR_RES_SKIPPED';
-  if ( $tinfo->{'disable'} )
-  {
-    mtr_report("[ disabled ]  $tinfo->{'comment'}");
-  }
-  elsif ( $tinfo->{'comment'} )
-  {
-    if ( $tinfo->{skip_detected_by_test} )
-    {
-      mtr_report("[ skip ].  $tinfo->{'comment'}");
-    }
-    else
-    {
-      mtr_report("[ skip ]  $tinfo->{'comment'}");
-    }
-  }
-  else
-  {
-    mtr_report("[ skip ]");
-  }
+
+  mtr_report_test($tinfo);
 }
 
 
-sub mtr_report_test_passed ($$) {
-  my ($tinfo, $use_timer)= @_;
-  _mtr_report_test_name($tinfo);
+sub mtr_report_test_passed ($) {
+  my ($tinfo)= @_;
 
-  my $timer=  "";
-  if ( $use_timer and -f "$::opt_vardir/log/timer" )
+  # Save the timer value
+  my $timer_str=  "";
+  if ( $timer and -f "$::opt_vardir/log/timer" )
   {
-    $timer= mtr_fromfile("$::opt_vardir/log/timer");
-    $tot_real_time += ($timer/1000);
-    $timer= sprintf "%12s", $timer;
+    $timer_str= mtr_fromfile("$::opt_vardir/log/timer");
+    $tinfo->{timer}= $timer_str;
   }
+
   # Set as passed unless already set
   if ( not defined $tinfo->{'result'} ){
     $tinfo->{'result'}= 'MTR_RES_PASSED';
   }
-  mtr_report("[ pass ]   $timer");
+
+  mtr_report_test($tinfo);
 }
 
 
-sub mtr_report_test_failed ($$) {
-  my ($tinfo, $logfile)= @_;
+sub mtr_report_test ($) {
+  my ($tinfo)= @_;
   _mtr_report_test_name($tinfo);
 
-  $tinfo->{'result'}= 'MTR_RES_FAILED';
-  my $test_failures= $tinfo->{'failures'} || 0;
-  $tinfo->{'failures'}=  $test_failures + 1;
-  if ( defined $tinfo->{'warnings'} )
-  {
-    mtr_report("[ fail ]  Found warnings in server log file!");
-    mtr_report($tinfo->{'warnings'});
-    return;
-  }
-  elsif ( defined $tinfo->{'timeout'} )
-  {
-    mtr_report("[ fail ]  timeout");
-    return;
-  }
-  else
-  {
-    mtr_report("[ fail ]");
-  }
+  if ($tinfo->{'result'} eq 'MTR_RES_FAILED'){
 
-  if ( $tinfo->{'comment'} )
-  {
-    # The test failure has been detected by mysql-test-run.pl
-    # when starting the servers or due to other error, the reason for
-    # failing the test is saved in "comment"
-    mtr_report("\nERROR: $tinfo->{'comment'}");
+    if ( defined $tinfo->{'warnings'} )
+    {
+      mtr_report("[ fail ]  Found warnings in server log file!");
+      mtr_report($tinfo->{'warnings'});
+      return;
+    }
+    if ( defined $tinfo->{'timeout'} )
+    {
+      mtr_report("[ fail ]  timeout");
+      return;
+    }
+    else
+    {
+      mtr_report("[ fail ]");
+    }
+
+    if ( $tinfo->{'comment'} )
+    {
+      # The test failure has been detected by mysql-test-run.pl
+      # when starting the servers or due to other error, the reason for
+      # failing the test is saved in "comment"
+      mtr_report("\nERROR: $tinfo->{'comment'}");
+    }
+    elsif ( $tinfo->{logfile} )
+    {
+      # Test failure was detected by test tool and its report
+      # about what failed has been saved to file. Display the report.
+      mtr_report("\n");
+      mtr_report($tinfo->{logfile}, "\n");
+
+    }
+    else
+    {
+      # Neither this script or the test tool has recorded info
+      # about why the test has failed. Should be debugged.
+      mtr_report("\nUnexpected termination, probably when starting mysqld");;
+    }
   }
-  elsif ( defined $logfile and -f $logfile )
+  elsif ($tinfo->{'result'} eq 'MTR_RES_SKIPPED')
   {
-    # Test failure was detected by test tool and its report
-    # about what failed has been saved to file. Display the report.
-    print "\n";
-    mtr_printfile($logfile);
-    print "\n";
+    if ( $tinfo->{'disable'} )
+    {
+      mtr_report("[ disabled ]  $tinfo->{'comment'}");
+    }
+    elsif ( $tinfo->{'comment'} )
+    {
+      if ( $tinfo->{skip_detected_by_test} )
+      {
+	mtr_report("[ skip ]. $tinfo->{'comment'}");
+      }
+      else
+      {
+	mtr_report("[ skip ]  $tinfo->{'comment'}");
+      }
+    }
+    else
+    {
+      mtr_report("[ skip ]");
+    }
   }
-  else
+  elsif ($tinfo->{'result'} eq 'MTR_RES_PASSED')
   {
-    # Neither this script or the test tool has recorded info
-    # about why the test has failed. Should be debugged.
-    mtr_report("\nUnexpected termination, probably when starting mysqld");;
+    my $timer_str= $tinfo->{timer} || "";
+    $tot_real_time += ($timer_str/1000);
+    mtr_report("[ pass ] ", sprintf("%5s", $timer_str));
+
+    # Show any problems check-testcase found
+    if ( defined $tinfo->{'check'} )
+    {
+      mtr_report($tinfo->{'check'});
+    }
   }
 }
 
@@ -172,22 +196,27 @@ sub mtr_report_stats ($) {
 
   foreach my $tinfo (@$tests)
   {
-    if ( $tinfo->{'result'} eq 'MTR_RES_SKIPPED' )
+    if ( $tinfo->{failures} )
+    {
+      # Test has failed at least one time
+      $tot_tests++;
+      $tot_failed++;
+    }
+    elsif ( $tinfo->{'result'} eq 'MTR_RES_SKIPPED' )
     {
+      # Test was skipped
       $tot_skiped++;
     }
     elsif ( $tinfo->{'result'} eq 'MTR_RES_PASSED' )
     {
+      # Test passed
       $tot_tests++;
       $tot_passed++;
     }
-    elsif ( $tinfo->{'result'} eq 'MTR_RES_FAILED' )
-    {
-      $tot_tests++;
-      $tot_failed++;
-    }
+
     if ( $tinfo->{'restarted'} )
     {
+      # Servers was restarted
       $tot_restarts++;
     }
 
@@ -207,7 +236,7 @@ sub mtr_report_stats ($) {
   # ----------------------------------------------------------------------
   print "The servers were restarted $tot_restarts times\n";
 
-  if ( $::opt_timer )
+  if ( $timer )
   {
     use English;
 
@@ -249,6 +278,14 @@ sub mtr_report_stats ($) {
   # Print a list of testcases that failed
   if ( $tot_failed != 0 )
   {
+
+    # Print each failed test, again
+    #foreach my $test ( @$tests ){
+    #  if ( $test->{failures} ) {
+    #    mtr_report_test($test);
+    #  }
+    #}
+
     my $ratio=  $tot_passed * 100 / $tot_tests;
     print "Failed $tot_failed/$tot_tests tests, ";
     printf("%.2f", $ratio);
@@ -262,7 +299,7 @@ sub mtr_report_stats ($) {
     foreach my $tinfo (@$tests)
     {
       my $tname= $tinfo->{'name'};
-      if ( $tinfo->{'result'} eq 'MTR_RES_FAILED' and ! $seen{$tname})
+      if ( $tinfo->{failures} and ! $seen{$tname})
       {
         print " $tname";
 	$seen{$tname}= 1;
@@ -280,7 +317,7 @@ sub mtr_report_stats ($) {
    }
   else
   {
-    print "All $tot_tests tests were successful.\n";
+    print "All $tot_tests tests were successful.\n\n";
   }
 
   if ( $tot_failed != 0 || $found_problems)
@@ -309,14 +346,11 @@ sub mtr_print_thick_line {
 
 sub mtr_print_header () {
   print "\n";
-  if ( $::opt_timer )
-  {
-    print "TEST                            RESULT        TIME (ms)\n";
-  }
-  else
-  {
-    print "TEST                            RESULT\n";
-  }
+  printf "TEST";
+  print " " x 38;
+  print "RESULT   ";
+  print "TIME (ms)" if $timer;
+  print "\n";
   mtr_print_line();
   print "\n";
 }
@@ -355,38 +389,50 @@ sub _timestamp {
 		 $tm->hour, $tm->min, $tm->sec, $diff);
 }
 
+# Always print message to screen
+sub mtr_print (@) {
+  print _name(), join(" ", @_), "\n";
+}
+
 
-# Print message to screen
+# Print message to screen if verbose is defined
 sub mtr_report (@) {
-  print join(" ", @_), "\n";
+  if (defined $verbose)
+  {
+    print _name(), join(" ", @_), "\n";
+  }
 }
 
 
 # Print warning to screen
 sub mtr_warning (@) {
-  print STDERR _timestamp(), "mysql-test-run: WARNING: ", join(" ", @_), "\n";
+  print STDERR _name(), _timestamp(),
+    "mysql-test-run: WARNING: ", join(" ", @_), "\n";
 }
 
 
 # Print error to screen and then exit
 sub mtr_error (@) {
-  print STDERR _timestamp(), "mysql-test-run: *** ERROR: ", join(" ", @_), "\n";
+  print STDERR _name(), _timestamp(),
+    "mysql-test-run: *** ERROR: ", join(" ", @_), "\n";
   exit(1);
 }
 
 
 sub mtr_debug (@) {
-  if ( $::opt_verbose > 1 )
+  if ( $verbose > 2 )
   {
-    print STDERR _timestamp(), "####: ", join(" ", @_), "\n";
+    print STDERR _name(),
+      _timestamp(), "####: ", join(" ", @_), "\n";
   }
 }
 
 
 sub mtr_verbose (@) {
-  if ( $::opt_verbose )
+  if ( $verbose )
   {
-    print STDERR _timestamp(), "> ",join(" ", @_),"\n";
+    print STDERR _name(), _timestamp(),
+      "> ",join(" ", @_),"\n";
   }
 }
 
@@ -394,9 +440,10 @@ sub mtr_verbose (@) {
 sub mtr_verbose_restart (@) {
   my ($server, @args)= @_;
   my $proc= $server->{proc};
-  if ( $::opt_verbose_restart )
+  if ( $verbose_restart )
   {
-    print STDERR _timestamp(), "> Restart $proc - ",join(" ", @args),"\n";
+    print STDERR _name(),_timestamp(),
+      "> Restart $proc - ",join(" ", @args),"\n";
   }
 }
 
diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl
index 3a769b9f550..b10a038d15a 100755
--- a/mysql-test/mysql-test-run.pl
+++ b/mysql-test/mysql-test-run.pl
@@ -41,6 +41,7 @@ use Getopt::Long;
 use My::File::Path; # Patched version of File::Path
 use File::Basename;
 use File::Copy;
+use File::Find;
 use File::Temp qw / tempdir /;
 use File::Spec::Functions qw / splitdir /;
 use My::Platform;
@@ -48,9 +49,12 @@ use My::SafeProcess;
 use My::ConfigFactory;
 use My::Options;
 use My::Find;
+use My::SysInfo;
 use mtr_cases;
 use mtr_report;
 use mtr_match;
+use IO::Socket::INET;
+use IO::Select;
 
 require "lib/mtr_process.pl";
 require "lib/mtr_io.pl";
@@ -88,7 +92,6 @@ my $DEFAULT_SUITES= "main,binlog,federated,rpl,rpl_ndb,ndb";
 my $opt_suites;
 
 our $opt_verbose= 0;  # Verbose output, enable with --verbose
-our $opt_verbose_restart= 0;  # Verbose output for restarts
 
 our $exe_mysql;
 our $exe_mysqladmin;
@@ -141,7 +144,7 @@ my $config; # The currently running config
 my $current_config_name; # The currently running config file template
 
 my $opt_baseport;
-my $opt_mtr_build_thread= $ENV{'MTR_BUILD_THREAD'} || "auto";
+my $opt_build_thread= $ENV{'MTR_BUILD_THREAD'} || "auto";
 
 my $opt_record;
 my $opt_report_features;
@@ -150,10 +153,10 @@ my $opt_mark_progress;
 
 my $opt_sleep;
 
-my $opt_testcase_timeout=    15; # 15 minutes
-my $opt_suite_timeout   =   180; # 3 hours
-my $opt_shutdown_timeout=    10; # 10 seconds
-my $opt_start_timeout   =   180; # 180 seconds
+my $opt_testcase_timeout=    15; # minutes
+my $opt_suite_timeout   =   180; # minutes
+my $opt_shutdown_timeout=    10; # seconds
+my $opt_start_timeout   =   180; # seconds
 
 my $opt_start;
 my $opt_start_dirty;
@@ -161,9 +164,9 @@ my $opt_repeat= 1;
 my $opt_retry= 3;
 my $opt_retry_failure= 2;
 
-my $opt_strace_client;
+my $opt_parallel;
 
-our $opt_timer= 1;
+my $opt_strace_client;
 
 our $opt_user;
 
@@ -194,37 +197,69 @@ our %mysqld_variables;
 my $source_dist= 0;
 
 my $opt_max_save_core= $ENV{MTR_MAX_SAVE_CORE} || 5;
-my $num_saved_cores= 0;  # Number of core files saved in vardir/log/ so far.
-
 my $opt_max_save_datadir= $ENV{MTR_MAX_SAVE_DATADIR} || 20;
-my $num_saved_datadir= 0;  # Number of datadirs saved in vardir/log/ so far.
+my $opt_max_test_fail= $ENV{MTR_MAX_TEST_FAIL} || 10;
 
 select(STDOUT);
 $| = 1; # Automatically flush STDOUT
 
 main();
 
+
 sub main {
+  report_option('verbose', 0);
 
-  command_line_setup();
+  # This is needed for test log evaluation in "gen-build-status-page"
+  # in all cases where the calling tool does not log the commands
+  # directly before it executes them, like "make test-force-pl" in RPM builds.
+  mtr_report("Logging: $0 ", join(" ", @ARGV));
 
-  mtr_report("Checking supported features...");
+  Getopt::Long::Configure("pass_through");
+  GetOptions('parallel=i' => \$opt_parallel) or usage("Can't read options");
 
-  check_ndbcluster_support(\%mysqld_variables);
-  check_ssl_support(\%mysqld_variables);
-  check_debug_support(\%mysqld_variables);
+  if ( not defined $opt_parallel ) {
+    # Try to find a suitable value for number of workers
+    my $sys_info= My::SysInfo->new();
 
-  executable_setup();
+    $opt_parallel= $sys_info->num_cpus();
+    for my $limit (2000, 1500, 1000, 500){
+      $opt_parallel-- if ($sys_info->min_bogomips() < $limit);
+    }
+    $opt_parallel= 1 if ($opt_parallel < 1);
+    mtr_report("Using parallel: $opt_parallel");
+  }
 
-  environment_setup();
+  # Create server socket on any free port
+  my $server = new IO::Socket::INET
+    (
+     LocalAddr => 'localhost',
+     Proto => 'tcp',
+     Listen => $opt_parallel,
+    );
+  mtr_error("Could not create testcase server port: $!") unless $server;
+  my $server_port = $server->sockport();
+  mtr_report("Using server port $server_port");
+
+  # Create child processes
+  my %children;
+  for my $child_num (1..$opt_parallel){
+    my $child_pid= My::SafeProcess::Base::_safe_fork();
+    if ($child_pid == 0){
+      $server= undef; # Close the server port in child
+      run_worker($server_port, $child_num);
+      exit(1);
+    }
 
-  if ( $opt_gcov )
-  {
+    $children{$child_pid}= 1;
+  }
+
+  command_line_setup(0);
+
+  if ( $opt_gcov ) {
     gcov_prepare();
   }
 
-  if (!$opt_suites)
-  {
+  if (!$opt_suites) {
     $opt_suites= $DEFAULT_SUITES;
 
     # Check for any extra suites to enable based on the path name
@@ -238,10 +273,9 @@ sub main {
        "mysql-6.0-ndb"                  => "ndb_team",
       );
 
-    foreach my $dir ( reverse splitdir($basedir) )
-    {
+    foreach my $dir ( reverse splitdir($basedir) ) {
       my $extra_suite= $extra_suites{$dir};
-      if (defined $extra_suite){
+      if (defined $extra_suite) {
 	mtr_report("Found extra suite: $extra_suite");
 	$opt_suites= "$extra_suite,$opt_suites";
 	last;
@@ -252,26 +286,377 @@ sub main {
   mtr_report("Collecting tests...");
   my $tests= collect_test_cases($opt_suites, \@opt_cases);
 
-  initialize_servers();
-
   if ( $opt_report_features ) {
     # Put "report features" as the first test to run
-    my $tinfo = {};
-    $tinfo->{'name'} = 'report_features';
-    $tinfo->{'result_file'} = undef; # Prints result
-    $tinfo->{'path'} = 'include/report-features.test';
-    $tinfo->{'master_opt'} = [];
-    $tinfo->{'slave_opt'} = [];
+    my $tinfo = My::Test->new
+      (
+       name           => 'report_features',
+       result_file    => undef, # Prints result
+       path           => 'include/report-features.test'.
+       master_opt     => [],
+       slave_opt      => [],
+      );
     unshift(@$tests, $tinfo);
   }
 
+  initialize_servers();
+
+  mtr_report();
+  mtr_print_thick_line();
+  mtr_print_header();
+
+  my $num_tests= @$tests;
+  my $completed= run_test_server($server, $tests, $opt_parallel);
+
+  # Send Ctrl-C to any children still running
+  kill("INT", keys(%children));
+
+  # Wait for childs to exit
+  foreach my $pid (keys %children)
+  {
+    my $ret_pid= waitpid($pid, 0);
+    if ($ret_pid != $pid){
+      mtr_report("Unknown process $ret_pid exited");
+    }
+    else {
+      delete $children{$ret_pid};
+    }
+  }
+
+  if ( not defined @$completed ) {
+    mtr_error("Test suite aborted");
+  }
+
+  if ( @$completed != $num_tests){
+
+    if ($opt_force){
+      # All test should have been run, print any that are still in $tests
+      foreach my $test ( @$tests ){
+	$test->print_test();
+      }
+    }
+
+    # Not all tests completed, failure
+    mtr_report();
+    mtr_report("Only ", int(@$completed), " of $num_tests completed.");
+    mtr_error("Not all tests completed");
+  }
+
+  mtr_print_line();
+
+  mtr_report_stats($completed);
+
+  exit(0);
+}
+
+
+sub run_test_server {
+  my ($server, $tests, $childs) = @_;
+
+  my $num_saved_cores= 0;  # Number of core files saved in vardir/log/ so far.
+  my $num_saved_datadir= 0;  # Number of datadirs saved in vardir/log/ so far.
+  my $num_failed_test= 0; # Number of tests failed so far
+
+  # Scheduler variables
+  my $max_ndb= $opt_parallel / 2;
+  $max_ndb = 4 if $max_ndb > 4;
+  $max_ndb = 1 if $max_ndb < 1;
+  my $num_ndb_tests= 0;
+
+  my $completed= [];
+  my %running;
+  my $result;
+
+  my $suite_timeout_proc= My::SafeProcess->timer($opt_suite_timeout * 60);
+
+  my $s= IO::Select->new();
+  $s->add($server);
+  while (1) {
+    my @ready = $s->can_read(1); # Wake up once every second
+    foreach my $sock (@ready) {
+      if ($sock == $server) {
+	# New client connected
+	my $child= $sock->accept();
+	mtr_verbose("Client connected");
+	$s->add($child);
+	print $child "HELLO\n";
+      }
+      else {
+	my $line= <$sock>;
+	if (!defined $line) {
+	  # Client disconnected
+	  mtr_verbose("Child closed socket");
+	  $s->remove($sock);
+	  if (--$childs == 0){
+	    $suite_timeout_proc->kill();
+	    return $completed;
+	  }
+	  next;
+	}
+	chomp($line);
+
+	if ($line eq 'TESTRESULT'){
+	  $result= My::Test::read_test($sock);
+	  # $result->print_test();
+
+	  # Report test status
+	  mtr_report_test($result);
+
+	  if ( $result->is_failed() ) {
+
+	    # Save the workers "savedir" in var/log
+	    my $worker_savedir= $result->{savedir};
+	    my $worker_savename= basename($worker_savedir);
+	    my $savedir= "$opt_vardir/log/$worker_savename";
+
+	    if ($opt_max_save_datadir > 0 &&
+		$num_saved_datadir >= $opt_max_save_datadir)
+	    {
+	      mtr_report(" - skipping '$worker_savedir/'");
+	      rmtree($worker_savedir);
+	    }
+	    else {
+	      mtr_report(" - saving '$worker_savedir/' to '$savedir/'");
+	      rename($worker_savedir, $savedir);
+
+	      if ($opt_max_save_core > 0) {
+		# Limit number of core files saved
+		find({ no_chdir => 1,
+		       wanted => sub {
+			 my $core_file= $File::Find::name;
+			 my $core_name= basename($core_file);
+
+			 if ($core_name =~ "core*"){
+			   if ($num_saved_cores >= $opt_max_save_core) {
+			     mtr_report(" - deleting '$core_name'",
+				      "($num_saved_cores/$opt_max_save_core)");
+			     unlink("$core_file");
+			   }
+			   else {
+			     mtr_report(" - found '$core_name'",
+				      "($num_saved_cores/$opt_max_save_core)");
+			   }
+			   ++$num_saved_cores;
+			 }
+		       }
+		     },
+		     $savedir);
+	      }
+	    }
+	    $num_saved_datadir++;
+
+	    if ( !$opt_force ) {
+	      # Test has failed, force is off
+	      $suite_timeout_proc->kill();
+	      push(@$completed, $result);
+	      return $completed;
+	    }
+	    elsif ($num_failed_test > 0 and
+		   $num_failed_test >= $opt_max_test_fail) {
+	      $suite_timeout_proc->kill();
+	      mtr_report("Too many tests($num_failed_test) failed!",
+			 "Terminating...");
+	      return undef;
+	    }
+	    $num_failed_test++;
+	  }
+
+	  # Retry test run after test failure
+	  my $retries= $result->{retries} || 2;
+	  my $test_has_failed= $result->{failures} || 0;
+	  if ($test_has_failed and $retries <= $opt_retry){
+	    # Test should be run one more time unless it has failed
+	    # too many times already
+	    my $failures= $result->{failures};
+	    if ($opt_retry > 1 and $failures >= $opt_retry_failure){
+	      mtr_report("\nTest has failed $failures times,",
+			 "no more retries!\n");
+	    }
+	    else {
+	      mtr_report("\nRetrying test, attempt($retries/$opt_retry)...\n");
+	      delete($result->{result});
+	      $result->{retries}= $retries+1;
+	      $result->write_test($sock, 'TESTCASE');
+	      next;
+	    }
+	  }
+
+	  # Repeat test $opt_repeat number of times
+	  my $repeat= $result->{repeat} || 1;
+	  if ($repeat < $opt_repeat)
+	  {
+	    $result->{retries}= 0;
+	    $result->{failures}= 0;
+	    delete($result->{result});
+	    $result->{repeat}= $repeat+1;
+	    $result->write_test($sock, 'TESTCASE');
+	    next;
+	  }
+
+	  # Remove from list of running
+	  mtr_error("'", $result->{name},"' is not known to be running")
+	    unless delete $running{$result->key()};
+
+	  # Update scheduler variables
+	  $num_ndb_tests-- if ($result->{ndb_test});
+
+	  # Save result in completed list
+	  push(@$completed, $result);
+
+	}
+	elsif ($line eq 'START'){
+	  ; # Send first test
+	}
+	else {
+	  mtr_error("Unknown response: '$line' from client");
+	}
+
+	# Find next test to schedule
+	# - Try to use same configuration as worker used last time
+	# - Limit number of parallel ndb tests
+
+	my $next;
+	my $second_best;
+	for(my $i= 0; $i <= $#$tests; $i++)
+	{
+	  my $t= $tests->[$i];
+
+	  if (run_testcase_check_skip_test($t)){
+	    # Move the test to completed list
+	    #mtr_report("skip - Moving test $i to completed");
+	    push(@$completed, splice(@$tests, $i, 1));
+	    redo; # Start over again
+	  }
+
+	  # Limit number of parallell NDB tests
+	  if ($t->{ndb_test} and $num_ndb_tests >= $max_ndb){
+	    #mtr_report("Skipping, num ndb is already at max, $num_ndb_tests");
+	    next;
+	  }
+
+	  # Prefer same configuration
+	  if (defined $result and
+	      $result->{template_path} eq $t->{template_path})
+	  {
+	    #mtr_report("Test uses same config => good match");
+	    # Test uses same config => good match
+	    $next= splice(@$tests, $i, 1);
+	    last;
+	  }
+
+	  # Second best choice is the first that does not fulfill
+	  # any of the above conditions
+	  if (!defined $second_best){
+	    #mtr_report("Setting second_best to $i");
+	    $second_best= $i;
+	  }
+	}
+
+	# Use second best choice if no other test has been found
+	if (!$next and defined $second_best){
+	  #mtr_report("Take second best choice $second_best");
+	  mtr_error("Internal error, second best too large")
+	    if $second_best >  $#$tests;
+	  $next= splice(@$tests, $second_best, 1);
+	}
+
+	if ($next) {
+	  #$next->print_test();
+	  $next->write_test($sock, 'TESTCASE');
+	  $running{$next->key()}= $next;
+	  $num_ndb_tests++ if ($next->{ndb_test});
+	}
+	else {
+	  # No more test, tell child to exit
+	  #mtr_report("Saying BYE to child");
+	  print $sock "BYE\n";
+	}
+      }
+    }
+
+    # ----------------------------------------------------
+    # Check if test suite timer expired
+    # ----------------------------------------------------
+    if ( ! $suite_timeout_proc->wait_one(0) )
+    {
+      mtr_report("Test suite timeout! Terminating...");
+      return undef;
+    }
+  }
+}
+
+
+sub run_worker ($) {
+  my ($server_port, $thread_num)= @_;
+
+  $SIG{INT}= sub { exit(1); };
+
+  report_option('name',"worker[$thread_num]");
+
+  # Connect to server
+  my $server = new IO::Socket::INET
+    (
+     PeerAddr => 'localhost',
+     PeerPort => $server_port,
+     Proto    => 'tcp'
+    );
+  mtr_error("Could not connect to server at port $server_port: $!")
+    unless $server;
+
+  # Read hello from server which it will send when shared
+  # resources have been setup
+  my $hello= <$server>;
+
+  command_line_setup($thread_num);
+
+  if ( $opt_gcov )
+  {
+    gcov_prepare();
+  }
+
+  setup_vardir();
+  mysql_install_db($thread_num);
+
   if ( using_extern() ) {
     create_config_file_for_extern(%opts_extern);
   }
 
-  run_tests($tests);
+  # Ask server for first test
+  print $server "START\n";
 
-  exit(0);
+  while(my $line= <$server>){
+    chomp($line);
+    if ($line eq 'TESTCASE'){
+      my $test= My::Test::read_test($server);
+      #$test->print_test();
+      run_testcase($test);
+      #$test->{result}= 'MTR_RES_PASSED';
+      # Send it back, now with results set
+      #$test->print_test();
+      $test->write_test($server, 'TESTRESULT');
+    }
+    elsif ($line eq 'BYE'){
+      mtr_report("Server said BYE");
+      exit(0);
+    }
+    else {
+      mtr_error("Could not understand server, '$line'");
+    }
+  }
+
+  stop_all_servers();
+
+  if ( $opt_gcov )
+  {
+    gcov_collect(); # collect coverage information
+  }
+
+  if ( $opt_gcov )
+  {
+    gcov_collect(); # collect coverage information
+  }
+
+  exit(1);
 }
 
 
@@ -281,14 +666,11 @@ sub ignore_option {
 }
 
 sub command_line_setup {
+  my ($thread_num)= @_;
+
   my $opt_comment;
   my $opt_usage;
 
-  # This is needed for test log evaluation in "gen-build-status-page"
-  # in all cases where the calling tool does not log the commands
-  # directly before it executes them, like "make test-force-pl" in RPM builds.
-  mtr_report("Logging: $0 ", join(" ", @ARGV));
-
   # Read the command line options
   # Note: Keep list, and the order, in sync with usage at end of this file
   Getopt::Long::Configure("pass_through");
@@ -328,7 +710,7 @@ sub command_line_setup {
 	     'skip-im'                  => \&ignore_option,
 
              # Specify ports
-	     'build-thread|mtr-build-thread=i' => \$opt_mtr_build_thread,
+	     'build-thread|mtr-build-thread=i' => \$opt_build_thread,
 
              # Test case authoring
              'record'                   => \$opt_record,
@@ -355,6 +737,7 @@ sub command_line_setup {
              'strace-client:s'          => \$opt_strace_client,
              'max-save-core=i'          => \$opt_max_save_core,
              'max-save-datadir=i'       => \$opt_max_save_datadir,
+             'max-test-fail=i'          => \$opt_max_test_fail,
 
              # Coverage, profiling etc
              'gcov'                     => \$opt_gcov,
@@ -387,10 +770,10 @@ sub command_line_setup {
              'report-features'          => \$opt_report_features,
              'comment=s'                => \$opt_comment,
              'fast'                     => \$opt_fast,
-             'reorder'                  => \&collect_option,
+             'reorder!'                 => \&collect_option,
              'enable-disabled'          => \&collect_option,
              'verbose+'                 => \$opt_verbose,
-             'verbose-restart'          => \$opt_verbose_restart,
+             'verbose-restart'          => \&report_option,
              'sleep=i'                  => \$opt_sleep,
              'start-dirty'              => \$opt_start_dirty,
              'start'                    => \$opt_start,
@@ -398,13 +781,14 @@ sub command_line_setup {
 	     'repeat=i'                 => \$opt_repeat,
 	     'retry=i'                  => \$opt_retry,
 	     'retry-failure=i'          => \$opt_retry_failure,
-             'timer!'                   => \$opt_timer,
+             'timer!'                   => \&report_option,
              'user=s'                   => \$opt_user,
              'testcase-timeout=i'       => \$opt_testcase_timeout,
              'suite-timeout=i'          => \$opt_suite_timeout,
              'shutdown-timeout=i'       => \$opt_shutdown_timeout,
              'warnings!'                => \$opt_warnings,
 	     'timestamp'                => \&report_option,
+	     'timediff'                 => \&report_option,
 
              'help|h'                   => \$opt_usage,
             ) or usage("Can't read options");
@@ -412,9 +796,21 @@ sub command_line_setup {
   usage("") if $opt_usage;
 
   # --------------------------------------------------------------------------
-  # Check mtr_build_thread and calculate baseport
+  # Setup verbosity
+  # --------------------------------------------------------------------------
+  if ($thread_num == 0){
+    # The server should by default have verbose on
+    report_option('verbose', $opt_verbose ? $opt_verbose : 0);
+  } else {
+    # Worker should by default have verbose off
+    report_option('verbose', $opt_verbose ? $opt_verbose : undef);
+  }
+
+  # --------------------------------------------------------------------------
+  # Check build_thread and calculate baseport
+  # Use auto build thread in all but first worker
   # --------------------------------------------------------------------------
-  set_mtr_build_thread_ports($opt_mtr_build_thread);
+  set_build_thread_ports($thread_num > 1 ? 'auto' : $opt_build_thread);
 
   if ( -d "../sql" )
   {
@@ -541,8 +937,9 @@ sub command_line_setup {
 
   # --------------------------------------------------------------------------
   # Check if we should speed up tests by trying to run on tmpfs
+  # - Dont check in workers
   # --------------------------------------------------------------------------
-  if ( defined $opt_mem )
+  if ( defined $opt_mem and $thread_num == 0)
   {
     mtr_error("Can't use --mem and --vardir at the same time ")
       if $opt_vardir;
@@ -558,7 +955,7 @@ sub command_line_setup {
     {
       if ( -d $fs )
       {
-	my $template= "var_${opt_mtr_build_thread}_XXXX";
+	my $template= "var_${opt_build_thread}_XXXX";
 	$opt_mem= tempdir( $template, DIR => $fs, CLEANUP => 0);
 	last;
       }
@@ -574,6 +971,11 @@ sub command_line_setup {
     $opt_vardir= $default_vardir;
   }
 
+  # If more than one parallel run, use a subdir of the selected var
+  if ($thread_num && $opt_parallel > 1) {
+    $opt_vardir.= "/".$thread_num;
+   }
+
   $path_vardir_trace= $opt_vardir;
   # Chop off any "c:", DBUG likes a unix path ex: c:/src/... => /src/...
   $path_vardir_trace=~ s/^\w://;
@@ -595,6 +997,11 @@ sub command_line_setup {
   $opt_tmpdir=       "$opt_vardir/tmp" unless $opt_tmpdir;
   $opt_tmpdir =~ s,/+$,,;       # Remove ending slash if any
 
+  # If more than one parallel run, use a subdir of the selected tmpdir
+  if ($thread_num && $opt_parallel > 1 and $opt_tmpdir ne "$opt_vardir/tmp") {
+    $opt_tmpdir.= "/".$thread_num;
+   }
+
   # --------------------------------------------------------------------------
   # fast option
   # --------------------------------------------------------------------------
@@ -672,6 +1079,11 @@ sub command_line_setup {
   }
 
   # --------------------------------------------------------------------------
+  # Set timeout values
+  # --------------------------------------------------------------------------
+  $opt_start_timeout*= $opt_parallel;
+
+  #
   # Check valgrind arguments
   # --------------------------------------------------------------------------
   if ( $opt_valgrind or $opt_valgrind_path or @valgrind_args)
@@ -727,6 +1139,16 @@ sub command_line_setup {
   $path_testlog=         "$opt_vardir/log/mysqltest.log";
   $path_current_testlog= "$opt_vardir/log/current_test";
 
+  mtr_report("Checking supported features...");
+
+  check_ndbcluster_support(\%mysqld_variables);
+  check_ssl_support(\%mysqld_variables);
+  check_debug_support(\%mysqld_variables);
+
+  executable_setup();
+
+  environment_setup();
+
 }
 
 
@@ -745,19 +1167,20 @@ sub command_line_setup {
 # http://www.ncftp.com/ncftpd/doc/misc/ephemeral_ports.html
 # But a fairly safe range seems to be 5001 - 32767
 #
-sub set_mtr_build_thread_ports($) {
-  my $mtr_build_thread= shift || 0;
+sub set_build_thread_ports($) {
+  my $build_thread= shift || 0;
 
-  if ( lc($mtr_build_thread) eq 'auto' ) {
-    print "Requesting build thread... ";
-    $mtr_build_thread=
+  if ( lc($build_thread) eq 'auto' ) {
+    mtr_report("Requesting build thread... ");
+    $build_thread=
       mtr_require_unique_id_and_wait("/tmp/mysql-test-ports", 200, 299);
-    print "got ".$mtr_build_thread."\n";
+    mtr_report(" - got $build_thread");
   }
-  $ENV{MTR_BUILD_THREAD}= $mtr_build_thread;
+  $ENV{MTR_BUILD_THREAD}= $build_thread;
+  $opt_build_thread= $build_thread;
 
   # Calculate baseport
-  $opt_baseport= $mtr_build_thread * 10 + 10000;
+  $opt_baseport= $build_thread * 10 + 10000;
   if ( $opt_baseport < 5001 or $opt_baseport + 9 >= 32767 )
   {
     mtr_error("MTR_BUILD_THREAD number results in a port",
@@ -765,7 +1188,7 @@ sub set_mtr_build_thread_ports($) {
               "($opt_baseport - $opt_baseport + 9)");
   }
 
-  mtr_report("Using MR_BUILD_THREAD $mtr_build_thread,",
+  mtr_report("Using MTR_BUILD_THREAD $build_thread,",
 	     "with reserved ports $opt_baseport..".($opt_baseport+9));
 
 }
@@ -1090,8 +1513,8 @@ sub environment_setup {
 
   $ENV{'UDF_EXAMPLE_LIB'}=
     ($lib_udf_example ? basename($lib_udf_example) : "");
-  $ENV{'UDF_EXAMPLE_LIB_OPT'}=
-    ($lib_udf_example ? "--plugin_dir=" . dirname($lib_udf_example) : "");
+  $ENV{'UDF_EXAMPLE_LIB_OPT'}= "--plugin-dir=".
+    ($lib_udf_example ? dirname($lib_udf_example) : "");
 
   # --------------------------------------------------------------------------
   # Add the path where mysqld will find ha_example.so
@@ -1102,9 +1525,9 @@ sub environment_setup {
 		      "$basedir/storage/example/.libs/ha_example.so",);
     $ENV{'EXAMPLE_PLUGIN'}=
       ($lib_example_plugin ? basename($lib_example_plugin) : "");
-    $ENV{'EXAMPLE_PLUGIN_OPT'}=
-      ($lib_example_plugin ?
-       "--plugin_dir=" . dirname($lib_example_plugin) : "");
+    $ENV{'EXAMPLE_PLUGIN_OPT'}= "--plugin-dir=".
+      ($lib_example_plugin ? dirname($lib_example_plugin) : "");
+
   }
 
   # --------------------------------------------------------------------------
@@ -1366,7 +1789,7 @@ sub remove_stale_vardir () {
 # Create var and the directories needed in var
 #
 sub setup_vardir() {
-  mtr_report("Creating var directory...");
+  mtr_report("Creating var directory '$opt_vardir'...");
 
   if ( $opt_vardir eq $default_vardir )
   {
@@ -1575,7 +1998,7 @@ sub ndbcluster_wait_started($$){
   mtr_init_args(\$args);
   mtr_add_arg($args, "--defaults-file=%s", $path_config_file);
   mtr_add_arg($args, "--defaults-group-suffix=%s", $cluster->suffix());
-  mtr_add_arg($args, "--timeout=60");
+  mtr_add_arg($args, "--timeout=%d", $opt_start_timeout);
 
   if ($ndb_waiter_extra_opt)
   {
@@ -1754,76 +2177,6 @@ sub ndbcluster_start ($) {
 }
 
 
-#
-# Run the collected tests
-#
-my $suite_timeout_proc;
-sub run_tests {
-  my ($tests)= @_;
-
-  mtr_report();
-  mtr_print_thick_line();
-  mtr_print_header();
-
-  $suite_timeout_proc= My::SafeProcess->timer($opt_suite_timeout* 60);
-  foreach my $tinfo ( @$tests )
-  {
-    if (run_testcase_check_skip_test($tinfo))
-    {
-      next;
-    }
-
-    for my $repeat (1..$opt_repeat){
-
-      if (run_testcase($tinfo))
-      {
-	# Testcase failed, enter retry mode
-	my $retries= 1;
-	while ($retries < $opt_retry){
-	  mtr_report("\nRetrying, attempt($retries/$opt_retry)...\n");
-
-	  if (run_testcase($tinfo) <= 0)
-	  {
-	    # Testcase suceeded
-
-	    my $test_has_failed= $tinfo->{failures} || 0;
-	    if (!$test_has_failed){
-	      last;
-	    }
-	  }
-	  else
-	  {
-	    # Testcase failed
-
-	    # Limit number of test failures
-	    my $failures= $tinfo->{failures};
-	    if ($opt_retry > 1 and $failures >= $opt_retry_failure){
-	      mtr_report("Test has failed $failures times, no more retries!\n");
-	      last;
-	    }
-	  }
-	  $retries++;
-	}
-      }
-    }
-  }
-  # Kill the test suite timer
-  $suite_timeout_proc->kill();
-
-  mtr_print_line();
-
-  stop_all_servers();
-
-  if ( $opt_gcov )
-  {
-    gcov_collect(); # collect coverage information
-  }
-
-  mtr_report_stats($tests);
-
-}
-
-
 sub create_config_file_for_extern {
   my %opts=
     (
@@ -1971,7 +2324,7 @@ sub initialize_servers {
       remove_stale_vardir();
       setup_vardir();
 
-      mysql_install_db();
+      mysql_install_db(0);
     }
   }
   check_running_as_root();
@@ -2028,6 +2381,7 @@ sub sql_to_bootstrap {
 
 
 sub mysql_install_db {
+  my ($thread_num)= @_;
   my $data_dir= "$opt_vardir/install.db";
 
   mtr_report("Installing system database...");
@@ -2064,6 +2418,8 @@ sub mysql_install_db {
   # ----------------------------------------------------------------------
   $ENV{'MYSQLD_BOOTSTRAP_CMD'}= "$exe_mysqld_bootstrap " . join(" ", @$args);
 
+  return if $thread_num > 0; # Only generate MYSQLD_BOOTSTRAP_CMD in workers
+
   # ----------------------------------------------------------------------
   # Create the bootstrap.sql file
   # ----------------------------------------------------------------------
@@ -2222,25 +2578,111 @@ sub do_before_run_mysqltest($)
 }
 
 
+#
+# Check all server for sideffects
+#
+# RETURN VALUE
+#  0 ok
+#  1 Check failed
+#  >1 Fatal errro
+
 sub check_testcase($$)
 {
   my ($tinfo, $mode)= @_;
-  my $result;
+  my $tname= $tinfo->{name};
 
-  # Parallell( mysqlds(), run_check_testcase, check_testcase_failed );
+  # Start the mysqltest processes in parallel to save time
+  # also makes it possible to wait for any process to exit during the check
+  my %started;
   foreach my $mysqld ( mysqlds() )
   {
     if ( defined $mysqld->{'proc'} )
     {
-      if (run_check_testcase($tinfo, $mode, $mysqld))
+      my $proc= start_check_testcase($tinfo, $mode, $mysqld);
+      $started{$proc->pid()}= $proc;
+    }
+  }
+
+  # Return immediately if no check proceess was started
+  return 0 unless ( keys %started );
+
+  while (1){
+    my $result;
+    my $proc= My::SafeProcess->wait_any();
+    mtr_report("Got $proc");
+
+    if ( delete $started{$proc->pid()} ) {
+
+      my $err_file= $proc->user_data();
+      my $base_file= mtr_match_extension($err_file, "err"); # Trim extension
+
+      # One check testcase process returned
+      my $res= $proc->exit_status();
+
+      if ( $res == 0){
+	# Check completed without problem
+
+	# Remove the .err file the check generated
+	unlink($err_file);
+
+	# Remove the .result file the check generated
+	if ( $mode eq 'after' ){
+	  unlink("$base_file.result");
+	}
+
+	if ( keys(%started) == 0){
+	  # All checks completed
+	  return 0;
+	}
+	# Wait for next process to exit
+	next;
+      }
+      else
       {
-	# Check failed, mark the test case with that info
-	$tinfo->{'check_testcase_failed'}= 1;
-	$result= 1;
+	if ( $mode eq "after" and $res == 1 )
+	{
+	  # Test failed, grab the report mysqltest has created
+	  my $report= mtr_grab_file($err_file);
+	  $tinfo->{check}.=
+	    "\nThe check of testcase '$tname' failed, this is the\n".
+	      "diff between before and after:\n";
+	  $tinfo->{check}.= $report;
+
+	  # Check failed, mark the test case with that info
+	  $tinfo->{'check_testcase_failed'}= 1;
+	  $result= 1;
+	}
+	elsif ( $res )
+	{
+	  my $report= mtr_grab_file($err_file);
+	  $tinfo->{comment}.=
+	    "Could not execute 'check-testcase' $mode testcase '$tname':\n";
+	  $tinfo->{comment}.= $report;
+
+	  $result= 2;
+	}
+
+	# Remove the .err file the check generated
+	unlink($err_file);
+
+	# Remove the .result file the check generated
+	unlink("$base_file.result");
+
       }
     }
+    else {
+      # Unknown process returned, most likley a crash, abort everything
+      $tinfo->{comment}=
+	"Unexpected process $proc returned during ".
+	"check testcase $mode test";
+      $result= 3;
+    }
+
+    # Kill any check processes still running
+    map($_->kill(), values(%started));
+
+    return $result;
   }
-  return $result;
 }
 
 
@@ -2391,16 +2833,15 @@ sub run_testcase ($) {
   # ----------------------------------------------------------------------
   if ( $opt_start or $opt_start_dirty )
   {
-    $suite_timeout_proc->kill();
-    mtr_report("\nStarted", started(all_servers()));
-    mtr_report("Waiting for server(s) to exit...");
+    mtr_print("\nStarted", started(all_servers()));
+    mtr_print("Waiting for server(s) to exit...");
     my $proc= My::SafeProcess->wait_any();
     if ( grep($proc eq $_, started(all_servers())) )
     {
-      mtr_report("Server $proc died");
+      mtr_print("Server $proc died");
       exit(1);
     }
-    mtr_report("Unknown process $proc died");
+    mtr_print("Unknown process $proc died");
     exit(1);
   }
 
@@ -2408,9 +2849,14 @@ sub run_testcase ($) {
 
   do_before_run_mysqltest($tinfo);
 
-  if ( $opt_check_testcases )
-  {
-    check_testcase($tinfo, "before")
+  if ( $opt_check_testcases and check_testcase($tinfo, "before") ){
+    # Failed to record state of server or server crashed
+    report_failure_and_restart($tinfo);
+
+    # Stop the test case timer
+    $test_timeout_proc->kill();
+
+    return 1;
   }
 
   my $test= start_mysqltest($tinfo);
@@ -2444,15 +2890,23 @@ sub run_testcase ($) {
 	}
 	else
 	{
-	  mtr_report_test_passed($tinfo, $opt_timer);
+	  mtr_report_test_passed($tinfo);
 	}
 
-	if ( $opt_check_testcases and check_testcase($tinfo, "after"))
+	my $check_res;
+	if ( $opt_check_testcases and
+	     $check_res= check_testcase($tinfo, "after"))
 	{
-	  # Stop all servers that are known to be running
-	  stop_all_servers();
-	  clean_datadir();
-	  mtr_report("Resuming tests...\n");
+	  if ($check_res == 1) {
+	    # Test case had sideeffects, not fatal error, just continue
+	    stop_all_servers();
+	    mtr_report("Resuming tests...\n");
+	  }
+	  else {
+	    # Test case check failed fatally, probably a server crashed
+	    report_failure_and_restart($tinfo);
+	    return 1;
+	  }
 	}
       }
       elsif ( $res == 62 )
@@ -2536,15 +2990,6 @@ sub run_testcase ($) {
       return 1;
     }
 
-    # ----------------------------------------------------
-    # Check if test suite timer expired
-    # ----------------------------------------------------
-    if ( $proc eq $suite_timeout_proc )
-    {
-      mtr_report("Test suite timeout! Terminating...");
-      exit(1);
-    }
-
     mtr_error("Unhandled process $proc exited");
   }
   mtr_error("Should never come here");
@@ -2557,7 +3002,7 @@ sub run_testcase ($) {
 #  0 OK
 #  1 Check failed
 #
-sub run_check_warnings ($$) {
+sub start_check_warnings ($$) {
   my $tinfo=    shift;
   my $mysqld=   shift;
 
@@ -2575,37 +3020,17 @@ sub run_check_warnings ($$) {
   mtr_add_arg($args, "--test-file=%s", "include/check-warnings.test");
 
   my $errfile= "$opt_vardir/tmp/$name.err";
-  my $res= My::SafeProcess->run
+  my $proc= My::SafeProcess->new
     (
      name          => $name,
      path          => $exe_mysqltest,
      error         => $errfile,
      output        => $errfile,
      args          => \$args,
+     user_data     => $errfile,
     );
-
-  if ( $res == 0 )
-  {
-    my $report= mtr_grab_file($errfile);
-    # Log to var/log/warnings file
-    mtr_tofile("$opt_vardir/log/warnings",
-               $tname."\n",
-               $report);
-
-    $res= 1;
-    $tinfo->{'warnings'}.= $report;
-  }
-  elsif ( $res == 62 ) {
-    # Test case was ok and called "skip"
-    $res= 0;
-  }
-  elsif ( $res )
-  {
-    mtr_report("\nCould not execute 'check-warnings' for testcase '$tname':");
-    mtr_printfile($errfile);
-    $res= 0; # Ignore error
-  }
-  return $res;
+  mtr_verbose("Started $proc");
+  return $proc;
 }
 
 
@@ -2617,17 +3042,94 @@ sub check_warnings ($) {
   my ($tinfo)= @_;
   my $res= 0;
 
+  my $tname= $tinfo->{name};
+
   # Clear previous warnings
-  $tinfo->{warnings}= undef;
+  delete($tinfo->{warnings});
 
-  # Parallell( mysqlds(), run_check_warning, check_warning_failed);
+  # Start the mysqltest processes in parallel to save time
+  # also makes it possible to wait for any process to exit during the check
+  my %started;
   foreach my $mysqld ( mysqlds() )
   {
-    if (run_check_warnings($tinfo, $mysqld)){
-      $res= 1;
-      mtr_report();
+    if ( defined $mysqld->{'proc'} )
+    {
+      my $proc= start_check_warnings($tinfo, $mysqld);
+      $started{$proc->pid()}= $proc;
+    }
+  }
+
+  # Return immediately if no check proceess was started
+  return 0 unless ( keys %started );
+
+  while (1){
+    my $result= 0;
+    my $proc= My::SafeProcess->wait_any();
+    mtr_report("Got $proc");
+
+    if ( delete $started{$proc->pid()} ) {
+      # One check warning process returned
+      my $res= $proc->exit_status();
+      my $err_file= $proc->user_data();
+
+      if ( $res == 0 or $res == 62 ){
+
+	if ( $res == 0 ) {
+	  # Check completed with problem
+	  my $report= mtr_grab_file($err_file);
+	  # Log to var/log/warnings file
+	  mtr_tofile("$opt_vardir/log/warnings",
+		     $tname."\n".$report);
+
+	  $tinfo->{'warnings'}.= $report;
+	  $result= 1;
+	}
+
+	if ( $res == 62 ) {
+	  # Test case was ok and called "skip"
+	  ;
+	}
+	# Remove the .err file the check generated
+	unlink($err_file);
+
+	if ( keys(%started) == 0){
+	  # All checks completed
+	  return $result;
+	}
+	# Wait for next process to exit
+	next;
+      }
+      else
+      {
+	my $report= mtr_grab_file($err_file);
+	$tinfo->{comment}.=
+	  "Could not execute 'check-warnings' for testcase '$tname':";
+	$tinfo->{comment}.= $report;
+
+	$result= 2;
+      }
+      # Remove the .err file the check generated
+      unlink($err_file);
+    }
+    else {
+      # Unknown process returned, most likley a crash, abort everything
+      $tinfo->{comment}=
+	"Unexpected process $proc returned during ".
+	"check warnings";
+      $result= 3;
     }
+
+    # Kill any check processes still running
+    map($_->kill(), values(%started));
+
+    return $result;
   }
+
+
+
+
+  return $res;
+
   return $res;
 }
 
@@ -2663,10 +3165,41 @@ sub check_expected_crash_and_restart {
 }
 
 
+# Remove all files and subdirectories of a directory
+sub clean_dir {
+  my ($dir)= @_;
+  mtr_verbose("clean_dir: $dir");
+  finddepth(
+	  { no_chdir => 1,
+	    wanted => sub {
+	      if (-d $_){
+		# A dir
+		if ($_ eq $dir){
+		  # The dir to clean
+		  return;
+		} else {
+		  mtr_verbose("rmdir: '$_'");
+		  rmdir($_) or mtr_warning("rmdir($_) failed: $!");
+		}
+	      } else {
+		# Hopefully a file
+		mtr_verbose("unlink: '$_'");
+		unlink($_) or mtr_warning("unlink($_) failed: $!");
+	      }
+	    }
+	  },
+	    $dir);
+}
+
+
 sub clean_datadir {
 
   mtr_verbose("Cleaning datadirs...");
 
+  if (started(all_servers()) != 0){
+    mtr_error("Trying to clean datadir before all servers stopped");
+  }
+
   foreach my $cluster ( clusters() )
   {
     my $cluster_dir= "$opt_vardir/".$cluster->{name};
@@ -2685,55 +3218,22 @@ sub clean_datadir {
   }
 
   # Remove all files in tmp and var/tmp
-  rmtree("$opt_vardir/tmp");
-  mkpath("$opt_vardir/tmp");
+  clean_dir("$opt_vardir/tmp");
   if ($opt_tmpdir ne "$opt_vardir/tmp"){
-    rmtree($opt_tmpdir);
-    mkpath($opt_tmpdir);
+    clean_dir($opt_tmpdir);
   }
 }
 
 
 #
-# Limit number of core files saved
-#
-sub limit_cores_after_failure ($) {
-  my ($datadir)= @_;
-
-  # Look for core files
-  foreach my $core_file ( glob("$datadir/core*") )
-  {
-    my $core_name= basename($core_file);
-    if ($opt_max_save_core > 0 && $num_saved_cores >= $opt_max_save_core) {
-      # Delete file to avoid saving it  when the datadir is later saved
-      mtr_report(" - deleting '$core_name'",
-		 "($num_saved_cores/$opt_max_save_core)");
-      unlink("$core_file");
-    }
-    else {
-      mtr_report(" - found '$core_name'",
-		 "($num_saved_cores/$opt_max_save_core)");
-    }
-    ++$num_saved_cores;
-  }
-}
-
-#
 # Save datadir before it's removed
 #
 sub save_datadir_after_failure($$) {
   my ($dir, $savedir)= @_;
 
-  if ($opt_max_save_datadir > 0 &&
-      $num_saved_datadir >= $opt_max_save_datadir)
-  {
-    mtr_report(" - skipping '$dir'");
-  }
-  else {
-    mtr_report(" - saving '$dir'");
-    my $dir_name= basename($dir);
-    rename("$dir", "$savedir/$dir_name");
-  }
+  mtr_report(" - saving '$dir'");
+  my $dir_name= basename($dir);
+  rename("$dir", "$savedir/$dir_name");
 }
 
 
@@ -2748,19 +3248,19 @@ sub after_failure ($) {
   $save_dir.= "-$tinfo->{combination}"
     if defined $tinfo->{combination};
 
+  # Save savedir  path for server
+  $tinfo->{savedir}= $save_dir;
+
   mkpath($save_dir) if ! -d $save_dir;
 
   # Save the used my.cnf file
   copy($path_config_file, $save_dir);
 
+  # Copy the tmp dir
+  copytree("$opt_vardir/tmp/", "$save_dir/tmp/");
+
   if ( clusters() ) {
     foreach my $cluster ( clusters() ) {
-
-      foreach my $server ( ndbds($cluster), ndb_mgmds($cluster) ) {
-	my $data_dir= $server->value('DataDir');
-	limit_cores_after_failure($data_dir);
-      }
-
       my $cluster_dir= "$opt_vardir/".$cluster->{name};
       save_datadir_after_failure($cluster_dir, $save_dir);
     }
@@ -2768,33 +3268,43 @@ sub after_failure ($) {
   else {
     foreach my $mysqld ( mysqlds() ) {
       my $data_dir= $mysqld->value('datadir');
-      limit_cores_after_failure($data_dir);
       save_datadir_after_failure(dirname($data_dir), $save_dir);
     }
   }
-
-  $num_saved_datadir++;
-
-  clean_datadir();
 }
 
 
 sub report_failure_and_restart ($) {
   my $tinfo= shift;
 
-  mtr_report_test_failed($tinfo, $path_current_testlog);
-  print "\n";
-  if ( $opt_force )
-  {
-    # Stop all servers that are known to be running
-    stop_all_servers();
+  stop_all_servers();
 
-    after_failure($tinfo);
-    mtr_report("Resuming tests...\n");
-    return;
+  $tinfo->{'result'}= 'MTR_RES_FAILED';
+
+  my $test_failures= $tinfo->{'failures'} || 0;
+  $tinfo->{'failures'}=  $test_failures + 1;
+
+
+  my $logfile= $path_current_testlog;
+  if ( $tinfo->{comment} )
+  {
+    # The test failure has been detected by mysql-test-run.pl
+    # when starting the servers or due to other error, the reason for
+    # failing the test is saved in "comment"
+    ;
+  }
+  elsif ( defined $logfile and -f $logfile )
+  {
+    # Test failure was detected by test tool and its report
+    # about what failed has been saved to file. Save the report
+    # in tinfo
+    $tinfo->{logfile}= mtr_fromfile($logfile);
   }
-  mtr_error("Test '$tinfo->{'name'}' failed.",
-	    "To continue, re-run with '--force'");
+
+  after_failure($tinfo);
+
+  mtr_report_test($tinfo);
+
 }
 
 
@@ -3007,7 +3517,7 @@ sub mysqld_start ($$) {
 
 sub stop_all_servers () {
 
-  mtr_report("Stopping all servers...");
+  mtr_verbose("Stopping all servers...");
 
   # Kill all started servers
   My::SafeProcess::shutdown(0, # shutdown timeout 0 => kill
@@ -3174,12 +3684,32 @@ sub started { return grep(defined $_, map($_->{proc}, @_));  }
 sub stopped { return grep(!defined $_, map($_->{proc}, @_)); }
 
 
+sub envsubst {
+  my $string= shift;
+
+  if ( ! defined $ENV{$string} )
+  {
+    mtr_error(".opt file references '$string' which is not set");
+  }
+
+  return $ENV{$string};
+}
+
+
 sub get_extra_opts {
   my ($mysqld, $tinfo)= @_;
 
-  return
+  my $opts=
     $mysqld->option("#!use-slave-opt") ?
       $tinfo->{slave_opt} : $tinfo->{master_opt};
+
+  # Expand environment variables
+  foreach my $opt ( @$opts )
+  {
+    $opt =~ s/\$\{(\w+)\}/envsubst($1)/ge;
+    $opt =~ s/\$(\w+)/envsubst($1)/ge;
+  }
+  return $opts;
 }
 
 
@@ -3283,9 +3813,18 @@ sub start_servers($) {
     }
 
     # Copy datadir from installed system db
-    copytree("$opt_vardir/install.db", $datadir)
+    for my $path ( "$opt_vardir", "$opt_vardir/..") {
+      my $install_db= "$path/install.db";
+      copytree($install_db, $datadir)
+	if -d $install_db;
+    }
+    mtr_error("Failed to copy system db to '$datadir'")
       unless -d $datadir;
 
+    # Create the servers tmpdir
+    my $tmpdir= $mysqld->value('tmpdir');
+    mkpath($tmpdir) unless -d $tmpdir;
+
     # Write start of testcase to log file
     mark_log($mysqld->value('log-error'), $tinfo);
 
@@ -3350,10 +3889,9 @@ sub start_servers($) {
 # After testcase, run and compare with the recorded file, they should be equal!
 #
 # RETURN VALUE
-#  0 OK
-#  1 Check failed
+#  The newly started process
 #
-sub run_check_testcase ($$$) {
+sub start_check_testcase ($$$) {
   my $tinfo=    shift;
   my $mode=     shift;
   my $mysqld=   shift;
@@ -3378,28 +3916,17 @@ sub run_check_testcase ($$$) {
     mtr_add_arg($args, "--record");
   }
   my $errfile= "$opt_vardir/tmp/$name.err";
-  my $res= My::SafeProcess->run
+  my $proc= My::SafeProcess->new
     (
      name          => $name,
      path          => $exe_mysqltest,
      error         => $errfile,
      args          => \$args,
+     user_data     => $errfile,
     );
 
-  if ( $mode eq "after" and $res == 1 )
-  {
-    mtr_report("\nThe check of testcase '$tname' failed, this is the\n",
-	       "diff between before and after:\n");
-    # Test failed, display the report mysqltest has created
-    mtr_printfile($errfile);
-  }
-  elsif ( $res )
-  {
-    mtr_report("\nCould not execute 'check-testcase' $mode testcase '$tname':");
-    mtr_printfile($errfile);
-    mtr_report();
-  }
-  return $res;
+  mtr_report("Started $proc");
+  return $proc;
 }
 
 
@@ -3457,10 +3984,7 @@ sub start_mysqltest ($) {
     mtr_add_arg($args, "$exe_mysqltest");
   }
 
-  if ( $opt_timer )
-  {
-    mtr_add_arg($args, "--timer-file=%s/log/timer", $opt_vardir);
-  }
+  mtr_add_arg($args, "--timer-file=%s/log/timer", $opt_vardir);
 
   if ( $opt_compress )
   {
@@ -3903,6 +4427,10 @@ Options for debugging the product
                         up disks for heavily crashing server). Defaults to
                         $opt_max_save_datadir, set to 0 for no limit. Set
                         it's default with MTR_MAX_SAVE_DATDIR
+  max-test-fail         Limit the number of test failurs before aborting
+                        the current test run. Defaults to
+                        $opt_max_test_fail, set to 0 for no limit. Set
+                        it's default with MTR_MAX_TEST_FAIL
 
 Options for valgrind