Merge with 5.1 to get in changes from MySQL 5.1.55

1 files changed, 178 insertions, 46 deletions

diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl
index 962f0553b01..4ec2a68a3de 100755
--- a/mysql-test/mysql-test-run.pl
+++ b/mysql-test/mysql-test-run.pl
@@ -1,6 +1,5 @@
 #!/usr/bin/perl
 # -*- cperl -*-
-
 #
 ##############################################################################
 #
@@ -614,6 +613,11 @@ sub run_test_server ($$$) {
 	    if ( !$opt_force ) {
 	      # Test has failed, force is off
 	      push(@$completed, $result);
+	      if ($result->{'dont_kill_server'})
+              {
+	        print $sock "BYE\n";
+	        next;
+              }
 	      return ("Failure", 1, $completed, $extra_warnings);
 	    }
 	    elsif ($opt_max_test_fail > 0 and
@@ -719,16 +723,6 @@ sub run_test_server ($$$) {
 	    next;
 	  }
 
-	  # Prefer same configuration, or just use next if --noreorder
-	  if (!$opt_reorder or (defined $result and
-	      $result->{template_path} eq $t->{template_path}))
-	  {
-	    #mtr_report("Test uses same config => good match");
-	    # Test uses same config => good match
-	    $next= splice(@$tests, $i, 1);
-	    last;
-	  }
-
 	  # From secondary choices, we prefer to pick a 'long-running' test if
           # possible; this helps avoid getting stuck with a few of those at the
           # end of high --parallel runs, with most workers being idle.
@@ -737,6 +731,34 @@ sub run_test_server ($$$) {
 	    #mtr_report("Setting second_best to $i");
 	    $second_best= $i;
 	  }
+
+	  # Smart allocation of next test within this thread.
+
+	  if ($opt_reorder and $opt_parallel > 1 and defined $result)
+	  {
+	    my $wid= $result->{worker};
+	    # Reserved for other thread, try next
+	    next if (defined $t->{reserved} and $t->{reserved} != $wid);
+	    if (! defined $t->{reserved})
+	    {
+	      # Force-restart not relevant when comparing *next* test
+	      $t->{criteria} =~ s/force-restart$/no-restart/;
+	      my $criteria= $t->{criteria};
+	      # Reserve similar tests for this worker, but not too many
+	      my $maxres= (@$tests - $i) / $opt_parallel + 1;
+	      for (my $j= $i+1; $j <= $i + $maxres; $j++)
+	      {
+		my $tt= $tests->[$j];
+		last unless defined $tt;
+		last if $tt->{criteria} ne $criteria;
+		$tt->{reserved}= $wid;
+	      }
+	    }
+	  }
+
+	  # At this point we have found next suitable test
+	  $next= splice(@$tests, $i, 1);
+	  last;
 	}
 
 	# Use second best choice if no other test has been found
@@ -745,12 +767,14 @@ sub run_test_server ($$$) {
 	  mtr_error("Internal error, second best too large($second_best)")
 	    if $second_best >  $#$tests;
 	  $next= splice(@$tests, $second_best, 1);
+	  delete $next->{reserved};
 	}
 
         xterm_stat(scalar(@$tests));
 
 	if ($next) {
-	  #$next->print_test();
+	  # We don't need this any more
+	  delete $next->{criteria};
 	  $next->write_test($sock, 'TESTCASE');
 	  $running{$next->key()}= $next;
 	  $num_ndb_tests++ if ($next->{ndb_test});
@@ -832,6 +856,11 @@ sub run_worker ($) {
       delete($test->{'comment'});
       delete($test->{'logfile'});
 
+      # A sanity check. Should this happen often we need to look at it.
+      if (defined $test->{reserved} && $test->{reserved} != $thread_num) {
+	my $tres= $test->{reserved};
+	mtr_warning("Test reserved for w$tres picked up by w$thread_num");
+      }
       $test->{worker} = $thread_num if $opt_parallel > 1;
 
       run_testcase($test, $server);
@@ -854,10 +883,14 @@ sub run_worker ($) {
       else {
         stop_all_servers($opt_shutdown_timeout);
       }
+      my $valgrind_reports= 0;
+      if ($opt_valgrind_mysqld) {
+        $valgrind_reports= valgrind_exit_reports();
+      }
       if ( $opt_gprof ) {
 	gprof_collect (find_mysqld($basedir), keys %gprof_dirs);
       }
-      exit(0);
+      exit($valgrind_reports);
     }
     else {
       mtr_error("Could not understand server, '$line'");
@@ -902,7 +935,7 @@ sub command_line_setup {
   my $opt_list_options;
 
   # Read the command line options
-  # Note: Keep list, and the order, in sync with usage at end of this file
+  # Note: Keep list in sync with usage at end of this file
   Getopt::Long::Configure("pass_through");
   my %options=(
              # Control what engine/variation to run
@@ -938,6 +971,7 @@ sub command_line_setup {
 	     'combination=s'            => \@opt_combinations,
              'skip-combinations'        => \&collect_option,
              'experimental=s'           => \@opt_experimentals,
+	     # skip-im is deprecated and silently ignored
 	     'skip-im'                  => \&ignore_option,
              'staging-run'              => \$opt_staging_run,
 
@@ -1039,8 +1073,9 @@ sub command_line_setup {
 	     'max-connections=i'        => \$opt_max_connections,
 
              'help|h'                   => \$opt_usage,
-             'list-options'             => \$opt_list_options,
-            );
+	     # list-options is internal, not listed in help
+	       'list-options'             => \$opt_list_options,
+	      );
 
   GetOptions(%options) or usage("Can't read options");
   usage("") if $opt_usage;
@@ -1596,16 +1631,16 @@ sub set_build_thread_ports($) {
   $ENV{MTR_BUILD_THREAD}= $build_thread;
 
   # Calculate baseport
-  $baseport= $build_thread * 10 + 10000;
-  if ( $baseport < 5001 or $baseport + 9 >= 32767 )
+  $baseport= $build_thread * 20 + 10000;
+  if ( $baseport < 5001 or $baseport + 19 >= 32767 )
   {
     mtr_error("MTR_BUILD_THREAD number results in a port",
               "outside 5001 - 32767",
-              "($baseport - $baseport + 9)");
+              "($baseport - $baseport + 19)");
   }
 
   mtr_report("Using MTR_BUILD_THREAD $build_thread,",
-	     "with reserved ports $baseport..".($baseport+9));
+	     "with reserved ports $baseport..".($baseport+19));
 
 }
 
@@ -2085,6 +2120,16 @@ sub environment_setup {
   $ENV{'DEFAULT_MASTER_PORT'}= $mysqld_variables{'master-port'} || 3306;
   $ENV{'MYSQL_TMP_DIR'}=      $opt_tmpdir;
   $ENV{'MYSQLTEST_VARDIR'}=   $opt_vardir;
+  
+  if (IS_WINDOWS)
+  {
+    $ENV{'SECURE_LOAD_PATH'}= $glob_mysql_test_dir."\\std_data";
+  }
+  else
+  {
+    $ENV{'SECURE_LOAD_PATH'}= $glob_mysql_test_dir."/std_data";
+  }
+    
 
   #
   # Some stupid^H^H^H^H^H^Hignorant network providers set up "wildcard DNS"
@@ -2287,10 +2332,10 @@ sub remove_stale_vardir () {
     # Running with "var" in some other place
     #
 
-    # Remove the var/ dir in mysql-test dir if any
-    # this could be an old symlink that shouldn't be there
-    mtr_verbose("Removing $default_vardir");
-    rmtree($default_vardir);
+    # Don't remove the var/ dir in mysql-test dir as it may be in
+    # use by another mysql-test-run run with --vardir
+    # mtr_verbose("Removing $default_vardir");
+    # rmtree($default_vardir);
 
     # Remove the "var" dir
     mtr_verbose("Removing $opt_vardir/");
@@ -2376,7 +2421,7 @@ sub setup_vardir() {
     $plugindir="$opt_vardir/plugins";
     unshift (@opt_extra_mysqld_opt, "--plugin-dir=$plugindir");
     mkpath($plugindir);
-    if (IS_WINDOWS)
+    if (IS_WINDOWS && !$opt_embedded_server)
     {
       for (<../storage/*$opt_vs_config/*.dll>,
            <../plugin/*$opt_vs_config/*.dll>,
@@ -3607,13 +3652,14 @@ sub find_analyze_request
 
 # The test can leave a file in var/tmp/ to signal
 # that all servers should be restarted
-sub restart_forced_by_test
+sub restart_forced_by_test($)
 {
+  my $file = shift;
   my $restart = 0;
   foreach my $mysqld ( mysqlds() )
   {
     my $datadir = $mysqld->value('datadir');
-    my $force_restart_file = "$datadir/mtr/force_restart";
+    my $force_restart_file = "$datadir/mtr/$file";
     if ( -f $force_restart_file )
     {
       mtr_verbose("Restart of servers forced by test");
@@ -3777,6 +3823,7 @@ sub run_testcase ($$) {
       # Generate new config file from template
       $config= My::ConfigFactory->new_config
 	( {
+           testname        => $tinfo->{name},
 	   basedir         => $basedir,
 	   testdir         => $glob_mysql_test_dir,
 	   template_path   => $tinfo->{template_path},
@@ -3948,7 +3995,7 @@ sub run_testcase ($$) {
       if ( $res == 0 )
       {
 	my $check_res;
-	if ( restart_forced_by_test() )
+	if ( restart_forced_by_test('force_restart') )
 	{
 	  stop_all_servers($opt_shutdown_timeout);
 	}
@@ -3987,8 +4034,11 @@ sub run_testcase ($$) {
 	find_testcase_skipped_reason($tinfo);
 	mtr_report_test_skipped($tinfo);
 	# Restart if skipped due to missing perl, it may have had side effects
-	stop_all_servers($opt_shutdown_timeout)
-	  if ($tinfo->{'comment'} =~ /^perl not found/);
+	if ( restart_forced_by_test('force_restart_if_skipped') ||
+             $tinfo->{'comment'} =~ /^perl not found/ )
+	{
+	  stop_all_servers($opt_shutdown_timeout);
+	}
       }
       elsif ( $res == 65 )
       {
@@ -4039,7 +4089,6 @@ sub run_testcase ($$) {
     # ----------------------------------------------------
     # Check if it was an expected crash
     # ----------------------------------------------------
-    SRVDIED:
     my $check_crash = check_expected_crash_and_restart($proc);
     if ($check_crash)
     {
@@ -4049,6 +4098,7 @@ sub run_testcase ($$) {
       next;
     }
 
+  SRVDIED:
     # ----------------------------------------------------
     # Stop the test case timer
     # ----------------------------------------------------
@@ -4312,6 +4362,7 @@ sub extract_warning_lines ($) {
      qr/Plugin 'ndbcluster' will be forced to shutdown/,
      qr/InnoDB: Error: in ALTER TABLE `test`.`t[12]`/,
      qr/InnoDB: Error: table `test`.`t[12]` does not exist in the InnoDB internal/,
+     qr/InnoDB: Warning: a long semaphore wait:/,
      qr/Slave: Unknown table 't1' Error_code: 1051/,
      qr/Slave SQL:.*(Error_code: [[:digit:]]+|Query:.*)/,
      qr/slave SQL thread aborted/,
@@ -4692,6 +4743,19 @@ sub after_failure ($) {
 
   mkpath($save_dir) if ! -d $save_dir;
 
+  #
+  # Create a log of files in vardir (good for buildbot)
+  #
+  if (!IS_WINDOWS)
+  {
+    my $Flog= IO::File->new("$opt_vardir/log/files.log", "w");
+    if ($Flog)
+    {
+      print $Flog scalar(`/bin/ls -Rl $opt_vardir/*`);
+      close($Flog);
+    }
+  }
+
   # Save the used config files
   my %config_files = config_files($tinfo);
   while (my ($file, $generate) = each %config_files) {
@@ -4711,7 +4775,12 @@ sub after_failure ($) {
 sub report_failure_and_restart ($) {
   my $tinfo= shift;
 
-  stop_all_servers();
+  if ($opt_valgrind_mysqld && ($tinfo->{'warnings'} || $tinfo->{'timeout'})) {
+    # In these cases we may want valgrind report from normal termination
+    $tinfo->{'dont_kill_server'}= 1;
+  }
+  # Shotdown properly if not to be killed (for valgrind)
+  stop_all_servers($tinfo->{'dont_kill_server'} ? $opt_shutdown_timeout : 0);
 
   $tinfo->{'result'}= 'MTR_RES_FAILED';
 
@@ -5098,17 +5167,6 @@ sub server_need_restart {
     }
   }
 
-  # Temporary re-enable the "always restart slave" hack
-  # this should be removed asap, but will require that each rpl
-  # testcase cleanup better after itself - ie. stop and reset
-  # replication
-  # Use the "#!use-slave-opt" marker to detect that this is a "slave"
-  # server
-  if ( $server->option("#!use-slave-opt") ){
-    mtr_verbose_restart($server, "Always restart slave(s)");
-    return 1;
-  }
-
   if ($server->name() =~ /^mysqld\./)
   {
 
@@ -5249,7 +5307,7 @@ sub start_servers($) {
   for (all_servers()) {
     next unless $_->{WAIT} and started($_);
     if ($_->{WAIT}->($_)) {
-      $tinfo->{comment}= "Failed to start ".$_->name();
+      $tinfo->{comment}= "Failed to start ".$_->name() . "\n";
       return 1;
     }
   }
@@ -5709,6 +5767,72 @@ sub strace_arguments {
   }
 }
 
+#
+# Search server logs for valgrind reports printed at mysqld termination
+#
+
+sub valgrind_exit_reports() {
+  my $found_err= 0;
+
+  foreach my $log_file (keys %mysqld_logs)
+  {
+    my @culprits= ();
+    my $valgrind_rep= "";
+    my $found_report= 0;
+    my $err_in_report= 0;
+
+    my $LOGF = IO::File->new($log_file)
+      or mtr_error("Could not open file '$log_file' for reading: $!");
+
+    while ( my $line = <$LOGF> )
+    {
+      if ($line =~ /^CURRENT_TEST: (.+)$/)
+      {
+        my $testname= $1;
+        # If we have a report, report it if needed and start new list of tests
+        if ($found_report)
+        {
+          if ($err_in_report)
+          {
+            mtr_print ("Valgrind report from $log_file after tests:\n",
+                        @culprits);
+            mtr_print_line();
+            print ("$valgrind_rep\n");
+            $err_in_report= 0;
+          }
+          # Make ready to collect new report
+          @culprits= ();
+          $found_report= 0;
+          $valgrind_rep= "";
+        }
+        push (@culprits, $testname);
+        next;
+      }
+      # This line marks the start of a valgrind report
+      $found_report= 1 if $line =~ /^==\d+== .* SUMMARY:/;
+
+      if ($found_report) {
+        $line=~ s/^==\d+== //;
+        $valgrind_rep .= $line;
+        $err_in_report= 1 if $line =~ /ERROR SUMMARY: [1-9]/;
+        $err_in_report= 1 if $line =~ /definitely lost: [1-9]/;
+        $err_in_report= 1 if $line =~ /possibly lost: [1-9]/;
+      }
+    }
+
+    $LOGF= undef;
+
+    if ($err_in_report) {
+      mtr_print ("Valgrind report from $log_file after tests:\n", @culprits);
+      mtr_print_line();
+      print ("$valgrind_rep\n");
+      $found_err= 1;
+    }
+  }
+
+  return $found_err;
+}
+
 
 #
 # Usage
@@ -5743,7 +5867,7 @@ Options to control what engine/variation to run
   parallel=#            How many parallell test should be run
   defaults-file=<config template> Use fixed config template for all
                         tests
-  defaults_extra_file=<config template> Extra config template to add to
+  defaults-extra-file=<config template> Extra config template to add to
                         all generated configs
   combination=<opt>     Use at least twice to run tests with specified
                         options to mysqld
@@ -5837,6 +5961,9 @@ Options for debugging the product
                         test(s)
   manual-ddd            Let user manually start mysqld in ddd, before running
                         test(s)
+  strace-client[=path]  Create strace output for mysqltest client, optionally
+                        specifying name and path to the trace program to use.
+                        Example: $0 --strace-client=ktrace
   max-save-core         Limit the number of core files saved (to avoid filling
                         up disks for heavily crashing server). Defaults to
                         $opt_max_save_core, set to 0 for no limit. Set
@@ -5876,7 +6003,7 @@ Options for strace
 Misc options
   user=USER             User for connecting to mysqld(default: $opt_user)
   comment=STR           Write STR to the output
-  notimer               Don't show test case execution time
+  timer                 Show test case execution time.
   verbose               More verbose output(use multiple times for even more)
   verbose-restart       Write when and why servers are restarted
   start                 Only initialize and start the servers, using the
@@ -5930,6 +6057,7 @@ Misc options
   gcov-src-dir=subdir   Colllect coverage only within the given subdirectory.
                         For example, if you're only developing the SQL layer, 
                         it makes sense to use --gcov-src-dir=sql
+  gprof                 Collect profiling information using gprof.
   experimental=<file>   Refer to list of tests considered experimental;
                         failures will be marked exp-fail instead of fail.
   report-features       First run a "test" that reports mysql features
@@ -5938,6 +6066,10 @@ Misc options
                         *previous* test started
   max-connections=N     Max number of open connection to server in mysqltest
 
+Some options that control enabling a feature for normal test runs,
+can be turned off by prepending 'no' to the option, e.g. --notimer.
+This applies to reorder, timer, check-testcases and warnings.
+
 HERE
   exit(1);