unify mtr handling of debuggers

"debugger" is anything that wraps execution of a target
binary (mysqld or mysqltest). Currently the list includes:
gdb, ddd, dbx, lldb, valgrind, strace, ktrace, rr,
devenv, windbg, vsjitdebugger.

for every debugger xxx, mtr will recognize four options:
--xxx, --boot-xxx, --manual-xxx, --client-xxx.
They all support an optional "=string" argument. String
being a semicolon-separated list of commands (e.g. for gdb)
or one (not semicolon-separated) command line of options
(e.g. for valgrind). Or both (e.g. --gdb='-quiet -nh;info files'

In embedded both --xxx and --client-xxx work.

Functionality changed/removed:
* --rr-args is gone
* --rr-dir is gone
* --manual-debug is gone
* --debugger={devenv|vc|windbg|vc_express|vsjitdebugger} is gone
* --strace-option is gone
* --stracer={strace|ktrace} is gone
* --valgrind only enables it for the server, not for everything
* --valgrind-all is gone
* --valgrind-mysqltest is gone
* --valgrind-mysqld is gone
* --valgrind-options is gone
* --valgrind-option is gone
* --valgrind-path is gone
* --callgrind is gone
* one cannot combine --valgrind --gdb anymore
* valgrind report doesn't add a fake test line to the output
* vc and vcexpress on windows are no longer supported

2 files changed, 276 insertions, 750 deletions

diff --git a/mysql-test/lib/My/Debugger.pm b/mysql-test/lib/My/Debugger.pm
new file mode 100644
index 00000000000..700124ec3cb
--- /dev/null
+++ b/mysql-test/lib/My/Debugger.pm
@@ -0,0 +1,247 @@
+package My::Debugger;
+
+use strict;
+use warnings;
+use Text::Wrap;
+use Cwd;
+
+# 1. options to support:
+#       --xxx[=ARGS]
+#       --manual-xxx[=ARGS]
+#       --client-xxx[=ARGS]
+#       --boot-xxx[=ARGS]
+#       TODO --manual-client-xxx[=ARGS]
+#       TODO --manual-boot-xxx[=ARGS]
+#       TODO --exec-xxx[=ARGS] (for $ENV{MYSQL}, etc)
+#
+#       ARGS is a semicolon-separated list of commands for the
+#       command file. If the first command starts from '-' it'll
+#       be for a command line, not for a command file.
+#
+# 2. terminal to use: xterm
+#       TODO MTR_TERM="xterm -title {title} -e {command}"
+#
+# 3. debugger combinations are *not allowed*
+#       (thus no --valgrind --gdb)
+#
+# 4. variables for the command line / file templates:
+#       {vardir}   -> vardir
+#       {exe} -> /path/to/binary/to/execute
+#       {args} -> command-line arguments, "-quoted
+#       {input}
+#       {type} -> client, mysqld.1, etc
+#       {script} -> vardir/tmp/{debugger}init.$type
+#       {log} -> vardir/log/$type.{debugger}
+#       {options} -> user options for the debugger.
+#
+#  if {options} isn't used, they're auto-placed before {exe}
+#     or at the end if no {exe}
+
+my %debuggers = (
+  gdb => {
+    term => 1,
+    options => '-x {script} {exe}',
+    script => 'set args {args} < {input}',
+  },
+  ddd => {
+    options => '--command {script} {exe}',
+    script => 'set args {args} < {input}',
+  },
+  dbx => {
+    term => 1,
+    options => '-c "stop in main; run {exe} {args} < {input}"',
+  },
+  devenv => {
+    options => '/debugexe {exe} {args}',
+  },
+  windbg => {
+    options => '{exe} {args}',
+  },
+  lldb => {
+    term => 1,
+    options => '-s {script} {exe}',
+    script => 'process launch --stop-at-entry {args}',
+  },
+  valgrind => {
+    options => '--tool=memcheck --show-reachable=yes --leak-check=yes --num-callers=16 --quiet --suppressions='.cwd().'/valgrind.supp {exe} {args} --loose-wait-for-pos-timeout=1500',
+    pre => sub {
+      my $debug_libraries_path= "/usr/lib/debug";
+      $ENV{LD_LIBRARY_PATH} .= ":$debug_libraries_path" if -d $debug_libraries_path;
+    }
+  },
+  strace => {
+    options => '-f -o {log} {exe} {args}',
+  },
+  rr => {
+    options => 'record -o {log} {exe} {args}',
+    pre => sub {
+      ::mtr_error('rr requires kernel.perf_event_paranoid <= 1')
+        if ::mtr_grab_file('/proc/sys/kernel/perf_event_paranoid') > 1;
+    }
+  },
+
+  # aliases
+  vsjitdebugger => 'windbg',
+  ktrace => 'strace',
+);
+
+my %opts;
+my %opt_vals;
+my $help = "\n\nOptions for running debuggers\n\n";
+
+for my $k (sort keys %debuggers) {
+  my $v = $debuggers{$k};
+  $v = $debuggers{$k} = $debuggers{$v} if not ref $v; # resolve aliases
+
+  sub register_opt($$) {
+    my ($name, $msg) = @_;
+    $opts{"$name=s"} = \$opt_vals{$name};
+    $help .= wrap(sprintf("  %-23s", $name), ' 'x25, "$msg under $name\n");
+  }
+
+  $v->{script} = '' unless $v->{script};
+  $v->{options} =~ s/(\{exe\}|$)/ {options} $&/ unless $v->{options} =~ /\{options\}/;
+
+  register_opt "$k" => "Start mysqld";
+  register_opt "client-$k" => "Start mysqltest client";
+  register_opt "boot-$k" => "Start bootstrap server";
+  register_opt "manual-$k" => "Before running test(s) let user manually start mysqld";
+}
+
+sub subst($%) {
+  use warnings FATAL => 'uninitialized';
+  my ($templ, %vars) = @_;
+  $templ =~ s/\{(\w+)\}/$vars{$1}/g;
+  $templ;
+}
+
+sub do_args($$$$$) {
+  my ($args, $exe, $input, $type, $opt) = @_;
+  my $k = $opt =~ /^(?:client|boot|manual)-(.*)$/ ? $1 : $opt;
+  my $v = $debuggers{$k};
+
+  # on windows mtr args are quoted (for system), otherwise not (for exec)
+  sub quote($) { $_[0] =~ / / ? "\"$_[0]\"" : $_[0] }
+  sub unquote($) { $_[0] =~ s/^"(.*)"$/$1/; $_[0] }
+  sub quote_from_mtr($) { ::IS_WINDOWS ? $_[0] : quote($_[0]) }
+  sub unquote_for_mtr($) { ::IS_WINDOWS ? $_[0] : unquote($_[0]) }
+
+  my %vars = (
+    vardir => $::opt_vardir,
+    exe => $$exe,
+    args => join(' ', map { quote_from_mtr $_ } @$$args, '--gdb'),
+    input => $input,
+    script => "$::opt_vardir/tmp/${k}init.$type",
+    log => "$::opt_vardir/log/$type.$k",
+    options => '',
+  );
+  my @params = split /;/, $opt_vals{$opt};
+  $vars{options} = shift @params  if @params and $params[0] =~ /^-/;
+
+  my $script = join "\n", @params;
+  if ($v->{script}) {
+    ::mtr_tofile($vars{script}, subst($v->{script}, %vars)."\n".$script);
+  } elsif ($script) {
+    die "$k is not using a script file, nowhere to write the script \n---\n$script\n---\n";
+  }
+
+  my $options = subst($v->{options}, %vars);
+  @$$args = map { unquote_for_mtr $_ } $options =~ /("[^"]+"|\S+)/g;
+
+  if ($opt =~ /^manual-/) {
+    print "\nTo start $k for $type, type in another window:\n";
+    print "$k $options\n";
+    $$exe= undef; # Indicate the exe should not be started
+  } elsif ($v->{term}) {
+    unshift @$$args, '-title', $type, '-e', $k;
+    $$exe = 'xterm';
+  } else {
+    $$exe = $k;
+  }
+}
+
+sub options() { %opts }
+sub help() { $help }
+
+sub fix_options(@) {
+  my $re=join '|', keys %opts;
+  $re =~ s/=s//g;
+  map { $_ . (/^--($re)$/ and '=;') } @_;
+}
+
+sub pre_setup() {
+  my $used;
+  for my $k (keys %debuggers) {
+    for my $opt ($k, "manual-$k", "boot-$k", "client-$k") {
+      if ($opt_vals{$opt})
+      {
+        $used = 1;
+        if ($debuggers{$k}->{pre}) {
+          $debuggers{$k}->{pre}->();
+          delete $debuggers{$k}->{pre};
+        }
+      }
+    }
+  }
+
+  if ($used) {
+    $ENV{ASAN_OPTIONS}= 'abort_on_error=1:'.($ENV{ASAN_OPTIONS} || '');
+    ::mtr_error("Can't use --extern when using debugger") if $ENV{USE_RUNNING_SERVER};
+
+    $::opt_retry= 1;
+    $::opt_retry_failure= 1;
+    $::opt_testcase_timeout= 7 * 24 * 60; # in minutes
+    $::opt_suite_timeout= 7 * 24 * 60;    # in minutes
+    $::opt_shutdown_timeout= 24 * 60 *60; # in seconds
+    $::opt_start_timeout= 24 * 60 * 60;   # in seconds
+  }
+}
+
+sub setup_boot_args($$$) {
+  my ($args, $exe, $input) = @_;
+  my $found;
+
+  for my $k (keys %debuggers) {
+    if ($opt_vals{"boot-$k"}) {
+      die "--boot-$k and --$found cannot be used at the same time\n" if $found;
+
+      $found="boot-$k";
+      do_args($args, $exe, $input, 'bootstrap', $found);
+    }
+  }
+}
+
+sub setup_client_args($$) {
+  my ($args, $exe) = @_;
+  my $found;
+  my $embedded = $::opt_embedded_server ? ' with --embedded' : '';
+
+  for my $k (keys %debuggers) {
+    my @opt_names=("client-$k");
+    push @opt_names, $k if $embedded;
+    for my $opt (@opt_names) {
+      if ($opt_vals{$opt}) {
+        die "--$opt and --$found cannot be used at the same time$embedded\n" if $found;
+        $found=$opt;
+        do_args($args, $exe, ::IS_WINDOWS ? 'NUL' : '/dev/null', 'client', $found);
+      }
+    }
+  }
+}
+
+sub setup_args($$$) {
+  my ($args, $exe, $type) = @_;
+  my $found;
+
+  for my $k (keys %debuggers) {
+    for my $opt ($k, "manual-$k") {
+      if ($opt_vals{$opt}) {
+        die "--$opt and --$found cannot be used at the same time\n" if $found;
+        $found=$opt;
+        do_args($args, $exe, ::IS_WINDOWS ? 'NUL' : '/dev/null', $type, $found);
+      }
+    }
+  }
+}
+
+1;
diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl
index 1c51acf7341..07f103ba91e 100755
--- a/mysql-test/mysql-test-run.pl
+++ b/mysql-test/mysql-test-run.pl
@@ -93,6 +93,7 @@ use My::Tee;
 use My::Find;
 use My::SysInfo;
 use My::CoreDump;
+use My::Debugger;
 use mtr_cases;
 use mtr_report;
 use mtr_match;
@@ -106,6 +107,9 @@ require "mtr_io.pl";
 require "mtr_gprof.pl";
 require "mtr_misc.pl";
 
+my $opt_valgrind;
+my $valgrind_reports= 0;
+
 $SIG{INT}= sub { mtr_error("Got ^C signal"); };
 $SIG{HUP}= sub { mtr_error("Hangup detected on controlling terminal"); };
 
@@ -257,28 +261,6 @@ our $opt_gcov;
 our $opt_gprof;
 our %gprof_dirs;
 
-our $glob_debugger= 0;
-our $opt_gdb;
-my $opt_rr;
-my $opt_rr_dir;
-my @rr_record_args;
-our $opt_client_gdb;
-my $opt_boot_gdb;
-my $opt_boot_rr;
-our $opt_dbx;
-our $opt_client_dbx;
-my $opt_boot_dbx;
-our $opt_ddd;
-our $opt_client_ddd;
-my $opt_boot_ddd;
-our $opt_manual_gdb;
-our $opt_manual_lldb;
-our $opt_manual_dbx;
-our $opt_manual_ddd;
-our $opt_manual_debug;
-our $opt_debugger;
-our $opt_client_debugger;
-
 my $config; # The currently running config
 my $current_config_name; # The currently running config file template
 
@@ -304,34 +286,23 @@ our $opt_report_times= 0;
 
 my $opt_sleep;
 
-my $opt_testcase_timeout= $ENV{MTR_TESTCASE_TIMEOUT} ||  15; # minutes
-my $opt_suite_timeout   = $ENV{MTR_SUITE_TIMEOUT}    || 360; # minutes
-my $opt_shutdown_timeout= $ENV{MTR_SHUTDOWN_TIMEOUT} ||  10; # seconds
-my $opt_start_timeout   = $ENV{MTR_START_TIMEOUT}    || 180; # seconds
+our $opt_retry= 1;
+our $opt_retry_failure= env_or_val(MTR_RETRY_FAILURE => 2);
+our $opt_testcase_timeout= $ENV{MTR_TESTCASE_TIMEOUT} ||  15; # minutes
+our $opt_suite_timeout   = $ENV{MTR_SUITE_TIMEOUT}    || 360; # minutes
+our $opt_shutdown_timeout= $ENV{MTR_SHUTDOWN_TIMEOUT} ||  10; # seconds
+our $opt_start_timeout   = $ENV{MTR_START_TIMEOUT}    || 180; # seconds
 
 sub suite_timeout { return $opt_suite_timeout * 60; };
 
 my $opt_wait_all;
 my $opt_user_args;
 my $opt_repeat= 1;
-my $opt_retry= 1;
-my $opt_retry_failure= env_or_val(MTR_RETRY_FAILURE => 2);
 my $opt_reorder= 1;
 my $opt_force_restart= 0;
 
 our $opt_user = "root";
 
-our $opt_valgrind= 0;
-my $opt_valgrind_mysqld= 0;
-my $opt_valgrind_mysqltest= 0;
-my @valgrind_args;
-my $opt_strace= 0;
-my $opt_stracer;
-my $opt_client_strace = 0;
-my @strace_args;
-my $opt_valgrind_path;
-my $valgrind_reports= 0;
-my $opt_callgrind;
 my %mysqld_logs;
 my $opt_debug_sync_timeout= 300; # Default timeout for WAIT_FOR actions.
 my $warn_seconds = 60;
@@ -1154,7 +1125,7 @@ sub run_worker ($) {
       }
       mark_time_used('restart');
       my $valgrind_reports= 0;
-      if ($opt_valgrind_mysqld) {
+      if ($opt_valgrind) {
         $valgrind_reports= valgrind_exit_reports();
 	print $server "VALGREP\n" if $valgrind_reports;
       }
@@ -1216,8 +1187,6 @@ sub print_global_resfile {
   resfile_global("debug", $opt_debug ? 1 : 0);
   resfile_global("gcov", $opt_gcov ? 1 : 0);
   resfile_global("gprof", $opt_gprof ? 1 : 0);
-  resfile_global("valgrind", $opt_valgrind ? 1 : 0);
-  resfile_global("callgrind", $opt_callgrind ? 1 : 0);
   resfile_global("mem", $opt_mem);
   resfile_global("tmpdir", $opt_tmpdir);
   resfile_global("vardir", $opt_vardir);
@@ -1307,30 +1276,6 @@ sub command_line_setup {
              'debug'                    => \$opt_debug,
              'debug-common'             => \$opt_debug_common,
              'debug-server'             => \$opt_debug_server,
-             'gdb=s'                    => \$opt_gdb,
-             'rr'                       => \$opt_rr,
-             'rr-arg=s'                 => \@rr_record_args,
-             'rr-dir=s'                 => \$opt_rr_dir,
-             'client-gdb=s'             => \$opt_client_gdb,
-             'manual-gdb'               => \$opt_manual_gdb,
-             'manual-lldb'              => \$opt_manual_lldb,
-	     'boot-gdb'                 => \$opt_boot_gdb,
-	     'boot-rr'                  => \$opt_boot_rr,
-             'manual-debug'             => \$opt_manual_debug,
-             'ddd'                      => \$opt_ddd,
-             'client-ddd'               => \$opt_client_ddd,
-             'manual-ddd'               => \$opt_manual_ddd,
-	     'boot-ddd'                 => \$opt_boot_ddd,
-             'dbx'                      => \$opt_dbx,
-	     'client-dbx'               => \$opt_client_dbx,
-	     'manual-dbx'               => \$opt_manual_dbx,
-	     'debugger=s'               => \$opt_debugger,
-	     'boot-dbx'                 => \$opt_boot_dbx,
-	     'client-debugger=s'        => \$opt_client_debugger,
-             'strace'              => \$opt_strace,
-             'strace-option=s'     => \@strace_args,
-             'client-strace'       => \$opt_client_strace,
-             'stracer=s'           => \$opt_stracer,
              'max-save-core=i'          => \$opt_max_save_core,
              'max-save-datadir=i'       => \$opt_max_save_datadir,
              'max-test-fail=i'          => \$opt_max_test_fail,
@@ -1339,23 +1284,6 @@ sub command_line_setup {
              # Coverage, profiling etc
              'gcov'                     => \$opt_gcov,
              'gprof'                    => \$opt_gprof,
-             'valgrind|valgrind-all'    => \$opt_valgrind,
-             'valgrind-mysqltest'       => \$opt_valgrind_mysqltest,
-             'valgrind-mysqld'          => \$opt_valgrind_mysqld,
-             'valgrind-options=s'       => sub {
-	       my ($opt, $value)= @_;
-	       # Deprecated option unless it's what we know pushbuild uses
-	       if ($value eq "--gen-suppressions=all --show-reachable=yes") {
-		 push(@valgrind_args, $_) for (split(' ', $value));
-		 return;
-	       }
-	       die("--valgrind-options=s is deprecated. Use ",
-		   "--valgrind-option=s, to be specified several",
-		   " times if necessary");
-	     },
-             'valgrind-option=s'        => \@valgrind_args,
-             'valgrind-path=s'          => \$opt_valgrind_path,
-	     'callgrind'                => \$opt_callgrind,
 	     'debug-sync-timeout=i'     => \$opt_debug_sync_timeout,
 
 	     # Directories
@@ -1404,12 +1332,13 @@ sub command_line_setup {
 	     # list-options is internal, not listed in help
 	     'list-options'             => \$opt_list_options,
              'skip-test-list=s'         => \@opt_skip_test_list,
-             'xml-report=s'             => \$opt_xml_report
+             'xml-report=s'             => \$opt_xml_report,
+
+             My::Debugger::options()
            );
 
   # fix options (that take an optional argument and *only* after = sign
-  my %fixopt = ( '--gdb' => '--gdb=#', '--client-gdb' => '--client-gdb=#' );
-  @ARGV = map { $fixopt{$_} or $_ } @ARGV;
+  @ARGV = My::Debugger::fix_options(@ARGV);
   GetOptions(%options) or usage("Can't read options");
   usage("") if $opt_usage;
   list_options(\%options) if $opt_list_options;
@@ -1734,39 +1663,6 @@ sub command_line_setup {
     {
       mtr_error("Can't use --extern with --embedded-server");
     }
-
-
-    if ($opt_gdb)
-    {
-      $opt_client_gdb= $opt_gdb;
-      $opt_gdb= undef;
-    }
-
-    if ($opt_ddd)
-    {
-      $opt_client_ddd= $opt_ddd;
-      $opt_ddd= undef;
-    }
-
-    if ($opt_dbx) {
-      mtr_warning("Silently converting --dbx to --client-dbx in embedded mode");
-      $opt_client_dbx= $opt_dbx;
-      $opt_dbx= undef;
-    }
-
-    if ($opt_debugger)
-    {
-      $opt_client_debugger= $opt_debugger;
-      $opt_debugger= undef;
-    }
-
-    if ( $opt_gdb || $opt_ddd || $opt_manual_gdb || $opt_manual_lldb || 
-         $opt_manual_ddd || $opt_manual_debug || $opt_debugger || $opt_dbx || 
-         $opt_manual_dbx)
-    {
-      mtr_error("You need to use the client debug options for the",
-		"embedded server. Ex: --client-gdb");
-    }
   }
 
   # --------------------------------------------------------------------------
@@ -1787,42 +1683,6 @@ sub command_line_setup {
   }
 
   # --------------------------------------------------------------------------
-  # Check debug related options
-  # --------------------------------------------------------------------------
-  if ( $opt_gdb || $opt_client_gdb || $opt_ddd || $opt_client_ddd || $opt_rr ||
-       $opt_manual_gdb || $opt_manual_lldb || $opt_manual_ddd ||
-       $opt_manual_debug || $opt_dbx || $opt_client_dbx || $opt_manual_dbx || 
-       $opt_debugger || $opt_client_debugger )
-  {
-    $ENV{ASAN_OPTIONS}= 'abort_on_error=1:'.($ENV{ASAN_OPTIONS} || '');
-    if ( using_extern() )
-    {
-      mtr_error("Can't use --extern when using debugger");
-    }
-    # Indicate that we are using debugger
-    $glob_debugger= 1;
-    $opt_retry= 1;
-    $opt_retry_failure= 1;
-    # Set one week timeout (check-testcase timeout will be 1/10th)
-    $opt_testcase_timeout= 7 * 24 * 60;
-    $opt_suite_timeout= 7 * 24 * 60;
-    # One day to shutdown
-    $opt_shutdown_timeout= 24 * 60;
-    # One day for PID file creation (this is given in seconds not minutes)
-    $opt_start_timeout= 24 * 60 * 60;
-    if ($opt_rr && open(my $fh, '<', '/proc/sys/kernel/perf_event_paranoid'))
-    {
-      my $perf_event_paranoid= <$fh>;
-      close $fh;
-      chomp $perf_event_paranoid;
-      if ($perf_event_paranoid == 0)
-      {
-        mtr_error("rr requires kernel.perf_event_paranoid set to 1");
-      }
-    }
-  }
-
-  # --------------------------------------------------------------------------
   # Modified behavior with --start options
   # --------------------------------------------------------------------------
   if ($opt_start or $opt_start_dirty or $opt_start_exit) {
@@ -1883,87 +1743,6 @@ sub command_line_setup {
 	    "for option --testsuite-timeout")
     if ($opt_suite_timeout <= 0);
 
-  # --------------------------------------------------------------------------
-  # Check valgrind arguments
-  # --------------------------------------------------------------------------
-  if ( $opt_valgrind or $opt_valgrind_path or @valgrind_args)
-  {
-    mtr_report("Turning on valgrind for all executables");
-    $opt_valgrind= 1;
-    $opt_valgrind_mysqld= 1;
-    $opt_valgrind_mysqltest= 1;
-  }
-  elsif ( $opt_valgrind_mysqld )
-  {
-    mtr_report("Turning on valgrind for mysqld(s) only");
-    $opt_valgrind= 1;
-  }
-  elsif ( $opt_valgrind_mysqltest )
-  {
-    mtr_report("Turning on valgrind for mysqltest and mysql_client_test only");
-    $opt_valgrind= 1;
-  }
-
-  if ($opt_valgrind)
-  {
-    # Increase the timeouts when running with valgrind
-    $opt_testcase_timeout*= 10;
-    $opt_suite_timeout*= 6;
-    $opt_start_timeout*= 10;
-    $warn_seconds*= 10;
-  }
-
-  if ( $opt_callgrind )
-  {
-    mtr_report("Turning on valgrind with callgrind for mysqld(s)");
-    $opt_valgrind= 1;
-    $opt_valgrind_mysqld= 1;
-
-    # Set special valgrind options unless options passed on command line
-    push(@valgrind_args, "--trace-children=yes")
-      unless @valgrind_args;
-    unshift(@valgrind_args, "--tool=callgrind");
-  }
-
-  # default to --tool=memcheck
-  if ($opt_valgrind && ! grep(/^--tool=/i, @valgrind_args))
-  {
-    # Set valgrind_option unless already defined
-    push(@valgrind_args, ("--show-reachable=yes", "--leak-check=yes",
-                          "--num-callers=16"))
-      unless @valgrind_args;
-    unshift(@valgrind_args, "--tool=memcheck");
-  }
-
-  if ( $opt_valgrind )
-  {
-    # Make valgrind run in quiet mode so it only print errors
-    push(@valgrind_args, "--quiet" );
-
-    push(@valgrind_args, "--suppressions=${glob_mysql_test_dir}/valgrind.supp")
-      if -f "$glob_mysql_test_dir/valgrind.supp";
-
-    mtr_report("Running valgrind with options \"",
-	       join(" ", @valgrind_args), "\"");
-  }
-
-  if (@strace_args || $opt_stracer)
-  {
-    $opt_strace=1;
-  }
-
-  # InnoDB does not bother to do individual de-allocations at exit. Instead it
-  # relies on a custom allocator to track every allocation, and frees all at
-  # once during exit.
-  # In XtraDB, an option use-sys-malloc is introduced (and on by default) to
-  # disable this (for performance). But this exposes Valgrind to all the
-  # missing de-allocations, so we need to disable it to at least get
-  # meaningful leak checking for the rest of the server.
-  if ($opt_valgrind_mysqld)
-  {
-    push(@opt_extra_mysqld_opt, "--loose-skip-innodb-use-sys-malloc");
-  }
-
   if ($opt_debug_common)
   {
     $opt_debug= 1;
@@ -2178,21 +1957,6 @@ sub executable_setup () {
 
   $exe_patch='patch' if `patch -v`;
 
-  #
-  # Check if libtool is available in this distribution/clone
-  # we need it when valgrinding or debugging non installed binary
-  # Otherwise valgrind will valgrind the libtool wrapper or bash
-  # and gdb will not find the real executable to debug
-  #
-  if ( -x "../libtool")
-  {
-    $exe_libtool= "../libtool";
-    if ($opt_valgrind or $glob_debugger or $opt_strace)
-    {
-      mtr_report("Using \"$exe_libtool\" when running valgrind, strace or debugger");
-    }
-  }
-
   # Look for the client binaries
   $exe_mysqladmin=     mtr_exe_exists("$path_client_bindir/mysqladmin");
   $exe_mysql=          mtr_exe_exists("$path_client_bindir/mysql");
@@ -2317,9 +2081,6 @@ sub mysql_client_test_arguments(){
 
   my $args;
   mtr_init_args(\$args);
-  if ( $opt_valgrind_mysqltest ) {
-    valgrind_arguments($args, \$exe);
-  }
   mtr_add_arg($args, "--defaults-file=%s", $path_config_file);
   mtr_add_arg($args, "--testcase");
   mtr_add_arg($args, "--vardir=$opt_vardir");
@@ -2365,6 +2126,8 @@ sub environment_setup {
 
   umask(022);
 
+  $ENV{'USE_RUNNING_SERVER'}= using_extern();
+
   my @ld_library_paths;
 
   if ($path_client_libdir)
@@ -2395,30 +2158,12 @@ sub environment_setup {
     }
   }
 
-  # --------------------------------------------------------------------------
-  # Valgrind need to be run with debug libraries otherwise it's almost
-  # impossible to add correct supressions, that means if "/usr/lib/debug"
-  # is available, it should be added to
-  # LD_LIBRARY_PATH
-  #
-  # But pthread is broken in libc6-dbg on Debian <= 3.1 (see Debian
-  # bug 399035, http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=399035),
-  # so don't change LD_LIBRARY_PATH on that platform.
-  # --------------------------------------------------------------------------
-  my $debug_libraries_path= "/usr/lib/debug";
-  my $deb_version;
-  if (  $opt_valgrind and -d $debug_libraries_path and
-        (! -e '/etc/debian_version' or
-	 ($deb_version=
-	    mtr_grab_file('/etc/debian_version')) !~ /^[0-9]+\.[0-9]$/ or
-         $deb_version > 3.1 ) )
-  {
-    push(@ld_library_paths, $debug_libraries_path);
-  }
-
   $ENV{'LD_LIBRARY_PATH'}= join(":", @ld_library_paths,
 				$ENV{'LD_LIBRARY_PATH'} ?
 				split(':', $ENV{'LD_LIBRARY_PATH'}) : ());
+
+  My::Debugger::pre_setup();
+
   mtr_debug("LD_LIBRARY_PATH: $ENV{'LD_LIBRARY_PATH'}");
 
   $ENV{'DYLD_LIBRARY_PATH'}= join(":", @ld_library_paths,
@@ -2453,7 +2198,6 @@ sub environment_setup {
   $ENV{'LC_CTYPE'}=           "C";
 
   $ENV{'LC_COLLATE'}=         "C";
-  $ENV{'USE_RUNNING_SERVER'}= using_extern();
   $ENV{'MYSQL_TEST_DIR'}=     $glob_mysql_test_dir;
   $ENV{'DEFAULT_MASTER_PORT'}= $mysqld_variables{'port'};
   $ENV{'MYSQL_TMP_DIR'}=      $opt_tmpdir;
@@ -2604,10 +2348,6 @@ sub environment_setup {
     $ENV{'INNOCHECKSUM'}= native_path($exe_innochecksum);
   }
 
-  # Create an environment variable to make it possible
-  # to detect that valgrind is being used from test cases
-  $ENV{'VALGRIND_TEST'}= $opt_valgrind;
-
   # Add dir of this perl to aid mysqltest in finding perl
   my $perldir= dirname($^X);
   my $pathsep= ":";
@@ -3404,25 +3144,7 @@ sub mysql_install_db {
 
   if (! -e $bootstrap_sql_file)
   {
-    if ($opt_boot_gdb) {
-      gdb_arguments(\$args, \$exe_mysqld_bootstrap, $mysqld->name(),
-        $bootstrap_sql_file);
-    }
-    if ($opt_boot_dbx) {
-      dbx_arguments(\$args, \$exe_mysqld_bootstrap, $mysqld->name(),
-        $bootstrap_sql_file);
-    }
-    if ($opt_boot_ddd) {
-      ddd_arguments(\$args, \$exe_mysqld_bootstrap, $mysqld->name(),
-        $bootstrap_sql_file);
-    }
-    if ($opt_boot_rr) {
-      $args= ["record", @rr_record_args, $exe_mysqld_bootstrap, @$args];
-      $exe_mysqld_bootstrap= "rr";
-      my $rr_dir= $opt_rr_dir ? $opt_rr_dir : "$opt_vardir/rr.boot";
-      $ENV{'_RR_TRACE_DIR'}= $rr_dir;
-      mkpath($rr_dir);
-    }
+    My::Debugger::setup_boot_args(\$args, \$exe_mysqld_bootstrap, $bootstrap_sql_file);
 
     my $path_sql= my_find_file($install_basedir,
              ["mysql", "sql/share", "share/mariadb",
@@ -5140,7 +4862,7 @@ sub after_failure ($) {
 sub report_failure_and_restart ($) {
   my $tinfo= shift;
 
-  if ($opt_valgrind_mysqld && ($tinfo->{'warnings'} || $tinfo->{'timeout'}) &&
+  if ($opt_valgrind && ($tinfo->{'warnings'} || $tinfo->{'timeout'}) &&
       $opt_core_on_failure == 0)
   {
     # In these cases we may want valgrind report from normal termination
@@ -5270,12 +4992,6 @@ sub mysqld_arguments ($$$) {
   # Check if "extra_opt" contains --log-bin
   my $skip_binlog= not grep /^--(loose-)?log-bin/, @$extra_opts;
 
-  # Indicate to mysqld it will be debugged in debugger
-  if ( $glob_debugger )
-  {
-    mtr_add_arg($args, "--gdb");
-  }
-
   my $found_skip_core= 0;
   foreach my $arg ( @$extra_opts )
   {
@@ -5324,7 +5040,6 @@ sub mysqld_start ($$) {
   mtr_verbose(My::Options::toStr("mysqld_start", @$extra_opts));
 
   my $exe= find_mysqld($mysqld->value('basedir'));
-  my $wait_for_pid_file= 1;
 
   mtr_error("Internal error: mysqld should never be started for embedded")
     if $opt_embedded_server;
@@ -5332,15 +5047,6 @@ sub mysqld_start ($$) {
   my $args;
   mtr_init_args(\$args);
 
-  if ( $opt_valgrind_mysqld and not $opt_gdb and not $opt_manual_gdb )
-  {
-    valgrind_arguments($args, \$exe);
-  }
-  if ( $opt_strace)
-  {
-    strace_arguments($args, \$exe, $mysqld->name());
-  }
-
   mtr_add_arg($args, "--defaults-group-suffix=%s", $mysqld->after('mysqld'));
 
   # Add any additional options from an in-test restart
@@ -5364,49 +5070,8 @@ sub mysqld_start ($$) {
   # options from *.opt and *.combination files.
   $ENV{'MYSQLD_LAST_CMD'}= "$exe  @$args";
 
-  if ( $opt_gdb || $opt_manual_gdb )
-  {
-    gdb_arguments(\$args, \$exe, $mysqld->name());
-  }
-  elsif ( $opt_manual_lldb )
-  {
-    lldb_arguments(\$args, \$exe, $mysqld->name());
-  }
-  elsif ( $opt_ddd || $opt_manual_ddd )
-  {
-    ddd_arguments(\$args, \$exe, $mysqld->name());
-  }
-  elsif ( $opt_dbx || $opt_manual_dbx ) {
-    dbx_arguments(\$args, \$exe, $mysqld->name());
-  }
-  elsif ( $opt_debugger )
-  {
-    debugger_arguments(\$args, \$exe, $mysqld->name());
-  }
-  elsif ( $opt_manual_debug )
-  {
-     print "\nStart " .$mysqld->name()." in your debugger\n" .
-           "dir: $glob_mysql_test_dir\n" .
-           "exe: $exe\n" .
-	   "args:  " . join(" ", @$args)  . "\n\n" .
-	   "Waiting ....\n";
-
-     # Indicate the exe should not be started
-    $exe= undef;
-  }
-  elsif ( $opt_rr )
-  {
-    $args= ["record", @rr_record_args, "$exe", @$args];
-    $exe= "rr";
-    my $rr_dir= $opt_rr_dir ? $opt_rr_dir : "$opt_vardir/rr". $mysqld->after('mysqld');
-    $ENV{'_RR_TRACE_DIR'}= $rr_dir;
-    mkpath($rr_dir);
-  }
-  else
-  {
-    # Default to not wait until pid file has been created
-    $wait_for_pid_file= 0;
-  }
+  My::Debugger::setup_args(\$args, \$exe, $mysqld->name());
+  $ENV{'VALGRIND_TEST'}= $opt_valgrind = int($exe && $exe eq 'valgrind');
 
   # Remove the old pidfile if any
   unlink($mysqld->value('pid-file'));
@@ -5455,11 +5120,8 @@ sub mysqld_start ($$) {
     mtr_verbose("Started $mysqld->{proc}");
   }
 
-  if ( $wait_for_pid_file &&
-       !sleep_until_file_created($mysqld->value('pid-file'),
-				 $opt_start_timeout,
-				 $mysqld->{'proc'},
-                                 $warn_seconds))
+  if (!sleep_until_file_created($mysqld->value('pid-file'),
+                      $opt_start_timeout, $mysqld->{'proc'}, $warn_seconds))
   {
     my $mname= $mysqld->name();
     mtr_error("Failed to start mysqld $mname with command $exe");
@@ -5906,13 +5568,6 @@ sub start_mysqltest ($) {
     mtr_add_arg($args, "--sleep=%d", $opt_sleep);
   }
 
-  if ( $opt_valgrind )
-  {
-    # We are running server under valgrind, which causes some replication
-    # test to be much slower, notable rpl_mdev6020.  Increase timeout.
-    mtr_add_arg($args, "--wait-for-pos-timeout=1500");
-  }
-
   if ( $opt_ssl )
   {
     # Turn on SSL for _all_ test cases if option --ssl was used
@@ -5945,31 +5600,6 @@ sub start_mysqltest ($) {
   # ----------------------------------------------------------------------
   $ENV{'MYSQL_TEST'}= mtr_args2str($exe_mysqltest, @$args);
 
-  # ----------------------------------------------------------------------
-  # Add arguments that should not go into the MYSQL_TEST env var
-  # ----------------------------------------------------------------------
-  if ( $opt_valgrind_mysqltest )
-  {
-    # Prefix the Valgrind options to the argument list.
-    # We do this here, since we do not want to Valgrind the nested invocations
-    # of mysqltest; that would mess up the stderr output causing test failure.
-    my @args_saved = @$args;
-    mtr_init_args(\$args);
-    valgrind_arguments($args, \$exe);
-    mtr_add_arg($args, "%s", $_) for @args_saved;
-  }
-
-  # ----------------------------------------------------------------------
-  # Prefix the strace options to the argument list.
-  # ----------------------------------------------------------------------
-  if ( $opt_client_strace )
-  {
-    my @args_saved = @$args;
-    mtr_init_args(\$args);
-    strace_arguments($args, \$exe, "mysqltest");
-    mtr_add_arg($args, "%s", $_) for @args_saved;
-  }
-
   if ($opt_force > 1)
   {
     mtr_add_arg($args, "--continue-on-error");
@@ -6005,21 +5635,7 @@ sub start_mysqltest ($) {
     }
   }
 
-  if ( $opt_client_gdb )
-  {
-    gdb_arguments(\$args, \$exe, "client");
-  }
-  elsif ( $opt_client_ddd )
-  {
-    ddd_arguments(\$args, \$exe, "client");
-  }
-  if ( $opt_client_dbx ) {
-    dbx_arguments(\$args, \$exe, "client");
-  }
-  elsif ( $opt_client_debugger )
-  {
-    debugger_arguments(\$args, \$exe, "client");
-  }
+  My::Debugger::setup_client_args(\$args, \$exe);
 
   my $proc= My::SafeProcess->new
     (
@@ -6035,291 +5651,6 @@ sub start_mysqltest ($) {
 }
 
 #
-# Modify the exe and args so that program is run in gdb in xterm
-#
-sub gdb_arguments {
-  my $args= shift;
-  my $exe=  shift;
-  my $type= shift;
-  my $input= shift;
-
-  my $gdb_init_file= "$opt_vardir/tmp/gdbinit.$type";
-
-  # Remove the old gdbinit file
-  unlink($gdb_init_file);
-
-  # Put $args into a single string
-  $input = $input ? "< $input" : "";
-
-  if ($type eq 'client') {
-    mtr_tofile($gdb_init_file,
-      join("\n",
-        "set args @$$args $input",
-        split /;/, $opt_client_gdb || ""
-        ));
-  } elsif ($opt_valgrind_mysqld) {
-    my $v = $$exe;
-    my $vargs = [];
-    valgrind_arguments($vargs, \$v);
-    mtr_tofile($gdb_init_file, <<EOF);
-shell @My::SafeProcess::safe_process_cmd --parent-pid=`pgrep -x gdb` -- $v --vgdb-error=0 @$vargs @$$args &
-shell sleep 1
-target remote | /usr/lib64/valgrind/../../bin/vgdb
-EOF
-  } else {
-    mtr_tofile($gdb_init_file,
-      join("\n",
-        "set args @$$args $input",
-        split /;/, $opt_gdb || ""
-        ));
-  }
-
-  if ( $opt_manual_gdb )
-  {
-     print "\nTo start gdb for $type, type in another window:\n";
-     print "gdb -cd $glob_mysql_test_dir -x $gdb_init_file $$exe\n";
-
-     # Indicate the exe should not be started
-     $$exe= undef;
-     return;
-  }
-
-  $$args= [];
-  mtr_add_arg($$args, "-title");
-  mtr_add_arg($$args, "$type");
-  mtr_add_arg($$args, "-e");
-
-  if ( $exe_libtool )
-  {
-    mtr_add_arg($$args, $exe_libtool);
-    mtr_add_arg($$args, "--mode=execute");
-  }
-
-  mtr_add_arg($$args, "gdb");
-  mtr_add_arg($$args, "-x");
-  mtr_add_arg($$args, "$gdb_init_file");
-  mtr_add_arg($$args, "$$exe");
-
-  $$exe= "xterm";
-}
-
-#
-# Modify the exe and args so that program is run in lldb
-#
-sub lldb_arguments {
-  my $args= shift;
-  my $exe= shift;
-  my $type= shift;
-  my $input= shift;
-
-  my $lldb_init_file= "$opt_vardir/tmp/lldbinit.$type";
-  unlink($lldb_init_file);
-
-  # Put $args into a single string
-  my $str= join(" ", @$$args);
-  $input = $input ? "< $input" : "";
-
-  # write init file for mysqld or client
-  mtr_tofile($lldb_init_file, "process launch --stop-at-entry -- $str $input\n");
-
-    print "\nTo start lldb for $type, type in another window:\n";
-    print "cd $glob_mysql_test_dir && lldb -s $lldb_init_file $$exe\n";
-
-    # Indicate the exe should not be started
-    $$exe= undef;
-    return;
-}
-
-#
-# Modify the exe and args so that program is run in ddd
-#
-sub ddd_arguments {
-  my $args= shift;
-  my $exe=  shift;
-  my $type= shift;
-  my $input= shift;
-
-  my $gdb_init_file= "$opt_vardir/tmp/gdbinit.$type";
-
-  # Remove the old gdbinit file
-  unlink($gdb_init_file);
-
-  # Put $args into a single string
-  my $str= join(" ", @$$args);
-  $input = $input ? "< $input" : "";
-
-  # write init file for mysqld or client
-  mtr_tofile($gdb_init_file, "file $$exe\nset args $str $input\n");
-
-  if ( $opt_manual_ddd )
-  {
-     print "\nTo start ddd for $type, type in another window:\n";
-     print "ddd -cd $glob_mysql_test_dir -x $gdb_init_file $$exe\n";
-
-     # Indicate the exe should not be started
-     $$exe= undef;
-     return;
-  }
-
-  my $save_exe= $$exe;
-  $$args= [];
-  if ( $exe_libtool )
-  {
-    $$exe= $exe_libtool;
-    mtr_add_arg($$args, "--mode=execute");
-    mtr_add_arg($$args, "ddd");
-  }
-  else
-  {
-    $$exe= "ddd";
-  }
-  mtr_add_arg($$args, "--command=$gdb_init_file");
-  mtr_add_arg($$args, "$save_exe");
-}
-
-
-#
-# Modify the exe and args so that program is run in dbx in xterm
-#
-sub dbx_arguments {
-  my $args= shift;
-  my $exe=  shift;
-  my $type= shift;
-  my $input= shift;
-
-  # Put $args into a single string
-  my $str= join " ", @$$args;
-  my $runline= $input ? "run $str < $input" : "run $str";
-
-  if ( $opt_manual_dbx ) {
-    print "\nTo start dbx for $type, type in another window:\n";
-    print "cd $glob_mysql_test_dir; dbx -c \"stop in main; " .
-          "$runline\" $$exe\n";
-
-    # Indicate the exe should not be started
-    $$exe= undef;
-    return;
-  }
-
-  $$args= [];
-  mtr_add_arg($$args, "-title");
-  mtr_add_arg($$args, "$type");
-  mtr_add_arg($$args, "-e");
-
-  if ( $exe_libtool ) {
-    mtr_add_arg($$args, $exe_libtool);
-    mtr_add_arg($$args, "--mode=execute");
-  }
-
-  mtr_add_arg($$args, "dbx");
-  mtr_add_arg($$args, "-c");
-  mtr_add_arg($$args, "stop in main; $runline");
-  mtr_add_arg($$args, "$$exe");
-
-  $$exe= "xterm";
-}
-
-
-#
-# Modify the exe and args so that program is run in the selected debugger
-#
-sub debugger_arguments {
-  my $args= shift;
-  my $exe=  shift;
-  my $debugger= $opt_debugger || $opt_client_debugger;
-
-  if ( $debugger =~ /vcexpress|vc|devenv/ )
-  {
-    # vc[express] /debugexe exe arg1 .. argn
-
-    # Add name of the exe and /debugexe before args
-    unshift(@$$args, "$$exe");
-    unshift(@$$args, "/debugexe");
-
-    # Set exe to debuggername
-    $$exe= $debugger;
-
-  }
-  elsif ( $debugger =~ /windbg|vsjitdebugger/ )
-  {
-    # windbg exe arg1 .. argn
-
-    # Add name of the exe before args
-    unshift(@$$args, "$$exe");
-
-    # Set exe to debuggername
-    $$exe= $debugger;
-
-  }
-  else
-  {
-    mtr_error("Unknown argument \"$debugger\" passed to --debugger");
-  }
-}
-
-#
-# Modify the exe and args so that program is run in valgrind
-#
-sub valgrind_arguments {
-  my $args= shift;
-  my $exe=  shift;
-
-  # Ensure the jemalloc works with mysqld
-  if ($$exe =~ /mysqld/)
-  {
-    my %somalloc=(
-      'system jemalloc' => 'libjemalloc*',
-      'bundled jemalloc' => 'NONE'
-    );
-    my ($syn) = $somalloc{$mysqld_variables{'version-malloc-library'}};
-    mtr_add_arg($args, '--soname-synonyms=somalloc=%s', $syn) if $syn;
-  }
-
-  # Add valgrind options, can be overridden by user
-  mtr_add_arg($args, '%s', $_) for (@valgrind_args);
-
-  mtr_add_arg($args, $$exe);
-
-  $$exe= $opt_valgrind_path || "valgrind";
-
-  if ($exe_libtool)
-  {
-    # Add "libtool --mode-execute" before the test to execute
-    # if running in valgrind(to avoid valgrinding bash)
-    unshift(@$args, "--mode=execute", $$exe);
-    $$exe= $exe_libtool;
-  }
-}
-
-#
-# Modify the exe and args so that program is run in strace
-#
-sub strace_arguments {
-  my $args= shift;
-  my $exe=  shift;
-  my $mysqld_name= shift;
-  my $output= sprintf("%s/log/%s.strace", $path_vardir_trace, $mysqld_name);
-
-  mtr_add_arg($args, "-f");
-  mtr_add_arg($args, "-o%s", $output);
-
-  # Add strace options
-  mtr_add_arg($args, '%s', $_) for (@strace_args);
-
-  mtr_add_arg($args, $$exe);
-
-  $$exe=  $opt_stracer || "strace";
-
-  if ($exe_libtool)
-  {
-    # Add "libtool --mode-execute" before the test to execute
-    # if running in valgrind(to avoid valgrinding bash)
-    unshift(@$args, "--mode=execute", $$exe);
-    $$exe= $exe_libtool;
-  }
-}
-
-#
 # Search server logs for valgrind reports printed at mysqld termination
 #
 sub valgrind_exit_reports() {
@@ -6400,7 +5731,7 @@ sub usage ($) {
 
   local $"= ','; # for @DEFAULT_SUITES below
 
-  print <<HERE;
+  print <<HERE . My::Debugger::help() . <<HERE;
 
 $0 [ OPTIONS ] [ TESTCASE ]
 
@@ -6527,32 +5858,11 @@ Options to run test on running server
 
 Options for debugging the product
 
-  boot-dbx              Start bootstrap server in dbx
-  boot-ddd              Start bootstrap server in ddd
-  boot-gdb              Start bootstrap server in gdb
-  client-dbx            Start mysqltest client in dbx
-  client-ddd            Start mysqltest client in ddd
-  client-debugger=NAME  Start mysqltest in the selected debugger
-  client-gdb            Start mysqltest client in gdb
-  dbx                   Start the mysqld(s) in dbx
-  ddd                   Start the mysqld(s) in ddd
   debug                 Dump trace output for all servers and client programs
   debug-common          Same as debug, but sets 'd' debug flags to
                         "query,info,error,enter,exit"
   debug-server          Use debug version of server, but without turning on
                         tracing
-  debugger=NAME         Start mysqld in the selected debugger
-  gdb[=gdb_arguments]   Start the mysqld(s) in gdb
-  manual-debug          Let user manually start mysqld in debugger, before
-                        running test(s)
-  manual-gdb            Let user manually start mysqld in gdb, before running
-                        test(s)
-  manual-ddd            Let user manually start mysqld in ddd, before running
-                        test(s)
-  manual-dbx            Let user manually start mysqld in dbx, before running
-                        test(s)
-  manual-lldb           Let user manually start mysqld in lldb, before running 
-                        test(s)
   max-save-core         Limit the number of core files saved (to avoid filling
                         up disks for heavily crashing server). Defaults to
                         $opt_max_save_core. Set its default with
@@ -6566,38 +5876,7 @@ Options for debugging the product
                         $opt_max_test_fail, set to 0 for no limit. Set
                         it's default with MTR_MAX_TEST_FAIL
   core-in-failure	Generate a core even if run server is run with valgrind
-
-Options for valgrind
-
-  valgrind              Run the "mysqltest" and "mysqld" executables using
-                        valgrind with default options
-  valgrind-all          Synonym for --valgrind
-  valgrind-mysqltest    Run the "mysqltest" and "mysql_client_test" executable
-                        with valgrind
-  valgrind-mysqld       Run the "mysqld" executable with valgrind
-  valgrind-options=ARGS Deprecated, use --valgrind-option
-  valgrind-option=ARGS  Option to give valgrind, replaces default option(s),
-                        can be specified more then once
-  valgrind-path=<EXE>   Path to the valgrind executable
-  callgrind             Instruct valgrind to use callgrind
-
-Options for strace
-
-  strace                Run the "mysqld" executables using strace. Default
-                        options are -f -o 'vardir'/log/'mysqld-name'.strace.
-  client-strace         Trace the "mysqltest".
-  strace-option=ARGS    Option to give strace, appends to existing options.
-  stracer=<EXE>         Specify name and path to the trace program to use.
-                        Default is "strace". Example: $0 --stracer=ktrace.
-
-Options for rr (Record and Replay)
-  rr                    Run the "mysqld" executables using rr. Default run
-                        option is "rr record mysqld mysqld_options"
-  boot-rr               Start bootstrap server in rr
-  rr-arg=ARG            Option to give rr record, can be specified more then once
-  rr-dir=DIR            The directory where rr recordings are stored. Defaults
-                        to 'vardir'/rr.0 (rr.boot for bootstrap instance and
-                        rr.1, ..., rr.N for slave instances).
+HERE
 
 Misc options
   user=USER             User for connecting to mysqld(default: $opt_user)