Partial fix for BUG#14106: IM: im_life_cycle and im_utils

tests fail on FreeBSD.

The patch contains of the following:
  - make Instance Manager, running in the daemon mode, dump
    the pid of angel-process in the special file;
  - default value of angel-pid-file-name is 'mysqlmanager.angel.pid';
  - if ordinary (IM) pid-file-name is specified in the configuration,
    angel-pid-file-name is updated according to the following
    rule: extension of the basename of pid-file-name is replaced by
    '.angel.pid.
    For example:
    - pid-file-name: /tmp/im.pid
      => angel-pid-file-name: /tmp/im.angel.pid
    - pid-file-name: /tmp/im.txt
      => angel-pid-file-name: /tmp/im.angel.pid
    - pid-file-name: /tmp/5.0/im
      => angel-pid-file-name: /tmp/5.0/im.angel.pid
  - add support for configuration option to customize angel
    pid file name;
  - fix test suite to use angel pid to kill Instance Manager
    by all means if something went wrong.

Background
----------

The problem is that on some OSes (FreeBSD for one) Instance
Manager does not get SIGTERM, so can not shutdown gracefully.
Test suite wasn't able to cope with it, so this leads to the
mess in test results.

The problem should be split into two:
  - fix signal handling;
  - fix test suite.

This patch fixes test suite so that it will be able to kill
uncooperative Instance Manager. In order to achieve this,
test suite needs to know PID of IM Angel process.


mysql-test/lib/mtr_process.pl:
  Added a function to send a signal to a process.
mysql-test/mysql-test-run.pl:
  Changed procedure of stopping Instance Manager.
  1. Try to stop IM normally (by sending SIGTERM);
  2. If one of IM-related processes is still alive,
  kill them all by SIGKILL and complain in the log.
server-tools/instance-manager/manager.cc:
  Made create_pid_file() available for the whole project.
server-tools/instance-manager/manager.h:
  Made create_pid_file() available for the whole project.
server-tools/instance-manager/mysqlmanager.cc:
  Dump PID of angel process into file.
server-tools/instance-manager/options.cc:
  Added an option to allow to customize angel pid file name.
server-tools/instance-manager/options.h:
  Added an option to allow to customize angel pid file name.

1 files changed, 73 insertions, 13 deletions

diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl
index 3a5fa2e7287..7613e7f087c 100755
--- a/mysql-test/mysql-test-run.pl
+++ b/mysql-test/mysql-test-run.pl
@@ -924,6 +924,7 @@ sub command_line_setup () {
    path_err =>        "$opt_vardir/log/im.err",
    path_log =>        "$opt_vardir/log/im.log",
    path_pid =>        "$opt_vardir/run/im.pid",
+   path_angel_pid =>  "$opt_vardir/run/im.angel.pid",
    path_sock =>       "$sockdir/im.sock",
    port =>            $im_port,
    start_timeout =>   $master->[0]->{'start_timeout'},
@@ -1179,6 +1180,7 @@ sub environment_setup () {
   $ENV{'NDB_STATUS_OK'}=      "YES";
 
   $ENV{'IM_PATH_PID'}=        $instance_manager->{path_pid};
+  $ENV{'IM_PATH_ANGEL_PID'}=  $instance_manager->{path_angel_pid};
   $ENV{'IM_PORT'}=            $instance_manager->{port};
 
   $ENV{'IM_MYSQLD1_SOCK'}=    $instance_manager->{instances}->[0]->{path_sock};
@@ -1803,6 +1805,7 @@ sub im_create_defaults_file($) {
 
 [manager]
 pid-file            = $instance_manager->{path_pid}
+angel-pid-file      = $instance_manager->{path_angel_pid}
 socket              = $instance_manager->{path_sock}
 port                = $instance_manager->{port}
 password-file       = $instance_manager->{password_file}
@@ -1827,7 +1830,7 @@ log-slow-queries    = $instance->{path_datadir}/mysqld$server_id.slow.log
 language            = $path_language
 character-sets-dir  = $path_charsetsdir
 basedir             = $path_my_basedir
-server_id           =$server_id
+server_id           = $server_id
 skip-stack-trace
 skip-innodb
 skip-bdb
@@ -2786,6 +2789,18 @@ sub im_start($$) {
 sub im_stop($) {
   my $instance_manager = shift;
 
+  # Obtain mysqld-process pids before we start stopping IM (it can delete pid
+  # files).
+
+  my @mysqld_pids = ();
+  my $instances = $instance_manager->{'instances'};
+
+  push(@mysqld_pids, mtr_get_pid_from_file($instances->[0]->{'path_pid'}))
+    if -r $instances->[0]->{'path_pid'};
+
+  push(@mysqld_pids, mtr_get_pid_from_file($instances->[1]->{'path_pid'}))
+    if -r $instances->[1]->{'path_pid'};
+
   # Re-read pid from the file, since during tests Instance Manager could have
   # been restarted, so its pid could have been changed.
 
@@ -2793,34 +2808,79 @@ sub im_stop($) {
     mtr_get_pid_from_file($instance_manager->{'path_pid'})
       if -f $instance_manager->{'path_pid'};
 
+  if (-f $instance_manager->{'path_angel_pid'})
+  {
+    $instance_manager->{'angel_pid'} =
+      mtr_get_pid_from_file($instance_manager->{'path_angel_pid'})
+  }
+  else
+  {
+    $instance_manager->{'angel_pid'} = undef;
+  }
+
   # Inspired from mtr_stop_mysqld_servers().
 
   start_reap_all();
 
-  # Create list of pids. We should stop Instance Manager and all started
-  # mysqld-instances. Some of them may be nonguarded, so IM will not stop them
-  # on shutdown.
+  # Try graceful shutdown.
 
-  my @pids = ( $instance_manager->{'pid'} );
-  my $instances = $instance_manager->{'instances'};
+  mtr_kill_process($instance_manager->{'pid'}, 'TERM', 10, 1);
+
+  # Check that all processes died.
+
+  my $clean_shutdown= 0;
 
-  if ( -r $instances->[0]->{'path_pid'} )
+  while (1)
   {
-    push(@pids, mtr_get_pid_from_file($instances->[0]->{'path_pid'}));
+    last if kill (0, $instance_manager->{'pid'});
+
+    last if (defined $instance_manager->{'angel_pid'}) &&
+            kill (0, $instance_manager->{'angel_pid'});
+
+    foreach my $pid (@mysqld_pids)
+    {
+      last if kill (0, $pid);
+    }
+
+    $clean_shutdown= 1;
+    last;
   }
 
-  if ( -r $instances->[1]->{'path_pid'} )
+  # Kill leftovers (the order is important).
+
+  unless ($clean_shutdown)
   {
-    push(@pids, mtr_get_pid_from_file($instances->[1]->{'path_pid'}));
-  }
+    mtr_kill_process($instance_manager->{'angel_pid'}, 'KILL', 10, 1)
+      if defined $instance_manager->{'angel_pid'};
+    
+    mtr_kill_process($instance_manager->{'pid'}, 'KILL', 10, 1);
 
-  # Kill processes.
+    # Shutdown managed mysqld-processes. Some of them may be nonguarded, so IM
+    # will not stop them on shutdown. So, we should firstly try to end them
+    # legally.
+
+    mtr_kill_processes(\@mysqld_pids);
+
+    # Complain in error log so that a warning will be shown.
+
+    my $errlog= "$opt_vardir/log/mysql-test-run.pl.err";
+
+    open (ERRLOG, ">>$errlog") ||
+      mtr_error("Can not open error log ($errlog)");
+
+    my $ts= localtime();
+    print ERRLOG
+      "Warning: [$ts] Instance Manager did not shutdown gracefully.\n";
+
+    close ERRLOG;
+  }
 
-  mtr_kill_processes(\@pids);
+  # That's all.
 
   stop_reap_all();
 
   $instance_manager->{'pid'} = undef;
+  $instance_manager->{'angel_pid'} = undef;
 }