daemon: Add support for process monitoring and restart.

6 files changed, 277 insertions, 2 deletions

diff --git a/lib/daemon.c b/lib/daemon.c
index 9895f7251..862911463 100644
--- a/lib/daemon.c
+++ b/lib/daemon.c
@@ -25,6 +25,7 @@
 #include "fatal-signal.h"
 #include "dirs.h"
 #include "lockfile.h"
+#include "process.h"
 #include "socket-util.h"
 #include "timeval.h"
 #include "util.h"
@@ -47,6 +48,10 @@ static bool chdir_ = true;
 /* File descriptor used by daemonize_start() and daemonize_complete(). */
 static int daemonize_fd = -1;
 
+/* --monitor: Should a supervisory process monitor the daemon and restart it if
+ * it dies due to an error signal? */
+static bool monitor;
+
 /* Returns the file name that would be used for a pidfile if 'name' were
  * provided to set_pidfile().  The caller must free the returned string. */
 char *
@@ -117,6 +122,14 @@ get_detach(void)
     return detach;
 }
 
+/* Sets up a following call to daemonize() to fork a supervisory process to
+ * monitor the daemon and restart it if it dies due to an error signal.  */
+void
+daemon_set_monitor(void)
+{
+    monitor = true;
+}
+
 /* If a pidfile has been configured and that pidfile already exists and is
  * locked by a running process, returns the pid of the running process.
  * Otherwise, returns 0. */
@@ -286,6 +299,69 @@ fork_notify_startup(int fd)
     }
 }
 
+static bool
+should_restart(int status)
+{
+    if (WIFSIGNALED(status)) {
+        static const int error_signals[] = {
+            SIGABRT, SIGALRM, SIGBUS, SIGFPE, SIGILL, SIGPIPE, SIGSEGV,
+            SIGXCPU, SIGXFSZ
+        };
+
+        size_t i;
+
+        for (i = 0; i < ARRAY_SIZE(error_signals); i++) {
+            if (error_signals[i] == WTERMSIG(status)) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
+static void
+monitor_daemon(pid_t daemon_pid)
+{
+    /* XXX Should limit the rate at which we restart the daemon. */
+    /* XXX Should log daemon's stderr output at startup time. */
+    const char *saved_program_name;
+
+    saved_program_name = program_name;
+    program_name = xasprintf("monitor(%s)", program_name);
+    for (;;) {
+        int retval;
+        int status;
+
+        do {
+            retval = waitpid(daemon_pid, &status, 0);
+        } while (retval == -1 && errno == EINTR);
+
+        if (retval == -1) {
+            ovs_fatal(errno, "waitpid failed");
+        } else if (retval == daemon_pid) {
+            char *status_msg = process_status_msg(status);
+            if (should_restart(status)) {
+                VLOG_ERR("%s daemon died unexpectedly (%s), restarting",
+                         saved_program_name, status_msg);
+                free(status_msg);
+
+                daemon_pid = fork_and_wait_for_startup(&daemonize_fd);
+                if (!daemon_pid) {
+                    break;
+                }
+            } else {
+                VLOG_INFO("%s daemon exited normally (%s), exiting",
+                          saved_program_name, status_msg);
+                exit(0);
+            }
+        }
+    }
+
+    /* Running in new daemon process. */
+    free((char *) program_name);
+    program_name = saved_program_name;
+}
+
 /* Close stdin, stdout, stderr.  If we're started from e.g. an SSH session,
  * then this keeps us from holding that session open artificially. */
 static void
@@ -314,6 +390,20 @@ daemonize_start(void)
             /* Running in parent process. */
             exit(0);
         }
+        /* Running in daemon or monitor process. */
+    }
+
+    if (monitor) {
+        int saved_daemonize_fd = daemonize_fd;
+        pid_t daemon_pid;
+
+        daemon_pid = fork_and_wait_for_startup(&daemonize_fd);
+        if (daemon_pid > 0) {
+            /* Running in monitor process. */
+            fork_notify_startup(saved_daemonize_fd);
+            close_standard_fds();
+            monitor_daemon(daemon_pid);
+        }
         /* Running in daemon process. */
     }
 
diff --git a/lib/daemon.h b/lib/daemon.h
index 8ded63f1a..1d630768e 100644
--- a/lib/daemon.h
+++ b/lib/daemon.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2009 Nicira Networks.
+ * Copyright (c) 2008, 2009, 2010 Nicira Networks.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,13 +26,15 @@ enum {
     OPT_NO_CHDIR,
     OPT_OVERWRITE_PIDFILE,
     OPT_PIDFILE,
+    OPT_MONITOR
 };
 
 #define DAEMON_LONG_OPTIONS                                          \
         {"detach",            no_argument, 0, OPT_DETACH},           \
         {"no-chdir",          no_argument, 0, OPT_NO_CHDIR},         \
         {"pidfile",           optional_argument, 0, OPT_PIDFILE},    \
-        {"overwrite-pidfile", no_argument, 0, OPT_OVERWRITE_PIDFILE}
+        {"overwrite-pidfile", no_argument, 0, OPT_OVERWRITE_PIDFILE},\
+        {"monitor",           no_argument, 0, OPT_MONITOR}
 
 #define DAEMON_OPTION_HANDLERS                  \
         case OPT_DETACH:                        \
@@ -49,6 +51,10 @@ enum {
                                                 \
         case OPT_OVERWRITE_PIDFILE:             \
             ignore_existing_pidfile();          \
+            break;                              \
+                                                \
+        case OPT_MONITOR:                       \
+            daemon_set_monitor();               \
             break;
 
 char *make_pidfile_name(const char *name);
@@ -58,6 +64,7 @@ void set_no_chdir(void);
 bool is_chdir_enabled(void);
 void set_detach(void);
 bool get_detach(void);
+void daemon_set_monitor(void);
 void daemonize(void);
 void daemonize_start(void);
 void daemonize_complete(void);
diff --git a/lib/daemon.man b/lib/daemon.man
index ea0561daf..c23937747 100644
--- a/lib/daemon.man
+++ b/lib/daemon.man
@@ -20,6 +20,17 @@ Causes \fB\*(PN\fR to detach itself from the foreground session and
 run as a background process.
 
 .TP
+\fB--monitor\fR
+Creates an additional process to monitor the \fB\*(PN\fR daemon.  If
+the daemon dies due to a signal that indicates a programming error
+(e.g. \fBSIGSEGV\fR, \fBSIGABRT\fR), then the monitor process starts a
+new copy of it.  If the daemon die or exits for another reason, the
+monitor process exits.
+.IP
+This option is normally used with \fB--detach\fR, but it also
+functions without it.
+
+.TP
 \fB--no-chdir\fR
 By default, when \fB--detach\fR is specified, \fB\*(PN\fR 
 changes its current working directory to the root directory after it 
diff --git a/tests/automake.mk b/tests/automake.mk
index 7a73a84f6..dc677eb1e 100644
--- a/tests/automake.mk
+++ b/tests/automake.mk
@@ -9,6 +9,7 @@ TESTSUITE_AT = \
 	tests/ovsdb-macros.at \
 	tests/lcov-pre.at \
 	tests/library.at \
+	tests/daemon.at \
 	tests/vconn.at \
 	tests/dir_name.at \
 	tests/aes128.at \
diff --git a/tests/daemon.at b/tests/daemon.at
new file mode 100644
index 000000000..1062229a0
--- /dev/null
+++ b/tests/daemon.at
@@ -0,0 +1,165 @@
+AT_BANNER([daemon unit tests])
+
+AT_SETUP([daemon])
+OVSDB_INIT([db])
+AT_CAPTURE_FILE([pid])
+AT_CAPTURE_FILE([expected])
+# Start the daemon and wait for the pidfile to get created
+# and that its contents are the correct pid.
+AT_CHECK([ovsdb-server --pidfile=$PWD/pid --remote=punix:socket --unixctl=$PWD/unixctl db& echo $! > expected], [0], [ignore], [ignore])
+OVS_WAIT_UNTIL([test -s pid], [kill `cat expected`])
+AT_CHECK(
+  [pid=`cat pid` && expected=`cat expected` && test "$pid" = "$expected"],
+  [0], [], [], [kill `cat expected`])
+AT_CHECK([kill -0 `cat pid`], [0], [], [], [kill `cat expected`])
+# Kill the daemon and make sure that the pidfile gets deleted.
+kill `cat expected`
+OVS_WAIT_WHILE([kill -0 `cat expected`])
+AT_CHECK([test ! -e pid])
+AT_CLEANUP
+
+AT_SETUP([daemon --monitor])
+OVSDB_INIT([db])
+AT_CAPTURE_FILE([pid])
+AT_CAPTURE_FILE([parent])
+AT_CAPTURE_FILE([parentpid])
+AT_CAPTURE_FILE([newpid])
+# Start the daemon and wait for the pidfile to get created.
+AT_CHECK([ovsdb-server --monitor --pidfile=$PWD/pid --remote=punix:socket --unixctl=$PWD/unixctl db& echo $! > parent], [0], [ignore], [ignore])
+OVS_WAIT_UNTIL([test -s pid], [kill `cat parent`])
+# Check that the pidfile names a running process,
+# and that the parent process of that process is our child process.
+AT_CHECK([kill -0 `cat pid`], [0], [], [], [kill `cat parent`])
+AT_CHECK([ps -o ppid= -p `cat pid` > parentpid],
+  [0], [], [], [kill `cat parent`])
+AT_CHECK(
+  [parentpid=`cat parentpid` && 
+   parent=`cat parent` && 
+   test $parentpid = $parent],
+  [0], [], [], [kill `cat parent`])
+# Kill the daemon process, making it look like a segfault,
+# and wait for a new child process to get spawned.
+AT_CHECK([cp pid oldpid], [0], [], [], [kill `cat parent`])
+AT_CHECK([kill -SEGV `cat pid`], [0], [], [ignore], [kill `cat parent`])
+OVS_WAIT_WHILE([kill -0 `cat oldpid`], [kill `cat parent`])
+OVS_WAIT_UNTIL([test -s pid && test `cat pid` != `cat oldpid`],
+  [kill `cat parent`])
+AT_CHECK([cp pid newpid], [0], [], [], [kill `cat parent`])
+# Check that the pidfile names a running process,
+# and that the parent process of that process is our child process.
+AT_CHECK([ps -o ppid= -p `cat pid` > parentpid],
+  [0], [], [], [kill `cat parent`])
+AT_CHECK(
+  [parentpid=`cat parentpid` && 
+   parent=`cat parent` && 
+   test $parentpid = $parent],
+  [0], [], [], [kill `cat parent`])
+# Kill the daemon process with SIGTERM, and wait for the daemon
+# and the monitor processes to go away and the pidfile to get deleted.
+AT_CHECK([kill `cat pid`], [0], [], [ignore], [kill `cat parent`])
+OVS_WAIT_WHILE([kill -0 `cat parent` || kill -0 `cat newpid` || test -e pid],
+  [kill `cat parent`])
+AT_CLEANUP
+
+AT_SETUP([daemon --detach])
+AT_CAPTURE_FILE([pid])
+OVSDB_INIT([db])
+# Start the daemon and make sure that the pidfile exists immediately.
+# We don't wait for the pidfile to get created because the daemon is
+# supposed to do so before the parent exits.
+AT_CHECK([ovsdb-server --detach --pidfile=$PWD/pid --remote=punix:socket --unixctl=$PWD/unixctl db], [0], [ignore], [ignore])
+AT_CHECK([test -s pid])
+AT_CHECK([kill -0 `cat pid`])
+# Kill the daemon and make sure that the pidfile gets deleted.
+cp pid saved-pid
+kill `cat pid`
+OVS_WAIT_WHILE([kill -0 `cat saved-pid`])
+AT_CHECK([test ! -e pid])
+AT_CLEANUP
+
+AT_SETUP([daemon --detach --monitor])
+m4_define([CHECK], 
+  [AT_CHECK([$1], [$2], [$3], [$4], [kill `cat daemon monitor`])])
+OVSDB_INIT([db])
+AT_CAPTURE_FILE([daemon])
+AT_CAPTURE_FILE([olddaemon])
+AT_CAPTURE_FILE([newdaemon])
+AT_CAPTURE_FILE([monitor])
+AT_CAPTURE_FILE([newmonitor])
+AT_CAPTURE_FILE([init])
+# Start the daemon and make sure that the pidfile exists immediately.
+# We don't wait for the pidfile to get created because the daemon is
+# supposed to do so before the parent exits.
+AT_CHECK([ovsdb-server --detach --pidfile=$PWD/daemon --monitor --remote=punix:socket --unixctl=$PWD/unixctl db], [0], [ignore], [ignore])
+AT_CHECK([test -s daemon])
+# Check that the pidfile names a running process,
+# and that the parent process of that process is a running process,
+# and that the parent process of that process is init.
+CHECK([kill -0 `cat daemon`])
+CHECK([ps -o ppid= -p `cat daemon` > monitor])
+CHECK([kill -0 `cat monitor`])
+CHECK([ps -o ppid= -p `cat monitor` > init])
+CHECK([test `cat init` = 1])
+# Kill the daemon process, making it look like a segfault,
+# and wait for a new daemon process to get spawned.
+CHECK([cp daemon olddaemon])
+CHECK([kill -SEGV `cat daemon`], [0], [ignore], [ignore])
+OVS_WAIT_WHILE([kill -0 `cat olddaemon`], [kill `cat olddaemon daemon`])
+OVS_WAIT_UNTIL([test -s daemon && test `cat daemon` != `cat olddaemon`],
+  [kill `cat olddaemon daemon`])
+CHECK([cp daemon newdaemon])
+# Check that the pidfile names a running process,
+# and that the parent process of that process is our child process.
+CHECK([kill -0 `cat daemon`])
+CHECK([diff olddaemon newdaemon], [1], [ignore])
+CHECK([ps -o ppid= -p `cat daemon` > newmonitor])
+CHECK([diff monitor newmonitor])
+CHECK([kill -0 `cat newmonitor`])
+CHECK([ps -o ppid= -p `cat newmonitor` > init])
+CHECK([test `cat init` = 1])
+# Kill the daemon process with SIGTERM, and wait for the daemon
+# and the monitor processes to go away and the pidfile to get deleted.
+CHECK([kill `cat daemon`], [0], [], [ignore])
+OVS_WAIT_WHILE(
+  [kill -0 `cat monitor` || kill -0 `cat newdaemon` || test -e daemon],
+  [kill `cat monitor newdaemon`])
+m4_undefine([CHECK])
+AT_CLEANUP
+
+AT_SETUP([daemon --detach startup errors])
+AT_CAPTURE_FILE([pid])
+OVSDB_INIT([db])
+OVS_CHECK_LCOV([ovsdb-server --detach --pidfile=$PWD/pid --unixctl=$PWD/nonexistent/unixctl db], [1], [], [stderr])
+AT_CHECK([grep 'ovsdb-server: could not initialize control socket' stderr],
+  [0], [ignore], [])
+AT_CHECK([test ! -s pid])
+AT_CLEANUP
+
+AT_SETUP([daemon --detach --monitor startup errors])
+AT_CAPTURE_FILE([pid])
+OVSDB_INIT([db])
+OVS_CHECK_LCOV([ovsdb-server --detach --pidfile=$PWD/pid --monitor --unixctl=$PWD/nonexistent/unixctl db], [1], [], [stderr])
+AT_CHECK([grep 'ovsdb-server: could not initialize control socket' stderr],
+  [0], [ignore], [])
+AT_CHECK([test ! -s pid])
+AT_CLEANUP
+
+AT_SETUP([daemon --detach closes standard fds])
+AT_CAPTURE_FILE([pid])
+OVSDB_INIT([db])
+AT_CHECK([(yes; echo $? > status) | ovsdb-server --detach --pidfile=$PWD/pid --unixctl=$PWD/unixctl db], [0], [], [stderr])
+AT_CHECK([kill `cat pid`])
+AT_CHECK([test -s status])
+AT_CHECK([kill -l `cat status`], [0], [PIPE
+])
+AT_CLEANUP
+
+AT_SETUP([daemon --detach --monitor closes standard fds])
+AT_CAPTURE_FILE([pid])
+OVSDB_INIT([db])
+AT_CHECK([(yes; echo $? > status) | ovsdb-server --detach --monitor --pidfile=$PWD/pid --unixctl=$PWD/unixctl db], [0], [], [stderr])
+AT_CHECK([kill `cat pid`])
+AT_CHECK([test -s status])
+AT_CHECK([kill -l `cat status`], [0], [PIPE
+])
+AT_CLEANUP
diff --git a/tests/testsuite.at b/tests/testsuite.at
index 6ec1a8094..93d7e6eb1 100644
--- a/tests/testsuite.at
+++ b/tests/testsuite.at
@@ -39,6 +39,7 @@ m4_include([tests/ovsdb-macros.at])
 m4_include([tests/lcov-pre.at])
 
 m4_include([tests/library.at])
+m4_include([tests/daemon.at])
 m4_include([tests/vconn.at])
 m4_include([tests/dir_name.at])
 m4_include([tests/aes128.at])