Robust handling of parent and child process deaths

Ensure that heat api child processes terminate when their parent
process dies unexpectedly.
Ensure that heat api child processes do not hang during clean shutdown.

Basically porting the same pipe handler code that exists in oslo_service.
This allows the child processes to detect their parent process has died
unexpectedly (ie: kill -9)

When a child process does a graceful shutdown it was observed to
sometimes never compelete.
This code waits up to a second before resorting to a more forceful shutdown
of the child process.

Change-Id: I8404f9727f41f37a9addb5541ae4d45fa1baba67
Story: 2003075
Task: 23126

1 files changed, 30 insertions, 0 deletions

diff --git a/heat/common/wsgi.py b/heat/common/wsgi.py
index 76d488e1a..15ec9e58e 100644
--- a/heat/common/wsgi.py
+++ b/heat/common/wsgi.py
@@ -339,6 +339,10 @@ class Server(object):
         signal.signal(signal.SIGTERM, self.kill_children)
         signal.signal(signal.SIGINT, self.kill_children)
         signal.signal(signal.SIGHUP, self.hup)
+
+        rfd, self.writepipe = os.pipe()
+        self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r')
+
         while len(self.children) < childs_num:
             self.run_child()
 
@@ -537,6 +541,26 @@ class Server(object):
             LOG.info('Started child %s', pid)
             self.children.add(pid)
 
+    def _pipe_watcher(self):
+        def _on_timeout_exit(*args):
+            LOG.info('Graceful shutdown timeout exceeded, '
+                     'instantaneous exiting')
+            os._exit(1)
+
+        # This will block until the write end is closed when the parent
+        # dies unexpectedly
+
+        self.readpipe.read(1)
+        LOG.info('Parent process has died unexpectedly, exiting')
+
+        # allow up to 1 second for sys.exit to gracefully shutdown
+        signal.signal(signal.SIGALRM, _on_timeout_exit)
+        signal.alarm(1)
+        # do the same as child_hup
+        eventlet.wsgi.is_accepting = False
+        self.sock.close()
+        sys.exit(1)
+
     def run_server(self):
         """Run a WSGI server."""
         eventlet.wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
@@ -544,6 +568,12 @@ class Server(object):
         eventlet.patcher.monkey_patch(all=False, socket=True)
         self.pool = eventlet.GreenPool(size=self.threads)
         socket_timeout = cfg.CONF.eventlet_opts.client_socket_timeout or None
+
+        # Close write to ensure only parent has it open
+        os.close(self.writepipe)
+        # Create greenthread to watch for parent to close pipe
+        eventlet.spawn_n(self._pipe_watcher)
+
         try:
             eventlet.wsgi.server(
                 self.sock,