diff options
author | Mitsuhiro Tanino <mitsuhiro.tanino@hds.com> | 2015-06-22 16:09:33 -0400 |
---|---|---|
committer | Mitsuhiro Tanino <mitsuhiro.tanino@hds.com> | 2015-06-22 16:17:32 -0400 |
commit | 13ce823686062d70b268b4d3888849adef07e4ff (patch) | |
tree | 0011d59e0263a682bc44cf0d51fce50627d4edee | |
parent | 0db1430757e1a33de7d3a20f4dd7048b4374d024 (diff) | |
download | oslo-incubator-13ce823686062d70b268b4d3888849adef07e4ff.tar.gz |
Graceful shutdown WSGI/RPC server
Currently, termination of WSGI application or RPC server immediately stops
service and so interrupts in-progress request.
Graceful handler for SIGTERM signal was added.
SIGINT signal handler was removed to allow instantaneous termination of
service.
DocImpact: graceful termination of process can be done by sending SIGTERM
signal to parent WSGI process. Graceful termination is not instantaneous.
To force instantaneous termination SIGINT signal must be sent.
Change-Id: If25f9565d832d1c36642ec3b1921563ef02890aa
Closes-bug: #1382390
Co-Author: Tiantian Gao <gtt116@gmail.com>
(cherry picked from commit fa9aa6b665f75e610f2b91a7d310f6499bd71770)
-rw-r--r-- | openstack/common/service.py | 34 | ||||
-rw-r--r-- | tests/unit/test_service.py | 51 |
2 files changed, 68 insertions, 17 deletions
diff --git a/openstack/common/service.py b/openstack/common/service.py index 9375bdc6..47b4887b 100644 --- a/openstack/common/service.py +++ b/openstack/common/service.py @@ -89,7 +89,6 @@ def _signo_to_signame(signo): def _set_signals_handler(handler): signal.signal(signal.SIGTERM, handler) - signal.signal(signal.SIGINT, handler) if _sighup_supported(): signal.signal(signal.SIGHUP, handler) @@ -216,6 +215,7 @@ class ProcessLauncher(object): self.sigcaught = None self.running = True self.wait_interval = wait_interval + self.launcher = None rfd, self.writepipe = os.pipe() self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r') self.handle_signal() @@ -238,20 +238,26 @@ class ProcessLauncher(object): LOG.info(_LI('Parent process has died unexpectedly, exiting')) + if self.launcher: + self.launcher.stop() + sys.exit(1) def _child_process_handle_signal(self): # Setup child signal handlers differently + + def _sigterm(*args): + signal.signal(signal.SIGTERM, signal.SIG_DFL) + self.launcher.stop() + def _sighup(*args): signal.signal(signal.SIGHUP, signal.SIG_DFL) raise SignalExit(signal.SIGHUP) # Parent signals with SIGTERM when it wants us to go away. - signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.signal(signal.SIGTERM, _sigterm) if _sighup_supported(): signal.signal(signal.SIGHUP, _sighup) - # Block SIGINT and let the parent send us a SIGTERM - signal.signal(signal.SIGINT, signal.SIG_IGN) def _child_wait_for_exit_or_signal(self, launcher): status = 0 @@ -272,8 +278,6 @@ class ProcessLauncher(object): except BaseException: LOG.exception(_LE('Unhandled exception')) status = 2 - finally: - launcher.stop() return status, signo @@ -312,13 +316,15 @@ class ProcessLauncher(object): pid = os.fork() if pid == 0: - launcher = self._child_process(wrap.service) + self.launcher = self._child_process(wrap.service) while True: self._child_process_handle_signal() - status, signo = self._child_wait_for_exit_or_signal(launcher) + status, signo = self._child_wait_for_exit_or_signal( + self.launcher) if not _is_sighup_and_daemon(signo): + self.launcher.wait() break - launcher.restart() + self.launcher.restart() os._exit(status) @@ -414,6 +420,13 @@ class ProcessLauncher(object): def stop(self): """Terminate child processes and wait on each.""" self.running = False + + LOG.debug("Stop services.") + for service in set( + [wrap.service for wrap in self.children.values()]): + service.stop() + + LOG.debug("Killing children.") for pid in self.children: try: os.kill(pid, signal.SIGTERM) @@ -470,7 +483,6 @@ class Services(object): # wait for graceful shutdown of services: for service in self.services: service.stop() - service.wait() # Each service has performed cleanup, now signal that the run_service # wrapper threads can now die: @@ -481,6 +493,8 @@ class Services(object): self.tg.stop() def wait(self): + for service in self.services: + service.wait() self.tg.wait() def restart(self): diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index f0fb5753..acfc9f31 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -382,17 +382,29 @@ class ProcessLauncherTest(test_base.BaseTestCase): launcher = service.ProcessLauncher() self.assertTrue(launcher.running) - launcher.children = [22, 222] + pid_nums = [22, 222] + fakeServiceWrapper = service.ServiceWrapper(service.Service(), 1) + launcher.children = {pid_nums[0]: fakeServiceWrapper, + pid_nums[1]: fakeServiceWrapper} with mock.patch('openstack.common.service.os.kill') as mock_kill: with mock.patch.object(launcher, '_wait_child') as _wait_child: - _wait_child.side_effect = lambda: launcher.children.pop() - launcher.stop() + + def fake_wait_child(): + pid = pid_nums.pop() + return launcher.children.pop(pid) + + _wait_child.side_effect = fake_wait_child + with mock.patch('openstack.common.service.Service.stop') as \ + mock_service_stop: + mock_service_stop.side_effect = lambda: None + launcher.stop() self.assertFalse(launcher.running) self.assertFalse(launcher.children) - self.assertEqual([mock.call(22, signal_mock.SIGTERM), - mock.call(222, signal_mock.SIGTERM)], + self.assertEqual([mock.call(222, signal_mock.SIGTERM), + mock.call(22, signal_mock.SIGTERM)], mock_kill.mock_calls) + mock_service_stop.assert_called_once_with() @mock.patch( "openstack.common.service.ProcessLauncher._signal_handlers_set", @@ -485,7 +497,7 @@ class ServiceTest(test_base.BaseTestCase): class EventletServerTest(test_base.BaseTestCase): - def test_shuts_down_on_sigterm_when_client_connected(self): + def run_server(self): server_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'eventlet_service.py') @@ -524,10 +536,35 @@ class EventletServerTest(test_base.BaseTestCase): # server or signal handlers. time.sleep(1) + return (server, conn) + + def test_shuts_down_on_sigint_when_client_connected(self): + server, conn = self.run_server() + + # send SIGINT to the server and wait for it to exit while client still + # connected. + server.send_signal(signal.SIGINT) + server.wait() + conn.close() + + def test_graceful_shuts_down_on_sigterm_when_client_connected(self): + server, conn = self.run_server() + # send SIGTERM to the server and wait for it to exit while client still # connected. server.send_signal(signal.SIGTERM) - server.wait() + server_wait_thread = threading.Thread( + target=lambda server: server.wait(), args=(server,)) + server_wait_thread.start() + + # server with graceful shutdown must wait forewer + # for closing connection by client + # but for test 3 seconds is enough + time.sleep(3) + + self.assertEqual(True, server_wait_thread.is_alive()) conn.close() + + server_wait_thread.join() |