diff options
| author | James E. Blair <jim@acmegating.com> | 2022-10-12 11:55:46 -0700 |
|---|---|---|
| committer | James E. Blair <jim@acmegating.com> | 2022-10-13 10:27:58 -0700 |
| commit | 8b2065c5bb2774ef5d892b528aa0ecb156a4b98c (patch) | |
| tree | 54636d51069b570a07bd5a14a99c604a99354eec | |
| parent | 4bda3e1969313d7046e54a4fbcb347217121aeeb (diff) | |
| download | zuul-8b2065c5bb2774ef5d892b528aa0ecb156a4b98c.tar.gz | |
Fix shutdown race in timer driver
If a reconfiguration happens after the timer connection is stopped,
we should not restart the apscheduler; that can lead to zombie
threads.
We should not perferm a reconfiguration after stopping the connection
anyway, so to correct that, change the shutdown sequence so that we
stop the layout update thread before we stop the connections.
Change-Id: If9c387dfc42fcba7de3969d81cf2c7917d7e4fa3
| -rw-r--r-- | zuul/driver/timer/__init__.py | 2 | ||||
| -rw-r--r-- | zuul/scheduler.py | 10 |
2 files changed, 8 insertions, 4 deletions
diff --git a/zuul/driver/timer/__init__.py b/zuul/driver/timer/__init__.py index 6eec5da7c..f415c6012 100644 --- a/zuul/driver/timer/__init__.py +++ b/zuul/driver/timer/__init__.py @@ -80,6 +80,8 @@ class TimerDriver(Driver, TriggerInterface): def reconfigure(self, tenant): self._removeJobs(tenant) + if self.stopped: + return if not self.apsched: # Handle possible reuse of the driver without connection objects. self.apsched = BackgroundScheduler() diff --git a/zuul/scheduler.py b/zuul/scheduler.py index d37ef9539..3377fa872 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -367,15 +367,17 @@ class Scheduler(threading.Thread): self.log.debug("Stopping nodepool") self.nodepool.stop() self.log.debug("Stopping connections") - # Connections may be waiting on the primed event + # Connections and layout update may be waiting on the primed event self.primed_event.set() + # Layout update can reconfigure connections, so make sure + # layout update is stopped first. + self.log.debug("Waiting for layout update thread") + self.layout_update_event.set() + self.layout_update_thread.join() self.stopConnections() self.log.debug("Stopping stats thread") self.stats_election.cancel() self.stats_thread.join() - self.log.debug("Waiting for layout update thread") - self.layout_update_event.set() - self.layout_update_thread.join() self.stopRepl() self._command_running = False self.log.debug("Stopping command socket") |