diff options
Diffstat (limited to 'lorrycontroller/removeghostjobs.py')
-rw-r--r-- | lorrycontroller/removeghostjobs.py | 65 |
1 files changed, 65 insertions, 0 deletions
diff --git a/lorrycontroller/removeghostjobs.py b/lorrycontroller/removeghostjobs.py new file mode 100644 index 0000000..2b2760c --- /dev/null +++ b/lorrycontroller/removeghostjobs.py @@ -0,0 +1,65 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import logging +import time + +import bottle + +import lorrycontroller + + +class RemoveGhostJobs(lorrycontroller.LorryControllerRoute): + + http_method = 'POST' + path = '/1.0/remove-ghost-jobs' + + def run(self, **kwargs): + logging.info('%s %s called', self.http_method, self.path) + + ghost_timeout = self.app_settings['ghost-timeout'] + ghosts = [] + with self.open_statedb() as statedb: + for job_id in statedb.get_running_jobs(): + if self.is_ghost_job(statedb, job_id, ghost_timeout): + self.exorcise_ghost_job(statedb, job_id) + ghosts.append(statedb.get_job_info(job_id)) + return { + 'killed-ghost-jobs': ghosts, + } + + def is_ghost_job(self, statedb, job_id, ghost_timeout): + updated = statedb.get_job_updated(job_id) + return self.now(statedb) - updated >= ghost_timeout + + def now(self, statedb): + return statedb.get_current_time() + + def exorcise_ghost_job(self, statedb, job_id): + logging.info('Job %s is a ghost job', job_id) + self.mark_job_to_be_killed_in_case_minion_appears(statedb, job_id) + self.mark_job_as_terminated(statedb, job_id) + + def mark_job_to_be_killed_in_case_minion_appears(self, statedb, job_id): + statedb.set_kill_job(job_id, True) + + def mark_job_as_terminated(self, statedb, job_id): + statedb.append_to_job_output( + job_id, '\nTERMINATED DUE TO GHOST TIMEOUT\n') + statedb.set_job_exit(job_id, 127, self.now(statedb), -1) + + job_info = statedb.get_job_info(job_id) + statedb.set_running_job(job_info['path'], None) |