1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
|
# Copyright (C) 2014 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import logging
import lorrycontroller
class RemoveGhostJobs(lorrycontroller.LorryControllerRoute):
http_method = 'POST'
path = '/1.0/remove-ghost-jobs'
def run(self, **kwargs):
logging.info('%s %s called', self.http_method, self.path)
ghost_timeout = self.app_settings['ghost-timeout']
ghosts = []
with self.open_statedb() as statedb:
for job_id in statedb.get_running_jobs():
if self.is_ghost_job(statedb, job_id, ghost_timeout):
self.exorcise_ghost_job(statedb, job_id)
ghosts.append(statedb.get_job_info(job_id))
return {
'killed-ghost-jobs': ghosts,
}
def is_ghost_job(self, statedb, job_id, ghost_timeout):
updated = statedb.get_job_updated(job_id)
return self.now(statedb) - updated >= ghost_timeout
def now(self, statedb):
return statedb.get_current_time()
def exorcise_ghost_job(self, statedb, job_id):
logging.info('Job %s is a ghost job', job_id)
self.mark_job_to_be_killed_in_case_minion_appears(statedb, job_id)
self.mark_job_as_terminated(statedb, job_id)
def mark_job_to_be_killed_in_case_minion_appears(self, statedb, job_id):
statedb.set_kill_job(job_id, True)
def mark_job_as_terminated(self, statedb, job_id):
statedb.append_to_job_output(
job_id, '\nTERMINATED DUE TO GHOST TIMEOUT\n')
statedb.set_job_exit(job_id, 127, self.now(statedb), -1)
job_info = statedb.get_job_info(job_id)
statedb.set_running_job(job_info['path'], None)
|
