Implement /1.0/remove-ghost-jobs

2 files changed, 66 insertions, 0 deletions

diff --git a/lorrycontroller/__init__.py b/lorrycontroller/__init__.py
index bc51b88..a65ff02 100644
--- a/lorrycontroller/__init__.py
+++ b/lorrycontroller/__init__.py
@@ -32,6 +32,7 @@ from movetopbottom import MoveToTop, MoveToBottom
 from stopjob import StopJob
 from listjobs import ListAllJobs, ListAllJobsHTML
 from showjob import ShowJob, ShowJobHTML, JobShower
+from removeghostjobs import RemoveGhostJobs
 from removejob import RemoveJob
 from lstroves import LsTroves, ForceLsTrove
 from pretendtime import PretendTime
diff --git a/lorrycontroller/removeghostjobs.py b/lorrycontroller/removeghostjobs.py
new file mode 100644
index 0000000..ceee3f6
--- /dev/null
+++ b/lorrycontroller/removeghostjobs.py
@@ -0,0 +1,65 @@
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import logging
+import time
+
+import bottle
+
+import lorrycontroller
+
+
+class RemoveGhostJobs(lorrycontroller.LorryControllerRoute):
+
+    http_method = 'POST'
+    path = '/1.0/remove-ghost-jobs'
+
+    def run(self, **kwargs):
+        logging.info('%s %s called', self.http_method, self.path)
+
+        ghost_timeout = self.app_settings['ghost-timeout']
+        ghosts = []
+        with self.open_statedb() as statedb:
+            for job_id in statedb.get_running_jobs():
+                if self.is_ghost_job(statedb, job_id, ghost_timeout):
+                    self.exorcise_ghost_job(statedb, job_id)
+                    ghosts.append(statedb.get_job_info(job_id))
+        return {
+            'killed-ghost-jobs': ghosts,
+            }
+
+    def is_ghost_job(self, statedb, job_id, ghost_timeout):
+        started, ended = statedb.get_job_started_and_ended(job_id)
+        return self.now(statedb) - started >= ghost_timeout
+
+    def now(self, statedb):
+        return statedb.get_current_time()
+
+    def exorcise_ghost_job(self, statedb, job_id):
+        logging.info('Job %s is a ghost job', job_id)
+        self.mark_job_to_be_killed_in_case_minion_appears(statedb, job_id)
+        self.mark_job_as_terminated(statedb, job_id)
+
+    def mark_job_to_be_killed_in_case_minion_appears(self, statedb, job_id):
+        statedb.set_kill_job(job_id, True)
+
+    def mark_job_as_terminated(self, statedb, job_id):
+        statedb.append_to_job_output(
+            job_id, '\nTERMINATED DUE TO GHOST TIMEOUT\n')
+        statedb.set_job_exit(job_id, 127, self.now(statedb), -1)
+
+        job_info = statedb.get_job_info(job_id)
+        statedb.set_running_job(job_info['path'], None)