Accept, store, and show MINION's disk usage info

11 files changed, 57 insertions, 19 deletions

diff --git a/lorrycontroller/__init__.py b/lorrycontroller/__init__.py
index 214024b..fec1866 100644
--- a/lorrycontroller/__init__.py
+++ b/lorrycontroller/__init__.py
@@ -21,7 +21,7 @@ from statedb import (
     TroveNotFoundError)
 from route import LorryControllerRoute
 from readconf import ReadConfiguration
-from status import Status, StatusHTML
+from status import Status, StatusHTML, StatusRenderer
 from listqueue import ListQueue
 from showlorry import ShowLorry, ShowLorryHTML
 from startstopqueue import StartQueue, StopQueue
@@ -31,7 +31,7 @@ from listrunningjobs import ListRunningJobs
 from movetopbottom import MoveToTop, MoveToBottom
 from stopjob import StopJob
 from listjobs import ListAllJobs, ListAllJobsHTML
-from showjob import ShowJob
+from showjob import ShowJob, ShowJobHTML, JobShower
 from removejob import RemoveJob
 from lstroves import LsTroves, ForceLsTrove
 from pretendtime import PretendTime
diff --git a/lorrycontroller/jobupdate.py b/lorrycontroller/jobupdate.py
index b32efdf..b6ee1fe 100644
--- a/lorrycontroller/jobupdate.py
+++ b/lorrycontroller/jobupdate.py
@@ -34,6 +34,7 @@ class JobUpdate(lorrycontroller.LorryControllerRoute):
         exit = bottle.request.forms.exit
         stdout = bottle.request.forms.stdout
         stderr = bottle.request.forms.stderr
+        disk_usage = bottle.request.forms.disk_usage
 
         logging.info('Job %s updated (exit=%s)', job_id, exit)
 
@@ -51,7 +52,8 @@ class JobUpdate(lorrycontroller.LorryControllerRoute):
                 now = statedb.get_current_time()
                 statedb.set_lorry_last_run(path, int(now))
                 statedb.set_running_job(path, None)
-                statedb.set_job_exit(job_id, exit, int(now))
+                statedb.set_job_exit(job_id, exit, int(now), disk_usage)
+                statedb.set_lorry_disk_usage(path, disk_usage)
             elif self.time_to_die(statedb, job_id, lorry_info):
                 logging.warning(
                     'Job %r has been running too long, '
diff --git a/lorrycontroller/listjobs.py b/lorrycontroller/listjobs.py
index a2020cc..eaffeef 100644
--- a/lorrycontroller/listjobs.py
+++ b/lorrycontroller/listjobs.py
@@ -45,7 +45,8 @@ class ListAllJobsHTML(lorrycontroller.LorryControllerRoute):
         now = statedb.get_current_time()
         values = {
             'job_infos': self.get_jobs(statedb),
-            'timestamp': time.strftime('%Y-%m-%d %H:%M:%S UTC', now),
+            'timestamp':
+                time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(now)),
             }
         return bottle.template(self._templates['list-jobs'], **values)
 
diff --git a/lorrycontroller/showjob.py b/lorrycontroller/showjob.py
index 3ebab8d..6f73ed6 100644
--- a/lorrycontroller/showjob.py
+++ b/lorrycontroller/showjob.py
@@ -29,6 +29,7 @@ class JobShower(object):
         exit = statedb.get_job_exit(job_id)
         output = statedb.get_job_output(job_id)
         started, ended = statedb.get_job_started_and_ended(job_id)
+        disk_usage = statedb.get_job_disk_usage(job_id)
         now = statedb.get_current_time()
 
         return {
@@ -37,15 +38,23 @@ class JobShower(object):
             'pid': statedb.get_job_minion_pid(job_id),
             'path': statedb.get_job_path(job_id),
             'exit': 'no' if exit is None else exit,
+            'disk_usage': disk_usage,
+            'disk_usage_nice': self.format_bytesize(disk_usage),
             'output': output,
-            'job-started': self.format_time(started),
-            'job-ended': self.format_time(ended),
+            'job_started': self.format_time(started),
+            'job_ended': self.format_time(ended),
             'timestamp': self.format_time(now),
             }
 
     def format_time(self, timestamp):
         return time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(timestamp))
 
+    def format_bytesize(self, num_bytes):
+        if num_bytes is None:
+            return 'unknown'
+        mebibyte = 2**20
+        return '%.1f MiB' % (float(num_bytes) / float(mebibyte))
+
 
 class ShowJob(lorrycontroller.LorryControllerRoute):
 
diff --git a/lorrycontroller/showlorry.py b/lorrycontroller/showlorry.py
index 599878c..fc336a5 100644
--- a/lorrycontroller/showlorry.py
+++ b/lorrycontroller/showlorry.py
@@ -51,7 +51,8 @@ class ShowLorryHTML(lorrycontroller.LorryControllerRoute):
         except lorrycontroller.LorryNotFoundError as e:
             bottle.abort(404, str(e))
 
-        renderer = StatusRenderer()
+        renderer = lorrycontroller.StatusRenderer()
+        shower = lorrycontroller.JobShower()
 
         lorry_obj = json.loads(lorry_info['text']).values()[0]
         lorry_info['url'] = lorry_obj['url']
@@ -63,11 +64,15 @@ class ShowLorryHTML(lorrycontroller.LorryControllerRoute):
             '%Y-%m-%d %H:%M:%S UTC',
             time.gmtime(lorry_info['last_run']))
         
+        lorry_info['disk_usage_nice'] = shower.format_bytesize(
+            lorry_info['disk_usage'])
+
+        now = statedb.get_current_time()
+
         due = lorry_info['last_run'] + lorry_info['interval']
-        lorry_info['due_nice'] = renderer.format_due_nicely(due)
+        lorry_info['due_nice'] = renderer.format_due_nicely(due, now)
         
-        now = statedb.get_current_time()
-        timestamp = time.strftime('%Y-%m-%d %H:%M:%S UTC', now)
+        timestamp = time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(now))
 
         parts = urlparse.urlparse(bottle.request.url)
         host, port = parts.netloc.split(':', 1)
diff --git a/lorrycontroller/statedb.py b/lorrycontroller/statedb.py
index db91782..f393e07 100644
--- a/lorrycontroller/statedb.py
+++ b/lorrycontroller/statedb.py
@@ -65,6 +65,7 @@ class StateDB(object):
             ('last_run', 'INT'),
             ('interval', 'INT'),
             ('lorry_timeout', 'INT'),
+            ('disk_usage', 'INT'),
             ]
         self.lorries_booleans = [
             'kill_job',
@@ -125,6 +126,7 @@ class StateDB(object):
             'ended INT, '
             'path TEXT, '
             'exit TEXT, '
+            'disk_usage INT, '
             'output TEXT)')
 
         # Table for holding max number of jobs running at once. If no
@@ -411,6 +413,15 @@ class StateDB(object):
             'UPDATE lorries SET last_run=? WHERE path=?',
             (last_run, path))
 
+    def set_lorry_disk_usage(self, path, disk_usage):
+        logging.debug(
+            'StateDB.set_lorry_disk_usage(%r, %r) called', path, disk_usage)
+        assert self.in_transaction
+        c = self.get_cursor()
+        c.execute(
+            'UPDATE lorries SET disk_usage=? WHERE path=?',
+            (disk_usage, path))
+
     def get_next_job_id(self):
         logging.debug('StateDB.get_next_job_id called')
         assert self.in_transaction
@@ -477,14 +488,22 @@ class StateDB(object):
         row = c.fetchone()
         return row[0]
 
-    def set_job_exit(self, job_id, exit, ended):
+    def set_job_exit(self, job_id, exit, ended, disk_usage):
         logging.debug(
-            'StateDB.set_job_exit(%r, %r, %r) called', job_id, exit, ended)
+            'StateDB.set_job_exit(%r, %r, %r, %r) called',
+            job_id, exit, ended, disk_usage)
         assert self.in_transaction
         c = self.get_cursor()
         c.execute(
-            'UPDATE jobs SET exit=?, ended=? WHERE job_id IS ?',
-            (exit, ended, job_id))
+            'UPDATE jobs SET exit=?, ended=?, disk_usage=? '
+            'WHERE job_id IS ?',
+            (exit, ended, disk_usage, job_id))
+
+    def get_job_disk_usage(self, job_id):
+        c = self.get_cursor()
+        c.execute('SELECT disk_usage FROM jobs WHERE job_id IS ?', (job_id,))
+        row = c.fetchone()
+        return row[0]
 
     def get_job_output(self, job_id):
         c = self.get_cursor()
diff --git a/templates/job.tpl b/templates/job.tpl
index cc418cf..da883af 100644
--- a/templates/job.tpl
+++ b/templates/job.tpl
@@ -9,6 +9,7 @@
 <p>Path of git repo: <code>{{path}}</code></p>
 <p>MINION: <code>{{host}}:{{pid}}</code></p>
 <p>Exit code: <code>{{exit}}</code></p>
+<p>Lorry disk usage (after job's finished): {{disk_usage_nice}}</p>
 <p>Output:</p>
 <pre>{{output}}</pre>
 <hr />
diff --git a/templates/list-jobs.tpl b/templates/list-jobs.tpl
index c9fe9f0..1d530aa 100644
--- a/templates/list-jobs.tpl
+++ b/templates/list-jobs.tpl
@@ -17,8 +17,8 @@
 </tr>
 % for job in job_infos:
 <tr>
-<td><a href="/1.0/job/{{job['job_id']}}">{{job['job_id']}}</a></td>
-<td><a href="/1.0/lorry/{{job['path']}}">{{job['path']}}</a></td>
+<td><a href="/1.0/job-html/{{job['job_id']}}">{{job['job_id']}}</a></td>
+<td><a href="/1.0/lorry-html/{{job['path']}}">{{job['path']}}</a></td>
 <td>{{job['exit']}}</td>
 </tr>
 % end
diff --git a/templates/lorry.tpl b/templates/lorry.tpl
index 7b6ccfa..fad85cd 100644
--- a/templates/lorry.tpl
+++ b/templates/lorry.tpl
@@ -18,6 +18,7 @@
 <tr> <th>Last run</th> <td>{{lorry['last_run_nice']}}</td> </tr>
 <tr> <th>Due</th> <td>{{lorry['due_nice']}}</td> </tr>
 <tr> <th>From Trove</th> <td>{{lorry['from_trovehost']}}</td> </tr>
+<tr> <th>Disk usage</th> <td>{{lorry['disk_usage_nice']}}</td> </tr>
 
 <tr> <th>Job?</th>
 %   if lorry['running_job']:
diff --git a/templates/status.tpl b/templates/status.tpl
index e283a76..80f1ff7 100644
--- a/templates/status.tpl
+++ b/templates/status.tpl
@@ -66,7 +66,7 @@
 %     for spec in run_queue:
 %         if spec['running_job'] is not None:
 <tr>
-<td><a href="/1.0/job/{{spec['running_job']}}">{{spec['running_job']}}</a></td>
+<td><a href="/1.0/job-html/{{spec['running_job']}}">{{spec['running_job']}}</a></td>
 <td><a href="/1.0/lorry-html/{{spec['path']}}">{{spec['path']}}</a></td>
 </tr>
 %         end
@@ -97,7 +97,7 @@
 <td>{{spec['interval_nice']}}</td>
 <td>{{spec['due_nice']}}</td>
 %   if spec['running_job']:
-<td><a href="/1.0/job/{{spec['running_job']}}">{{spec['running_job']}}</a></td>
+<td><a href="/1.0/job-html/{{spec['running_job']}}">{{spec['running_job']}}</a></td>
 %   else:
 <td></td>
 %   end
diff --git a/yarns.webapp/040-running-jobs.yarn b/yarns.webapp/040-running-jobs.yarn
index e67cd4b..1ffe79d 100644
--- a/yarns.webapp/040-running-jobs.yarn
+++ b/yarns.webapp/040-running-jobs.yarn
@@ -199,7 +199,7 @@ Start the job.
 Check that the job info contains a start time.
 
     WHEN admin makes request GET /1.0/job/1
-    THEN response has job-started set
+    THEN response has job_started set
 
 Pretend it is now much later, or at least later than the timeout specified.