Support generating HTML status messages

4 files changed, 284 insertions, 1 deletions

diff --git a/lorry-controller b/lorry-controller
index e3f7d36..b3c2c45 100755
--- a/lorry-controller
+++ b/lorry-controller
@@ -12,6 +12,7 @@ import re
 
 from lorrycontroller.confparser import LorryControllerConfig
 from lorrycontroller.workingstate import WorkingStateManager
+from lorrycontroller.htmlstatus import HTMLStatusManager
 
 
 defaults = {
@@ -51,6 +52,10 @@ class LorryController(cliapp.Application):
                              'Log file name for lorry if wanted',
                              metavar='LORRYLOG',
                              default=None)
+        self.settings.string(['html-file'],
+                             'HTML filename for lorry controller status',
+                             metavar='HTMLFILE',
+                             default=None)
 
     def process_args(self, args):
         logging.info("Starting to control lorry")
@@ -83,9 +88,16 @@ class LorryController(cliapp.Application):
             raise SystemExit(4)
 
         self.conf = LorryControllerConfig(self, 'git/lorry-controller.conf')
+        self.html = HTMLStatusManager(self)
+        if self.settings['dry-run']:
+            self.html.series = 0
+        self.html.write_out_status()
+        self.conf.parse_config()
 
         with WorkingStateManager(self) as mgr:
             # Update any troves
+            self.html.set_mgr(mgr)
+            self.html.bump_state()
             self.conf.update_troves(mgr)
             prev_lorries = set(mgr.lorry_state.keys())
             cur_lorries = set(self.conf.lorries.keys())
@@ -94,8 +106,10 @@ class LorryController(cliapp.Application):
                     len(prev_lorries), len(cur_lorries)))
 
             # 1. Handle deletes for any old lorries we no longer want
+            self.html.bump_state()
             logging.info("Delete any old lorries...")
             for dead_lorry in prev_lorries - cur_lorries:
+                self.html.set_processing(dead_lorry)
                 logging.info("Dead lorry: %s" % dead_lorry)
                 conf_uuid = mgr.lorry_state[dead_lorry]['conf']
                 if conf_uuid in self.conf.configs:
@@ -134,8 +148,10 @@ class LorryController(cliapp.Application):
                 del mgr.lorry_state[dead_lorry]
 
             # 2. Handle creates for any new lorries we now want
+            self.html.bump_state()
             logging.info("Create any new lorries...")
             for new_lorry in cur_lorries - prev_lorries:
+                self.html.set_processing(new_lorry)
                 logging.info("New lorry: %s" % new_lorry)
                 lorry = self.conf.lorries[new_lorry]
                 conf_uuid = lorry['controller-uuid']
@@ -170,6 +186,7 @@ class LorryController(cliapp.Application):
 
             # 3. For every lorry we have, update the settings if necessary.
             #    and reset the next-due as appropriate.
+            self.html.bump_state()
             logging.info("Update active lorry configurations...")
             updated_count = 0
             for upd_lorry in cur_lorries:
@@ -217,6 +234,7 @@ class LorryController(cliapp.Application):
                 lorried += 1
                 logging.info("Running %d/%d. Lorrying: %s" % (
                         lorried, len(lorries_to_run),lorry))
+                self.html.set_processing(lorry)
                 with mgr.runner(lorry) as runner:
                     runner.run_lorry(*self.lorrycmd)
                 while state['next-due'] <= now:
@@ -235,6 +253,7 @@ class LorryController(cliapp.Application):
                 logging.info("Lorried %d.  %s due in %d seconds" % (
                         lorried, what_early_due, int(earliest_due - now)))
             logging.info("All done.")
+        self.html.bump_state()
 
     def rungit(self, args):
         self.runcmd(['git']+args, cwd=os.path.join(self.settings['work-area'],
diff --git a/lorrycontroller/confparser.py b/lorrycontroller/confparser.py
index 00a9fe1..71becd7 100644
--- a/lorrycontroller/confparser.py
+++ b/lorrycontroller/confparser.py
@@ -30,11 +30,14 @@ class LorryControllerConfig(object):
 
     def __init__(self, app, confpath):
         self.app = app
+        self.confpath = confpath
         self.lorries = {}
         self.configs = {}
         self.duetimes = {}
         self.troves = []
-        confpath = os.path.join(app.settings['work-area'], confpath)
+
+    def parse_config(self):
+        confpath = os.path.join(self.app.settings['work-area'], self.confpath)
         logging.info("Parsing configuration: %s" % confpath)
         try:
             with open(confpath, "r") as fh:
@@ -260,6 +263,7 @@ class LorryControllerConfig(object):
     def update_troves(self, statemgr):
         # Now that we have a state manager we can look at the trove data.
         for trove in self.troves:
+            self.app.html.set_processing(trove['uuid'])
             trove_state = statemgr.get_trove(trove['uuid'])
             self.update_trove(trove, trove_state)
 
diff --git a/lorrycontroller/htmlstatus.py b/lorrycontroller/htmlstatus.py
new file mode 100644
index 0000000..aae75c7
--- /dev/null
+++ b/lorrycontroller/htmlstatus.py
@@ -0,0 +1,258 @@
+# Copyright (C) 2012  Codethink Limited
+#
+
+import os
+import time
+from cgi import escape
+
+state_names = [
+    "Initialisation",
+    "Load Troves",
+    "Remove old repos",
+    "Create new repos",
+    "Process Lorries",
+    "Finished"
+    ]
+
+
+class HTMLStatusManager(object):
+    '''Manage the HTML status page for lorry-controller.'''
+
+
+    def __init__(self, app):
+        self.app = app
+        self.state = 0
+        self.series = None
+        self.filename = self.app.settings['html-file']
+        self.mgr = None
+        self.processing = None
+        self.processing_time = None
+        self.failing = None
+        self.all_lorries_ever = set()
+
+    def set_failing(self, failmsg):
+        self.failing = failmsg
+        self.write_out_status()
+
+    def set_mgr(self, mgr):
+        self.mgr = mgr
+
+    def set_processing(self, proc):
+        self.processing = proc
+        self.processing_time = time.time()
+        self.write_out_status()
+
+    def bump_state(self):
+        self.state = self.state + 1
+        self.processing = None
+        self.write_out_status()
+
+    def write_out_status(self):
+        if self.filename is None: return
+        try:
+            with open(self.filename + ".new", "w") as ofh:
+                ofh.write("<!DOCTYPE html>\n")
+                ofh.write(self.gen_html())
+                ofh.write("\n")
+            target = self.filename
+            if self.series is not None:
+                target += ".%d" % self.series
+                self.series += 1
+            os.rename(self.filename + ".new", target)
+        except:
+            os.unlink(self.filename + ".new")
+            raise
+
+    def gen_html(self):
+        head = self.gen_head()
+        body = self.gen_body()
+        return self.tag("html", content=head+"\n"+body, gap=True)
+
+    def gen_head(self):
+        title = self.tag("title", content="Lorry Controller")
+        css = self.tag("link", href="trove.css", rel="stylesheet",
+                       type="text/css")
+        script = self.tag("script", tyle="text/JavaScript", content='''
+<!--
+function reloadAfter(timeout) {
+   setTimeout("location.reload(true);", timeout);
+}
+// -->
+''')
+        return self.tag("head", content=title+css+script)
+
+    def gen_body(self):
+        # 1. Rough header, branded as trove
+        curtime = time.asctime(time.gmtime())
+        link = "/"
+        if self.series is not None:
+            link = self.filename + ".%d" % (self.series + 1)
+        header = '''
+<table id='header'><tr><td class='logo' rowspan='2'>
+<a href='%s'><img src='trove.png' alt='trove logo'/></a></td>
+<td class='main'>Status of Lorry Controller</td></tr>
+<tr><td class='sub'>Updated at %s</td></tr></table>
+''' % (link, curtime)
+        # 2. List of steps and where we are currently
+        steps = self.gen_steps()
+
+        # 4. Main content
+        content = self.gen_content()
+
+        # 5. footer
+        footer = self.gen_footer()
+        
+        # magic args
+        kwargs = {
+            "onload": "JavaScript:reloadAfter(5000);",
+            }
+        if self.state >= (len(state_names)-1):
+            kwargs["onload"] = "JavaScript:reloadAfter(10000);",
+        return self.tag("body", content=self.tag(
+                "div", content=(header+steps+content+footer),
+                Class="lorrycontroller"), **kwargs)
+
+    def gen_content(self):
+        if self.failing is not None:
+            return self.tag("div", Class="failure", content=self.failing)
+        # 1. Troves known
+        troves = self.gen_troves()
+        # 2. Lorries known
+        lorries = self.gen_lorries()
+
+        return self.tag("div", Class="content", content=
+                        self.tag("div", id="troves", content=troves) +
+                        self.tag("div", id="lorries", content=lorries))
+
+    def gen_troves(self):
+        troves = []
+        now = time.time()
+        for trove in self.app.conf.troves:
+            troveinfo = {}
+            if self.mgr is not None:
+                troveinfo = self.mgr.trove_state.get(trove['uuid'], {})
+            uuid = self.tag("td", content=escape(trove['uuid']))
+            state = "Up to date"
+            if self.processing == trove['uuid']:
+                state = "Processing since " + \
+                    time.asctime(time.gmtime(self.processing_time))
+            elif troveinfo.get('next-vls', now - 1) < now:
+                state = "Due to be checked"
+            state = self.tag("td", content=escape(state))
+            nextdue = self.tag("td", content=escape(time.asctime(
+                        time.gmtime(troveinfo.get('next-vls', now - 1)))))
+            lorrycount = len([l for l in self.app.conf.lorries.itervalues()
+                              if l['controller-uuid'] == trove['uuid']])
+            lorrycount = self.tag("td", content=str(lorrycount))
+
+            troves.append(self.tag("tr", content=
+                                   uuid+state+nextdue+lorrycount))
+        if len(troves) == 0:
+            content = "No troves detected"
+        else:
+            header = self.tag("tr", content=
+                              self.tag("th", content="Trove UUID") +
+                              self.tag("th", content="Status") +
+                              self.tag("th", content="Next due") +
+                              self.tag("th", content="Lorries created"))
+            content = self.tag("table", content=
+                               header + "".join(troves))
+
+        return content
+
+    def gen_lorries(self):
+        lorries = []
+        now = time.time()
+        all_lorry_names = set(self.app.conf.lorries.keys())
+        if self.mgr is not None:
+            all_lorry_names.update(set(self.mgr.lorry_state.keys()))
+        self.all_lorries_ever.update(all_lorry_names)
+        all_lorry_names = list(self.all_lorries_ever)
+        all_lorry_names.sort()
+        for lorry_name in all_lorry_names:
+            lorry = self.app.conf.lorries.get(lorry_name, None)
+            dead_lorry = False
+            dead_and_gone = False
+            if lorry is None:
+                lorrystate = self.mgr.lorry_state.get(lorry_name, None)
+                if lorrystate is None:
+                    dead_and_gone = True
+                    lorry = {}
+                else:
+                    lorry = lorrystate['lorry']
+                dead_lorry = True
+            lorryinfo = {}
+            if self.mgr is not None:
+                lorryinfo = self.mgr.lorry_state.get(lorry_name, {})
+            uuid = self.tag("td", content=
+                            escape(lorry.get('controller-uuid', 'Dead')))
+            state = "Waiting until " + \
+                time.asctime(time.gmtime(lorryinfo.get('next-due', now - 1)))
+            if self.processing == lorry_name:
+                state = "Processing since " + \
+                    time.asctime(time.gmtime(self.processing_time))
+            elif lorryinfo.get('next-due', now - 1) < now:
+                state = "Due to be checked"
+                if self.mgr is not None:
+                    if self.mgr.lorry_state.get(lorry_name, None) is None:
+                        state = "Needs creating"
+            if dead_lorry:
+                state = "Dead - To be removed"
+                if dead_and_gone:
+                    state = "Dead"
+                if self.processing == lorry_name:
+                    state = "Removing since " + \
+                        time.asctime(time.gmtime(self.processing_time))
+            state = self.tag("td", content=escape(state))
+            lastresult = self.tag("td", content=self.tag(
+                    "pre", content=escape(lorryinfo.get('result', '-'))))
+            nextdue = self.tag("td", content=escape(time.asctime(
+                        time.gmtime(lorryinfo.get('next-due', now - 1)))))
+            lorryname = self.tag("td", content=escape(lorry_name))
+            lorries.append(self.tag("tr", content=
+                                   lorryname+uuid+state+lastresult+nextdue))
+        if len(lorries) == 0:
+            content = "No lorries detected yet"
+        else:
+            header = self.tag("tr", content=
+                              self.tag("th", content="Lorry Name") +
+                              self.tag("th", content="Comes From") +
+                              self.tag("th", content="Status") +
+                              self.tag("th", content="Last result") +
+                              self.tag("th", content="Next due"))
+            content = self.tag("table", content=
+                               header + "".join(lorries))
+
+        return content
+
+
+    def gen_footer(self):
+        curtime = time.asctime(time.gmtime())
+        return self.tag("div", Class="footer", content=
+                        "Generated by Lorry Controller at " + curtime)
+
+    def gen_steps(self):
+        steps = []
+        for idx in xrange(len(state_names)):
+            if idx < self.state:
+                Class = "donestep"
+            elif idx == self.state:
+                Class = "activestep"
+            else:
+                Class = "pendingstep"
+            steps.append(self.tag("span", Class=Class,
+                                  content=state_names[idx]))
+        return self.tag("table", Class="steps", content=
+                        self.tag("tr", content=
+                                 self.tag("td", content=
+                                          "".join(steps))))
+
+    def tag(self, tagname, content=None, gap=False, **kwargs):
+        tagval = " ".join([tagname] + 
+                          ["%s=%r" % (k.lower(), v) for k, v in kwargs.iteritems()])
+        gap = "\n" if gap else ""
+        if content is None:
+            return "<%s />" % tagval
+        else:
+            return "<%s>%s%s%s</%s>" % (tagval, gap, content, gap, tagname)
+
diff --git a/lorrycontroller/workingstate.py b/lorrycontroller/workingstate.py
index 082647e..8e4b20e 100644
--- a/lorrycontroller/workingstate.py
+++ b/lorrycontroller/workingstate.py
@@ -38,8 +38,10 @@ class LorryFileRunner(object):
         exit, out, err = self.mgr.app.maybe_runcmd(cmdargs)
         if exit == 0:
             logging.debug("Lorry of %s succeeded: %s" % (self.lorryname, out))
+            self.mgr.lorry_state[self.lorryname]['result'] = "OK"
         else:
             logging.warn("Lorry of %s failed: %s" % (self.lorryname, err))
+            self.mgr.lorry_state[self.lorryname]['result'] = err
 
 class WorkingStateManager(object):
     '''Manage the working state of lorry-controller'''