From 4334f9d810ea6e448ad4c452c8fd7fd99487c693 Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Thu, 11 Oct 2012 17:40:59 +0100 Subject: Support generating HTML status messages --- lorry-controller | 19 +++ lorrycontroller/confparser.py | 6 +- lorrycontroller/htmlstatus.py | 258 ++++++++++++++++++++++++++++++++++++++++ lorrycontroller/workingstate.py | 2 + 4 files changed, 284 insertions(+), 1 deletion(-) create mode 100644 lorrycontroller/htmlstatus.py diff --git a/lorry-controller b/lorry-controller index e3f7d36..b3c2c45 100755 --- a/lorry-controller +++ b/lorry-controller @@ -12,6 +12,7 @@ import re from lorrycontroller.confparser import LorryControllerConfig from lorrycontroller.workingstate import WorkingStateManager +from lorrycontroller.htmlstatus import HTMLStatusManager defaults = { @@ -51,6 +52,10 @@ class LorryController(cliapp.Application): 'Log file name for lorry if wanted', metavar='LORRYLOG', default=None) + self.settings.string(['html-file'], + 'HTML filename for lorry controller status', + metavar='HTMLFILE', + default=None) def process_args(self, args): logging.info("Starting to control lorry") @@ -83,9 +88,16 @@ class LorryController(cliapp.Application): raise SystemExit(4) self.conf = LorryControllerConfig(self, 'git/lorry-controller.conf') + self.html = HTMLStatusManager(self) + if self.settings['dry-run']: + self.html.series = 0 + self.html.write_out_status() + self.conf.parse_config() with WorkingStateManager(self) as mgr: # Update any troves + self.html.set_mgr(mgr) + self.html.bump_state() self.conf.update_troves(mgr) prev_lorries = set(mgr.lorry_state.keys()) cur_lorries = set(self.conf.lorries.keys()) @@ -94,8 +106,10 @@ class LorryController(cliapp.Application): len(prev_lorries), len(cur_lorries))) # 1. Handle deletes for any old lorries we no longer want + self.html.bump_state() logging.info("Delete any old lorries...") for dead_lorry in prev_lorries - cur_lorries: + self.html.set_processing(dead_lorry) logging.info("Dead lorry: %s" % dead_lorry) conf_uuid = mgr.lorry_state[dead_lorry]['conf'] if conf_uuid in self.conf.configs: @@ -134,8 +148,10 @@ class LorryController(cliapp.Application): del mgr.lorry_state[dead_lorry] # 2. Handle creates for any new lorries we now want + self.html.bump_state() logging.info("Create any new lorries...") for new_lorry in cur_lorries - prev_lorries: + self.html.set_processing(new_lorry) logging.info("New lorry: %s" % new_lorry) lorry = self.conf.lorries[new_lorry] conf_uuid = lorry['controller-uuid'] @@ -170,6 +186,7 @@ class LorryController(cliapp.Application): # 3. For every lorry we have, update the settings if necessary. # and reset the next-due as appropriate. + self.html.bump_state() logging.info("Update active lorry configurations...") updated_count = 0 for upd_lorry in cur_lorries: @@ -217,6 +234,7 @@ class LorryController(cliapp.Application): lorried += 1 logging.info("Running %d/%d. Lorrying: %s" % ( lorried, len(lorries_to_run),lorry)) + self.html.set_processing(lorry) with mgr.runner(lorry) as runner: runner.run_lorry(*self.lorrycmd) while state['next-due'] <= now: @@ -235,6 +253,7 @@ class LorryController(cliapp.Application): logging.info("Lorried %d. %s due in %d seconds" % ( lorried, what_early_due, int(earliest_due - now))) logging.info("All done.") + self.html.bump_state() def rungit(self, args): self.runcmd(['git']+args, cwd=os.path.join(self.settings['work-area'], diff --git a/lorrycontroller/confparser.py b/lorrycontroller/confparser.py index 00a9fe1..71becd7 100644 --- a/lorrycontroller/confparser.py +++ b/lorrycontroller/confparser.py @@ -30,11 +30,14 @@ class LorryControllerConfig(object): def __init__(self, app, confpath): self.app = app + self.confpath = confpath self.lorries = {} self.configs = {} self.duetimes = {} self.troves = [] - confpath = os.path.join(app.settings['work-area'], confpath) + + def parse_config(self): + confpath = os.path.join(self.app.settings['work-area'], self.confpath) logging.info("Parsing configuration: %s" % confpath) try: with open(confpath, "r") as fh: @@ -260,6 +263,7 @@ class LorryControllerConfig(object): def update_troves(self, statemgr): # Now that we have a state manager we can look at the trove data. for trove in self.troves: + self.app.html.set_processing(trove['uuid']) trove_state = statemgr.get_trove(trove['uuid']) self.update_trove(trove, trove_state) diff --git a/lorrycontroller/htmlstatus.py b/lorrycontroller/htmlstatus.py new file mode 100644 index 0000000..aae75c7 --- /dev/null +++ b/lorrycontroller/htmlstatus.py @@ -0,0 +1,258 @@ +# Copyright (C) 2012 Codethink Limited +# + +import os +import time +from cgi import escape + +state_names = [ + "Initialisation", + "Load Troves", + "Remove old repos", + "Create new repos", + "Process Lorries", + "Finished" + ] + + +class HTMLStatusManager(object): + '''Manage the HTML status page for lorry-controller.''' + + + def __init__(self, app): + self.app = app + self.state = 0 + self.series = None + self.filename = self.app.settings['html-file'] + self.mgr = None + self.processing = None + self.processing_time = None + self.failing = None + self.all_lorries_ever = set() + + def set_failing(self, failmsg): + self.failing = failmsg + self.write_out_status() + + def set_mgr(self, mgr): + self.mgr = mgr + + def set_processing(self, proc): + self.processing = proc + self.processing_time = time.time() + self.write_out_status() + + def bump_state(self): + self.state = self.state + 1 + self.processing = None + self.write_out_status() + + def write_out_status(self): + if self.filename is None: return + try: + with open(self.filename + ".new", "w") as ofh: + ofh.write("\n") + ofh.write(self.gen_html()) + ofh.write("\n") + target = self.filename + if self.series is not None: + target += ".%d" % self.series + self.series += 1 + os.rename(self.filename + ".new", target) + except: + os.unlink(self.filename + ".new") + raise + + def gen_html(self): + head = self.gen_head() + body = self.gen_body() + return self.tag("html", content=head+"\n"+body, gap=True) + + def gen_head(self): + title = self.tag("title", content="Lorry Controller") + css = self.tag("link", href="trove.css", rel="stylesheet", + type="text/css") + script = self.tag("script", tyle="text/JavaScript", content=''' + +''') + return self.tag("head", content=title+css+script) + + def gen_body(self): + # 1. Rough header, branded as trove + curtime = time.asctime(time.gmtime()) + link = "/" + if self.series is not None: + link = self.filename + ".%d" % (self.series + 1) + header = ''' + + + +''' % (link, curtime) + # 2. List of steps and where we are currently + steps = self.gen_steps() + + # 4. Main content + content = self.gen_content() + + # 5. footer + footer = self.gen_footer() + + # magic args + kwargs = { + "onload": "JavaScript:reloadAfter(5000);", + } + if self.state >= (len(state_names)-1): + kwargs["onload"] = "JavaScript:reloadAfter(10000);", + return self.tag("body", content=self.tag( + "div", content=(header+steps+content+footer), + Class="lorrycontroller"), **kwargs) + + def gen_content(self): + if self.failing is not None: + return self.tag("div", Class="failure", content=self.failing) + # 1. Troves known + troves = self.gen_troves() + # 2. Lorries known + lorries = self.gen_lorries() + + return self.tag("div", Class="content", content= + self.tag("div", id="troves", content=troves) + + self.tag("div", id="lorries", content=lorries)) + + def gen_troves(self): + troves = [] + now = time.time() + for trove in self.app.conf.troves: + troveinfo = {} + if self.mgr is not None: + troveinfo = self.mgr.trove_state.get(trove['uuid'], {}) + uuid = self.tag("td", content=escape(trove['uuid'])) + state = "Up to date" + if self.processing == trove['uuid']: + state = "Processing since " + \ + time.asctime(time.gmtime(self.processing_time)) + elif troveinfo.get('next-vls', now - 1) < now: + state = "Due to be checked" + state = self.tag("td", content=escape(state)) + nextdue = self.tag("td", content=escape(time.asctime( + time.gmtime(troveinfo.get('next-vls', now - 1))))) + lorrycount = len([l for l in self.app.conf.lorries.itervalues() + if l['controller-uuid'] == trove['uuid']]) + lorrycount = self.tag("td", content=str(lorrycount)) + + troves.append(self.tag("tr", content= + uuid+state+nextdue+lorrycount)) + if len(troves) == 0: + content = "No troves detected" + else: + header = self.tag("tr", content= + self.tag("th", content="Trove UUID") + + self.tag("th", content="Status") + + self.tag("th", content="Next due") + + self.tag("th", content="Lorries created")) + content = self.tag("table", content= + header + "".join(troves)) + + return content + + def gen_lorries(self): + lorries = [] + now = time.time() + all_lorry_names = set(self.app.conf.lorries.keys()) + if self.mgr is not None: + all_lorry_names.update(set(self.mgr.lorry_state.keys())) + self.all_lorries_ever.update(all_lorry_names) + all_lorry_names = list(self.all_lorries_ever) + all_lorry_names.sort() + for lorry_name in all_lorry_names: + lorry = self.app.conf.lorries.get(lorry_name, None) + dead_lorry = False + dead_and_gone = False + if lorry is None: + lorrystate = self.mgr.lorry_state.get(lorry_name, None) + if lorrystate is None: + dead_and_gone = True + lorry = {} + else: + lorry = lorrystate['lorry'] + dead_lorry = True + lorryinfo = {} + if self.mgr is not None: + lorryinfo = self.mgr.lorry_state.get(lorry_name, {}) + uuid = self.tag("td", content= + escape(lorry.get('controller-uuid', 'Dead'))) + state = "Waiting until " + \ + time.asctime(time.gmtime(lorryinfo.get('next-due', now - 1))) + if self.processing == lorry_name: + state = "Processing since " + \ + time.asctime(time.gmtime(self.processing_time)) + elif lorryinfo.get('next-due', now - 1) < now: + state = "Due to be checked" + if self.mgr is not None: + if self.mgr.lorry_state.get(lorry_name, None) is None: + state = "Needs creating" + if dead_lorry: + state = "Dead - To be removed" + if dead_and_gone: + state = "Dead" + if self.processing == lorry_name: + state = "Removing since " + \ + time.asctime(time.gmtime(self.processing_time)) + state = self.tag("td", content=escape(state)) + lastresult = self.tag("td", content=self.tag( + "pre", content=escape(lorryinfo.get('result', '-')))) + nextdue = self.tag("td", content=escape(time.asctime( + time.gmtime(lorryinfo.get('next-due', now - 1))))) + lorryname = self.tag("td", content=escape(lorry_name)) + lorries.append(self.tag("tr", content= + lorryname+uuid+state+lastresult+nextdue)) + if len(lorries) == 0: + content = "No lorries detected yet" + else: + header = self.tag("tr", content= + self.tag("th", content="Lorry Name") + + self.tag("th", content="Comes From") + + self.tag("th", content="Status") + + self.tag("th", content="Last result") + + self.tag("th", content="Next due")) + content = self.tag("table", content= + header + "".join(lorries)) + + return content + + + def gen_footer(self): + curtime = time.asctime(time.gmtime()) + return self.tag("div", Class="footer", content= + "Generated by Lorry Controller at " + curtime) + + def gen_steps(self): + steps = [] + for idx in xrange(len(state_names)): + if idx < self.state: + Class = "donestep" + elif idx == self.state: + Class = "activestep" + else: + Class = "pendingstep" + steps.append(self.tag("span", Class=Class, + content=state_names[idx])) + return self.tag("table", Class="steps", content= + self.tag("tr", content= + self.tag("td", content= + "".join(steps)))) + + def tag(self, tagname, content=None, gap=False, **kwargs): + tagval = " ".join([tagname] + + ["%s=%r" % (k.lower(), v) for k, v in kwargs.iteritems()]) + gap = "\n" if gap else "" + if content is None: + return "<%s />" % tagval + else: + return "<%s>%s%s%s" % (tagval, gap, content, gap, tagname) + diff --git a/lorrycontroller/workingstate.py b/lorrycontroller/workingstate.py index 082647e..8e4b20e 100644 --- a/lorrycontroller/workingstate.py +++ b/lorrycontroller/workingstate.py @@ -38,8 +38,10 @@ class LorryFileRunner(object): exit, out, err = self.mgr.app.maybe_runcmd(cmdargs) if exit == 0: logging.debug("Lorry of %s succeeded: %s" % (self.lorryname, out)) + self.mgr.lorry_state[self.lorryname]['result'] = "OK" else: logging.warn("Lorry of %s failed: %s" % (self.lorryname, err)) + self.mgr.lorry_state[self.lorryname]['result'] = err class WorkingStateManager(object): '''Manage the working state of lorry-controller''' -- cgit v1.2.1