From 1d4eba57f93d0ed561bd05053079f3256ed645bd Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Tue, 2 Oct 2012 09:56:34 +0100 Subject: Support due-time and decide what to run and when --- lorry-controller | 87 +++++++++++++++++++++++++++++++++++++++++-- lorrycontroller/confparser.py | 16 ++++++++ 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/lorry-controller b/lorry-controller index 10bad19..06dcea1 100755 --- a/lorry-controller +++ b/lorry-controller @@ -6,9 +6,12 @@ import cliapp import logging import os +import time defaults = { - 'work-area': '/home/lorry/controller-area' + 'work-area': '/home/lorry/controller-area', + 'config-name': 'lorry-controller.conf', + 'lorry': 'lorry', } from lorrycontroller.confparser import LorryControllerConfig @@ -21,6 +24,15 @@ class LorryController(cliapp.Application): 'path to the area for the controller to work in', metavar='PATH', default=defaults['work-area']) + self.settings.string(['lorry'], + 'path to the lorry binary to use', + metavar='LORRY', + default=defaults['lorry']) + self.settings.string(['config-name'], + 'configuration leafname. Defaults to ' + 'lorry-controller.conf', + metavar='CONFNAME', + default=defaults['config-name']) def process_args(self, args): logging.info("Starting to control lorry") @@ -41,7 +53,8 @@ class LorryController(cliapp.Application): self.rungit(['reset', '--hard', 'origin/master']) self.rungit(['clean', '-fdx']) - if not os.path.exists('git/lorry-controller.conf'): + if not os.path.exists(os.path.join('git', + self.settings['config-name'])): logging.error("Unable to find lorry-controller.conf in git") raise SystemExit(4) @@ -49,8 +62,76 @@ class LorryController(cliapp.Application): 'git/lorry-controller.conf') with WorkingStateManager(self) as mgr: - pass + prev_lorries = set(mgr.lorry_state.keys()) + cur_lorries = set(self.conf.lorries.keys()) + logging.debug("Starting processing. Previously %d lorries " + "were handled. We currently have %d defined." % ( + len(prev_lorries), len(cur_lorries))) + # 1. Handle deletes for any old lorries we no longer want + logging.debug("Delete any old lorries...") + for dead_lorry in prev_lorries - cur_lorries: + logging.debug("Dead lorry: %s" % dead_lorry) + should_delete = mgr.lorry_state[dead_lorry]['conf']['delete'] + # TODO: also handle 'unchanged' + if should_delete == "always": + logging.debug("TODO: Delete from Trove") + del mgr.lorry_state[dead_lorry] + + # 2. Handle creates for any new lorries we now want + logging.debug("Create any new lorries...") + for new_lorry in cur_lorries - prev_lorries: + logging.debug("New lorry: %s" % new_lorry) + conf = self.conf.configs[new_lorry] + lorry = self.conf.lorries[new_lorry] + nextdue = self.conf.duetimes[new_lorry] + should_create = conf['create'] == "always" + if should_create: + logging.debug("TODO: Create in Trove") + mgr.lorry_state[new_lorry] = { + 'conf': conf, + 'lorry': lorry, + 'next-due': nextdue, + } + + # 3. For every lorry we have, update the settings if necessary. + # and reset the next-due as appropriate. + logging.debug("Update active lorry configurations...") + for lorry in cur_lorries: + if mgr.lorry_state[lorry]['lorry'] != self.conf.lorries[lorry]: + conf = self.conf.configs[new_lorry] + lorry = self.conf.lorries[new_lorry] + nextdue = self.conf.duetimes[new_lorry] + mgr.lorry_state[new_lorry] = { + 'conf': conf, + 'lorry': lorry, + 'next-due': nextdue, + } + + # 3. Iterate all active lorries and see if they're due + logging.debug("Iterate active lorries looking for work...") + now = time.time() + lorried = 0 + earliest_due = None + what_early_due = "" + for lorry in cur_lorries: + state = mgr.lorry_state[lorry] + due = state['next-due'] + if now >= due: + logging.debug("Running Lorry for: %s" % lorry) + # TODO: Actually run the lorry + while state['next-due'] <= now: + state['next-due'] += state['conf']['interval-parsed'] + lorried += 1 + + due = state['next-due'] + if earliest_due is None or due < earliest_due: + earliest_due = due + what_early_due = lorry + + logging.debug("Lorried %d. %s due in %d seconds" % ( + lorried, what_early_due, int(earliest_due - now))) + logging.debug("All done.") def rungit(self, args): self.runcmd(['git']+args, cwd=os.path.join(self.settings['work-area'], 'git')) diff --git a/lorrycontroller/confparser.py b/lorrycontroller/confparser.py index 4a3ec8f..d23cf4c 100644 --- a/lorrycontroller/confparser.py +++ b/lorrycontroller/confparser.py @@ -6,6 +6,7 @@ import logging import re import glob import os +import time default_values = [ ( u'serial', 0 ), @@ -30,6 +31,8 @@ class LorryControllerConfig(object): def __init__(self, settings, confpath): self.settings = settings self.lorries = {} + self.configs = {} + self.duetimes = {} confpath = os.path.join(settings['work-area'], confpath) logging.debug("Parsing configuration: %s" % confpath) with open(confpath, "r") as fh: @@ -111,6 +114,7 @@ class LorryControllerConfig(object): len(my_lorries)) logging.debug("Loading lorries into memory, please wait...") + my_lorry_names = set() for lorry in my_lorries: try: with open(lorry, "r") as fh: @@ -120,11 +124,23 @@ class LorryControllerConfig(object): if self.lorries.get(fullname, None) is not None: self._give_up("Lorry repeated: %s" % fullname) content['serial'] = entry['serial'] + my_lorry_names.add(fullname) self.lorries[fullname] = content + self.configs[fullname] = entry except Exception, e: logging.debug("Unable to parse %s, because of %s. Moving on" % (lorry, e)) + # Now calculate the 'next due' time for every lorry we just parsed + starttime = time.time() - 1 + endtime = starttime + entry['interval-parsed'] + step = 0 + if entry['stagger']: + step = (endtime - starttime) / len(my_lorry_names) + for lorry_name in my_lorry_names: + self.duetimes[lorry_name] = starttime + starttime += step + logging.debug("Now loaded %d lorries" % len(self.lorries.keys())) def _give_up(self, *args, **kwargs): -- cgit v1.2.1