diff options
author | Daniel Silverstone <daniel.silverstone@codethink.co.uk> | 2012-10-03 16:35:52 +0100 |
---|---|---|
committer | Daniel Silverstone <daniel.silverstone@codethink.co.uk> | 2012-10-03 16:50:35 +0100 |
commit | fb032af65309be0770cc5b75e1730123ad6f3734 (patch) | |
tree | b509fbf7f85cce2fc560842a3938cb663fe877fc | |
parent | e19f1e813de714c17d148b0a7440650ef1b8bf3e (diff) | |
download | lorry-controller-fb032af65309be0770cc5b75e1730123ad6f3734.tar.gz |
Beginnings of rework to remove serial and make behaviour better in general
-rwxr-xr-x | lorry-controller | 53 | ||||
-rw-r--r-- | lorry-controller.conf | 10 | ||||
-rw-r--r-- | lorrycontroller/confparser.py | 15 | ||||
-rw-r--r-- | lorrycontroller/workingstate.py | 3 |
4 files changed, 53 insertions, 28 deletions
diff --git a/lorry-controller b/lorry-controller index 2a18d8c..89e20d8 100755 --- a/lorry-controller +++ b/lorry-controller @@ -90,7 +90,8 @@ class LorryController(cliapp.Application): logging.debug("Delete any old lorries...") for dead_lorry in prev_lorries - cur_lorries: logging.debug("Dead lorry: %s" % dead_lorry) - should_delete = mgr.lorry_state[dead_lorry]['conf']['delete'] + conf_uuid = mgr.lorry_state[dead_lorry]['conf'] + should_delete = self.conf.configs[conf_uuid]['delete'] # TODO: also handle 'unchanged' if should_delete == "always": logging.debug("TODO: Delete from Trove") @@ -100,8 +101,9 @@ class LorryController(cliapp.Application): logging.debug("Create any new lorries...") for new_lorry in cur_lorries - prev_lorries: logging.debug("New lorry: %s" % new_lorry) - conf = self.conf.configs[new_lorry] lorry = self.conf.lorries[new_lorry] + conf_uuid = lorry['controller-uuid'] + conf = self.conf.configs[conf_uuid] nextdue = self.conf.duetimes[new_lorry] should_create = conf['create'] == "always" store_state = True @@ -113,7 +115,7 @@ class LorryController(cliapp.Application): store_state = False if store_state: mgr.lorry_state[new_lorry] = { - 'conf': conf, + 'conf': conf_uuid, 'lorry': lorry, 'next-due': nextdue, } @@ -124,16 +126,21 @@ class LorryController(cliapp.Application): # 3. For every lorry we have, update the settings if necessary. # and reset the next-due as appropriate. logging.debug("Update active lorry configurations...") - for lorry in cur_lorries: - if mgr.lorry_state[lorry]['lorry'] != self.conf.lorries[lorry]: - conf = self.conf.configs[new_lorry] - lorry = self.conf.lorries[new_lorry] - nextdue = self.conf.duetimes[new_lorry] - mgr.lorry_state[new_lorry] = { - 'conf': conf, + updated_count = 0 + for upd_lorry in cur_lorries: + if mgr.lorry_state[upd_lorry]['lorry'] != \ + self.conf.lorries[upd_lorry]: + lorry = self.conf.lorries[upd_lorry] + conf_uuid = lorry['controller-uuid'] + nextdue = self.conf.duetimes[upd_lorry] + mgr.lorry_state[upd_lorry] = { + 'conf': conf_uuid, 'lorry': lorry, 'next-due': nextdue, } + updated += 1 + logging.debug("Result: %d/%d lorries needed updating" % ( + updated_count, len(cur_lorries))) # 3. Iterate all active lorries and see if they're due logging.debug("Iterate active lorries looking for work...") @@ -141,21 +148,35 @@ class LorryController(cliapp.Application): lorried = 0 earliest_due = None what_early_due = "" + lorries_to_run = [] for lorry in cur_lorries: state = mgr.lorry_state[lorry] + conf_uuid = state['conf'] + conf = self.conf.configs[conf_uuid] due = state['next-due'] if now >= due: - logging.debug("Running Lorry for: %s" % lorry) - with mgr.runner(lorry) as runner: - runner.run_lorry(*self.lorrycmd) - while state['next-due'] <= now: - state['next-due'] += state['conf']['interval-parsed'] - lorried += 1 + lorries_to_run.append(lorry) + lorries_to_run.sort() + for lorry in lorries_to_run: + state = mgr.lorry_state[lorry] + conf_uuid = state['conf'] + conf = self.conf.configs[conf_uuid] + due = state['next-due'] + lorried += 1 + logging.debug("Running %d/%d. Lorrying: %s" % ( + lorried, len(lorries_to_run),lorry)) + with mgr.runner(lorry) as runner: + runner.run_lorry(*self.lorrycmd) + while state['next-due'] <= now: + state['next-due'] += conf['interval-parsed'] + for lorry in cur_lorries: + state = mgr.lorry_state[lorry] due = state['next-due'] if earliest_due is None or due < earliest_due: earliest_due = due what_early_due = lorry + if earliest_due is None: logging.debug("Lorried %d. No idea what's next." % lorried) else: diff --git a/lorry-controller.conf b/lorry-controller.conf index 378f45a..b3f8449 100644 --- a/lorry-controller.conf +++ b/lorry-controller.conf @@ -1,8 +1,7 @@ [ { "type": "trove", - "uuid": "SomethingUniqueToIdentifyThisStanzaShouldItChange", - "serial": 1, + "uuid": "default-staggered-short", "trovehost": "git.baserock.org", "ls-interval": "1H", "prefixmap": { @@ -12,8 +11,9 @@ "ignore": [ "baserock/lorries" ], - "create": "always", + "create": "never", "destroy": "unchanged", - "interval": "2H" + "interval": "30M", + "stagger": true } -]
\ No newline at end of file +] diff --git a/lorrycontroller/confparser.py b/lorrycontroller/confparser.py index 8179a61..b78dc54 100644 --- a/lorrycontroller/confparser.py +++ b/lorrycontroller/confparser.py @@ -9,7 +9,6 @@ import os import time default_values = [ - ( u'serial', 0 ), ( u'create', u'never' ), ( u'destroy', u'never' ), ( u'interval', u'1m' ), @@ -55,7 +54,7 @@ class LorryControllerConfig(object): # Set the defaults for key, defval in default_values: entry[key] = entry.get(key, defval) - # And validate the generic values such as serial + # And validate the generic values self._validate__generics(entry) # Now validate the rest validator = getattr(self, '_validate_' + entry['type'], None) @@ -65,9 +64,12 @@ class LorryControllerConfig(object): validator(entry) def _validate__generics(self, entry): - '''Validate the generic entries such as 'serial'.''' + '''Validate the generic entries such as 'uuid'.''' if type(entry.get('uuid', None)) != unicode: self._give_up("UUID missing, cannot reconcile without it!") + if entry['uuid'] in self.configs: + self._give_up("UUID is not unique") + self.configs[entry['uuid']] = entry for key, defval in default_values: if type(defval) != type(entry[key]): self._give_up("Invalid type for '%s': %r" % (key, entry[key])) @@ -129,10 +131,9 @@ class LorryControllerConfig(object): fullname = os.path.join(entry['prefix'], name) if self.lorries.get(fullname, None) is not None: self._give_up("Lorry repeated: %s" % fullname) - content['serial'] = entry['serial'] + content['controller-uuid'] = entry['uuid'] my_lorry_names.add(fullname) self.lorries[fullname] = content - self.configs[fullname] = entry except Exception, e: logging.debug("Unable to parse %s, because of %s. Moving on" % (lorry, e)) @@ -210,7 +211,8 @@ class LorryControllerConfig(object): lorry = { "type": "git", "url": "ssh://git@%s/%s.git" % (trove['trovehost'], - remotereponame) + remotereponame), + "controller-uuid": trove['uuid'] } if localreponame in self.lorries: logging.warn("Skipping %s (%s from %s) because we already " @@ -218,7 +220,6 @@ class LorryControllerConfig(object): localreponame, remotereponame, trove['trovehost'])) else: self.lorries[localreponame] = lorry - self.configs[localreponame] = trove lorries_made.add(localreponame) # 3. Now schedule all those lorries in case they're new diff --git a/lorrycontroller/workingstate.py b/lorrycontroller/workingstate.py index 1ef2e9b..68d2ef9 100644 --- a/lorrycontroller/workingstate.py +++ b/lorrycontroller/workingstate.py @@ -64,6 +64,9 @@ class WorkingStateManager(object): logging.debug("Loading state file: %s" % self.lorry_state_file) with open(self.lorry_state_file, "r") as fh: self.lorry_state = json.load(fh) + for lorry_name, dct in self.lorry_state.iteritems(): + if type(dct['conf']) == dict: + dct['conf'] = dct['uuid'] else: self.lorry_state = dict() |