diff options
Diffstat (limited to 'lorry-controller')
| -rwxr-xr-x | lorry-controller | 355 |
1 files changed, 0 insertions, 355 deletions
diff --git a/lorry-controller b/lorry-controller deleted file mode 100755 index 0ae4ceb..0000000 --- a/lorry-controller +++ /dev/null @@ -1,355 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import cliapp -import json -import logging -import os -import time -import re -import urllib -import urllib2 - - -from lorrycontroller.confparser import LorryControllerConfig -from lorrycontroller.workingstate import WorkingStateManager -from lorrycontroller.htmlstatus import HTMLStatusManager - - -defaults = { - 'work-area': '/home/lorry/controller-area', - 'config-name': 'lorry-controller.conf', - 'lorry': 'lorry', -} - - -token_finder = re.compile("([0-9a-f]{40})") - - -class LorryController(cliapp.Application): - - def add_settings(self): - self.settings.string(['work-area'], - 'path to the area for the controller to work in', - metavar='PATH', - default=defaults['work-area']) - self.settings.boolean(['dry-run'], - "do a dry-run and don't actually do anything " - "beyond updating the git tree", - default=False) - self.settings.string(['lorry'], - 'path to the lorry binary to use', - metavar='LORRY', - default=defaults['lorry']) - self.settings.string(['config-name'], - 'configuration leafname. Defaults to ' - 'lorry-controller.conf', - metavar='CONFNAME', - default=defaults['config-name']) - self.settings.boolean(['lorry-verbose'], - 'Whether to pass --verbose to lorry', - default=False) - self.settings.string(['lorry-log'], - 'Log file name for lorry if wanted', - metavar='LORRYLOG', - default=None) - self.settings.string(['html-file'], - 'HTML filename for lorry controller status', - metavar='HTMLFILE', - default=None) - - def process_args(self, args): - logging.info("Starting to control lorry") - try: - os.chdir(self.settings['work-area']) - except OSError, e: - logging.error("Unable to chdir() to %s" % - self.settings['work-area']) - raise SystemExit(2) - if not os.path.isdir("git"): - logging.error("Unable to find git checkout") - raise SystemExit(3) - if not os.path.isdir("work"): - os.mkdir("work") - - logging.info("Updating configuration checkout") - self.rungit(['remote', 'update', 'origin']) - self.rungit(['reset', '--hard', 'origin/master']) - self.rungit(['clean', '-fdx']) - - self.lorrycmd=[self.settings['lorry']] - if self.settings['lorry-verbose']: - self.lorrycmd += ["--verbose"] - if self.settings['lorry-log'] is not None: - self.lorrycmd += ["--log", self.settings['lorry-log']] - - if not os.path.exists(os.path.join('git', - self.settings['config-name'])): - logging.error("Unable to find lorry-controller.conf in git") - raise SystemExit(4) - - if os.path.isfile('git/proxy.conf'): - self.set_proxy('git/proxy.conf') - logging.info('Loaded proxy information') - self.conf = LorryControllerConfig(self, 'git/lorry-controller.conf') - self.html = HTMLStatusManager(self) - if self.settings['dry-run']: - self.html.series = 0 - self.html.write_out_status() - self.conf.parse_config() - - with WorkingStateManager(self) as mgr: - # Update any troves - self.html.set_mgr(mgr) - self.html.bump_state() - self.conf.update_troves(mgr) - prev_lorries = set(mgr.lorry_state.keys()) - cur_lorries = set(self.conf.lorries.keys()) - logging.info("Starting processing. Previously %d lorries " - "were handled. We currently have %d defined." % ( - len(prev_lorries), len(cur_lorries))) - - # 1. Handle deletes for any old lorries we no longer want - self.html.bump_state() - logging.info("Delete any old lorries...") - for dead_lorry in prev_lorries - cur_lorries: - self.html.set_processing(dead_lorry) - logging.info("Dead lorry: %s" % dead_lorry) - conf_uuid = mgr.lorry_state[dead_lorry]['conf'] - if conf_uuid in self.conf.configs: - should_delete = self.conf.configs[conf_uuid]['destroy'] - else: - # Could not find UUID in config, switch to 'never' - should_delete = "never" - want_destroy = (should_delete == "always") - if should_delete == "unchanged": - exit, out, err = self.maybe_runcmd( - ['git', 'ls-remote', 'ssh://git@localhost/%s.git' % - dead_lorry], dry=True) - if exit != 0: - logging.error("Unable to ls-remote to decide if " - "unchanged. Assuming it is changed.") - else: - logging.debug("TODO: Should decide if unchanged!") - - if want_destroy: - exit, out, err = self.maybe_runcmd(['ssh', 'git@localhost', - 'destroy', dead_lorry], - dry=True) - if exit != 0: - logging.error("Unable to destroy %s" % dead_lorry) - else: - token = token_finder.match(out).group(1) - exit, out, err = self.maybe_runcmd( - ['ssh', 'git@localhost', 'destroy', dead_lorry, - token]) - if exit != 0: - logging.error("Unable to destroy %s despite having" - " the token %s" % - (dead_lorry, token)) - else: - logging.debug("Destroyed") - del mgr.lorry_state[dead_lorry] - - # 2. Handle creates for any new lorries we now want - self.html.bump_state() - logging.info("Create any new lorries...") - for new_lorry in cur_lorries - prev_lorries: - self.html.set_processing(new_lorry) - logging.info("New lorry: %s" % new_lorry) - lorry = self.conf.lorries[new_lorry] - conf_uuid = lorry['controller-uuid'] - conf = self.conf.configs[conf_uuid] - nextdue = self.conf.duetimes[new_lorry] - # Make new lorries overdue. - nextdue -= conf['interval-parsed'] - should_create = conf['create'] == "always" - store_state = True - if should_create: - exit, out, err = self.maybe_runcmd(["ssh", "git@localhost", - "create", new_lorry]) - if exit != 0: - if ' already exists' in err: - logging.warn("Repository %s already exists" % - new_lorry) - else: - logging.error("Unable to create repository %s" % - new_lorry) - logging.error(err) - store_state = False - if store_state: - self.maybe_runcmd(["ssh", "git@localhost", "set-head", - new_lorry, lorry['source-HEAD']]) - mgr.lorry_state[new_lorry] = { - 'destroy': conf['destroy'], - 'conf': conf_uuid, - 'lorry': lorry, - 'next-due': nextdue, - } - else: - # Remove this from cur_lorries so we don't run it - cur_lorries.remove(new_lorry) - - # 3. For every lorry we have, update the settings if necessary. - # and reset the next-due as appropriate. - self.html.bump_state() - logging.info("Update active lorry configurations...") - updated_count = 0 - for upd_lorry in cur_lorries: - if mgr.lorry_state[upd_lorry]['lorry'] != \ - self.conf.lorries[upd_lorry]: - lorry = self.conf.lorries[upd_lorry] - old_lorry = mgr.lorry_state[upd_lorry]["lorry"] - if lorry["source-HEAD"] != \ - old_lorry.get("source-HEAD", "refs/heads/master"): - self.maybe_runcmd(['ssh', 'git@localhost', 'set-head', - upd_lorry, lorry["source-HEAD"]]) - conf_uuid = lorry['controller-uuid'] - conf = self.conf.configs[conf_uuid] - nextdue = self.conf.duetimes[upd_lorry] - mgr.lorry_state[upd_lorry] = { - 'destroy': conf['destroy'], - 'conf': conf_uuid, - 'lorry': lorry, - 'next-due': nextdue, - } - updated_count += 1 - logging.info("Result: %d/%d lorries needed updating" % ( - updated_count, len(cur_lorries))) - - # 3. Iterate all active lorries and see if they're due - logging.info("Iterate active lorries looking for work...") - now = time.time() - lorried = 0 - earliest_due = None - what_early_due = "" - lorries_to_run = [] - for lorry in cur_lorries: - state = mgr.lorry_state[lorry] - conf_uuid = state['conf'] - conf = self.conf.configs[conf_uuid] - due = state['next-due'] - if now >= due: - lorries_to_run.append(lorry) - lorries_to_run.sort() - for lorry in lorries_to_run: - state = mgr.lorry_state[lorry] - conf_uuid = state['conf'] - conf = self.conf.configs[conf_uuid] - due = state['next-due'] - lorried += 1 - logging.info("Running %d/%d. Lorrying: %s" % ( - lorried, len(lorries_to_run),lorry)) - self.html.set_processing(lorry) - # Before we run lorry, make sure that Git doesn't verify - # SSL certificates. This is a workaround for the fact that - # we don't yet have a solution for proper SSL certificates - # in Trove yet. - os.environ['GIT_SSL_NO_VERIFY'] = 'true' - with mgr.runner(lorry) as runner: - runner.run_lorry(*self.lorrycmd) - while state['next-due'] <= now: - state['next-due'] += conf['interval-parsed'] - - for lorry in cur_lorries: - state = mgr.lorry_state[lorry] - due = state['next-due'] - if earliest_due is None or due < earliest_due: - earliest_due = due - what_early_due = lorry - - if earliest_due is None: - logging.info("Lorried %d. No idea what's next." % lorried) - else: - logging.info("Lorried %d. %s due in %d seconds" % ( - lorried, what_early_due, int(earliest_due - now))) - logging.info("All done.") - self.html.bump_state() - - def rungit(self, args): - self.runcmd(['git']+args, cwd=os.path.join(self.settings['work-area'], - 'git')) - - def maybe_http_request(self, url, auth=None, dry=False): - """If not a dry run, make an HTTP request and return its output.""" - if (not self.settings['dry-run']) or dry: - return self.http_request(url, auth) - else: - logging.debug('DRY-RUN: Not sending a request to %s' % url) - return 0, 'DRY-RUN', 'DRY-RUN' - - def maybe_runcmd(self, cmdline, dry=False, *args, **kwargs): - if (not self.settings['dry-run']) or dry: - return self.runcmd_unchecked(cmdline, *args, **kwargs) - else: - logging.debug("DRY-RUN: Not running %r" % cmdline) - return 0, 'DRY-RUN', 'DRY-RUN' - - def http_request(self, url, auth=None): - """Make an HTTP request to the given url, return the output. - - Make an HTTP request to `url`. If the request succeeds (response code - 200) then return an exit code 0, the data from the response and the - response code. Otherwise return the response code, any data in the - repsonse and a string containing the response code. - - """ - request = urllib2.Request(url, None, {}) - if auth: - password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() - password_mgr.add_password( - None, url, auth['username'], auth['password']) - auth_handler = urllib2.HTTPBasicAuthHandler(password_mgr) - opener = urllib2.build_opener(auth_handler) - response = opener.open(url) - else: - response = urllib2.urlopen(request) - code = response.getcode() - if code == 200: - return 0, response.read(), '200' - else: - return code, response.read(), str(code) - - def set_proxy(self, proxy_def): - """Tell urllib2 to use a proxy for http action by lorry-controller. - - Load the proxy information from the JSON file given by proxy_def, then - set urllib2's url opener to open urls via an authenticated proxy. - - """ - with open(proxy_def, 'r') as proxy_info: - proxy = json.load(proxy_info) - - # set the required environment variables - hostname = urllib.quote(proxy['hostname']) - user = '%s:%s' % (proxy['username'], proxy['password']) - url = '%s:%s' % (hostname, proxy['port']) - os.environ['http_proxy'] = 'http://%s@%s' % (user, url) - os.environ['https_proxy'] = 'https://%s@%s' % (user, url) - - # create a ProxyHandler - proxies = {'http_proxy': 'http://%s@%s' % (user, url), - 'https_proxy': 'https://%s@%s' % (user, url)} - proxy_handler = urllib2.ProxyHandler(proxies) - - # install an opener to use the proxy - opener = urllib2.build_opener(proxy_handler) - urllib2.install_opener(opener) - -if __name__ == '__main__': - LorryController(version='1').run() |