#!/usr/bin/env python # # Copyright (C) 2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import cliapp import json import logging import os import time import re import urllib import urllib2 from lorrycontroller.confparser import LorryControllerConfig from lorrycontroller.workingstate import WorkingStateManager from lorrycontroller.htmlstatus import HTMLStatusManager defaults = { 'work-area': '/home/lorry/controller-area', 'config-name': 'lorry-controller.conf', 'lorry': 'lorry', } token_finder = re.compile("([0-9a-f]{40})") class LorryController(cliapp.Application): def add_settings(self): self.settings.string(['work-area'], 'path to the area for the controller to work in', metavar='PATH', default=defaults['work-area']) self.settings.boolean(['dry-run'], "do a dry-run and don't actually do anything " "beyond updating the git tree", default=False) self.settings.string(['lorry'], 'path to the lorry binary to use', metavar='LORRY', default=defaults['lorry']) self.settings.string(['config-name'], 'configuration leafname. Defaults to ' 'lorry-controller.conf', metavar='CONFNAME', default=defaults['config-name']) self.settings.boolean(['lorry-verbose'], 'Whether to pass --verbose to lorry', default=False) self.settings.string(['lorry-log'], 'Log file name for lorry if wanted', metavar='LORRYLOG', default=None) self.settings.string(['html-file'], 'HTML filename for lorry controller status', metavar='HTMLFILE', default=None) def process_args(self, args): logging.info("Starting to control lorry") try: os.chdir(self.settings['work-area']) except OSError, e: logging.error("Unable to chdir() to %s" % self.settings['work-area']) raise SystemExit(2) if not os.path.isdir("git"): logging.error("Unable to find git checkout") raise SystemExit(3) if not os.path.isdir("work"): os.mkdir("work") logging.info("Updating configuration checkout") self.rungit(['remote', 'update', 'origin']) self.rungit(['reset', '--hard', 'origin/master']) self.rungit(['clean', '-fdx']) self.lorrycmd=[self.settings['lorry']] if self.settings['lorry-verbose']: self.lorrycmd += ["--verbose"] if self.settings['lorry-log'] is not None: self.lorrycmd += ["--log", self.settings['lorry-log']] if not os.path.exists(os.path.join('git', self.settings['config-name'])): logging.error("Unable to find lorry-controller.conf in git") raise SystemExit(4) if os.path.isfile('git/proxy.conf'): self.set_proxy('git/proxy.conf') logging.info('Loaded proxy information') self.conf = LorryControllerConfig(self, 'git/lorry-controller.conf') self.html = HTMLStatusManager(self) if self.settings['dry-run']: self.html.series = 0 self.html.write_out_status() self.conf.parse_config() with WorkingStateManager(self) as mgr: # Update any troves self.html.set_mgr(mgr) self.html.bump_state() self.conf.update_troves(mgr) prev_lorries = set(mgr.lorry_state.keys()) cur_lorries = set(self.conf.lorries.keys()) logging.info("Starting processing. Previously %d lorries " "were handled. We currently have %d defined." % ( len(prev_lorries), len(cur_lorries))) # 1. Handle deletes for any old lorries we no longer want self.html.bump_state() logging.info("Delete any old lorries...") for dead_lorry in prev_lorries - cur_lorries: self.html.set_processing(dead_lorry) logging.info("Dead lorry: %s" % dead_lorry) conf_uuid = mgr.lorry_state[dead_lorry]['conf'] if conf_uuid in self.conf.configs: should_delete = self.conf.configs[conf_uuid]['destroy'] else: # Could not find UUID in config, switch to 'never' should_delete = "never" want_destroy = (should_delete == "always") if should_delete == "unchanged": exit, out, err = self.maybe_runcmd( ['git', 'ls-remote', 'ssh://git@localhost/%s.git' % dead_lorry], dry=True) if exit != 0: logging.error("Unable to ls-remote to decide if " "unchanged. Assuming it is changed.") else: logging.debug("TODO: Should decide if unchanged!") if want_destroy: exit, out, err = self.maybe_runcmd(['ssh', 'git@localhost', 'destroy', dead_lorry], dry=True) if exit != 0: logging.error("Unable to destroy %s" % dead_lorry) else: token = token_finder.match(out).group(1) exit, out, err = self.maybe_runcmd( ['ssh', 'git@localhost', 'destroy', dead_lorry, token]) if exit != 0: logging.error("Unable to destroy %s despite having" " the token %s" % (dead_lorry, token)) else: logging.debug("Destroyed") del mgr.lorry_state[dead_lorry] # 2. Handle creates for any new lorries we now want self.html.bump_state() logging.info("Create any new lorries...") for new_lorry in cur_lorries - prev_lorries: self.html.set_processing(new_lorry) logging.info("New lorry: %s" % new_lorry) lorry = self.conf.lorries[new_lorry] conf_uuid = lorry['controller-uuid'] conf = self.conf.configs[conf_uuid] nextdue = self.conf.duetimes[new_lorry] # Make new lorries overdue. nextdue -= conf['interval-parsed'] should_create = conf['create'] == "always" store_state = True if should_create: exit, out, err = self.maybe_runcmd(["ssh", "git@localhost", "create", new_lorry]) if exit != 0: if ' already exists' in err: logging.warn("Repository %s already exists" % new_lorry) else: logging.error("Unable to create repository %s" % new_lorry) logging.error(err) store_state = False if store_state: self.maybe_runcmd(["ssh", "git@localhost", "set-head", new_lorry, lorry['source-HEAD']]) mgr.lorry_state[new_lorry] = { 'destroy': conf['destroy'], 'conf': conf_uuid, 'lorry': lorry, 'next-due': nextdue, } else: # Remove this from cur_lorries so we don't run it cur_lorries.remove(new_lorry) # 3. For every lorry we have, update the settings if necessary. # and reset the next-due as appropriate. self.html.bump_state() logging.info("Update active lorry configurations...") updated_count = 0 for upd_lorry in cur_lorries: if mgr.lorry_state[upd_lorry]['lorry'] != \ self.conf.lorries[upd_lorry]: lorry = self.conf.lorries[upd_lorry] old_lorry = mgr.lorry_state[upd_lorry]["lorry"] if lorry["source-HEAD"] != \ old_lorry.get("source-HEAD", "refs/heads/master"): self.maybe_runcmd(['ssh', 'git@localhost', 'set-head', upd_lorry, lorry["source-HEAD"]]) conf_uuid = lorry['controller-uuid'] conf = self.conf.configs[conf_uuid] nextdue = self.conf.duetimes[upd_lorry] mgr.lorry_state[upd_lorry] = { 'destroy': conf['destroy'], 'conf': conf_uuid, 'lorry': lorry, 'next-due': nextdue, } updated_count += 1 logging.info("Result: %d/%d lorries needed updating" % ( updated_count, len(cur_lorries))) # 3. Iterate all active lorries and see if they're due logging.info("Iterate active lorries looking for work...") now = time.time() lorried = 0 earliest_due = None what_early_due = "" lorries_to_run = [] for lorry in cur_lorries: state = mgr.lorry_state[lorry] conf_uuid = state['conf'] conf = self.conf.configs[conf_uuid] due = state['next-due'] if now >= due: lorries_to_run.append(lorry) lorries_to_run.sort() for lorry in lorries_to_run: state = mgr.lorry_state[lorry] conf_uuid = state['conf'] conf = self.conf.configs[conf_uuid] due = state['next-due'] lorried += 1 logging.info("Running %d/%d. Lorrying: %s" % ( lorried, len(lorries_to_run),lorry)) self.html.set_processing(lorry) # Before we run lorry, make sure that Git doesn't verify # SSL certificates. This is a workaround for the fact that # we don't yet have a solution for proper SSL certificates # in Trove yet. os.environ['GIT_SSL_NO_VERIFY'] = 'true' with mgr.runner(lorry) as runner: runner.run_lorry(*self.lorrycmd) while state['next-due'] <= now: state['next-due'] += conf['interval-parsed'] for lorry in cur_lorries: state = mgr.lorry_state[lorry] due = state['next-due'] if earliest_due is None or due < earliest_due: earliest_due = due what_early_due = lorry if earliest_due is None: logging.info("Lorried %d. No idea what's next." % lorried) else: logging.info("Lorried %d. %s due in %d seconds" % ( lorried, what_early_due, int(earliest_due - now))) logging.info("All done.") self.html.bump_state() def rungit(self, args): self.runcmd(['git']+args, cwd=os.path.join(self.settings['work-area'], 'git')) def maybe_http_request(self, url, dry=False): """If not a dry run, make an HTTP request and return its output.""" if (not self.settings['dry-run']) or dry: return self.http_request(url) else: logging.debug('DRY-RUN: Not sending a request to %s' % url) return 0, 'DRY-RUN', 'DRY-RUN' def maybe_runcmd(self, cmdline, dry=False, *args, **kwargs): if (not self.settings['dry-run']) or dry: return self.runcmd_unchecked(cmdline, *args, **kwargs) else: logging.debug("DRY-RUN: Not running %r" % cmdline) return 0, 'DRY-RUN', 'DRY-RUN' def http_request(self, url): """Make an HTTP request to the given url, return the output. Make an HTTP request to `url`. If the request succeeds (response code 200) then return an exit code 0, the data from the response and the response code. Otherwise return the response code, any data in the repsonse and a string containing the response code. """ request = urllib2.Request(url, None, {}) response = urllib2.urlopen(request) code = response.getcode() if code == 200: return 0, response.read(), '200' else: return code, response.read(), str(code) def set_proxy(self, proxy_def): """Tell urllib2 to use a proxy for http action by lorry-controller. Load the proxy information from the JSON file given by proxy_def, then set urllib2's url opener to open urls via an authenticated proxy. """ with open(proxy_def, 'r') as proxy_info: proxy = json.load(proxy_info) # set the required environment variables hostname = urllib.quote(proxy['hostname']) user = '%s:%s' % (proxy['username'], proxy['password']) url = '%s:%s' % (hostname, proxy['port']) os.environ['http_proxy'] = 'http://%s@%s' % (user, url) os.environ['https_proxy'] = 'https://%s@%s' % (user, url) # create a ProxyHandler proxies = {'http_proxy': 'http://%s@%s' % (user, url), 'https_proxy': 'https://%s@%s' % (user, url)} proxy_handler = urllib2.ProxyHandler(proxies) # install an opener to use the proxy opener = urllib2.build_opener(proxy_handler) urllib2.install_opener(opener) if __name__ == '__main__': LorryController(version='1').run()