#!/usr/bin/python # -*- coding: utf-8 -*- # Copyright (C) 2015 The Qt Company Ltd. # Contact: http://www.qt.io/licensing/ # # You may use this file under the terms of the 3-clause BSD license. # See the file LICENSE from this package for details. # # Checkouts have to have the remote: origin codereview-bot:qt/qtbase.git # Run with --dir and --addr # python qdoc-bot.py --dir /home/USER/dev/qt-qdoc/ --addr codereview-bot # --dir is where the sources are checked out # --addr codereview-bot # With .ssh/config: # Host codereview-bot # Hostname codereview.qt-project.org # Port 29418 # User qt_docbot # IdentityFile /home/USER/.ssh/docbot_id_rsa import os import subprocess import sys import logging import re config = None # configuration object, it is initialized in main and passed to every worker process created by multiprocessing.Pool #{ #"change": { #"branch": "2.6", #"id": "I765c9e165f9e7a5e0928401f5c5254397ad9be0b", #"number": "33424", #"owner": { #"email": "Friedemann.Kleint@nokia.com", #"name": "Friedemann Kleint" #}, #"project": "qt-creator/qt-creator", #"subject": "WIP: CDB: Fix some dumpers.", #"url": "https://codereview.qt-project.org/33424" #}, #"patchSet": { #"number": "1", #"ref": "refs/changes/24/33424/1", #"revision": "c4dd516729ff8f9b7cb1e13d9c8f09560cf2c078", #"uploader": { #"email": "Friedemann.Kleint@nokia.com", #"name": "Friedemann Kleint" #} #}, #"type": "patchset-created", #"uploader": { #"email": "Friedemann.Kleint@nokia.com", #"name": "Friedemann Kleint" #} #} def pasteKDE(data): import httplib, urllib, json params = urllib.urlencode({'paste_data': data, 'paste_lang': 'text', 'api_submit': True, 'mode': 'json', 'paste_project':'Qt', 'paste_user': 'Qt Doc bot', 'paste_private': 'yes', 'paste_expire': 2629744 # around 1 month }) headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"} conn = httplib.HTTPConnection("paste.kde.org") try: conn.request("POST", "", params, headers) response = json.loads(conn.getresponse().read()) logging.debug("Making a paste %s", response) return "http://pastebin.kde.org/" + str(response['result']['id']) + "/" + str(response['result']['hash']) finally: conn.close() #make sub-qdoc in src/tools def run_qdoc(module_name): output = [] try: environment = os.environ environment["CCACHE_BASEDIR"] = os.getcwd() environment["QT_HASH_SEED"] = "1234" environment["QMAKEFLAGS"] = "CONFIG+=force_bootstrap" for cmd in ["/bin/sh ./configure --opensource --confirm-license --release".split() , "make sub-src-qmake_all".split() , ["/bin/sh", "-c", "if [ -f src/tools/Makefile ] ; then make -Csrc/tools sub-qdoc; else make -Csrc sub-qdoc ; fi"] # build qdoc ]: logging.debug("Executing command: %s", cmd) try: proc = subprocess.Popen(cmd, env=environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) buildOutput = proc.communicate()[0] except Exception, e: logging.error("Unknown error. Command execution failed: %s", str(e)) raise e if proc.returncode: raise subprocess.CalledProcessError(proc.returncode, cmd, buildOutput) cmd = "make html_docs".split() output.append(subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environment).communicate()[1]) def smart_concatenate(r, x): # concatenate multi-line comments if x.startswith(" ["): r[-1] = r[-1] + x # add comment else: r.append(x) # new error return r output[-1] = reduce(smart_concatenate, output[-1].split('\n'), []) output[-1].sort() output[-1] = "\n".join(output[-1]) except KeyError: pass return "\n".join(output) #ssh codereview.qt-project.org gerrit review --project qt/qtbase --message "hello" 33219,44 def post_review(event, message, score): project = event["change"]["project"] sha1 = event["patchSet"]["revision"] command = ["ssh", config.gerrit_address, "gerrit", "review", "--project", project, "--code-review", str(score), "--tag", "autogenerated:qdoc"] if len(message) > 0: command += "--message", "\"" + message + "\"", command += sha1, logging.debug(command) if not config.sandbox: subprocess.check_call(command) elif event["change"]["number"] == '48230': subprocess.check_call(command) def remove_moved_doc_errors(fixes, errors): for fix in fixes: pos = fix.find(":") if pos >= 0: begin = "+" + fix[1:pos] end = fix[fix.find(":", pos+1):] logging.debug(end) for i in range(len(errors)): if errors[i].startswith(begin) and errors[i].endswith(end): del errors[i] break; message = "" for line in errors: message += line + "\n " return message def review_output(event, output_no_patch, output_with_patch, score_on_negative_review): try: import difflib project = event["change"]["project"] patch_set = event["patchSet"] url = event["change"]["url"] subject = event["change"]["subject"] new_error_count = output_with_patch.count("\n") - output_no_patch.count("\n") if new_error_count == 0: logging.info("NO DOC ERRORS: %s, %s", subject, url) #post_review(event, "", 1) return 0 logging.debug("DIFF for %s: %s", subject, url) message = "" score = 0 if new_error_count == 1: message += "This change introduces one new documentation error." score = score_on_negative_review elif new_error_count > 1: message += "This change introduces " + str(new_error_count) + " new documentation errors." score = score_on_negative_review elif new_error_count == -1: message += "This change removes a documentation error. Thank you!" score = 0 else: message = "This change removes " + str(abs(new_error_count)) + " documentation errors. Thank you!" score = 0 message += "\n\n " fixes = [] errors = [] line_count = 0 for line in difflib.unified_diff(output_no_patch.split('\n'), output_with_patch.split('\n'), fromfile='before', tofile='after', n=0): line_count += 1 if line_count > 2: if line[0] == "-": fixes.append(line) elif line[0] == "+": errors.append(line) new_errors = remove_moved_doc_errors(fixes, errors) if len(new_errors) > 0: message += "New qdoc warnings:\n \n " message += new_errors message += "\n\nThis comment was generated by the qtdoc bot.\n\nPlease contact Jedrzej Nowacki (nierob) or Frederik Gladhorn (fregl) if you have questions or think the script gives you false positives." logging.debug("Review result: %s", message) post_review(event, message, score) return score except: logging.error("Unexpected error: %s", sys.exc_info()) def fetch_change(project, patch_set, change_number): try: gerrit_url = config.gerrit_address + ":" + project # the mainlines need to be updated, so that we keep the updated refs and fetches don't start from scratch every time mainlines = subprocess.check_output(["git", "config", "remote.origin.fetch"]).rstrip() fetch_cmd = ["git", "fetch", "-f", "origin", patch_set["ref"]+":refs/changes/"+change_number, mainlines] num_tries = 0 while True: try: p = subprocess.check_call(fetch_cmd) return 0 except subprocess.CalledProcessError, e: # Try to fetch again - one problem is that running git fetch will fail if run simultaneously num_tries += 1 logging.warn("Fetching subprocess failed, trying again: %s", e) import time time.sleep(30) if num_tries > 11: raise e except subprocess.CalledProcessError, e: logging.error("Fetching subprocess failed too many times: %s", e) return -1 #git fetch https://codereview.qt-project.org/p/qt/qtbase refs/changes/46/32446/6 && git checkout FETCH_HEAD def process_event(event_string): # load the event data import time start_time = time.time() import json import tempfile import shutil try: event = json.loads(event_string) except ValueError, e: logging.error("JSON loading problem: %s\nJSON data: %s", e, event_string) return -1 if event["type"] != "patchset-created": return 0 # go to the right project checkout project = event["change"]["project"] patch_set = event["patchSet"] change_number = event["change"]["number"] module_name = project.split('/')[-1] if module_name not in ["qtbase"]: # TODO for now we care only about qtbase logging.info("IGNORED MODULE: %s %s", module_name, patch_set) return 0 logging.info("MODULE NAME: %s %s", module_name, patch_set) source_path = config.watcher_working_dir + "/" + project try: os.chdir(source_path) except OSError, e: logging.info("Unknown project: %s, trying to clone it", project) tmp_dir = tempfile.mkdtemp(prefix="qdoc_code_") os.chdir(tmp_dir) try: logging.debug("Clonning a new project (%s) to a tmp_dir (%s)", project, tmp_dir) cmd_git_clone = ["git", "clone", "ssh://" + config.gerrit_address + "/" + project, "."] subprocess.check_call(cmd_git_clone) except Error, e: logging.error("Could not clone a new project (%s) to a tmp_dir (%s), error message:", project, tmp_dir, e) shutil.rmtree(tmp_dir) return -1 try: root_project_dir = config.watcher_working_dir + "/" + project.split('/')[:-1][0] logging.debug("Creating destination folder (%s) for the new project (%s)", root_project_dir, project) try: os.mkdir(root_project_dir) except OSError, e: if e.errno != 17: # if error is different then "File exist" logging.error("Can not make project root directory: %s ", e) raise e logging.debug("Moving the clone (%s) from tmp dir (%s) to the destination folder (%s)", project, tmp_dir, source_path) os.rename(tmp_dir, source_path) except Error, e: logging.info("Initialization of a new project (%s) failed because it was initialized before, probably by an other process", project) logging.debug("Exception caught during rename operation: %s", e) shutil.rmtree(tmp_dir) os.chdir(source_path) logging.info("New project (%s) was initialized correctly", project) if fetch_change(project, patch_set, change_number) != 0: return -1 try: #make own copy tmp_dir = tempfile.mkdtemp(prefix="qdoc_") logging.info("CREATING TMP DIR: %s", tmp_dir) try: os.chdir(tmp_dir) cmd_git_clone = ["git", "clone", source_path, module_name] subprocess.check_call(cmd_git_clone) os.chdir(module_name) cmd_git_reset = ["git", "checkout", event["patchSet"]["revision"], "-b", "tmp"] subprocess.check_call(cmd_git_reset) score_on_negative_review = -1 header = subprocess.check_output(["git", "log", '--pretty=oneline', '-n 1']) if (re.search(r'\bWIP\b', header) or header.find('***') >= 0): score_on_negative_review = 0 # Respect work in progress try: output_with_patch = run_qdoc(module_name) except subprocess.CalledProcessError, e: logging.debug("RUNNING QDOC FAILED: %s", e) msg = "Qt Doc Bot tried to build the change on Linux, without success. Most likely because of a bug in the patch. " msg += "Please verify that a clean build works." msg += "\n\nRunning command\n " + str(e.cmd) + "\nfailed." if e.output != None: link = pasteKDE(e.output) msg += "\n\nBuild log can be found here: " msg += link post_review(event, msg, -1) return -1 #reseting to parent and cleaning (especially needed for docs removal) cmd_git_clean = ["git", "clean", "-fdxq"] subprocess.check_call(cmd_git_clean) cmd_git_reset = ["git", "reset", "--hard", "HEAD^"] subprocess.check_call(cmd_git_reset) output_no_patch = run_qdoc(module_name) logging.info("DONE RUNNING QDOC") return review_output(event, output_no_patch, output_with_patch, score_on_negative_review) finally: # TODO reuse tmp dir logging.info("REMOVING TMP DIR: %s", tmp_dir) shutil.rmtree(tmp_dir) logging.debug("WORKER EXECUTION TIME: %s", time.time() - start_time) except subprocess.CalledProcessError, e: logging.error("QDoc sanity failed because of an internal error: %s", e) return -1 def watcher(): while True: try: ssh = subprocess.Popen(["ssh", config.gerrit_address, "gerrit", "stream-events"], stdout=subprocess.PIPE) import multiprocessing logging.info("STARTING WATCHER") def multiprocess_config_assign(c): config = c processPool = multiprocessing.Pool(processes=config.workers_count, initializer=multiprocess_config_assign, initargs=(config,), maxtasksperchild=5) os.chdir(config.watcher_working_dir) event_string = ssh.stdout.readline() while len(event_string): try: processPool.apply_async(func=process_event, args=(event_string,)) finally: #clean up working dir is it neccessery? os.chdir(config.watcher_working_dir) event_string = ssh.stdout.readline() # we can get here if connection was broken somehow for example because of a timeout logging.error("WATCHER RESTART because an empty response was received") try: if ssh.returncode is None: # try to terminate the connection, it is better then wait for gc to do it. logging.warn("WATCHER received empty response but the connection was not closed, about to terminate it before restart") ssh.terminate() else: logging.warn("WATCHER was terminated with error code: %s", ssh.returncode) except: pass except: logging.critical("WATCHER RESTART because of: %s", sys.exc_info()) finally: # an error occurred let's restart in 15 sec import time time.sleep(15) try: ssh.kill() except: # swallow all errors, we do not care anymore pass if __name__== "__main__": class Config(object): pass config = Config() import argparse parser = argparse.ArgumentParser(prog="QDoc bot", description='It listens to gerrit events and tries to catch all new documentation errors', epilog="QDoc bot takes into consideration different environment variables (like TMP or TEMP). It also may use ccache and icecc if configured correctly") parser.add_argument('-j', type=int, dest="workers_count", default=None, help='Workers count, how many parallel builds can be done at the same time. Default is equals to CPU count.') parser.add_argument('--addr', type=str, dest="gerrit_address", metavar="gerrit.example.org", default="codereview.qt-project.org", help='Address of the gerrit instance. Default is codereview.qt-project.org') parser.add_argument('--dir', type=str, dest="watcher_working_dir", metavar="/home/user/dev/qdoc-watcher", required=True, help='Set working directory which contains all neccessery checkouts') parser.add_argument('--log-level', type=str, dest="logging_level", choices=["debug", "info", "warning", "error", "critical"], default="INFO", help='Set verbose level') parser.add_argument('--sandbox', dest="sandbox", action="store_true", default=False, help='The option should be used for debugging only. It simulates execution of the bot, post reviews are sent only for 33219 and 33224 changes') args = parser.parse_args(namespace=config) logging.basicConfig(format='(PID: %(process)d) %(asctime)s %(levelname)s: %(message)s', level=getattr(logging, config.logging_level.upper(), None)) watcher()