path: root/git-hooks/qdoc-bot

#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright (C) 2015 The Qt Company Ltd.
# Contact: http://www.qt.io/licensing/
#
# You may use this file under the terms of the 3-clause BSD license.
# See the file LICENSE from this package for details.
#

# Checkouts have to have the remote: origin codereview-bot:qt/qtbase.git

# Run with --dir and --addr
# python qdoc-bot.py --dir /home/USER/dev/qt-qdoc/ --addr codereview-bot
# --dir is where the sources are checked out
# --addr codereview-bot
# With .ssh/config:
# Host codereview-bot
#    Hostname codereview.qt-project.org
#    Port 29418
#    User qt_docbot
#    IdentityFile /home/USER/.ssh/docbot_id_rsa



import os
import subprocess
import sys
import logging
import re

config = None # configuration object, it is initialized in main and passed to every worker process created by multiprocessing.Pool

#{
    #"change": {
        #"branch": "2.6",
        #"id": "I765c9e165f9e7a5e0928401f5c5254397ad9be0b",
        #"number": "33424",
        #"owner": {
            #"email": "Friedemann.Kleint@nokia.com",
            #"name": "Friedemann Kleint"
        #},
        #"project": "qt-creator/qt-creator",
        #"subject": "WIP: CDB: Fix some dumpers.",
        #"url": "https://codereview.qt-project.org/33424"
        #},
    #"patchSet": {
        #"number": "1",
        #"ref": "refs/changes/24/33424/1",
        #"revision": "c4dd516729ff8f9b7cb1e13d9c8f09560cf2c078",
        #"uploader": {
            #"email": "Friedemann.Kleint@nokia.com",
            #"name": "Friedemann Kleint"
        #}
    #},
    #"type": "patchset-created",
    #"uploader": {
        #"email": "Friedemann.Kleint@nokia.com",
        #"name": "Friedemann Kleint"
    #}
#}



def pasteKDE(data):
    import httplib, urllib, json
    params = urllib.urlencode({'paste_data': data,
                               'paste_lang': 'text',
                               'api_submit': True,
                               'mode': 'json',
                               'paste_project':'Qt',
                               'paste_user': 'Qt Doc bot',
                               'paste_private': 'yes',
                               'paste_expire': 2629744 # around 1 month
                              })
    headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}
    conn = httplib.HTTPConnection("paste.kde.org")
    try:
        conn.request("POST", "", params, headers)
        response = json.loads(conn.getresponse().read())
        logging.debug("Making a paste %s", response)
        return "http://pastebin.kde.org/" + str(response['result']['id']) + "/" + str(response['result']['hash'])
    finally:
        conn.close()

#make sub-qdoc in src/tools
def run_qdoc(module_name):
    output = []
    try:
        environment = os.environ
        environment["CCACHE_BASEDIR"] =  os.getcwd()
        environment["QT_HASH_SEED"] = "1234"
        environment["QMAKEFLAGS"] = "CONFIG+=force_bootstrap"
        for cmd in ["/bin/sh ./configure --opensource --confirm-license --release".split()
                   , "make sub-src-qmake_all".split()
                   , ["/bin/sh", "-c", "if [ -f src/tools/Makefile ] ; then make -Csrc/tools sub-qdoc; else make -Csrc sub-qdoc ; fi"] # build qdoc
                   ]:
            logging.debug("Executing command: %s", cmd)
            try:
                proc = subprocess.Popen(cmd, env=environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                buildOutput = proc.communicate()[0]
            except Exception, e:
                logging.error("Unknown error. Command execution failed: %s", str(e))
                raise e

            if proc.returncode:
                raise subprocess.CalledProcessError(proc.returncode, cmd, buildOutput)

        cmd = "make html_docs".split()
        output.append(subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environment).communicate()[1])
        def smart_concatenate(r, x):
            # concatenate multi-line comments
            if x.startswith("    ["):
                r[-1] = r[-1] + x  # add comment
            else:
                r.append(x)        # new error
            return r
        output[-1] = reduce(smart_concatenate, output[-1].split('\n'), [])
        output[-1].sort()
        output[-1] = "\n".join(output[-1])
    except KeyError:
        pass
    return "\n".join(output)

#ssh codereview.qt-project.org gerrit review --project qt/qtbase --message "hello" 33219,44
def post_review(event, message, score):
    project = event["change"]["project"]
    sha1 = event["patchSet"]["revision"]
    command = ["ssh", config.gerrit_address, "gerrit", "review", "--project", project,
               "--code-review", str(score), "--tag", "autogenerated:qdoc"]
    if len(message) > 0:
        command += "--message", "\"" + message + "\"",
    command += sha1,
    logging.debug(command)
    if not config.sandbox:
        subprocess.check_call(command)
    elif event["change"]["number"] == '48230':
        subprocess.check_call(command)

def remove_moved_doc_errors(fixes, errors):
    for fix in fixes:
        pos = fix.find(":")
        if pos >= 0:
            begin = "+" + fix[1:pos]
            end = fix[fix.find(":", pos+1):]
            logging.debug(end)
            for i in range(len(errors)):
                if errors[i].startswith(begin) and errors[i].endswith(end):
                    del errors[i]
                    break;
    message = ""
    for line in errors:
        message += line + "\n "
    return message

def review_output(event, output_no_patch, output_with_patch, score_on_negative_review):
    try:
        import difflib
        project = event["change"]["project"]
        patch_set = event["patchSet"]
        url = event["change"]["url"]
        subject = event["change"]["subject"]
        new_error_count = output_with_patch.count("\n") - output_no_patch.count("\n")
        if new_error_count == 0:
            logging.info("NO DOC ERRORS: %s, %s", subject, url)
            #post_review(event, "", 1)
            return 0
        logging.debug("DIFF for %s: %s", subject, url)
        message = ""
        score = 0
        if new_error_count == 1:
            message += "This change introduces one new documentation error."
            score = score_on_negative_review
        elif new_error_count > 1:
            message += "This change introduces " + str(new_error_count) + " new documentation errors."
            score = score_on_negative_review
        elif new_error_count == -1:
            message += "This change removes a documentation error. Thank you!"
            score = 0
        else:
            message = "This change removes " + str(abs(new_error_count)) + " documentation errors. Thank you!"
            score = 0

        message += "\n\n "
        fixes = []
        errors = []
        line_count = 0
        for line in difflib.unified_diff(output_no_patch.split('\n'), output_with_patch.split('\n'), fromfile='before', tofile='after', n=0):
            line_count += 1
            if line_count > 2:
                if line[0] == "-":
                    fixes.append(line)
                elif line[0] == "+":
                    errors.append(line)
        new_errors = remove_moved_doc_errors(fixes, errors)
        if len(new_errors) > 0:
            message += "New qdoc warnings:\n \n "
            message += new_errors
        message += "\n\nThis comment was generated by the qtdoc bot.\n\nPlease contact Jedrzej Nowacki (nierob) or Frederik Gladhorn (fregl) if you have questions or think the script gives you false positives."
        logging.debug("Review result: %s", message)
        post_review(event, message, score)
        return score
    except:
        logging.error("Unexpected error: %s", sys.exc_info())


def fetch_change(project, patch_set, change_number):
    try:
        gerrit_url = config.gerrit_address + ":" + project
        # the mainlines need to be updated, so that we keep the updated refs and fetches don't start from scratch every time
        mainlines = subprocess.check_output(["git", "config", "remote.origin.fetch"]).rstrip()
        fetch_cmd = ["git", "fetch", "-f", "origin", patch_set["ref"]+":refs/changes/"+change_number, mainlines]
        num_tries = 0
        while True:
            try:
                p = subprocess.check_call(fetch_cmd)
                return 0

            except subprocess.CalledProcessError, e:
                # Try to fetch again - one problem is that running git fetch will fail if run simultaneously
                num_tries += 1
                logging.warn("Fetching subprocess failed, trying again: %s", e)
                import time
                time.sleep(30)
                if num_tries > 11:
                    raise e

    except subprocess.CalledProcessError, e:
        logging.error("Fetching subprocess failed too many times: %s", e)
        return -1


#git fetch https://codereview.qt-project.org/p/qt/qtbase refs/changes/46/32446/6 && git checkout FETCH_HEAD
def process_event(event_string):
    # load the event data
    import time
    start_time = time.time()
    import json
    import tempfile
    import shutil

    try:
        event = json.loads(event_string)
    except ValueError, e:
        logging.error("JSON loading problem: %s\nJSON data: %s", e, event_string)
        return -1

    if event["type"] != "patchset-created":
        return 0

    # go to the right project checkout
    project = event["change"]["project"]
    patch_set = event["patchSet"]
    change_number = event["change"]["number"]

    module_name = project.split('/')[-1]
    if module_name not in ["qtbase"]: # TODO for now we care only about qtbase
        logging.info("IGNORED MODULE: %s %s", module_name, patch_set)
        return 0

    logging.info("MODULE NAME: %s %s", module_name, patch_set)

    source_path = config.watcher_working_dir + "/" + project
    try:
        os.chdir(source_path)
    except OSError, e:
        logging.info("Unknown project: %s, trying to clone it", project)
        tmp_dir = tempfile.mkdtemp(prefix="qdoc_code_")
        os.chdir(tmp_dir)
        try:
            logging.debug("Clonning a new project (%s) to a tmp_dir (%s)", project, tmp_dir)
            cmd_git_clone = ["git", "clone", "ssh://" + config.gerrit_address + "/" + project, "."]
            subprocess.check_call(cmd_git_clone)
        except Error, e:
            logging.error("Could not clone a new project (%s) to a tmp_dir (%s), error message:", project, tmp_dir, e)
            shutil.rmtree(tmp_dir)
            return -1
        try:
            root_project_dir = config.watcher_working_dir + "/" + project.split('/')[:-1][0]
            logging.debug("Creating destination folder (%s) for the new project (%s)", root_project_dir, project)
            try:
                os.mkdir(root_project_dir)
            except OSError, e:
                if e.errno != 17: # if error is different then "File exist"
                    logging.error("Can not make project root directory: %s ", e)
                    raise e
            logging.debug("Moving the clone (%s) from tmp dir (%s) to the destination folder (%s)", project, tmp_dir, source_path)
            os.rename(tmp_dir, source_path)
        except Error, e:
            logging.info("Initialization of a new project (%s) failed because it was initialized before, probably by an other process", project)
            logging.debug("Exception caught during rename operation: %s", e)
            shutil.rmtree(tmp_dir)
        os.chdir(source_path)
        logging.info("New project (%s) was initialized correctly", project)

    if fetch_change(project, patch_set, change_number) != 0:
        return -1

    try:
        #make own copy
        tmp_dir = tempfile.mkdtemp(prefix="qdoc_")
        logging.info("CREATING TMP DIR: %s", tmp_dir)
        try:
            os.chdir(tmp_dir)

            cmd_git_clone = ["git", "clone", source_path, module_name]
            subprocess.check_call(cmd_git_clone)

            os.chdir(module_name)

            cmd_git_reset = ["git", "checkout", event["patchSet"]["revision"], "-b", "tmp"]
            subprocess.check_call(cmd_git_reset)

            score_on_negative_review = -1
            header = subprocess.check_output(["git", "log", '--pretty=oneline', '-n 1'])
            if  (re.search(r'\bWIP\b', header) or header.find('***') >= 0):
                score_on_negative_review = 0 # Respect work in progress

            try:
                output_with_patch = run_qdoc(module_name)
            except subprocess.CalledProcessError, e:
                logging.debug("RUNNING QDOC FAILED: %s", e)
                msg = "Qt Doc Bot tried to build the change on Linux, without success. Most likely because of a bug in the patch. "
                msg += "Please verify that a clean build works."
                msg += "\n\nRunning command\n " + str(e.cmd) + "\nfailed."
                if e.output != None:
                    link = pasteKDE(e.output)
                    msg += "\n\nBuild log can be found here: "
                    msg += link
                post_review(event, msg, -1)
                return -1

            #reseting to parent and cleaning (especially needed for docs removal)
            cmd_git_clean = ["git", "clean", "-fdxq"]
            subprocess.check_call(cmd_git_clean)
            cmd_git_reset = ["git", "reset", "--hard", "HEAD^"]
            subprocess.check_call(cmd_git_reset)

            output_no_patch = run_qdoc(module_name)

            logging.info("DONE RUNNING QDOC")
            return review_output(event, output_no_patch, output_with_patch, score_on_negative_review)

        finally:
            # TODO reuse tmp dir
            logging.info("REMOVING TMP DIR: %s", tmp_dir)
            shutil.rmtree(tmp_dir)
            logging.debug("WORKER EXECUTION TIME: %s", time.time() - start_time)
    except subprocess.CalledProcessError, e:
        logging.error("QDoc sanity failed because of an internal error: %s", e)
        return -1


def watcher():
    while True:
        try:
            ssh = subprocess.Popen(["ssh", config.gerrit_address, "gerrit", "stream-events"], stdout=subprocess.PIPE)
            import multiprocessing
            logging.info("STARTING WATCHER")
            def multiprocess_config_assign(c):
                config = c
            processPool = multiprocessing.Pool(processes=config.workers_count,
                                               initializer=multiprocess_config_assign,
                                               initargs=(config,),
                                               maxtasksperchild=5)
            os.chdir(config.watcher_working_dir)
            event_string = ssh.stdout.readline()
            while len(event_string):
                try:
                    processPool.apply_async(func=process_event, args=(event_string,))
                finally:
                    #clean up working dir is it neccessery?
                    os.chdir(config.watcher_working_dir)
                event_string = ssh.stdout.readline()
            # we can get here if connection was broken somehow for example because of a timeout
            logging.error("WATCHER RESTART because an empty response was received")
            try:
                if ssh.returncode is None:
                    # try to terminate the connection, it is better then wait for gc to do it.
                    logging.warn("WATCHER received empty response but the connection was not closed, about to terminate it before restart")
                    ssh.terminate()
                else:
                    logging.warn("WATCHER was terminated with error code: %s", ssh.returncode)
            except:
                pass
        except:
            logging.critical("WATCHER RESTART because of: %s", sys.exc_info())
        finally:
            # an error occurred let's restart in 15 sec
            import time
            time.sleep(15)
            try:
                ssh.kill()
            except: # swallow all errors, we do not care anymore
                pass


if __name__== "__main__":
    class Config(object): pass
    config = Config()
    import argparse
    parser = argparse.ArgumentParser(prog="QDoc bot",
                   description='It listens to gerrit events and tries to catch all new documentation errors',
                   epilog="QDoc bot takes into consideration different environment variables (like TMP or TEMP). It also may use ccache and icecc if configured correctly")
    parser.add_argument('-j',
                   type=int,
                   dest="workers_count",
                   default=None,
                   help='Workers count, how many parallel builds can be done at the same time. Default is equals to CPU count.')
    parser.add_argument('--addr',
                   type=str,
                   dest="gerrit_address",
                   metavar="gerrit.example.org",
                   default="codereview.qt-project.org",
                   help='Address of the gerrit instance. Default is codereview.qt-project.org')
    parser.add_argument('--dir',
                   type=str,
                   dest="watcher_working_dir",
                   metavar="/home/user/dev/qdoc-watcher",
                   required=True,
                   help='Set working directory which contains all neccessery checkouts')
    parser.add_argument('--log-level',
                   type=str,
                   dest="logging_level",
                   choices=["debug", "info", "warning", "error", "critical"],
                   default="INFO",
                   help='Set verbose level')
    parser.add_argument('--sandbox',
                   dest="sandbox",
                   action="store_true",
                   default=False,
                   help='The option should be used for debugging only. It simulates execution of the bot, post reviews are sent only for 33219 and 33224 changes')

    args = parser.parse_args(namespace=config)
    logging.basicConfig(format='(PID: %(process)d) %(asctime)s %(levelname)s: %(message)s', level=getattr(logging, config.logging_level.upper(), None))
    watcher()