diff options
| -rw-r--r-- | automation/clang-format/Dockerfile | 26 | ||||
| -rwxr-xr-x | automation/clang-format/run_clang_format.sh (renamed from automation/taskcluster/scripts/run_clang_format.sh) | 12 | ||||
| -rw-r--r-- | automation/clang-format/setup.sh | 43 | ||||
| -rw-r--r-- | automation/taskcluster/graph/src/extend.js | 2 | ||||
| -rwxr-xr-x | mach | 148 |
5 files changed, 225 insertions, 6 deletions
diff --git a/automation/clang-format/Dockerfile b/automation/clang-format/Dockerfile new file mode 100644 index 000000000..163c9b8fa --- /dev/null +++ b/automation/clang-format/Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:16.04 +MAINTAINER Franziskus Kiefer <franziskuskiefer@gmail.com> + +RUN useradd -d /home/worker -s /bin/bash -m worker +WORKDIR /home/worker + +# Install dependencies. +ADD setup.sh /tmp/setup.sh +RUN bash /tmp/setup.sh + +# Change user. +USER worker + +# Env variables. +ENV HOME /home/worker +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME worker +ENV HOSTNAME taskcluster-worker +ENV LANG en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 +ENV HOST localhost +ENV DOMSUF localdomain + +# Entrypoint. +ENTRYPOINT ["/home/worker/nss/automation/clang-format/run_clang_format.sh"] diff --git a/automation/taskcluster/scripts/run_clang_format.sh b/automation/clang-format/run_clang_format.sh index 05e63173f..df29a085f 100755 --- a/automation/taskcluster/scripts/run_clang_format.sh +++ b/automation/clang-format/run_clang_format.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash -source $(dirname "$0")/tools.sh - -set +x +if [[ $(id -u) -eq 0 ]]; then + # Drop privileges by re-running this script. + # Note: this mangles arguments, better to avoid running scripts as root. + exec su worker -c "$0 $*" +fi # Apply clang-format on the provided folder and verify that this doesn't change any file. # If any file differs after formatting, the script eventually exits with 1. @@ -22,13 +24,13 @@ blacklist=( "./.hg" \ ) -top="$PWD/$(dirname $0)/../../.." +top="$(dirname $0)/../.." cd "$top" if [ $# -gt 0 ]; then dirs=("$@") else - dirs=($(find . ! -path . \( ! -regex '.*/' \) -maxdepth 2 -mindepth 1 -type d)) + dirs=($(find . -maxdepth 2 -mindepth 1 -type d ! -path . \( ! -regex '.*/' \))) fi format_folder() diff --git a/automation/clang-format/setup.sh b/automation/clang-format/setup.sh new file mode 100644 index 000000000..f719f1973 --- /dev/null +++ b/automation/clang-format/setup.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -v -e -x + +# Update packages. +export DEBIAN_FRONTEND=noninteractive +apt-get -y update && apt-get -y upgrade + +# Install packages. +apt_packages=() +apt_packages+=('ca-certificates') +apt_packages+=('curl') +apt_packages+=('xz-utils') +apt_packages+=('mercurial') +apt_packages+=('git') +apt-get install -y --no-install-recommends ${apt_packages[@]} + +# Download clang. +curl -L http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -o clang.tar.xz +curl -L http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig -o clang.tar.xz.sig +# Verify the signature. +gpg --keyserver pool.sks-keyservers.net --recv-keys B6C8F98282B944E3B0D5C2530FC3042E345AD05D +gpg --verify clang.tar.xz.sig +# Install into /usr/local/. +tar xJvf *.tar.xz -C /usr/local --strip-components=1 + +# Cleanup. +function cleanup() { + rm -f clang.tar.xz clang.tar.xz.sig +} +trap cleanup ERR EXIT + +locale-gen en_US.UTF-8 +dpkg-reconfigure locales + +# Cleanup. +rm -rf ~/.ccache ~/.cache +apt-get autoremove -y +apt-get clean +apt-get autoclean + +# We're done. Remove this script. +rm $0 diff --git a/automation/taskcluster/graph/src/extend.js b/automation/taskcluster/graph/src/extend.js index d9f6938c0..d541a1a3b 100644 --- a/automation/taskcluster/graph/src/extend.js +++ b/automation/taskcluster/graph/src/extend.js @@ -746,7 +746,7 @@ async function scheduleTools() { command: [ "/bin/bash", "-c", - "bin/checkout.sh && nss/automation/taskcluster/scripts/run_clang_format.sh" + "bin/checkout.sh && nss/automation/clang-format/run_clang_format.sh" ] })); @@ -0,0 +1,148 @@ +#!/usr/bin/env python +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +################################################################################ +# +# This is a collection of helper tools to get stuff done in NSS. +# + +import sys +import argparse +import subprocess +import os +import platform +from hashlib import sha256 + +cwd = os.path.dirname(os.path.abspath(__file__)) + + +class cfAction(argparse.Action): + docker_command = ["docker"] + + def __call__(self, parser, args, values, option_string=None): + if "noroot" not in values: + self.setDockerCommand() + else: + values.remove("noroot") + + # First check if we can run docker. + try: + with open(os.devnull, "w") as f: + subprocess.check_call( + self.docker_command + ["images"], stdout=f) + except: + print("Please install docker and start the docker daemon.") + sys.exit(1) + + docker_image = 'clang-format-service:latest' + cf_docker_folder = cwd + "/automation/clang-format" + + # Build the image if necessary. + if self.filesChanged(cf_docker_folder): + self.buildImage(docker_image, cf_docker_folder) + + # Check if we have the docker image. + try: + command = self.docker_command + [ + "image", "inspect", "clang-format-service:latest" + ] + with open(os.devnull, "w") as f: + subprocess.check_call(command, stdout=f) + except: + print("I have to build the docker image first.") + self.buildImage(docker_image, cf_docker_folder) + + command = self.docker_command + [ + 'run', '-v', cwd + ':/home/worker/nss', '--rm', '-ti', docker_image + ] + # The clang format script returns 1 if something's to do. We don't care. + subprocess.call(command + values) + + def filesChanged(self, path): + hash = sha256() + for dirname, dirnames, files in os.walk(path): + for file in files: + with open(os.path.join(dirname, file), "rb") as f: + hash.update(f.read()) + chk_file = cwd + "/out/.chk" + old_chk = "" + new_chk = hash.hexdigest() + if os.path.exists(chk_file): + with open(chk_file) as f: + old_chk = f.readline() + if old_chk != new_chk: + with open(chk_file, "w+") as f: + f.write(new_chk) + return True + return False + + def buildImage(self, docker_image, cf_docker_folder): + command = self.docker_command + [ + "build", "-t", docker_image, cf_docker_folder + ] + subprocess.check_call(command) + return + + def setDockerCommand(self): + if platform.system() == "Linux": + self.docker_command = ["sudo"] + self.docker_command + + +class buildAction(argparse.Action): + def __call__(self, parser, args, values, option_string=None): + cwd = os.path.dirname(os.path.abspath(__file__)) + subprocess.check_call([cwd + "/build.sh"] + values) + + +class testAction(argparse.Action): + def runTest(self, test, cycles="standard"): + cwd = os.path.dirname(os.path.abspath(__file__)) + domsuf = os.getenv('DOMSUF', "localdomain") + env = {"NSS_TESTS": test, "NSS_CYCLES": cycles, "DOMSUF": domsuf} + command = cwd + "/tests/all.sh" + subprocess.check_call(command, env=env) + + def __call__(self, parser, args, values, option_string=None): + self.runTest(values) + + +def parse_arguments(): + parser = argparse.ArgumentParser( + description='NSS helper script. ' + + 'Make sure to separate sub-command arguments with --.') + subparsers = parser.add_subparsers() + + parser_build = subparsers.add_parser( + 'build', help='All arguments are passed to build.sh') + parser_build.add_argument( + 'build_args', nargs='*', help="build arguments", action=buildAction) + + parser_cf = subparsers.add_parser( + 'clang-format', + help='Run clang-format on all folders or provide a folder to format.') + parser_cf.add_argument( + 'cf_args', + nargs='*', + help="clang-format folders and noroot if you don't want to use sudo", + action=cfAction) + + parser_test = subparsers.add_parser( + 'tests', help='Run tests through tests/all.sh.') + tests = [ + "cipher", "lowhash", "libpkix", "cert", "dbtests", "tools", "fips", + "sdr", "crmf", "smime", "ssl", "ocsp", "merge", "pkits", "ec", + "gtests", "ssl_gtests" + ] + parser_test.add_argument( + 'test', choices=tests, help="Available tests", action=testAction) + return parser.parse_args() + + +def main(): + parse_arguments() + + +if __name__ == '__main__': + main() |