diff options
| -rw-r--r-- | .taskcluster.yml | 58 | ||||
| -rw-r--r-- | automation/taskcluster/docker/Dockerfile | 27 | ||||
| -rw-r--r-- | automation/taskcluster/docker/bin/checkout.sh | 11 | ||||
| -rw-r--r-- | automation/taskcluster/docker/setup.sh | 52 | ||||
| -rw-r--r-- | automation/taskcluster/graph/build.js | 138 | ||||
| -rw-r--r-- | automation/taskcluster/graph/graph.yml | 254 | ||||
| -rwxr-xr-x | automation/taskcluster/scripts/build.sh | 20 | ||||
| -rwxr-xr-x | automation/taskcluster/scripts/extend_task_graph.sh | 13 | ||||
| -rwxr-xr-x | automation/taskcluster/scripts/run_clang_format.sh | 18 | ||||
| -rwxr-xr-x | automation/taskcluster/scripts/run_tests.sh | 20 |
10 files changed, 611 insertions, 0 deletions
diff --git a/.taskcluster.yml b/.taskcluster.yml new file mode 100644 index 000000000..8bcc14431 --- /dev/null +++ b/.taskcluster.yml @@ -0,0 +1,58 @@ +--- +metadata: + name: "NSS Continuous Integration" + description: "Build NSS and run tests in various configurations" + owner: "mozilla-taskcluster-maintenance@mozilla.com" + source: "{{{source}}}" + +tasks: + - provisionerId: "aws-provisioner-v1" + workerType: "gecko-decision" + + scopes: + - "queue:route:tc-treeherder-stage.nss.{{revision}}" + - "queue:route:tc-treeherder.nss.{{revision}}" + - "scheduler:extend-task-graph:*" + + routes: + - "tc-treeherder-stage.nss.{{revision}}" + - "tc-treeherder.nss.{{revision}}" + + metadata: + name: "NSS Decision Task" + description: "Extends the task graph with everything we need" + owner: "mozilla-taskcluster-maintenance@mozilla.com" + source: "{{{source}}}" + tags: + createdForUser: {{owner}} + + payload: + maxRunTime: 1800 + image: "ttaubert/nss-ci:0.0.12" + command: + - bash + - -cx + - > + bin/checkout.sh && + nss/automation/taskcluster/scripts/extend_task_graph.sh + env: + TC_DOCKER_IMAGE: "ttaubert/nss-ci:0.0.12" + TC_PROVISIONER_ID: "aws-provisioner-v1" + TC_WORKER_TYPE: "hg-worker" + NSS_HEAD_REPOSITORY: '{{{url}}}' + NSS_HEAD_REVISION: '{{revision}}' + + graphs: + - /home/worker/artifacts/graph.json + + artifacts: + public: + type: "directory" + path: "/home/worker/artifacts" + expires: "{{#from_now}}1 hour{{/from_now}}" + + extra: + treeherder: + symbol: D + revision: '{{revision}}' + revision_hash: '{{revision_hash}}' diff --git a/automation/taskcluster/docker/Dockerfile b/automation/taskcluster/docker/Dockerfile new file mode 100644 index 000000000..675d858dc --- /dev/null +++ b/automation/taskcluster/docker/Dockerfile @@ -0,0 +1,27 @@ +FROM ubuntu:16.04 +MAINTAINER Tim Taubert <ttaubert@mozilla.com> + +RUN useradd -d /home/worker -s /bin/bash -m worker +WORKDIR /home/worker + +# Install non-build specific dependencies. +ADD setup.sh /tmp/setup.sh +RUN bash /tmp/setup.sh + +# Add build and test scripts. +ADD bin /home/worker/bin +RUN chmod +x /home/worker/bin/* + +# Set variables usually configured at login. +ENV HOME /home/worker +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME worker +ENV HOSTNAME taskcluster-worker +ENV LANG en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 +env HOST localhost +env DOMSUF localdomain + +# Set a default command for debugging. +CMD ["/bin/bash", "--login"] diff --git a/automation/taskcluster/docker/bin/checkout.sh b/automation/taskcluster/docker/bin/checkout.sh new file mode 100644 index 000000000..3b3b0094f --- /dev/null +++ b/automation/taskcluster/docker/bin/checkout.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -v -e -x + +if [ $(id -u) = 0 ]; then + # Drop privileges by re-running this script. + exec su worker $0 +fi + +# Clone NSS. +hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss diff --git a/automation/taskcluster/docker/setup.sh b/automation/taskcluster/docker/setup.sh new file mode 100644 index 000000000..da0c1126e --- /dev/null +++ b/automation/taskcluster/docker/setup.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +set -v -e -x + +apt_packages=() +apt_packages+=('build-essential') +apt_packages+=('ca-certificates') +apt_packages+=('curl') +apt_packages+=('mercurial') +apt_packages+=('npm') +apt_packages+=('git') +apt_packages+=('zlib1g-dev') + +# 32-bit builds +apt_packages+=('lib32z1-dev') +apt_packages+=('gcc-multilib') +apt_packages+=('g++-multilib') + +# Install prerequisites. +apt-get -y update +export DEBIAN_FRONTEND=noninteractive +apt-get install -y --no-install-recommends curl apt-utils + +# clang-format-3.8 +apt_packages+=('clang-format-3.8') +curl http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add - +echo "deb http://llvm.org/apt/xenial/ llvm-toolchain-xenial-3.8 main" > /etc/apt/sources.list.d/docker.list + +# gcc 6 +apt_packages+=('g++-6') +apt_packages+=('g++-6-multilib') +apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 60C317803A41BA51845E371A1E9377A2BA9EF27F +echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu xenial main" > /etc/apt/sources.list.d/toolchain.list + +# Install all other packages. +apt-get -y update +apt-get install -y --no-install-recommends ${apt_packages[@]} + +# 32-bit builds +ln -s /usr/include/x86_64-linux-gnu/zconf.h /usr/include + +locale-gen en_US.UTF-8 +dpkg-reconfigure locales + +# Install required Node modules. +su -c "npm install flatmap js-yaml merge slugid" worker + +# Cleanup. +rm -rf ~/.ccache ~/.cache +apt-get clean +apt-get autoclean +rm $0 diff --git a/automation/taskcluster/graph/build.js b/automation/taskcluster/graph/build.js new file mode 100644 index 000000000..44baa9028 --- /dev/null +++ b/automation/taskcluster/graph/build.js @@ -0,0 +1,138 @@ +var fs = require("fs"); +var path = require("path"); +var yaml = require("js-yaml"); +var merge = require("merge"); +var slugid = require("slugid"); +var flatmap = require("flatmap"); +var taskids = {}; + +// TODO +function taskid(id) { + if (!(id in taskids)) { + taskids[id] = slugid.v4(); + } + return taskids[id]; +} + +// TODO +function from_now(hours) { + var d = new Date(); + d.setHours(d.getHours() + (hours || 0)); + return d.toJSON(); +} + +// TODO +function build_task(id, def) { + var task, retvals = [{ + taskId: taskid(id), + task: task = { + payload: { + image: process.env.TC_DOCKER_IMAGE, + maxRunTime: 3600 + }, + metadata: { + owner: process.env.GITHUB_HEAD_USER_EMAIL, + source: process.env.GITHUB_HEAD_REPO_URL + } + } + }]; + + // Fill in some basic data. + task.created = from_now(0); + task.deadline = from_now(24); + task.provisionerId = process.env.TC_PROVISIONER_ID || "aws-provisioner-v1"; + task.workerType = process.env.TC_WORKER_TYPE || "github-worker"; + task.schedulerId = "task-graph-scheduler"; + + // Clone definition. + def = merge.recursive(true, {}, def); + + // Extend task definition. + while (def.extends) { + var base = def.extends; + delete def.extends; + + var template = doc.templates[base]; + def = merge.recursive(true, template, def); + + if ("name" in template) { + def.name = template.name + " | " + def.name; + } + } + + // Fill in attributes. + task.metadata.name = def.name; + task.metadata.description = def.description; + task.payload.command = def.command; + task.payload.env = def.env || {}; + + // Forward some GitHub env variables. + task.payload.env.NSS_HEAD_REPOSITORY = process.env.NSS_HEAD_REPOSITORY; + task.payload.env.NSS_HEAD_REVISION = process.env.NSS_HEAD_REVISION; + + // Register artifacts. + if (def.artifact) { + task.payload.artifacts = { + "public": { + "type": "directory", + "path": "/home/worker/artifacts", + "expires": from_now(1) + } + }; + } + + // Create subtasks. + if ("subtasks" in def) { + def.subtasks.forEach(function (sid) { + if (!(sid in doc.templates)) { + throw new Error("Can't find template '" + sid + "'"); + } + + var subtasks = build_task(id + "_" + sid, doc.templates[sid]); + + // TODO + subtasks.forEach(function (subtask) { + subtask.task.metadata.name = task.metadata.name + " | " + subtask.task.metadata.name; + subtask.task.payload.env = merge.recursive(true, task.payload.env, subtask.task.payload.env); + + // TODO + if (!subtask.task.metadata.description) { + subtask.task.metadata.description = task.metadata.description; + } + + // TODO + if (!subtask.requires) { + subtask.requires = [taskid(id)]; + subtask.task.payload.env.TC_PARENT_TASK_ID = taskid(id); + } + }); + + // Append subtasks. + retvals = retvals.concat(subtasks); + }); + } + + return retvals; +} + +// Load the tasks definition file. +var source = fs.readFileSync(path.join(__dirname, "./graph.yml"), "utf-8"); +var doc = yaml.load(source); + +// Build the graph. +var graph = {tasks: flatmap(Object.keys(doc.graph), function (id) { + return build_task(id, doc.graph[id]); +})}; + +// Clean up env variables. +graph.tasks.forEach(function (task) { + var env = task.task.payload.env; + Object.keys(env).forEach(function (name) { + if (env[name] === "") { + delete env[name]; + } + }); +}); + +// Output the final graph. +process.stdout.write(JSON.stringify(graph, null, 2)); diff --git a/automation/taskcluster/graph/graph.yml b/automation/taskcluster/graph/graph.yml new file mode 100644 index 000000000..754fee182 --- /dev/null +++ b/automation/taskcluster/graph/graph.yml @@ -0,0 +1,254 @@ +templates: + builder: + name: "NSS" + description: "Build NSS & NSPR" + artifact: "dist.tar.bz2" + command: + - "/bin/bash" + - "-c" + - "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh" + env: + NSS_ENABLE_TLS_1_3: 1 +# subtasks: +# - cert +# - chains +# - cipher +# - crmf +# - dbtests +# - ec +# - fips +# - gtests +# - libpkix +# - lowhash +# - merge +# - ocsp +# - pkits +# - sdr +# - smime +# - ssl +# - tools + + test_runner: + command: + - "/bin/bash" + - "-c" + - "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh" + + cert: + name: "cert tests" + description: "Run NSS cert tests" + extends: test_runner + env: + NSS_TESTS: "cert" + + chains: + name: "chains tests" + description: "Run NSS chains tests" + extends: test_runner + env: + NSS_TESTS: "chains" + + cipher: + name: "cipher tests" + description: "Run NSS cipher tests" + extends: test_runner + env: + NSS_TESTS: "cipher" + + crmf: + name: "crmf tests" + description: "Run NSS crmf tests" + extends: test_runner + env: + NSS_TESTS: "crmf" + + dbtests: + name: "dbtests" + description: "Run NSS dbtests" + extends: test_runner + env: + NSS_TESTS: "dbtests" + + ec: + name: "EC tests" + description: "Run NSS EC tests" + extends: test_runner + env: + NSS_TESTS: "ec" + + fips: + name: "FIPS tests" + description: "Run NSS FIPS tests" + extends: test_runner + env: + NSS_TESTS: "fips" + + gtests: + name: "Gtests" + description: "Run NSS Gtests" + extends: test_runner + env: + NSS_TESTS: "ssl_gtests gtests" + + libpkix: + name: "libpkix tests" + description: "Run NSS libpkix tests" + extends: test_runner + env: + NSS_TESTS: "libpkix" + + lowhash: + name: "lowhash tests" + description: "Run NSS lowhash tests" + extends: test_runner + env: + NSS_TESTS: "lowhash" + + merge: + name: "merge tests" + description: "Run NSS merge tests" + extends: test_runner + env: + NSS_TESTS: "merge" + + ocsp: + name: "ocsp tests" + description: "Run NSS ocsp tests" + extends: test_runner + env: + NSS_TESTS: "ocsp" + + pkits: + name: "pkits tests" + description: "Run NSS pkits tests" + extends: test_runner + env: + NSS_TESTS: "pkits" + + sdr: + name: "sdr tests" + description: "Run NSS sdr tests" + extends: test_runner + env: + NSS_TESTS: "sdr" + + smime: + name: "smime tests" + description: "Run NSS smime tests" + extends: test_runner + env: + NSS_TESTS: "smime" + + ssl: # Update this when TLS v1.3 doesn't fail these anymore. + name: "ssl tests" + description: "Run NSS ssl tests" + artifact: "dist.tar.bz2" + command: + - "/bin/bash" + - "-c" + - "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh" + env: + NSS_ENABLE_TLS_1_3: "" # Remove this. + NSS_TESTS: "ssl" + subtasks: + - ssl1 + - ssl2 + - ssl3 + - ssl4 + + ssl1: + name: "cycle=standard" + extends: test_runner + env: + NSS_CYCLES: "standard" + ssl2: + name: "cycle=pkix" + extends: test_runner + env: + NSS_CYCLES: "pkix" + ssl3: + name: "cycle=upgradedb" + extends: test_runner + env: + NSS_CYCLES: "upgradedb" + ssl4: + name: "cycle=sharedb" + extends: test_runner + env: + NSS_CYCLES: "sharedb" + + tools: + name: "tools tests" + description: "Run NSS tools tests" + extends: test_runner + env: + NSS_TESTS: "tools" + +graph: + build-32-debug-asan-gcc5: + name: "Linux 32 (gcc5, debug, ASan)" + extends: builder + env: + USE_ASAN: 1 + + build-32-opt-gcc5: + name: "Linux 32 (gcc5, opt)" + extends: builder + env: + BUILD_OPT: 1 + + build-64-debug-asan-gcc5: + name: "Linux 64 (gcc5, debug, ASan)" + extends: builder + env: + USE_ASAN: 1 + USE_64: 1 + + build-64-opt-gcc5: + name: "Linux 64 (gcc5, opt)" + extends: builder + env: + BUILD_OPT: 1 + USE_64: 1 + + build-32-debug-asan-gcc6: + name: "Linux 32 (gcc6, debug, ASan)" + extends: builder + env: + USE_ASAN: 1 + CCC: g++-6 + CC: gcc-6 + + build-32-opt-gcc6: + name: "Linux 32 (gcc6, opt)" + extends: builder + env: + BUILD_OPT: 1 + CCC: g++-6 + CC: gcc-6 + + build-64-debug-asan-gcc6: + name: "Linux 64 (gcc6, debug, ASan)" + extends: builder + env: + USE_ASAN: 1 + USE_64: 1 + CCC: g++-6 + CC: gcc-6 + + build-64-opt-gcc6: + name: "Linux 64 (gcc6, opt)" + extends: builder + env: + BUILD_OPT: 1 + USE_64: 1 + CCC: g++-6 + CC: gcc-6 + + clang-format: + name: "NSS | clang-format-3.8" + description: "Validate source code formatting" + command: + - "/bin/bash" + - "-c" + - "bin/checkout.sh && nss/automation/taskcluster/scripts/run_clang_format.sh nss/lib/ssl" diff --git a/automation/taskcluster/scripts/build.sh b/automation/taskcluster/scripts/build.sh new file mode 100755 index 000000000..54032c1da --- /dev/null +++ b/automation/taskcluster/scripts/build.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -v -e -x + +if [ $(id -u) = 0 ]; then + # Drop privileges by re-running this script. + exec su worker $0 +fi + +# Clone NSPR if needed. +if [ ! -d "nspr" ]; then + hg clone https://hg.mozilla.org/projects/nspr +fi + +# Build. +cd nss && make nss_build_all + +# Package. +mkdir -p /home/worker/artifacts +tar cvfjh /home/worker/artifacts/dist.tar.bz2 ../dist diff --git a/automation/taskcluster/scripts/extend_task_graph.sh b/automation/taskcluster/scripts/extend_task_graph.sh new file mode 100755 index 000000000..8419716b4 --- /dev/null +++ b/automation/taskcluster/scripts/extend_task_graph.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -v -e -x + +if [ $(id -u) = 0 ]; then + # Drop privileges by re-running this script. + exec su worker $0 +fi + +mkdir -p /home/worker/artifacts + +# Build the task graph definition. +nodejs nss/automation/taskcluster/graph/build.js > /home/worker/artifacts/graph.json diff --git a/automation/taskcluster/scripts/run_clang_format.sh b/automation/taskcluster/scripts/run_clang_format.sh new file mode 100755 index 000000000..246270f88 --- /dev/null +++ b/automation/taskcluster/scripts/run_clang_format.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Apply clang-format 3.8 on the provided folder and verify that this doesn't change any file. +# If any file differs after formatting, the script eventually exits with 1. +# Any differences between formatted and unformatted files is printed to stdout to give a hint what's wrong. + +STATUS=0 +for i in $(find $1 -type f -name '*.[ch]' -print); do + if ! clang-format-3.8 $i | diff $i -; then + echo "Sorry, $i is not formatted properly. Please use clang-format 3.8 on your patch before landing." + STATUS=1 + fi +done +exit $STATUS diff --git a/automation/taskcluster/scripts/run_tests.sh b/automation/taskcluster/scripts/run_tests.sh new file mode 100755 index 000000000..fc0f32115 --- /dev/null +++ b/automation/taskcluster/scripts/run_tests.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -v -e -x + +if [ $(id -u) = 0 ]; then + # Stupid Docker. + echo "127.0.0.1 localhost.localdomain" >> /etc/hosts + + # Drop privileges by re-running this script. + exec su worker $0 +fi + +# Fetch artifact if needed. +if [ ! -d "dist" ]; then + curl --retry 3 -Lo dist.tar.bz2 https://queue.taskcluster.net/v1/task/$TC_PARENT_TASK_ID/artifacts/public/dist.tar.bz2 + tar xvjf dist.tar.bz2 +fi + +# Run tests. +cd nss/tests && ./all.sh |