summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTim Taubert <ttaubert@mozilla.com>2016-09-12 11:02:28 -0700
committerTim Taubert <ttaubert@mozilla.com>2016-09-12 11:02:28 -0700
commit42b6c9e5bc50508cc0c7d2b38a81b8734a6684e5 (patch)
tree680984400d04f2c72cede29896af51a2f8298ecf
parent10d4b3330eb5b4c014bc7d0520bdd750cccb93e1 (diff)
downloadnss-hg-42b6c9e5bc50508cc0c7d2b38a81b8734a6684e5.tar.gz
Bug 1301374 - Rewrite decision task to use imperative style r=me,npotb
-rw-r--r--automation/taskcluster/decision_task.yml10
-rw-r--r--automation/taskcluster/graph/arm/_build_base.yml40
-rw-r--r--automation/taskcluster/graph/arm/_test_base.yml27
-rw-r--r--automation/taskcluster/graph/arm/build32-debug.yml29
-rw-r--r--automation/taskcluster/graph/build.js147
-rw-r--r--automation/taskcluster/graph/image_builder.js148
-rw-r--r--automation/taskcluster/graph/image_builder.yml49
-rw-r--r--automation/taskcluster/graph/linux/_build_base.yml39
-rw-r--r--automation/taskcluster/graph/linux/_test_base.yml26
-rw-r--r--automation/taskcluster/graph/linux/build32-debug.yml116
-rw-r--r--automation/taskcluster/graph/linux/build32-opt.yml124
-rw-r--r--automation/taskcluster/graph/linux/build64-asan.yml38
-rw-r--r--automation/taskcluster/graph/linux/build64-debug.yml146
-rw-r--r--automation/taskcluster/graph/linux/build64-opt.yml129
-rw-r--r--automation/taskcluster/graph/package.json8
-rw-r--r--automation/taskcluster/graph/src/context_hash.js40
-rw-r--r--automation/taskcluster/graph/src/extend.js362
-rw-r--r--automation/taskcluster/graph/src/image_builder.js62
-rw-r--r--automation/taskcluster/graph/src/index.js14
-rw-r--r--automation/taskcluster/graph/src/merge.js10
-rw-r--r--automation/taskcluster/graph/src/queue.js235
-rw-r--r--automation/taskcluster/graph/src/try_syntax.js (renamed from automation/taskcluster/graph/try_syntax.js)87
-rw-r--r--automation/taskcluster/graph/tests/chains.yml14
-rw-r--r--automation/taskcluster/graph/tests/cipher.yml13
-rw-r--r--automation/taskcluster/graph/tests/crmf.yml14
-rw-r--r--automation/taskcluster/graph/tests/db.yml13
-rw-r--r--automation/taskcluster/graph/tests/ec.yml13
-rw-r--r--automation/taskcluster/graph/tests/fips.yml14
-rw-r--r--automation/taskcluster/graph/tests/gtests.yml13
-rw-r--r--automation/taskcluster/graph/tests/lowhash.yml13
-rw-r--r--automation/taskcluster/graph/tests/merge.yml13
-rw-r--r--automation/taskcluster/graph/tests/sdr.yml13
-rw-r--r--automation/taskcluster/graph/tests/smime.yml13
-rw-r--r--automation/taskcluster/graph/tests/ssl.yml65
-rw-r--r--automation/taskcluster/graph/tests/tools.yml13
-rw-r--r--automation/taskcluster/graph/tools/_build_base.yml29
-rw-r--r--automation/taskcluster/graph/tools/clang-format.yml15
-rw-r--r--automation/taskcluster/graph/tools/scan-build.yml26
-rw-r--r--automation/taskcluster/graph/windows/_build_base.yml37
-rw-r--r--automation/taskcluster/graph/windows/_test_base.yml24
-rw-r--r--automation/taskcluster/graph/windows/build64-debug.yml55
-rw-r--r--automation/taskcluster/graph/windows/build64-opt.yml57
-rw-r--r--automation/taskcluster/graph/yaml.js51
-rwxr-xr-xautomation/taskcluster/scripts/build.sh8
-rwxr-xr-xautomation/taskcluster/scripts/extend_task_graph.sh4
-rwxr-xr-xautomation/taskcluster/scripts/gen_certs.sh29
-rwxr-xr-xautomation/taskcluster/scripts/run_tests.sh9
-rw-r--r--automation/taskcluster/scripts/tools.sh12
-rw-r--r--automation/taskcluster/windows/build.sh8
-rw-r--r--automation/taskcluster/windows/gen_certs.sh19
50 files changed, 842 insertions, 1651 deletions
diff --git a/automation/taskcluster/decision_task.yml b/automation/taskcluster/decision_task.yml
index 48f9abef0..9d56c9bcd 100644
--- a/automation/taskcluster/decision_task.yml
+++ b/automation/taskcluster/decision_task.yml
@@ -77,14 +77,8 @@ tasks:
bin/checkout.sh &&
nss/automation/taskcluster/scripts/extend_task_graph.sh
- artifacts:
- public:
- type: "directory"
- path: "/home/worker/artifacts"
- expires: "{{#from_now}}7 days{{/from_now}}"
-
- graphs:
- - /home/worker/artifacts/graph.json
+ features:
+ taskclusterProxy: true
extra:
treeherder:
diff --git a/automation/taskcluster/graph/arm/_build_base.yml b/automation/taskcluster/graph/arm/_build_base.yml
deleted file mode 100644
index cef0fd48b..000000000
--- a/automation/taskcluster/graph/arm/_build_base.yml
+++ /dev/null
@@ -1,40 +0,0 @@
----
-reruns: 2
-
-task:
- created: !from_now 0
- deadline: !from_now 24
- provisionerId: localprovisioner
- workerType: nss-rpi
- schedulerId: task-graph-scheduler
-
- metadata:
- owner: !env TC_OWNER
- source: !env TC_SOURCE
-
- payload:
- maxRunTime: 7200
- image: ttaubert/nss-rpi-ci:0.0.3
-
- artifacts:
- public:
- type: directory
- path: /home/worker/artifacts
- expires: !from_now 24
-
- command:
- - "/bin/bash"
- - "-c"
- - "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh"
-
- env:
- NSS_HEAD_REPOSITORY: !env NSS_HEAD_REPOSITORY
- NSS_HEAD_REVISION: !env NSS_HEAD_REVISION
- GCC_VERSION: gcc-5
- GXX_VERSION: g++-5
-
- extra:
- treeherder:
- tier: 3 # hide jobs by default
- jobKind: build
- symbol: B
diff --git a/automation/taskcluster/graph/arm/_test_base.yml b/automation/taskcluster/graph/arm/_test_base.yml
deleted file mode 100644
index 5f7fd68d6..000000000
--- a/automation/taskcluster/graph/arm/_test_base.yml
+++ /dev/null
@@ -1,27 +0,0 @@
----
-reruns: 2
-
-task:
- created: !from_now 0
- deadline: !from_now 24
- provisionerId: localprovisioner
- workerType: nss-rpi
- schedulerId: task-graph-scheduler
-
- metadata:
- owner: !env TC_OWNER
- source: !env TC_SOURCE
-
- payload:
- maxRunTime: 7200
- image: ttaubert/nss-rpi-ci:0.0.3
-
- command:
- - "/bin/bash"
- - "-c"
- - "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh"
-
- extra:
- treeherder:
- tier: 3 # hide jobs by default
- jobKind: test
diff --git a/automation/taskcluster/graph/arm/build32-debug.yml b/automation/taskcluster/graph/arm/build32-debug.yml
deleted file mode 100644
index e9c61c091..000000000
--- a/automation/taskcluster/graph/arm/build32-debug.yml
+++ /dev/null
@@ -1,29 +0,0 @@
----
-- task:
- metadata:
- name: "Linux 32 (ARM, debug)"
- description: "Linux 32 (ARM, debug)"
-
- extra:
- treeherder:
- build:
- platform: linux32
- machine:
- platform: linux32
- collection:
- arm-debug: true
-
- tests:
- - chains
- - cipher
- - crmf
- - db
- - ec
- - fips
- - gtests
- - lowhash
- - merge
- - sdr
- - smime
- - ssl
- - tools
diff --git a/automation/taskcluster/graph/build.js b/automation/taskcluster/graph/build.js
deleted file mode 100644
index 21d17248f..000000000
--- a/automation/taskcluster/graph/build.js
+++ /dev/null
@@ -1,147 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-var fs = require("fs");
-var path = require("path");
-var merge = require("merge");
-var slugid = require("slugid");
-var flatmap = require("flatmap");
-
-var yaml = require("./yaml");
-var try_syntax = require("./try_syntax");
-var image_builder = require("./image_builder");
-
-// Default values for debugging.
-var TC_OWNER = process.env.TC_OWNER || "{{tc_owner}}";
-var TC_SOURCE = process.env.TC_SOURCE || "{{tc_source}}";
-var TC_PROJECT = process.env.TC_PROJECT || "{{tc_project}}";
-var TC_COMMENT = process.env.TC_COMMENT || "{{tc_comment}}";
-var NSS_PUSHLOG_ID = process.env.NSS_PUSHLOG_ID || "{{nss_pushlog_id}}";
-var NSS_HEAD_REVISION = process.env.NSS_HEAD_REVISION || "{{nss_head_rev}}";
-
-// Add base information to the given task.
-function decorateTask(task) {
- // Assign random task id.
- task.taskId = slugid.v4();
-
- // TreeHerder routes.
- task.task.routes = [
- "tc-treeherder-stage.v2." + TC_PROJECT + "." + NSS_HEAD_REVISION + "." + NSS_PUSHLOG_ID,
- "tc-treeherder.v2." + TC_PROJECT + "." + NSS_HEAD_REVISION + "." + NSS_PUSHLOG_ID
- ];
-}
-
-// Generate all tasks for a given build.
-function generateBuildTasks(platform, file) {
- var dir = path.join(__dirname, "./" + platform);
-
- // Parse base definitions.
- var buildBase = yaml.parse(path.join(dir, "_build_base.yml"), {});
- var testBase = yaml.parse(path.join(dir, "_test_base.yml"), {});
-
- return flatmap(yaml.parse(path.join(dir, file)), function (task) {
- // Merge base build task definition with the current one.
- var tasks = [task = merge.recursive(true, buildBase, task)];
-
- // Add base info.
- decorateTask(task);
-
- // Generate test tasks.
- if (task.tests) {
- // The base definition for all tests of this platform.
- var base = merge.recursive(true, {
- requires: [task.taskId],
-
- task: {
- payload: {
- env: {
- TC_PARENT_TASK_ID: task.taskId
- }
- }
- }
- }, testBase);
-
- // Generate and append test task definitions.
- tasks = tasks.concat(flatmap(task.tests, function (name) {
- return generateTestTasks(name, base, task);
- }));
-
- // |tests| is not part of the schema.
- delete task.tests;
- }
-
- return tasks;
- });
-}
-
-// Generate all tasks for a given test.
-function generateTestTasks(name, base, task) {
- // Load test definitions.
- var dir = path.join(__dirname, "./tests");
- var tests = yaml.parse(path.join(dir, name + ".yml"));
-
- return tests.map(function (test) {
- // Merge test with base definition.
- test = merge.recursive(true, base, test);
-
- // Add base info.
- decorateTask(test);
-
- // We only want to carry over environment variables...
- test.task.payload.env =
- merge.recursive(true, task.task.payload.env,
- test.task.payload.env);
-
- // ...and TreeHerder configuration data.
- test.task.extra.treeherder =
- merge.recursive(true, task.task.extra.treeherder,
- test.task.extra.treeherder);
-
- return test;
- });
-}
-
-// Generate all tasks for a given platform.
-function generatePlatformTasks(platform) {
- var dir = path.join(__dirname, "./" + platform);
- var buildBase = yaml.parse(path.join(dir, "_build_base.yml"), {});
- var testBase = yaml.parse(path.join(dir, "_test_base.yml"), {});
-
- // Parse all build tasks.
- return flatmap(fs.readdirSync(dir), function (file) {
- if (!file.startsWith("_") && file.endsWith(".yml")) {
- var tasks = generateBuildTasks(platform, file);
-
- // Convert env variables to strings.
- tasks.forEach(function (task) {
- var env = task.task.payload.env || {};
- Object.keys(env).forEach(function (name) {
- if (typeof(env[name]) != "undefined") {
- env[name] = env[name] + "";
- }
- });
- });
-
- return tasks;
- }
- });
-}
-
-// Construct the task graph.
-var graph = {
- tasks: flatmap(["linux", "windows", "arm", "tools"], generatePlatformTasks)
-};
-
-// Filter tasks when try syntax is given.
-if (TC_PROJECT == "nss-try") {
- graph.tasks = try_syntax.filterTasks(graph.tasks, TC_COMMENT);
-}
-
-// Inject the image builder tasks and dependencies.
-image_builder.asyncTweakTasks(graph.tasks).then(function (tasks) {
- graph.tasks = tasks;
-
- // Output the final graph.
- process.stdout.write(JSON.stringify(graph, null, 2));
-});
diff --git a/automation/taskcluster/graph/image_builder.js b/automation/taskcluster/graph/image_builder.js
deleted file mode 100644
index 4e32f3706..000000000
--- a/automation/taskcluster/graph/image_builder.js
+++ /dev/null
@@ -1,148 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-var fs = require("fs");
-var path = require("path");
-var crypto = require("crypto");
-var slugid = require("slugid");
-var flatmap = require("flatmap");
-var taskcluster = require("taskcluster-client");
-
-var yaml = require("./yaml");
-
-// Default values for debugging.
-var TC_PROJECT = process.env.TC_PROJECT || "{{tc_project}}";
-var NSS_PUSHLOG_ID = process.env.NSS_PUSHLOG_ID || "{{nss_pushlog_id}}";
-var NSS_HEAD_REVISION = process.env.NSS_HEAD_REVISION || "{{nss_head_rev}}";
-
-// Add base information to the given task.
-function decorateTask(task) {
- // Assign random task id.
- task.taskId = slugid.v4();
-
- // TreeHerder routes.
- task.task.routes = [
- "tc-treeherder-stage.v2." + TC_PROJECT + "." + NSS_HEAD_REVISION + "." + NSS_PUSHLOG_ID,
- "tc-treeherder.v2." + TC_PROJECT + "." + NSS_HEAD_REVISION + "." + NSS_PUSHLOG_ID
- ];
-}
-
-// Compute the SHA-256 digest.
-function sha256(data) {
- var hash = crypto.createHash("sha256");
- hash.update(data);
- return hash.digest("hex");
-}
-
-// Recursively collect a list of all files of a given directory.
-function collectFilesInDirectory(dir) {
- return flatmap(fs.readdirSync(dir), function (entry) {
- var entry_path = path.join(dir, entry);
-
- if (fs.lstatSync(entry_path).isDirectory()) {
- return collectFilesInDirectory(entry_path);
- }
-
- return [entry_path];
- });
-}
-
-// Compute a context hash for the given context path.
-function computeContextHash(context_path) {
- var root = path.join(__dirname, "../../..");
- var dir = path.join(root, context_path);
- var files = collectFilesInDirectory(dir).sort();
- var hashes = files.map(function (file) {
- return sha256(file + "|" + fs.readFileSync(file, "utf-8"));
- });
-
- return sha256(hashes.join(","));
-}
-
-// Generates the image-builder task description.
-function generateImageBuilderTask(context_path) {
- var task = yaml.parse(path.join(__dirname, "image_builder.yml"), {});
-
- // Add base info.
- decorateTask(task);
-
- // Add info for docker image building.
- task.task.payload.env.CONTEXT_PATH = context_path;
- task.task.payload.env.HASH = computeContextHash(context_path);
-
- return task;
-}
-
-// Returns a Promise<bool> that tells whether the task with the given id
-// has a public/image.tar artifact with a ready-to-use docker image.
-function asyncTaskHasImageArtifact(taskId) {
- var queue = new taskcluster.Queue();
-
- return queue.listLatestArtifacts(taskId).then(function (result) {
- return result.artifacts.some(function (artifact) {
- return artifact.name == "public/image.tar";
- });
- }, function () {
- return false;
- });
-}
-
-// Returns a Promise<task-id|null> with either a task id or null, depending
-// on whether we could find a task in the given namespace with a docker image.
-function asyncFindTaskWithImageArtifact(ns) {
- var index = new taskcluster.Index();
-
- return index.findTask(ns).then(function (result) {
- return asyncTaskHasImageArtifact(result.taskId).then(function (has_image) {
- return has_image ? result.taskId : null;
- });
- }, function () {
- return null;
- });
-}
-
-// Tweak the given list of tasks by injecting the image-builder task
-// and setting the right dependencies where needed.
-function asyncTweakTasks(tasks) {
- var id = "linux";
- var cx_path = "automation/taskcluster/docker";
- var hash = computeContextHash(cx_path);
- var ns = "docker.images.v1." + TC_PROJECT + "." + id + ".hash." + hash;
- var additional_tasks = [];
-
- // Check whether the docker image was already built.
- return asyncFindTaskWithImageArtifact(ns).then(function (taskId) {
- var builder_task;
-
- if (!taskId) {
- // No docker image found, add a task to build one.
- builder_task = generateImageBuilderTask(cx_path);
- taskId = builder_task.taskId;
-
- // Add a route so we can find the task later again.
- builder_task.task.routes.push("index." + ns);
- additional_tasks.push(builder_task);
- }
-
- tasks.forEach(function (task) {
- if (task.task.payload.image == cx_path) {
- task.task.payload.image = {
- path: "public/image.tar",
- type: "task-image",
- taskId: taskId
- };
-
- // Add a dependency only for top-level tasks (builds & tools) and only
- // if we added an image building task. Otherwise we don't need to wait.
- if (builder_task && !task.requires) {
- task.requires = [taskId];
- }
- }
- });
-
- return additional_tasks.concat(tasks);
- });
-}
-
-module.exports.asyncTweakTasks = asyncTweakTasks;
diff --git a/automation/taskcluster/graph/image_builder.yml b/automation/taskcluster/graph/image_builder.yml
deleted file mode 100644
index 417646b83..000000000
--- a/automation/taskcluster/graph/image_builder.yml
+++ /dev/null
@@ -1,49 +0,0 @@
----
-reruns: 2
-
-task:
- created: !from_now 0
- deadline: !from_now 24
- provisionerId: aws-provisioner-v1
- workerType: hg-worker
- schedulerId: task-graph-scheduler
-
- metadata:
- name: Image Builder
- description: Image Builder
- owner: !env TC_OWNER
- source: !env TC_SOURCE
-
- payload:
- maxRunTime: 3600
- image: taskcluster/image_builder:0.1.5
-
- artifacts:
- public/image.tar:
- type: file
- path: /artifacts/image.tar
- expires: !from_now 8760
-
- command:
- - "/bin/bash"
- - "-c"
- - "/home/worker/bin/build_image.sh"
-
- env:
- HEAD_REPOSITORY: !env NSS_HEAD_REPOSITORY
- BASE_REPOSITORY: !env NSS_HEAD_REPOSITORY
- HEAD_REV: !env NSS_HEAD_REVISION
- HEAD_REF: !env NSS_HEAD_REVISION
- PROJECT: !env TC_PROJECT
-
- features:
- dind: true
-
- extra:
- treeherder:
- build:
- platform: nss-decision
- machine:
- platform: nss-decision
- jobKind: build
- symbol: I
diff --git a/automation/taskcluster/graph/linux/_build_base.yml b/automation/taskcluster/graph/linux/_build_base.yml
deleted file mode 100644
index 3f23d4be2..000000000
--- a/automation/taskcluster/graph/linux/_build_base.yml
+++ /dev/null
@@ -1,39 +0,0 @@
----
-reruns: 2
-
-task:
- created: !from_now 0
- deadline: !from_now 24
- provisionerId: aws-provisioner-v1
- workerType: hg-worker
- schedulerId: task-graph-scheduler
-
- metadata:
- owner: !env TC_OWNER
- source: !env TC_SOURCE
-
- payload:
- maxRunTime: 3600
- image: automation/taskcluster/docker
-
- artifacts:
- public:
- type: directory
- path: /home/worker/artifacts
- expires: !from_now 24
-
- command:
- - "/bin/bash"
- - "-c"
- - "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh"
-
- env:
- NSS_HEAD_REPOSITORY: !env NSS_HEAD_REPOSITORY
- NSS_HEAD_REVISION: !env NSS_HEAD_REVISION
- GCC_VERSION: gcc-5
- GXX_VERSION: g++-5
-
- extra:
- treeherder:
- jobKind: build
- symbol: B
diff --git a/automation/taskcluster/graph/linux/_test_base.yml b/automation/taskcluster/graph/linux/_test_base.yml
deleted file mode 100644
index 7b4f9337b..000000000
--- a/automation/taskcluster/graph/linux/_test_base.yml
+++ /dev/null
@@ -1,26 +0,0 @@
----
-reruns: 2
-
-task:
- created: !from_now 0
- deadline: !from_now 24
- provisionerId: aws-provisioner-v1
- workerType: hg-worker
- schedulerId: task-graph-scheduler
-
- metadata:
- owner: !env TC_OWNER
- source: !env TC_SOURCE
-
- payload:
- maxRunTime: 3600
- image: automation/taskcluster/docker
-
- command:
- - "/bin/bash"
- - "-c"
- - "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh"
-
- extra:
- treeherder:
- jobKind: test
diff --git a/automation/taskcluster/graph/linux/build32-debug.yml b/automation/taskcluster/graph/linux/build32-debug.yml
deleted file mode 100644
index c75138fc4..000000000
--- a/automation/taskcluster/graph/linux/build32-debug.yml
+++ /dev/null
@@ -1,116 +0,0 @@
----
-- task:
- metadata:
- name: "Linux 32 (debug)"
- description: "Linux 32 (debug)"
-
- extra:
- treeherder:
- build:
- platform: linux32
- machine:
- platform: linux32
- collection:
- debug: true
-
- tests:
- - chains
- - cipher
- - crmf
- - db
- - ec
- - fips
- - gtests
- - lowhash
- - merge
- - sdr
- - smime
- - ssl
- - tools
-
-- task:
- metadata:
- name: "Linux 32 (debug, clang-3.8)"
- description: "Linux 32 (debug, clang-3.8)"
-
- payload:
- env:
- GCC_VERSION: clang
- GXX_VERSION: clang++
-
- extra:
- treeherder:
- build:
- platform: linux32
- machine:
- platform: linux32
- collection:
- debug: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: clang-3.8
-
-- task:
- metadata:
- name: "Linux 32 (debug, gcc-4.8)"
- description: "Linux 32 (debug, gcc-4.8)"
-
- payload:
- env:
- GCC_VERSION: gcc-4.8
- GXX_VERSION: g++-4.8
-
- extra:
- treeherder:
- build:
- platform: linux32
- machine:
- platform: linux32
- collection:
- debug: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: gcc-4.8
-
-- task:
- metadata:
- name: "Linux 32 (debug, gcc-6.1)"
- description: "Linux 32 (debug, gcc-6.1)"
-
- payload:
- env:
- GCC_VERSION: gcc-6
- GXX_VERSION: g++-6
-
- extra:
- treeherder:
- build:
- platform: linux32
- machine:
- platform: linux32
- collection:
- debug: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: gcc-6.1
-
-- task:
- metadata:
- name: "Linux 32 (debug, NSS_NO_PKCS11_BYPASS=1)"
- description: "Linux 32 (debug, NSS_NO_PKCS11_BYPASS=1)"
-
- payload:
- env:
- NSS_NO_PKCS11_BYPASS: 1
-
- extra:
- treeherder:
- build:
- platform: linux32
- machine:
- platform: linux32
- collection:
- debug: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: noPkcs11Bypass
diff --git a/automation/taskcluster/graph/linux/build32-opt.yml b/automation/taskcluster/graph/linux/build32-opt.yml
deleted file mode 100644
index c71b901b5..000000000
--- a/automation/taskcluster/graph/linux/build32-opt.yml
+++ /dev/null
@@ -1,124 +0,0 @@
----
-- task:
- metadata:
- name: "Linux 32 (opt)"
- description: "Linux 32 (opt)"
-
- payload:
- env:
- BUILD_OPT: 1
-
- extra:
- treeherder:
- build:
- platform: linux32
- machine:
- platform: linux32
- collection:
- opt: true
-
- tests:
- - chains
- - cipher
- - crmf
- - db
- - ec
- - fips
- - gtests
- - lowhash
- - merge
- - sdr
- - smime
- - ssl
- - tools
-
-- task:
- metadata:
- name: "Linux 32 (opt, clang-3.8)"
- description: "Linux 32 (opt, clang-3.8)"
-
- payload:
- env:
- GCC_VERSION: clang
- GXX_VERSION: clang++
- BUILD_OPT: 1
-
- extra:
- treeherder:
- build:
- platform: linux32
- machine:
- platform: linux32
- collection:
- opt: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: clang-3.8
-
-- task:
- metadata:
- name: "Linux 32 (opt, gcc-4.8)"
- description: "Linux 32 (opt, gcc-4.8)"
-
- payload:
- env:
- GCC_VERSION: gcc-4.8
- GXX_VERSION: g++-4.8
- BUILD_OPT: 1
-
- extra:
- treeherder:
- build:
- platform: linux32
- machine:
- platform: linux32
- collection:
- opt: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: gcc-4.8
-
-- task:
- metadata:
- name: "Linux 32 (opt, gcc-6.1)"
- description: "Linux 32 (opt, gcc-6.1)"
-
- payload:
- env:
- GCC_VERSION: gcc-6
- GXX_VERSION: g++-6
- BUILD_OPT: 1
-
- extra:
- treeherder:
- build:
- platform: linux32
- machine:
- platform: linux32
- collection:
- opt: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: gcc-6.1
-
-- task:
- metadata:
- name: "Linux 32 (opt, NSS_NO_PKCS11_BYPASS=1)"
- description: "Linux 32 (opt, NSS_NO_PKCS11_BYPASS=1)"
-
- payload:
- env:
- NSS_NO_PKCS11_BYPASS: 1
- BUILD_OPT: 1
-
- extra:
- treeherder:
- build:
- platform: linux32
- machine:
- platform: linux32
- collection:
- opt: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: noPkcs11Bypass
diff --git a/automation/taskcluster/graph/linux/build64-asan.yml b/automation/taskcluster/graph/linux/build64-asan.yml
deleted file mode 100644
index d7fca39e2..000000000
--- a/automation/taskcluster/graph/linux/build64-asan.yml
+++ /dev/null
@@ -1,38 +0,0 @@
----
-- task:
- metadata:
- name: "Linux 64 (ASan, debug)"
- description: "Linux 64 (ASan, debug)"
-
- payload:
- env:
- GCC_VERSION: clang
- GXX_VERSION: clang++
- NSS_DISABLE_ARENA_FREE_LIST: 1
- NSS_DISABLE_UNLOAD: 1
- USE_ASAN: 1
- USE_64: 1
-
- extra:
- treeherder:
- build:
- platform: linux64
- machine:
- platform: linux64
- collection:
- asan: true
-
- tests:
- - chains
- - cipher
- - crmf
- - db
- - ec
- - fips
- - gtests
- - lowhash
- - merge
- - sdr
- - smime
- - ssl
- - tools
diff --git a/automation/taskcluster/graph/linux/build64-debug.yml b/automation/taskcluster/graph/linux/build64-debug.yml
deleted file mode 100644
index 197727bb3..000000000
--- a/automation/taskcluster/graph/linux/build64-debug.yml
+++ /dev/null
@@ -1,146 +0,0 @@
----
-- task:
- metadata:
- name: "Linux 64 (debug)"
- description: "Linux 64 (debug)"
-
- payload:
- env:
- USE_64: 1
-
- extra:
- treeherder:
- build:
- platform: linux64
- machine:
- platform: linux64
- collection:
- debug: true
-
- tests:
- - chains
- - cipher
- - crmf
- - db
- - ec
- - fips
- - gtests
- - lowhash
- - merge
- - sdr
- - smime
- - ssl
- - tools
-
-- task:
- metadata:
- name: "Linux 64 (debug, clang-3.8)"
- description: "Linux 64 (debug, clang-3.8)"
-
- payload:
- env:
- GCC_VERSION: clang
- GXX_VERSION: clang++
- USE_64: 1
-
- extra:
- treeherder:
- build:
- platform: linux64
- machine:
- platform: linux64
- collection:
- debug: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: clang-3.8
-
-- task:
- metadata:
- name: "Linux 64 (debug, gcc-4.8)"
- description: "Linux 64 (debug, gcc-4.8)"
-
- payload:
- env:
- GCC_VERSION: gcc-4.8
- GXX_VERSION: g++-4.8
- USE_64: 1
-
- extra:
- treeherder:
- build:
- platform: linux64
- machine:
- platform: linux64
- collection:
- debug: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: gcc-4.8
-
-- task:
- metadata:
- name: "Linux 64 (debug, gcc-6.1)"
- description: "Linux 64 (debug, gcc-6.1)"
-
- payload:
- env:
- GCC_VERSION: gcc-6
- GXX_VERSION: g++-6
- USE_64: 1
-
- extra:
- treeherder:
- build:
- platform: linux64
- machine:
- platform: linux64
- collection:
- debug: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: gcc-6.1
-
-- task:
- metadata:
- name: "Linux 64 (debug, NSS_NO_PKCS11_BYPASS=1)"
- description: "Linux 64 (debug, NSS_NO_PKCS11_BYPASS=1)"
-
- payload:
- env:
- NSS_NO_PKCS11_BYPASS: 1
- USE_64: 1
-
- extra:
- treeherder:
- build:
- platform: linux64
- machine:
- platform: linux64
- collection:
- debug: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: noPkcs11Bypass
-
-- task:
- metadata:
- name: "Linux 64 (debug, NSS_DISABLE_LIBPKIX=1)"
- description: "Linux 64 (debug, NSS_DISABLE_LIBPKIX=1)"
-
- payload:
- env:
- NSS_DISABLE_LIBPKIX: 1
- USE_64: 1
-
- extra:
- treeherder:
- build:
- platform: linux64
- machine:
- platform: linux64
- collection:
- debug: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: noLibpkix
diff --git a/automation/taskcluster/graph/linux/build64-opt.yml b/automation/taskcluster/graph/linux/build64-opt.yml
deleted file mode 100644
index ff57c5312..000000000
--- a/automation/taskcluster/graph/linux/build64-opt.yml
+++ /dev/null
@@ -1,129 +0,0 @@
----
-- task:
- metadata:
- name: "Linux 64 (opt)"
- description: "Linux 64 (opt)"
-
- payload:
- env:
- BUILD_OPT: 1
- USE_64: 1
-
- extra:
- treeherder:
- build:
- platform: linux64
- machine:
- platform: linux64
- collection:
- opt: true
-
- tests:
- - chains
- - cipher
- - crmf
- - db
- - ec
- - fips
- - gtests
- - lowhash
- - merge
- - sdr
- - smime
- - ssl
- - tools
-
-- task:
- metadata:
- name: "Linux 64 (opt, clang-3.8)"
- description: "Linux 64 (opt, clang-3.8)"
-
- payload:
- env:
- GCC_VERSION: clang
- GXX_VERSION: clang++
- BUILD_OPT: 1
- USE_64: 1
-
- extra:
- treeherder:
- build:
- platform: linux64
- machine:
- platform: linux64
- collection:
- opt: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: clang-3.8
-
-- task:
- metadata:
- name: "Linux 64 (opt, gcc-4.8)"
- description: "Linux 64 (opt, gcc-4.8)"
-
- payload:
- env:
- GCC_VERSION: gcc-4.8
- GXX_VERSION: g++-4.8
- BUILD_OPT: 1
- USE_64: 1
-
- extra:
- treeherder:
- build:
- platform: linux64
- machine:
- platform: linux64
- collection:
- opt: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: gcc-4.8
-
-- task:
- metadata:
- name: "Linux 64 (opt, gcc-6.1)"
- description: "Linux 64 (opt, gcc-6.1)"
-
- payload:
- env:
- GCC_VERSION: gcc-6
- GXX_VERSION: g++-6
- BUILD_OPT: 1
- USE_64: 1
-
- extra:
- treeherder:
- build:
- platform: linux64
- machine:
- platform: linux64
- collection:
- opt: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: gcc-6.1
-
-- task:
- metadata:
- name: "Linux 64 (opt, NSS_NO_PKCS11_BYPASS=1)"
- description: "Linux 64 (opt, NSS_NO_PKCS11_BYPASS=1)"
-
- payload:
- env:
- NSS_NO_PKCS11_BYPASS: 1
- BUILD_OPT: 1
- USE_64: 1
-
- extra:
- treeherder:
- build:
- platform: linux64
- machine:
- platform: linux64
- collection:
- opt: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: noPkcs11Bypass
diff --git a/automation/taskcluster/graph/package.json b/automation/taskcluster/graph/package.json
index e58ab0b0d..d866adebc 100644
--- a/automation/taskcluster/graph/package.json
+++ b/automation/taskcluster/graph/package.json
@@ -4,7 +4,15 @@
"private": true,
"author": "Tim Taubert <ttaubert@mozilla.com>",
"description": "Decision Task for NSS",
+ "scripts": {
+ "compile": "babel-compile -p taskcluster src:lib",
+ "install": "npm run compile"
+ },
"dependencies": {
+ "babel-cli": "^6.14.0",
+ "babel-compile": "^2.0.0",
+ "babel-preset-taskcluster": "^3.0.0",
+ "babel-runtime": "^6.11.6",
"flatmap": "0.0.3",
"intersect": "^1.0.1",
"js-yaml": "^3.6.1",
diff --git a/automation/taskcluster/graph/src/context_hash.js b/automation/taskcluster/graph/src/context_hash.js
new file mode 100644
index 000000000..67bf25784
--- /dev/null
+++ b/automation/taskcluster/graph/src/context_hash.js
@@ -0,0 +1,40 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import fs from "fs";
+import path from "path";
+import crypto from "crypto";
+import flatmap from "flatmap";
+
+// Compute the SHA-256 digest.
+function sha256(data) {
+ let hash = crypto.createHash("sha256");
+ hash.update(data);
+ return hash.digest("hex");
+}
+
+// Recursively collect a list of all files of a given directory.
+function collectFilesInDirectory(dir) {
+ return flatmap(fs.readdirSync(dir), entry => {
+ let entry_path = path.join(dir, entry);
+
+ if (fs.lstatSync(entry_path).isDirectory()) {
+ return collectFilesInDirectory(entry_path);
+ }
+
+ return [entry_path];
+ });
+}
+
+// Compute a context hash for the given context path.
+export default function (context_path) {
+ let root = path.join(__dirname, "../../../..");
+ let dir = path.join(root, context_path);
+ let files = collectFilesInDirectory(dir).sort();
+ let hashes = files.map(file => {
+ return sha256(file + "|" + fs.readFileSync(file, "utf-8"));
+ });
+
+ return sha256(hashes.join(","));
+}
diff --git a/automation/taskcluster/graph/src/extend.js b/automation/taskcluster/graph/src/extend.js
new file mode 100644
index 000000000..633baa2ac
--- /dev/null
+++ b/automation/taskcluster/graph/src/extend.js
@@ -0,0 +1,362 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import merge from "./merge";
+import * as queue from "./queue";
+
+const LINUX_IMAGE = {name: "linux", path: "automation/taskcluster/docker"};
+
+const WINDOWS_CHECKOUT_CMD =
+ "bash -c \"hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || " +
+ "(sleep 2; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss) || " +
+ "(sleep 5; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss)\"";
+
+/*****************************************************************************/
+
+queue.filter(task => {
+ // Remove extra builds on ASan and ARM.
+ if (task.collection == "asan" || task.collection == "arm-debug") {
+ return task.group != "Builds";
+ }
+
+ return true;
+});
+
+queue.map(task => {
+ if (task.collection == "asan") {
+ // CRMF and FIPS tests still leak, unfortunately.
+ if (task.tests == "crmf" || task.tests == "fips") {
+ task.env.ASAN_OPTIONS = "detect_leaks=0";
+ }
+ }
+
+ if (task.collection == "arm-debug") {
+ // These tests take quite some time on our poor ARM devices.
+ if (task.tests == "chains" || (task.tests == "ssl" && task.cycle == "standard")) {
+ task.maxRunTime = 14400;
+ }
+ }
+
+ return task;
+});
+
+/*****************************************************************************/
+
+export default async function main() {
+ await scheduleLinux("Linux 32 (opt)", {
+ env: {BUILD_OPT: "1"},
+ platform: "linux32",
+ image: LINUX_IMAGE
+ });
+
+ await scheduleLinux("Linux 32 (debug)", {
+ platform: "linux32",
+ collection: "debug",
+ image: LINUX_IMAGE
+ });
+
+ await scheduleLinux("Linux 64 (opt)", {
+ env: {USE_64: "1", BUILD_OPT: "1"},
+ platform: "linux64",
+ image: LINUX_IMAGE
+ });
+
+ await scheduleLinux("Linux 64 (debug)", {
+ env: {USE_64: "1"},
+ platform: "linux64",
+ collection: "debug",
+ image: LINUX_IMAGE
+ });
+
+ await scheduleLinux("Linux 64 (ASan, debug)", {
+ env: {
+ NSS_DISABLE_ARENA_FREE_LIST: "1",
+ NSS_DISABLE_UNLOAD: "1",
+ GCC_VERSION: "clang",
+ GXX_VERSION: "clang++",
+ USE_ASAN: "1",
+ USE_64: "1"
+ },
+ platform: "linux64",
+ collection: "asan",
+ image: LINUX_IMAGE
+ });
+
+ await scheduleWindows("Windows 2012 64 (opt)", {
+ env: {BUILD_OPT: "1"}
+ });
+
+ await scheduleWindows("Windows 2012 64 (debug)", {
+ collection: "debug"
+ });
+
+ await scheduleTools();
+
+ await scheduleLinux("Linux 32 (ARM, debug)", {
+ image: "ttaubert/nss-rpi-ci:0.0.3",
+ provisioner: "localprovisioner",
+ collection: "arm-debug",
+ workerType: "nss-rpi",
+ platform: "linux32",
+ maxRunTime: 7200,
+ tier: 3
+ });
+}
+
+/*****************************************************************************/
+
+async function scheduleLinux(name, base) {
+ // Build base definition.
+ let build_base = merge(base, {
+ command: [
+ "/bin/bash",
+ "-c",
+ "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh"
+ ],
+ artifacts: {
+ public: {
+ expires: 24 * 7,
+ type: "directory",
+ path: "/home/worker/artifacts"
+ }
+ },
+ kind: "build",
+ symbol: "B"
+ });
+
+ // The task that builds NSPR+NSS.
+ let task_build = queue.scheduleTask(merge(build_base, {name}));
+
+ // The task that generates certificates.
+ let task_cert = queue.scheduleTask(merge(build_base, {
+ name: "Certificates",
+ command: [
+ "/bin/bash",
+ "-c",
+ "bin/checkout.sh && nss/automation/taskcluster/scripts/gen_certs.sh"
+ ],
+ parent: task_build,
+ symbol: "Certs"
+ }));
+
+ // Schedule tests.
+ scheduleTests(task_build, task_cert, merge(base, {
+ command: [
+ "/bin/bash",
+ "-c",
+ "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh"
+ ]
+ }));
+
+ // Extra builds.
+ let extra_base = merge({group: "Builds"}, build_base);
+ queue.scheduleTask(merge(extra_base, {
+ name: `${name} w/ clang-3.8`,
+ env: {
+ GCC_VERSION: "clang",
+ GXX_VERSION: "clang++"
+ },
+ symbol: "clang-3.8"
+ }));
+
+ queue.scheduleTask(merge(extra_base, {
+ name: `${name} w/ gcc-4.8`,
+ env: {
+ GCC_VERSION: "gcc-4.8",
+ GXX_VERSION: "g++-4.8"
+ },
+ symbol: "gcc-4.8"
+ }));
+
+ queue.scheduleTask(merge(extra_base, {
+ name: `${name} w/ gcc-6.1`,
+ env: {
+ GCC_VERSION: "gcc-6",
+ GXX_VERSION: "g++-6"
+ },
+ symbol: "gcc-6.1"
+ }));
+
+ queue.scheduleTask(merge(extra_base, {
+ name: `${name} w/ NSS_NO_PKCS11_BYPASS=1`,
+ env: {NSS_NO_PKCS11_BYPASS: "1"},
+ symbol: "noPkcs11Bypass"
+ }));
+
+ return queue.submit();
+}
+
+/*****************************************************************************/
+
+async function scheduleWindows(name, base) {
+ base = merge(base, {
+ workerType: "nss-win2012r2",
+ platform: "windows2012-64",
+ env: {
+ PATH: "c:\\mozilla-build\\python;c:\\mozilla-build\\msys\\local\\bin;" +
+ "c:\\mozilla-build\\7zip;c:\\mozilla-build\\info-zip;" +
+ "c:\\mozilla-build\\python\\Scripts;c:\\mozilla-build\\yasm;" +
+ "c:\\mozilla-build\\msys\\bin;c:\\Windows\\system32;" +
+ "c:\\mozilla-build\\upx391w;c:\\mozilla-build\\moztools-x64\\bin;" +
+ "c:\\mozilla-build\\wget",
+ DOMSUF: "localdomain",
+ HOST: "localhost",
+ USE_64: "1"
+ }
+ });
+
+ // Build base definition.
+ let build_base = merge(base, {
+ command: [
+ WINDOWS_CHECKOUT_CMD,
+ "bash -c nss/automation/taskcluster/windows/build.sh"
+ ],
+ artifacts: [{
+ expires: 24 * 7,
+ type: "directory",
+ path: "public\\build"
+ }],
+ kind: "build",
+ symbol: "B"
+ });
+
+ // The task that builds NSPR+NSS.
+ let task_build = queue.scheduleTask(merge(build_base, {name}));
+
+ // The task that generates certificates.
+ let task_cert = queue.scheduleTask(merge(build_base, {
+ name: "Certificates",
+ command: [
+ WINDOWS_CHECKOUT_CMD,
+ "bash -c nss/automation/taskcluster/windows/gen_certs.sh"
+ ],
+ parent: task_build,
+ symbol: "Certs"
+ }));
+
+ // Schedule tests.
+ scheduleTests(task_build, task_cert, merge(base, {
+ command: [
+ WINDOWS_CHECKOUT_CMD,
+ "bash -c nss/automation/taskcluster/windows/run_tests.sh"
+ ]
+ }));
+
+ // Extra builds.
+ let extra_base = merge({group: "Builds"}, build_base);
+ queue.scheduleTask(merge(extra_base, {
+ name: `${name} w/ NSS_NO_PKCS11_BYPASS=1`,
+ env: {NSS_NO_PKCS11_BYPASS: "1"},
+ symbol: "noPkcs11Bypass"
+ }));
+
+ return queue.submit();
+}
+
+/*****************************************************************************/
+
+function scheduleTests(task_build, task_cert, test_base) {
+ test_base = merge({kind: "test"}, test_base);
+
+ // Schedule tests that do NOT need certificates.
+ let no_cert_base = merge(test_base, {parent: task_build});
+ queue.scheduleTask(merge(no_cert_base, {
+ name: "Gtests", symbol: "Gtest", tests: "ssl_gtests gtests"
+ }));
+ queue.scheduleTask(merge(no_cert_base, {
+ name: "Chains tests", symbol: "Chains", tests: "chains"
+ }));
+ queue.scheduleTask(merge(no_cert_base, {
+ name: "Cipher tests", symbol: "Cipher", tests: "cipher"
+ }));
+ queue.scheduleTask(merge(no_cert_base, {
+ name: "EC tests", symbol: "EC", tests: "ec"
+ }));
+ queue.scheduleTask(merge(no_cert_base, {
+ name: "Lowhash tests", symbol: "Lowhash", tests: "lowhash"
+ }));
+ queue.scheduleTask(merge(no_cert_base, {
+ name: "SDR tests", symbol: "SDR", tests: "sdr"
+ }));
+
+ // Schedule tests that need certificates.
+ let cert_base = merge(test_base, {parent: task_cert});
+ queue.scheduleTask(merge(cert_base, {
+ name: "CRMF tests", symbol: "CRMF", tests: "crmf"
+ }));
+ queue.scheduleTask(merge(cert_base, {
+ name: "DB tests", symbol: "DB", tests: "dbtests"
+ }));
+ queue.scheduleTask(merge(cert_base, {
+ name: "FIPS tests", symbol: "FIPS", tests: "fips"
+ }));
+ queue.scheduleTask(merge(cert_base, {
+ name: "Merge tests", symbol: "Merge", tests: "merge"
+ }));
+ queue.scheduleTask(merge(cert_base, {
+ name: "S/MIME tests", symbol: "SMIME", tests: "smime"
+ }));
+ queue.scheduleTask(merge(cert_base, {
+ name: "Tools tests", symbol: "Tools", tests: "tools"
+ }));
+
+ // SSL tests, need certificates too.
+ let ssl_base = merge(cert_base, {tests: "ssl"});
+ queue.scheduleTask(merge(ssl_base, {
+ name: "SSL tests (standard)", symbol: "standard", cycle: "standard"
+ }));
+ queue.scheduleTask(merge(ssl_base, {
+ name: "SSL tests (pkix)", symbol: "pkix", cycle: "pkix"
+ }));
+ queue.scheduleTask(merge(ssl_base, {
+ name: "SSL tests (sharedb)", symbol: "sharedb", cycle: "sharedb"
+ }));
+ queue.scheduleTask(merge(ssl_base, {
+ name: "SSL tests (upgradedb)", symbol: "upgradedb", cycle: "upgradedb"
+ }));
+}
+
+/*****************************************************************************/
+
+async function scheduleTools() {
+ let base = {
+ image: LINUX_IMAGE,
+ platform: "nss-tools",
+ kind: "test"
+ };
+
+ queue.scheduleTask(merge(base, {
+ symbol: "clang-format-3.8",
+ name: "clang-format-3.8",
+ command: [
+ "/bin/bash",
+ "-c",
+ "bin/checkout.sh && nss/automation/taskcluster/scripts/run_clang_format.sh"
+ ]
+ }));
+
+ queue.scheduleTask(merge(base, {
+ symbol: "scan-build-3.8",
+ name: "scan-build-3.8",
+ env: {
+ USE_64: "1",
+ GCC_VERSION: "clang",
+ GXX_VERSION: "clang++"
+ },
+ artifacts: {
+ public: {
+ expires: 24,
+ type: "directory",
+ path: "/home/worker/artifacts"
+ }
+ },
+ command: [
+ "/bin/bash",
+ "-c",
+ "bin/checkout.sh && nss/automation/taskcluster/scripts/run_scan_build.sh"
+ ]
+ }));
+
+ return queue.submit();
+}
diff --git a/automation/taskcluster/graph/src/image_builder.js b/automation/taskcluster/graph/src/image_builder.js
new file mode 100644
index 000000000..093eacb44
--- /dev/null
+++ b/automation/taskcluster/graph/src/image_builder.js
@@ -0,0 +1,62 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import * as queue from "./queue";
+import context_hash from "./context_hash";
+import taskcluster from "taskcluster-client";
+
+async function taskHasImageArtifact(taskId) {
+ let queue = new taskcluster.Queue();
+ let {artifacts} = await queue.listLatestArtifacts(taskId);
+ return artifacts.some(artifact => artifact.name == "public/image.tar");
+}
+
+async function findTaskWithImageArtifact(ns) {
+ let index = new taskcluster.Index();
+ let {taskId} = await index.findTask(ns);
+ let has_image = await taskHasImageArtifact(taskId);
+ return has_image ? taskId : null;
+}
+
+export async function findTask({name, path}) {
+ let hash = await context_hash(path);
+ let ns = `docker.images.v1.${process.env.TC_PROJECT}.${name}.hash.${hash}`;
+ return findTaskWithImageArtifact(ns).catch(() => null);
+}
+
+export async function buildTask({name, path}) {
+ let hash = await context_hash(path);
+ let ns = `docker.images.v1.${process.env.TC_PROJECT}.${name}.hash.${hash}`;
+
+ return {
+ name: "Image Builder",
+ image: "taskcluster/image_builder:0.1.5",
+ routes: ["index." + ns],
+ env: {
+ HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY,
+ BASE_REPOSITORY: process.env.NSS_HEAD_REPOSITORY,
+ HEAD_REV: process.env.NSS_HEAD_REVISION,
+ HEAD_REF: process.env.NSS_HEAD_REVISION,
+ PROJECT: process.env.TC_PROJECT,
+ CONTEXT_PATH: path,
+ HASH: hash
+ },
+ artifacts: {
+ "public/image.tar": {
+ type: "file",
+ expires: 24 * 365,
+ path: "/artifacts/image.tar"
+ }
+ },
+ command: [
+ "/bin/bash",
+ "-c",
+ "/home/worker/bin/build_image.sh"
+ ],
+ platform: "nss-decision",
+ features: ["dind"],
+ kind: "build",
+ symbol: "I"
+ };
+}
diff --git a/automation/taskcluster/graph/src/index.js b/automation/taskcluster/graph/src/index.js
new file mode 100644
index 000000000..4153e1b18
--- /dev/null
+++ b/automation/taskcluster/graph/src/index.js
@@ -0,0 +1,14 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import * as try_syntax from "./try_syntax";
+import extend from "./extend";
+
+// Init try syntax filter.
+if (process.env.TC_PROJECT == "nss-try") {
+ try_syntax.initFilter();
+}
+
+// Extend the task graph.
+extend().catch(console.error);
diff --git a/automation/taskcluster/graph/src/merge.js b/automation/taskcluster/graph/src/merge.js
new file mode 100644
index 000000000..17043dd8e
--- /dev/null
+++ b/automation/taskcluster/graph/src/merge.js
@@ -0,0 +1,10 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import {recursive as merge} from "merge";
+
+// We always want to clone.
+export default function (...args) {
+ return merge(true, ...args);
+}
diff --git a/automation/taskcluster/graph/src/queue.js b/automation/taskcluster/graph/src/queue.js
new file mode 100644
index 000000000..dbeb2622c
--- /dev/null
+++ b/automation/taskcluster/graph/src/queue.js
@@ -0,0 +1,235 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import {clone} from "merge";
+import merge from "./merge";
+import slugid from "slugid";
+import taskcluster from "taskcluster-client";
+import * as image_builder from "./image_builder";
+
+let maps = [];
+let filters = [];
+
+let tasks = new Map();
+let image_tasks = new Map();
+
+let queue = new taskcluster.Queue({
+ baseUrl: "http://taskcluster/queue/v1/"
+});
+
+function fromNow(hours) {
+ let d = new Date();
+ d.setHours(d.getHours() + (hours|0));
+ return d.toJSON();
+}
+
+function parseRoutes(routes) {
+ return [
+ `tc-treeherder.v2.${process.env.TC_PROJECT}.${process.env.NSS_HEAD_REVISION}.${process.env.NSS_PUSHLOG_ID}`,
+ ...routes
+ ];
+}
+
+function parseFeatures(list) {
+ return list.reduce((map, feature) => {
+ map[feature] = true;
+ return map;
+ }, {});
+}
+
+function parseArtifacts(artifacts) {
+ let copy = clone(artifacts);
+ Object.keys(copy).forEach(key => {
+ copy[key].expires = fromNow(copy[key].expires);
+ });
+ return copy;
+}
+
+function parseCollection(name) {
+ let collection = {};
+ collection[name] = true;
+ return collection;
+}
+
+function parseTreeherder(def) {
+ let treeherder = {
+ build: {
+ platform: def.platform
+ },
+ machine: {
+ platform: def.platform
+ },
+ symbol: def.symbol,
+ jobKind: def.kind
+ };
+
+ if (def.group) {
+ treeherder.groupSymbol = def.group;
+ }
+
+ if (def.collection) {
+ treeherder.collection = parseCollection(def.collection);
+ }
+
+ if (def.tier) {
+ treeherder.tier = def.tier;
+ }
+
+ return treeherder;
+}
+
+function convertTask(def) {
+ let dependencies = def.dependencies || [];
+
+ let env = merge({
+ NSS_HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY,
+ NSS_HEAD_REVISION: process.env.NSS_HEAD_REVISION
+ }, def.env || {});
+
+ if (def.parent) {
+ dependencies.push(def.parent);
+ env.TC_PARENT_TASK_ID = def.parent;
+ }
+
+ if (def.tests) {
+ env.NSS_TESTS = def.tests;
+ }
+
+ if (def.cycle) {
+ env.NSS_CYCLES = def.cycle;
+ }
+
+ let payload = {
+ env,
+ command: def.command,
+ maxRunTime: def.maxRunTime || 3600
+ };
+
+ if (def.image) {
+ payload.image = def.image;
+ }
+
+ if (def.features) {
+ payload.features = parseFeatures(def.features);
+ }
+
+ if (def.artifacts) {
+ payload.artifacts = parseArtifacts(def.artifacts);
+ }
+
+ return {
+ provisionerId: def.provisioner || "aws-provisioner-v1",
+ workerType: def.workerType || "hg-worker",
+ schedulerId: "task-graph-scheduler",
+
+ created: fromNow(0),
+ deadline: fromNow(24),
+
+ dependencies,
+ routes: parseRoutes(def.routes || []),
+
+ metadata: {
+ name: def.name,
+ description: def.name,
+ owner: process.env.TC_OWNER,
+ source: process.env.TC_SOURCE
+ },
+
+ payload,
+
+ extra: {
+ treeherder: parseTreeherder(def)
+ }
+ };
+}
+
+export function map(fun) {
+ maps.push(fun);
+}
+
+export function filter(fun) {
+ filters.push(fun);
+}
+
+export function scheduleTask(def) {
+ let taskId = slugid.v4();
+ tasks.set(taskId, merge({}, def));
+ return taskId;
+}
+
+export async function submit() {
+ let promises = [];
+
+ for (let [taskId, task] of tasks) {
+ // Allow filtering tasks before we schedule them.
+ if (!filters.every(filter => filter(task))) {
+ continue;
+ }
+
+ // Allow changing tasks before we schedule them.
+ maps.forEach(map => { task = map(merge({}, task)) });
+
+ let log_id = `${task.name} @ ${task.platform}[${task.collection || "opt"}]`;
+ console.log(`+ Submitting ${log_id}.`);
+
+ // Convert the task definition.
+ task = await convertTask(task);
+
+ // Convert the docker image definition.
+ let image_def = task.payload.image;
+ if (image_def && image_def.hasOwnProperty("path")) {
+ let key = `${image_def.name}:${image_def.path}`;
+ let data = {};
+
+ // Check the cache first.
+ if (image_tasks.has(key)) {
+ data = image_tasks.get(key);
+ } else {
+ data.taskId = await image_builder.findTask(image_def);
+ data.isPending = !data.taskId;
+
+ // No task found.
+ if (data.isPending) {
+ let image_task = await image_builder.buildTask(image_def);
+
+ // Schedule a new image builder task immediately.
+ data.taskId = slugid.v4();
+
+ try {
+ await queue.createTask(data.taskId, convertTask(image_task));
+ } catch (e) {
+ console.error("! FAIL: Scheduling image builder task failed.");
+ continue; /* Skip this task on failure. */
+ }
+ }
+
+ // Store in cache.
+ image_tasks.set(key, data);
+ }
+
+ if (data.isPending) {
+ task.dependencies.push(data.taskId);
+ }
+
+ task.payload.image = {
+ path: "public/image.tar",
+ taskId: data.taskId,
+ type: "task-image"
+ };
+ }
+
+ // Schedule the task.
+ promises.push(queue.createTask(taskId, task).catch(err => {
+ console.error(`! FAIL: Scheduling ${log_id} failed.`, err);
+ }));
+ }
+
+ // Wait for all requests to finish.
+ if (promises.length) {
+ await Promise.all(promises);
+ console.log("=== Total:", promises.length, "tasks. ===");
+ }
+
+ tasks.clear();
+}
diff --git a/automation/taskcluster/graph/try_syntax.js b/automation/taskcluster/graph/src/try_syntax.js
index b2fbfbefa..032d4515c 100644
--- a/automation/taskcluster/graph/try_syntax.js
+++ b/automation/taskcluster/graph/src/try_syntax.js
@@ -2,8 +2,9 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-var intersect = require("intersect");
-var parse_args = require("minimist");
+import * as queue from "./queue";
+import intersect from "intersect";
+import parse_args from "minimist";
function parseOptions(opts) {
opts = parse_args(opts.split(/\s+/), {
@@ -13,7 +14,7 @@ function parseOptions(opts) {
});
// Parse build types (d=debug, o=opt).
- var builds = intersect(opts.build.split(""), ["d", "o"]);
+ let builds = intersect(opts.build.split(""), ["d", "o"]);
// If the given value is nonsense default to debug and opt builds.
if (builds.length == 0) {
@@ -21,8 +22,8 @@ function parseOptions(opts) {
}
// Parse platforms.
- var allPlatforms = ["linux", "linux64", "linux64-asan", "win64", "arm"];
- var platforms = intersect(opts.platform.split(/\s*,\s*/), allPlatforms);
+ let allPlatforms = ["linux", "linux64", "linux64-asan", "win64", "arm"];
+ let platforms = intersect(opts.platform.split(/\s*,\s*/), allPlatforms);
// If the given value is nonsense or "none" default to all platforms.
if (platforms.length == 0 && opts.platform != "none") {
@@ -30,9 +31,9 @@ function parseOptions(opts) {
}
// Parse unit tests.
- var allUnitTests = ["crmf", "chains", "cipher", "db", "ec", "fips", "gtest",
+ let allUnitTests = ["crmf", "chains", "cipher", "db", "ec", "fips", "gtest",
"lowhash", "merge", "sdr", "smime", "tools", "ssl"];
- var unittests = intersect(opts.unittests.split(/\s*,\s*/), allUnitTests);
+ let unittests = intersect(opts.unittests.split(/\s*,\s*/), allUnitTests);
// If the given value is "all" run all tests.
// If it's nonsense then don't run any tests.
@@ -43,8 +44,8 @@ function parseOptions(opts) {
}
// Parse tools.
- var allTools = ["clang-format", "scan-build"];
- var tools = intersect(opts.tools.split(/\s*,\s*/), allTools);
+ let allTools = ["clang-format", "scan-build"];
+ let tools = intersect(opts.tools.split(/\s*,\s*/), allTools);
// If the given value is "all" run all tools.
// If it's nonsense then don't run any tools.
@@ -63,36 +64,20 @@ function parseOptions(opts) {
};
}
-function filterTasks(tasks, comment) {
- // Check for try syntax in changeset comment.
- var match = comment.match(/^\s*try:\s*(.*)\s*$/);
- if (!match) {
- return tasks;
- }
-
- var opts = parseOptions(match[1]);
-
- return tasks.filter(function (task) {
- var env = task.task.payload.env || {};
- var th = task.task.extra.treeherder;
- var machine = th.machine.platform;
- var coll = th.collection || {};
- var found;
-
+function filter(opts) {
+ return function (task) {
// Filter tools. We can immediately return here as those
// are not affected by platform or build type selectors.
- if (machine == "nss-tools") {
- return opts.tools.some(function (tool) {
- var symbol = th.symbol.toLowerCase();
- return symbol.startsWith(tool);
+ if (task.platform == "nss-tools") {
+ return opts.tools.some(tool => {
+ return task.symbol.toLowerCase().startsWith(tool);
});
}
// Filter unit tests.
- if (env.NSS_TESTS && env.TC_PARENT_TASK_ID) {
- found = opts.unittests.some(function (test) {
- var symbol = (th.groupSymbol || th.symbol).toLowerCase();
- return symbol.startsWith(test);
+ if (task.tests) {
+ let found = opts.unittests.some(test => {
+ return (task.group || task.symbol).toLowerCase().startsWith(test);
});
if (!found) {
@@ -101,13 +86,15 @@ function filterTasks(tasks, comment) {
}
// Filter extra builds.
- if (th.groupSymbol == "Builds" && !opts.extra) {
+ if (task.group == "Builds" && !opts.extra) {
return false;
}
+ let coll = name => name == (task.collection || "opt");
+
// Filter by platform.
- found = opts.platforms.some(function (platform) {
- var aliases = {
+ let found = opts.platforms.some(platform => {
+ let aliases = {
"linux": "linux32",
"linux64-asan": "linux64",
"win64": "windows2012-64",
@@ -115,15 +102,15 @@ function filterTasks(tasks, comment) {
};
// Check the platform name.
- var keep = machine == (aliases[platform] || platform);
+ let keep = (task.platform == (aliases[platform] || platform));
// Additional checks.
if (platform == "linux64-asan") {
- keep &= coll.asan;
+ keep &= coll("asan");
} else if (platform == "arm") {
- keep &= (coll["arm-opt"] || coll["arm-debug"]);
+ keep &= coll("arm-opt") || coll("arm-debug");
} else {
- keep &= (coll.opt || coll.debug);
+ keep &= coll("opt") || coll("debug");
}
return keep;
@@ -134,10 +121,20 @@ function filterTasks(tasks, comment) {
}
// Finally, filter by build type.
- var isDebug = coll.debug || coll.asan || coll["arm-debug"];
- return (isDebug && opts.builds.indexOf("d") > -1) ||
- (!isDebug && opts.builds.indexOf("o") > -1);
- });
+ let isDebug = coll("debug") || coll("asan") || coll("arm-debug");
+ return (isDebug && opts.builds.includes("d")) ||
+ (!isDebug && opts.builds.includes("o"));
+ }
}
-module.exports.filterTasks = filterTasks;
+export function initFilter() {
+ let comment = process.env.TC_COMMENT || "";
+
+ // Check for try syntax in changeset comment.
+ let match = comment.match(/^\s*try:\s*(.*)\s*$/);
+
+ // Add try syntax filter.
+ if (match) {
+ queue.filter(filter(parseOptions(match[1])));
+ }
+}
diff --git a/automation/taskcluster/graph/tests/chains.yml b/automation/taskcluster/graph/tests/chains.yml
deleted file mode 100644
index b553d8d39..000000000
--- a/automation/taskcluster/graph/tests/chains.yml
+++ /dev/null
@@ -1,14 +0,0 @@
----
-- task:
- metadata:
- name: Chains tests
- description: Chains tests
-
- payload:
- maxRunTime: 14400
- env:
- NSS_TESTS: chains
-
- extra:
- treeherder:
- symbol: Chains
diff --git a/automation/taskcluster/graph/tests/cipher.yml b/automation/taskcluster/graph/tests/cipher.yml
deleted file mode 100644
index e1d0f9faa..000000000
--- a/automation/taskcluster/graph/tests/cipher.yml
+++ /dev/null
@@ -1,13 +0,0 @@
----
-- task:
- metadata:
- name: Cipher tests
- description: Cipher tests
-
- payload:
- env:
- NSS_TESTS: cipher
-
- extra:
- treeherder:
- symbol: Cipher
diff --git a/automation/taskcluster/graph/tests/crmf.yml b/automation/taskcluster/graph/tests/crmf.yml
deleted file mode 100644
index a29a9c5e2..000000000
--- a/automation/taskcluster/graph/tests/crmf.yml
+++ /dev/null
@@ -1,14 +0,0 @@
----
-- task:
- metadata:
- name: CRMF tests
- description: CRMF tests
-
- payload:
- env:
- ASAN_OPTIONS: detect_leaks=0
- NSS_TESTS: crmf
-
- extra:
- treeherder:
- symbol: CRMF
diff --git a/automation/taskcluster/graph/tests/db.yml b/automation/taskcluster/graph/tests/db.yml
deleted file mode 100644
index b5ee77653..000000000
--- a/automation/taskcluster/graph/tests/db.yml
+++ /dev/null
@@ -1,13 +0,0 @@
----
-- task:
- metadata:
- name: DB tests
- description: DB tests
-
- payload:
- env:
- NSS_TESTS: dbtests
-
- extra:
- treeherder:
- symbol: DB
diff --git a/automation/taskcluster/graph/tests/ec.yml b/automation/taskcluster/graph/tests/ec.yml
deleted file mode 100644
index c6e21917e..000000000
--- a/automation/taskcluster/graph/tests/ec.yml
+++ /dev/null
@@ -1,13 +0,0 @@
----
-- task:
- metadata:
- name: EC tests
- description: EC tests
-
- payload:
- env:
- NSS_TESTS: ec
-
- extra:
- treeherder:
- symbol: EC
diff --git a/automation/taskcluster/graph/tests/fips.yml b/automation/taskcluster/graph/tests/fips.yml
deleted file mode 100644
index 58484c7b2..000000000
--- a/automation/taskcluster/graph/tests/fips.yml
+++ /dev/null
@@ -1,14 +0,0 @@
----
-- task:
- metadata:
- name: FIPS tests
- description: FIPS tests
-
- payload:
- env:
- ASAN_OPTIONS: detect_leaks=0
- NSS_TESTS: fips
-
- extra:
- treeherder:
- symbol: FIPS
diff --git a/automation/taskcluster/graph/tests/gtests.yml b/automation/taskcluster/graph/tests/gtests.yml
deleted file mode 100644
index d4a355a24..000000000
--- a/automation/taskcluster/graph/tests/gtests.yml
+++ /dev/null
@@ -1,13 +0,0 @@
----
-- task:
- metadata:
- name: GTests
- description: GTests
-
- payload:
- env:
- NSS_TESTS: ssl_gtests gtests
-
- extra:
- treeherder:
- symbol: GTest
diff --git a/automation/taskcluster/graph/tests/lowhash.yml b/automation/taskcluster/graph/tests/lowhash.yml
deleted file mode 100644
index 097a92377..000000000
--- a/automation/taskcluster/graph/tests/lowhash.yml
+++ /dev/null
@@ -1,13 +0,0 @@
----
-- task:
- metadata:
- name: Lowhash tests
- description: Lowhash tests
-
- payload:
- env:
- NSS_TESTS: lowhash
-
- extra:
- treeherder:
- symbol: Lowhash
diff --git a/automation/taskcluster/graph/tests/merge.yml b/automation/taskcluster/graph/tests/merge.yml
deleted file mode 100644
index 7db008a1d..000000000
--- a/automation/taskcluster/graph/tests/merge.yml
+++ /dev/null
@@ -1,13 +0,0 @@
----
-- task:
- metadata:
- name: Merge tests
- description: Merge tests
-
- payload:
- env:
- NSS_TESTS: merge
-
- extra:
- treeherder:
- symbol: Merge
diff --git a/automation/taskcluster/graph/tests/sdr.yml b/automation/taskcluster/graph/tests/sdr.yml
deleted file mode 100644
index 12d881eda..000000000
--- a/automation/taskcluster/graph/tests/sdr.yml
+++ /dev/null
@@ -1,13 +0,0 @@
----
-- task:
- metadata:
- name: SDR tests
- description: SDR tests
-
- payload:
- env:
- NSS_TESTS: sdr
-
- extra:
- treeherder:
- symbol: SDR
diff --git a/automation/taskcluster/graph/tests/smime.yml b/automation/taskcluster/graph/tests/smime.yml
deleted file mode 100644
index f5c3ab8fb..000000000
--- a/automation/taskcluster/graph/tests/smime.yml
+++ /dev/null
@@ -1,13 +0,0 @@
----
-- task:
- metadata:
- name: S/MIME tests
- description: S/MIME tests
-
- payload:
- env:
- NSS_TESTS: smime
-
- extra:
- treeherder:
- symbol: SMIME
diff --git a/automation/taskcluster/graph/tests/ssl.yml b/automation/taskcluster/graph/tests/ssl.yml
deleted file mode 100644
index da66c4384..000000000
--- a/automation/taskcluster/graph/tests/ssl.yml
+++ /dev/null
@@ -1,65 +0,0 @@
----
-- task:
- metadata:
- name: "SSL tests (standard)"
- description: "SSL tests (standard)"
-
- payload:
- maxRunTime: 14400
- env:
- NSS_CYCLES: standard
- NSS_TESTS: ssl
-
- extra:
- treeherder:
- symbol: standard
- groupSymbol: SSL
- groupName: SSL tests
-
-- task:
- metadata:
- name: "SSL tests (pkix)"
- description: "SSL tests (pkix)"
-
- payload:
- env:
- NSS_CYCLES: pkix
- NSS_TESTS: ssl
-
- extra:
- treeherder:
- symbol: pkix
- groupSymbol: SSL
- groupName: SSL tests
-
-- task:
- metadata:
- name: "SSL tests (sharedb)"
- description: "SSL tests (sharedb)"
-
- payload:
- env:
- NSS_CYCLES: sharedb
- NSS_TESTS: ssl
-
- extra:
- treeherder:
- symbol: sharedb
- groupSymbol: SSL
- groupName: SSL tests
-
-- task:
- metadata:
- name: "SSL tests (upgradedb)"
- description: "SSL tests (upgradedb)"
-
- payload:
- env:
- NSS_CYCLES: upgradedb
- NSS_TESTS: ssl
-
- extra:
- treeherder:
- symbol: upgradedb
- groupSymbol: SSL
- groupName: SSL tests
diff --git a/automation/taskcluster/graph/tests/tools.yml b/automation/taskcluster/graph/tests/tools.yml
deleted file mode 100644
index 2edcce01a..000000000
--- a/automation/taskcluster/graph/tests/tools.yml
+++ /dev/null
@@ -1,13 +0,0 @@
----
-- task:
- metadata:
- name: Tools tests
- description: Tools tests
-
- payload:
- env:
- NSS_TESTS: tools
-
- extra:
- treeherder:
- symbol: Tools
diff --git a/automation/taskcluster/graph/tools/_build_base.yml b/automation/taskcluster/graph/tools/_build_base.yml
deleted file mode 100644
index cd7244b97..000000000
--- a/automation/taskcluster/graph/tools/_build_base.yml
+++ /dev/null
@@ -1,29 +0,0 @@
----
-reruns: 0
-
-task:
- created: !from_now 0
- deadline: !from_now 24
- provisionerId: aws-provisioner-v1
- workerType: hg-worker
- schedulerId: task-graph-scheduler
-
- metadata:
- owner: !env TC_OWNER
- source: !env TC_SOURCE
-
- payload:
- maxRunTime: 3600
- image: automation/taskcluster/docker
-
- env:
- NSS_HEAD_REPOSITORY: !env NSS_HEAD_REPOSITORY
- NSS_HEAD_REVISION: !env NSS_HEAD_REVISION
-
- extra:
- treeherder:
- build:
- platform: nss-tools
- machine:
- platform: nss-tools
- jobKind: test
diff --git a/automation/taskcluster/graph/tools/clang-format.yml b/automation/taskcluster/graph/tools/clang-format.yml
deleted file mode 100644
index 16c18dd2c..000000000
--- a/automation/taskcluster/graph/tools/clang-format.yml
+++ /dev/null
@@ -1,15 +0,0 @@
----
-- task:
- metadata:
- name: clang-format-3.8
- description: clang-format-3.8
-
- payload:
- command:
- - "/bin/bash"
- - "-c"
- - "bin/checkout.sh && nss/automation/taskcluster/scripts/run_clang_format.sh"
-
- extra:
- treeherder:
- symbol: clang-format-3.8
diff --git a/automation/taskcluster/graph/tools/scan-build.yml b/automation/taskcluster/graph/tools/scan-build.yml
deleted file mode 100644
index 8733d845d..000000000
--- a/automation/taskcluster/graph/tools/scan-build.yml
+++ /dev/null
@@ -1,26 +0,0 @@
----
-- task:
- metadata:
- name: scan-build-3.8
- description: scan-build-3.8
-
- payload:
- artifacts:
- public:
- type: directory
- path: /home/worker/artifacts
- expires: !from_now 24
-
- command:
- - "/bin/bash"
- - "-c"
- - "bin/checkout.sh && nss/automation/taskcluster/scripts/run_scan_build.sh"
-
- env:
- GCC_VERSION: clang
- GXX_VERSION: clang++
- USE_64: 1
-
- extra:
- treeherder:
- symbol: scan-build-3.8
diff --git a/automation/taskcluster/graph/windows/_build_base.yml b/automation/taskcluster/graph/windows/_build_base.yml
deleted file mode 100644
index a75a7a1af..000000000
--- a/automation/taskcluster/graph/windows/_build_base.yml
+++ /dev/null
@@ -1,37 +0,0 @@
----
-reruns: 2
-
-task:
- created: !from_now 0
- deadline: !from_now 24
- provisionerId: aws-provisioner-v1
- workerType: nss-win2012r2
- schedulerId: task-graph-scheduler
-
- metadata:
- owner: !env TC_OWNER
- source: !env TC_SOURCE
-
- payload:
- maxRunTime: 3600
-
- artifacts:
- - type: directory
- path: "public\\build"
- expires: !from_now 24
-
- command:
- - "bash -c \"hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || (sleep 2; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss) || (sleep 5; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss)\""
- - "bash -c nss/automation/taskcluster/windows/build.sh"
-
- env:
- PATH: "c:\\mozilla-build\\python;c:\\mozilla-build\\msys\\local\\bin;c:\\mozilla-build\\7zip;c:\\mozilla-build\\info-zip;c:\\mozilla-build\\python\\Scripts;c:\\mozilla-build\\yasm;c:\\mozilla-build\\msys\\bin;c:\\Windows\\system32;c:\\mozilla-build\\upx391w;c:\\mozilla-build\\moztools-x64\\bin;c:\\mozilla-build\\wget"
- NSS_HEAD_REPOSITORY: !env NSS_HEAD_REPOSITORY
- NSS_HEAD_REVISION: !env NSS_HEAD_REVISION
- DOMSUF: localdomain
- HOST: localhost
-
- extra:
- treeherder:
- jobKind: build
- symbol: B
diff --git a/automation/taskcluster/graph/windows/_test_base.yml b/automation/taskcluster/graph/windows/_test_base.yml
deleted file mode 100644
index 0ffaabc75..000000000
--- a/automation/taskcluster/graph/windows/_test_base.yml
+++ /dev/null
@@ -1,24 +0,0 @@
----
-reruns: 2
-
-task:
- created: !from_now 0
- deadline: !from_now 24
- provisionerId: aws-provisioner-v1
- workerType: nss-win2012r2
- schedulerId: task-graph-scheduler
-
- metadata:
- owner: !env TC_OWNER
- source: !env TC_SOURCE
-
- payload:
- maxRunTime: 3600
-
- command:
- - "hg clone -r %NSS_HEAD_REVISION% %NSS_HEAD_REPOSITORY% nss"
- - "bash -c nss/automation/taskcluster/windows/run_tests.sh"
-
- extra:
- treeherder:
- jobKind: test
diff --git a/automation/taskcluster/graph/windows/build64-debug.yml b/automation/taskcluster/graph/windows/build64-debug.yml
deleted file mode 100644
index ccc47f2d7..000000000
--- a/automation/taskcluster/graph/windows/build64-debug.yml
+++ /dev/null
@@ -1,55 +0,0 @@
----
-- task:
- metadata:
- name: "Windows 2012 64 (debug)"
- description: "Windows 2012 64 (debug)"
-
- payload:
- env:
- USE_64: 1
-
- extra:
- treeherder:
- build:
- platform: windows2012-64
- machine:
- platform: windows2012-64
- collection:
- debug: true
-
- tests:
- - chains
- - cipher
- - crmf
- - db
- - ec
- - fips
- - gtests
- - lowhash
- - merge
- - sdr
- - smime
- - ssl
- - tools
-
-- task:
- metadata:
- name: "Windows 2012 64 (debug, NSS_NO_PKCS11_BYPASS=1)"
- description: "Windows 2012 64 (debug, NSS_NO_PKCS11_BYPASS=1)"
-
- payload:
- env:
- NSS_NO_PKCS11_BYPASS: 1
- USE_64: 1
-
- extra:
- treeherder:
- build:
- platform: windows2012-64
- machine:
- platform: windows2012-64
- collection:
- debug: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: noPkcs11Bypass
diff --git a/automation/taskcluster/graph/windows/build64-opt.yml b/automation/taskcluster/graph/windows/build64-opt.yml
deleted file mode 100644
index ad4d46e31..000000000
--- a/automation/taskcluster/graph/windows/build64-opt.yml
+++ /dev/null
@@ -1,57 +0,0 @@
----
-- task:
- metadata:
- name: "Windows 2012 64 (opt)"
- description: "Windows 2012 64 (opt)"
-
- payload:
- env:
- BUILD_OPT: 1
- USE_64: 1
-
- extra:
- treeherder:
- build:
- platform: windows2012-64
- machine:
- platform: windows2012-64
- collection:
- opt: true
-
- tests:
- - chains
- - cipher
- - crmf
- - db
- - ec
- - fips
- - gtests
- - lowhash
- - merge
- - sdr
- - smime
- - ssl
- - tools
-
-- task:
- metadata:
- name: "Windows 2012 64 (opt, NSS_NO_PKCS11_BYPASS=1)"
- description: "Windows 2012 64 (opt, NSS_NO_PKCS11_BYPASS=1)"
-
- payload:
- env:
- NSS_NO_PKCS11_BYPASS: 1
- BUILD_OPT: 1
- USE_64: 1
-
- extra:
- treeherder:
- build:
- platform: windows2012-64
- machine:
- platform: windows2012-64
- collection:
- opt: true
- groupSymbol: Builds
- groupName: Various builds
- symbol: noPkcs11Bypass
diff --git a/automation/taskcluster/graph/yaml.js b/automation/taskcluster/graph/yaml.js
deleted file mode 100644
index dd4f6ba47..000000000
--- a/automation/taskcluster/graph/yaml.js
+++ /dev/null
@@ -1,51 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-var fs = require("fs");
-var yaml = require("js-yaml");
-
-// Register custom YAML types.
-var YAML_SCHEMA = yaml.Schema.create([
- // Point in time at $now + x hours.
- new yaml.Type('!from_now', {
- kind: "scalar",
-
- resolve: function (data) {
- return true;
- },
-
- construct: function (data) {
- var d = new Date();
- d.setHours(d.getHours() + (data|0));
- return d.toJSON();
- }
- }),
-
- // Environment variables.
- new yaml.Type('!env', {
- kind: "scalar",
-
- resolve: function (data) {
- return true;
- },
-
- construct: function (data) {
- return process.env[data] || "{{" + data.toLowerCase() + "}}";
- }
- })
-]);
-
-// Parse a given YAML file.
-function parse(file, fallback) {
- // Return fallback if the file doesn't exist.
- if (!fs.existsSync(file) && fallback) {
- return fallback;
- }
-
- // Otherwise, read the file or fail.
- var source = fs.readFileSync(file, "utf-8");
- return yaml.load(source, {schema: YAML_SCHEMA});
-}
-
-module.exports.parse = parse;
diff --git a/automation/taskcluster/scripts/build.sh b/automation/taskcluster/scripts/build.sh
index a6a202164..c44a321d8 100755
--- a/automation/taskcluster/scripts/build.sh
+++ b/automation/taskcluster/scripts/build.sh
@@ -18,12 +18,6 @@ hg_clone https://hg.mozilla.org/projects/nspr nspr default
# Build.
make -C nss nss_build_all
-# Generate certificates.
-NSS_TESTS=cert NSS_CYCLES="standard pkix sharedb" $(dirname $0)/run_tests.sh
-
-# Reset test counter so that test runs pick up our certificates.
-echo 1 > tests_results/security/localhost
-
# Package.
mkdir artifacts
-tar cvfjh artifacts/dist.tar.bz2 dist tests_results
+tar cvfjh artifacts/dist.tar.bz2 dist
diff --git a/automation/taskcluster/scripts/extend_task_graph.sh b/automation/taskcluster/scripts/extend_task_graph.sh
index abbd4ecd7..5a3fb8d98 100755
--- a/automation/taskcluster/scripts/extend_task_graph.sh
+++ b/automation/taskcluster/scripts/extend_task_graph.sh
@@ -12,5 +12,5 @@ mkdir -p /home/worker/artifacts
# Install Node.JS dependencies.
cd nss/automation/taskcluster/graph/ && npm install
-# Build the task graph definition.
-nodejs build.js > /home/worker/artifacts/graph.json
+# Extend the task graph.
+node lib/index.js
diff --git a/automation/taskcluster/scripts/gen_certs.sh b/automation/taskcluster/scripts/gen_certs.sh
new file mode 100755
index 000000000..56b690d67
--- /dev/null
+++ b/automation/taskcluster/scripts/gen_certs.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+source $(dirname $0)/tools.sh
+
+if [ $(id -u) = 0 ]; then
+ # Set compiler.
+ switch_compilers
+
+ # Stupid Docker.
+ echo "127.0.0.1 localhost.localdomain" >> /etc/hosts
+
+ # Drop privileges by re-running this script.
+ exec su worker $0
+fi
+
+# Fetch artifact if needed.
+fetch_dist
+
+# Generate certificates.
+NSS_TESTS=cert NSS_CYCLES="standard pkix sharedb" $(dirname $0)/run_tests.sh
+
+# Reset test counter so that test runs pick up our certificates.
+echo 1 > tests_results/security/localhost
+
+# Package.
+mkdir artifacts
+tar cvfjh artifacts/dist.tar.bz2 dist tests_results
diff --git a/automation/taskcluster/scripts/run_tests.sh b/automation/taskcluster/scripts/run_tests.sh
index b917a9d48..d8341ee82 100755
--- a/automation/taskcluster/scripts/run_tests.sh
+++ b/automation/taskcluster/scripts/run_tests.sh
@@ -2,9 +2,9 @@
set -v -e -x
-if [ $(id -u) = 0 ]; then
- source $(dirname $0)/tools.sh
+source $(dirname $0)/tools.sh
+if [ $(id -u) = 0 ]; then
# Set compiler.
switch_compilers
@@ -16,10 +16,7 @@ if [ $(id -u) = 0 ]; then
fi
# Fetch artifact if needed.
-if [ ! -d "dist" ]; then
- curl --retry 3 -Lo dist.tar.bz2 https://queue.taskcluster.net/v1/task/$TC_PARENT_TASK_ID/artifacts/public/dist.tar.bz2
- tar xvjf dist.tar.bz2
-fi
+fetch_dist
# Run tests.
cd nss/tests && ./all.sh
diff --git a/automation/taskcluster/scripts/tools.sh b/automation/taskcluster/scripts/tools.sh
index 57f45d6be..632c909c2 100644
--- a/automation/taskcluster/scripts/tools.sh
+++ b/automation/taskcluster/scripts/tools.sh
@@ -27,3 +27,15 @@ hg_clone() {
done
exit 1
}
+
+fetch_dist() {
+ url=https://queue.taskcluster.net/v1/task/$TC_PARENT_TASK_ID/artifacts/public/dist.tar.bz2
+ if [ ! -d "dist" ]; then
+ for i in 0 2 5; do
+ sleep $i
+ curl --retry 3 -Lo dist.tar.bz2 $url && tar xvjf dist.tar.bz2 && return
+ rm -fr dist.tar.bz2 dist
+ done
+ exit 1
+ fi
+}
diff --git a/automation/taskcluster/windows/build.sh b/automation/taskcluster/windows/build.sh
index c275df54d..6c8a47470 100644
--- a/automation/taskcluster/windows/build.sh
+++ b/automation/taskcluster/windows/build.sh
@@ -11,11 +11,5 @@ hg_clone https://hg.mozilla.org/projects/nspr nspr default
# Build.
make -C nss nss_build_all
-# Generate certificates.
-NSS_TESTS=cert NSS_CYCLES="standard pkix sharedb" nss/tests/all.sh
-
-# Reset test counter so that test runs pick up our certificates.
-echo 1 > tests_results/security/localhost
-
# Package.
-7z a public/build/dist.7z dist tests_results
+7z a public/build/dist.7z dist
diff --git a/automation/taskcluster/windows/gen_certs.sh b/automation/taskcluster/windows/gen_certs.sh
new file mode 100644
index 000000000..ead16bbc3
--- /dev/null
+++ b/automation/taskcluster/windows/gen_certs.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+# Set up the toolchain.
+source $(dirname $0)/setup.sh
+
+# Fetch artifact.
+wget -t 3 --retry-connrefused -w 5 --random-wait https://queue.taskcluster.net/v1/task/$TC_PARENT_TASK_ID/artifacts/public/build/dist.7z -O dist.7z
+7z x dist.7z
+
+# Generate certificates.
+NSS_TESTS=cert NSS_CYCLES="standard pkix sharedb" nss/tests/all.sh
+
+# Reset test counter so that test runs pick up our certificates.
+echo 1 > tests_results/security/localhost
+
+# Package.
+7z a public/build/dist.7z dist tests_results