diff options
author | Tim Taubert <ttaubert@mozilla.com> | 2016-09-12 11:02:28 -0700 |
---|---|---|
committer | Tim Taubert <ttaubert@mozilla.com> | 2016-09-12 11:02:28 -0700 |
commit | 42b6c9e5bc50508cc0c7d2b38a81b8734a6684e5 (patch) | |
tree | 680984400d04f2c72cede29896af51a2f8298ecf | |
parent | 10d4b3330eb5b4c014bc7d0520bdd750cccb93e1 (diff) | |
download | nss-hg-42b6c9e5bc50508cc0c7d2b38a81b8734a6684e5.tar.gz |
Bug 1301374 - Rewrite decision task to use imperative style r=me,npotb
50 files changed, 842 insertions, 1651 deletions
diff --git a/automation/taskcluster/decision_task.yml b/automation/taskcluster/decision_task.yml index 48f9abef0..9d56c9bcd 100644 --- a/automation/taskcluster/decision_task.yml +++ b/automation/taskcluster/decision_task.yml @@ -77,14 +77,8 @@ tasks: bin/checkout.sh && nss/automation/taskcluster/scripts/extend_task_graph.sh - artifacts: - public: - type: "directory" - path: "/home/worker/artifacts" - expires: "{{#from_now}}7 days{{/from_now}}" - - graphs: - - /home/worker/artifacts/graph.json + features: + taskclusterProxy: true extra: treeherder: diff --git a/automation/taskcluster/graph/arm/_build_base.yml b/automation/taskcluster/graph/arm/_build_base.yml deleted file mode 100644 index cef0fd48b..000000000 --- a/automation/taskcluster/graph/arm/_build_base.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -reruns: 2 - -task: - created: !from_now 0 - deadline: !from_now 24 - provisionerId: localprovisioner - workerType: nss-rpi - schedulerId: task-graph-scheduler - - metadata: - owner: !env TC_OWNER - source: !env TC_SOURCE - - payload: - maxRunTime: 7200 - image: ttaubert/nss-rpi-ci:0.0.3 - - artifacts: - public: - type: directory - path: /home/worker/artifacts - expires: !from_now 24 - - command: - - "/bin/bash" - - "-c" - - "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh" - - env: - NSS_HEAD_REPOSITORY: !env NSS_HEAD_REPOSITORY - NSS_HEAD_REVISION: !env NSS_HEAD_REVISION - GCC_VERSION: gcc-5 - GXX_VERSION: g++-5 - - extra: - treeherder: - tier: 3 # hide jobs by default - jobKind: build - symbol: B diff --git a/automation/taskcluster/graph/arm/_test_base.yml b/automation/taskcluster/graph/arm/_test_base.yml deleted file mode 100644 index 5f7fd68d6..000000000 --- a/automation/taskcluster/graph/arm/_test_base.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -reruns: 2 - -task: - created: !from_now 0 - deadline: !from_now 24 - provisionerId: localprovisioner - workerType: nss-rpi - schedulerId: task-graph-scheduler - - metadata: - owner: !env TC_OWNER - source: !env TC_SOURCE - - payload: - maxRunTime: 7200 - image: ttaubert/nss-rpi-ci:0.0.3 - - command: - - "/bin/bash" - - "-c" - - "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh" - - extra: - treeherder: - tier: 3 # hide jobs by default - jobKind: test diff --git a/automation/taskcluster/graph/arm/build32-debug.yml b/automation/taskcluster/graph/arm/build32-debug.yml deleted file mode 100644 index e9c61c091..000000000 --- a/automation/taskcluster/graph/arm/build32-debug.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -- task: - metadata: - name: "Linux 32 (ARM, debug)" - description: "Linux 32 (ARM, debug)" - - extra: - treeherder: - build: - platform: linux32 - machine: - platform: linux32 - collection: - arm-debug: true - - tests: - - chains - - cipher - - crmf - - db - - ec - - fips - - gtests - - lowhash - - merge - - sdr - - smime - - ssl - - tools diff --git a/automation/taskcluster/graph/build.js b/automation/taskcluster/graph/build.js deleted file mode 100644 index 21d17248f..000000000 --- a/automation/taskcluster/graph/build.js +++ /dev/null @@ -1,147 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -var fs = require("fs"); -var path = require("path"); -var merge = require("merge"); -var slugid = require("slugid"); -var flatmap = require("flatmap"); - -var yaml = require("./yaml"); -var try_syntax = require("./try_syntax"); -var image_builder = require("./image_builder"); - -// Default values for debugging. -var TC_OWNER = process.env.TC_OWNER || "{{tc_owner}}"; -var TC_SOURCE = process.env.TC_SOURCE || "{{tc_source}}"; -var TC_PROJECT = process.env.TC_PROJECT || "{{tc_project}}"; -var TC_COMMENT = process.env.TC_COMMENT || "{{tc_comment}}"; -var NSS_PUSHLOG_ID = process.env.NSS_PUSHLOG_ID || "{{nss_pushlog_id}}"; -var NSS_HEAD_REVISION = process.env.NSS_HEAD_REVISION || "{{nss_head_rev}}"; - -// Add base information to the given task. -function decorateTask(task) { - // Assign random task id. - task.taskId = slugid.v4(); - - // TreeHerder routes. - task.task.routes = [ - "tc-treeherder-stage.v2." + TC_PROJECT + "." + NSS_HEAD_REVISION + "." + NSS_PUSHLOG_ID, - "tc-treeherder.v2." + TC_PROJECT + "." + NSS_HEAD_REVISION + "." + NSS_PUSHLOG_ID - ]; -} - -// Generate all tasks for a given build. -function generateBuildTasks(platform, file) { - var dir = path.join(__dirname, "./" + platform); - - // Parse base definitions. - var buildBase = yaml.parse(path.join(dir, "_build_base.yml"), {}); - var testBase = yaml.parse(path.join(dir, "_test_base.yml"), {}); - - return flatmap(yaml.parse(path.join(dir, file)), function (task) { - // Merge base build task definition with the current one. - var tasks = [task = merge.recursive(true, buildBase, task)]; - - // Add base info. - decorateTask(task); - - // Generate test tasks. - if (task.tests) { - // The base definition for all tests of this platform. - var base = merge.recursive(true, { - requires: [task.taskId], - - task: { - payload: { - env: { - TC_PARENT_TASK_ID: task.taskId - } - } - } - }, testBase); - - // Generate and append test task definitions. - tasks = tasks.concat(flatmap(task.tests, function (name) { - return generateTestTasks(name, base, task); - })); - - // |tests| is not part of the schema. - delete task.tests; - } - - return tasks; - }); -} - -// Generate all tasks for a given test. -function generateTestTasks(name, base, task) { - // Load test definitions. - var dir = path.join(__dirname, "./tests"); - var tests = yaml.parse(path.join(dir, name + ".yml")); - - return tests.map(function (test) { - // Merge test with base definition. - test = merge.recursive(true, base, test); - - // Add base info. - decorateTask(test); - - // We only want to carry over environment variables... - test.task.payload.env = - merge.recursive(true, task.task.payload.env, - test.task.payload.env); - - // ...and TreeHerder configuration data. - test.task.extra.treeherder = - merge.recursive(true, task.task.extra.treeherder, - test.task.extra.treeherder); - - return test; - }); -} - -// Generate all tasks for a given platform. -function generatePlatformTasks(platform) { - var dir = path.join(__dirname, "./" + platform); - var buildBase = yaml.parse(path.join(dir, "_build_base.yml"), {}); - var testBase = yaml.parse(path.join(dir, "_test_base.yml"), {}); - - // Parse all build tasks. - return flatmap(fs.readdirSync(dir), function (file) { - if (!file.startsWith("_") && file.endsWith(".yml")) { - var tasks = generateBuildTasks(platform, file); - - // Convert env variables to strings. - tasks.forEach(function (task) { - var env = task.task.payload.env || {}; - Object.keys(env).forEach(function (name) { - if (typeof(env[name]) != "undefined") { - env[name] = env[name] + ""; - } - }); - }); - - return tasks; - } - }); -} - -// Construct the task graph. -var graph = { - tasks: flatmap(["linux", "windows", "arm", "tools"], generatePlatformTasks) -}; - -// Filter tasks when try syntax is given. -if (TC_PROJECT == "nss-try") { - graph.tasks = try_syntax.filterTasks(graph.tasks, TC_COMMENT); -} - -// Inject the image builder tasks and dependencies. -image_builder.asyncTweakTasks(graph.tasks).then(function (tasks) { - graph.tasks = tasks; - - // Output the final graph. - process.stdout.write(JSON.stringify(graph, null, 2)); -}); diff --git a/automation/taskcluster/graph/image_builder.js b/automation/taskcluster/graph/image_builder.js deleted file mode 100644 index 4e32f3706..000000000 --- a/automation/taskcluster/graph/image_builder.js +++ /dev/null @@ -1,148 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -var fs = require("fs"); -var path = require("path"); -var crypto = require("crypto"); -var slugid = require("slugid"); -var flatmap = require("flatmap"); -var taskcluster = require("taskcluster-client"); - -var yaml = require("./yaml"); - -// Default values for debugging. -var TC_PROJECT = process.env.TC_PROJECT || "{{tc_project}}"; -var NSS_PUSHLOG_ID = process.env.NSS_PUSHLOG_ID || "{{nss_pushlog_id}}"; -var NSS_HEAD_REVISION = process.env.NSS_HEAD_REVISION || "{{nss_head_rev}}"; - -// Add base information to the given task. -function decorateTask(task) { - // Assign random task id. - task.taskId = slugid.v4(); - - // TreeHerder routes. - task.task.routes = [ - "tc-treeherder-stage.v2." + TC_PROJECT + "." + NSS_HEAD_REVISION + "." + NSS_PUSHLOG_ID, - "tc-treeherder.v2." + TC_PROJECT + "." + NSS_HEAD_REVISION + "." + NSS_PUSHLOG_ID - ]; -} - -// Compute the SHA-256 digest. -function sha256(data) { - var hash = crypto.createHash("sha256"); - hash.update(data); - return hash.digest("hex"); -} - -// Recursively collect a list of all files of a given directory. -function collectFilesInDirectory(dir) { - return flatmap(fs.readdirSync(dir), function (entry) { - var entry_path = path.join(dir, entry); - - if (fs.lstatSync(entry_path).isDirectory()) { - return collectFilesInDirectory(entry_path); - } - - return [entry_path]; - }); -} - -// Compute a context hash for the given context path. -function computeContextHash(context_path) { - var root = path.join(__dirname, "../../.."); - var dir = path.join(root, context_path); - var files = collectFilesInDirectory(dir).sort(); - var hashes = files.map(function (file) { - return sha256(file + "|" + fs.readFileSync(file, "utf-8")); - }); - - return sha256(hashes.join(",")); -} - -// Generates the image-builder task description. -function generateImageBuilderTask(context_path) { - var task = yaml.parse(path.join(__dirname, "image_builder.yml"), {}); - - // Add base info. - decorateTask(task); - - // Add info for docker image building. - task.task.payload.env.CONTEXT_PATH = context_path; - task.task.payload.env.HASH = computeContextHash(context_path); - - return task; -} - -// Returns a Promise<bool> that tells whether the task with the given id -// has a public/image.tar artifact with a ready-to-use docker image. -function asyncTaskHasImageArtifact(taskId) { - var queue = new taskcluster.Queue(); - - return queue.listLatestArtifacts(taskId).then(function (result) { - return result.artifacts.some(function (artifact) { - return artifact.name == "public/image.tar"; - }); - }, function () { - return false; - }); -} - -// Returns a Promise<task-id|null> with either a task id or null, depending -// on whether we could find a task in the given namespace with a docker image. -function asyncFindTaskWithImageArtifact(ns) { - var index = new taskcluster.Index(); - - return index.findTask(ns).then(function (result) { - return asyncTaskHasImageArtifact(result.taskId).then(function (has_image) { - return has_image ? result.taskId : null; - }); - }, function () { - return null; - }); -} - -// Tweak the given list of tasks by injecting the image-builder task -// and setting the right dependencies where needed. -function asyncTweakTasks(tasks) { - var id = "linux"; - var cx_path = "automation/taskcluster/docker"; - var hash = computeContextHash(cx_path); - var ns = "docker.images.v1." + TC_PROJECT + "." + id + ".hash." + hash; - var additional_tasks = []; - - // Check whether the docker image was already built. - return asyncFindTaskWithImageArtifact(ns).then(function (taskId) { - var builder_task; - - if (!taskId) { - // No docker image found, add a task to build one. - builder_task = generateImageBuilderTask(cx_path); - taskId = builder_task.taskId; - - // Add a route so we can find the task later again. - builder_task.task.routes.push("index." + ns); - additional_tasks.push(builder_task); - } - - tasks.forEach(function (task) { - if (task.task.payload.image == cx_path) { - task.task.payload.image = { - path: "public/image.tar", - type: "task-image", - taskId: taskId - }; - - // Add a dependency only for top-level tasks (builds & tools) and only - // if we added an image building task. Otherwise we don't need to wait. - if (builder_task && !task.requires) { - task.requires = [taskId]; - } - } - }); - - return additional_tasks.concat(tasks); - }); -} - -module.exports.asyncTweakTasks = asyncTweakTasks; diff --git a/automation/taskcluster/graph/image_builder.yml b/automation/taskcluster/graph/image_builder.yml deleted file mode 100644 index 417646b83..000000000 --- a/automation/taskcluster/graph/image_builder.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- -reruns: 2 - -task: - created: !from_now 0 - deadline: !from_now 24 - provisionerId: aws-provisioner-v1 - workerType: hg-worker - schedulerId: task-graph-scheduler - - metadata: - name: Image Builder - description: Image Builder - owner: !env TC_OWNER - source: !env TC_SOURCE - - payload: - maxRunTime: 3600 - image: taskcluster/image_builder:0.1.5 - - artifacts: - public/image.tar: - type: file - path: /artifacts/image.tar - expires: !from_now 8760 - - command: - - "/bin/bash" - - "-c" - - "/home/worker/bin/build_image.sh" - - env: - HEAD_REPOSITORY: !env NSS_HEAD_REPOSITORY - BASE_REPOSITORY: !env NSS_HEAD_REPOSITORY - HEAD_REV: !env NSS_HEAD_REVISION - HEAD_REF: !env NSS_HEAD_REVISION - PROJECT: !env TC_PROJECT - - features: - dind: true - - extra: - treeherder: - build: - platform: nss-decision - machine: - platform: nss-decision - jobKind: build - symbol: I diff --git a/automation/taskcluster/graph/linux/_build_base.yml b/automation/taskcluster/graph/linux/_build_base.yml deleted file mode 100644 index 3f23d4be2..000000000 --- a/automation/taskcluster/graph/linux/_build_base.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -reruns: 2 - -task: - created: !from_now 0 - deadline: !from_now 24 - provisionerId: aws-provisioner-v1 - workerType: hg-worker - schedulerId: task-graph-scheduler - - metadata: - owner: !env TC_OWNER - source: !env TC_SOURCE - - payload: - maxRunTime: 3600 - image: automation/taskcluster/docker - - artifacts: - public: - type: directory - path: /home/worker/artifacts - expires: !from_now 24 - - command: - - "/bin/bash" - - "-c" - - "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh" - - env: - NSS_HEAD_REPOSITORY: !env NSS_HEAD_REPOSITORY - NSS_HEAD_REVISION: !env NSS_HEAD_REVISION - GCC_VERSION: gcc-5 - GXX_VERSION: g++-5 - - extra: - treeherder: - jobKind: build - symbol: B diff --git a/automation/taskcluster/graph/linux/_test_base.yml b/automation/taskcluster/graph/linux/_test_base.yml deleted file mode 100644 index 7b4f9337b..000000000 --- a/automation/taskcluster/graph/linux/_test_base.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -reruns: 2 - -task: - created: !from_now 0 - deadline: !from_now 24 - provisionerId: aws-provisioner-v1 - workerType: hg-worker - schedulerId: task-graph-scheduler - - metadata: - owner: !env TC_OWNER - source: !env TC_SOURCE - - payload: - maxRunTime: 3600 - image: automation/taskcluster/docker - - command: - - "/bin/bash" - - "-c" - - "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh" - - extra: - treeherder: - jobKind: test diff --git a/automation/taskcluster/graph/linux/build32-debug.yml b/automation/taskcluster/graph/linux/build32-debug.yml deleted file mode 100644 index c75138fc4..000000000 --- a/automation/taskcluster/graph/linux/build32-debug.yml +++ /dev/null @@ -1,116 +0,0 @@ ---- -- task: - metadata: - name: "Linux 32 (debug)" - description: "Linux 32 (debug)" - - extra: - treeherder: - build: - platform: linux32 - machine: - platform: linux32 - collection: - debug: true - - tests: - - chains - - cipher - - crmf - - db - - ec - - fips - - gtests - - lowhash - - merge - - sdr - - smime - - ssl - - tools - -- task: - metadata: - name: "Linux 32 (debug, clang-3.8)" - description: "Linux 32 (debug, clang-3.8)" - - payload: - env: - GCC_VERSION: clang - GXX_VERSION: clang++ - - extra: - treeherder: - build: - platform: linux32 - machine: - platform: linux32 - collection: - debug: true - groupSymbol: Builds - groupName: Various builds - symbol: clang-3.8 - -- task: - metadata: - name: "Linux 32 (debug, gcc-4.8)" - description: "Linux 32 (debug, gcc-4.8)" - - payload: - env: - GCC_VERSION: gcc-4.8 - GXX_VERSION: g++-4.8 - - extra: - treeherder: - build: - platform: linux32 - machine: - platform: linux32 - collection: - debug: true - groupSymbol: Builds - groupName: Various builds - symbol: gcc-4.8 - -- task: - metadata: - name: "Linux 32 (debug, gcc-6.1)" - description: "Linux 32 (debug, gcc-6.1)" - - payload: - env: - GCC_VERSION: gcc-6 - GXX_VERSION: g++-6 - - extra: - treeherder: - build: - platform: linux32 - machine: - platform: linux32 - collection: - debug: true - groupSymbol: Builds - groupName: Various builds - symbol: gcc-6.1 - -- task: - metadata: - name: "Linux 32 (debug, NSS_NO_PKCS11_BYPASS=1)" - description: "Linux 32 (debug, NSS_NO_PKCS11_BYPASS=1)" - - payload: - env: - NSS_NO_PKCS11_BYPASS: 1 - - extra: - treeherder: - build: - platform: linux32 - machine: - platform: linux32 - collection: - debug: true - groupSymbol: Builds - groupName: Various builds - symbol: noPkcs11Bypass diff --git a/automation/taskcluster/graph/linux/build32-opt.yml b/automation/taskcluster/graph/linux/build32-opt.yml deleted file mode 100644 index c71b901b5..000000000 --- a/automation/taskcluster/graph/linux/build32-opt.yml +++ /dev/null @@ -1,124 +0,0 @@ ---- -- task: - metadata: - name: "Linux 32 (opt)" - description: "Linux 32 (opt)" - - payload: - env: - BUILD_OPT: 1 - - extra: - treeherder: - build: - platform: linux32 - machine: - platform: linux32 - collection: - opt: true - - tests: - - chains - - cipher - - crmf - - db - - ec - - fips - - gtests - - lowhash - - merge - - sdr - - smime - - ssl - - tools - -- task: - metadata: - name: "Linux 32 (opt, clang-3.8)" - description: "Linux 32 (opt, clang-3.8)" - - payload: - env: - GCC_VERSION: clang - GXX_VERSION: clang++ - BUILD_OPT: 1 - - extra: - treeherder: - build: - platform: linux32 - machine: - platform: linux32 - collection: - opt: true - groupSymbol: Builds - groupName: Various builds - symbol: clang-3.8 - -- task: - metadata: - name: "Linux 32 (opt, gcc-4.8)" - description: "Linux 32 (opt, gcc-4.8)" - - payload: - env: - GCC_VERSION: gcc-4.8 - GXX_VERSION: g++-4.8 - BUILD_OPT: 1 - - extra: - treeherder: - build: - platform: linux32 - machine: - platform: linux32 - collection: - opt: true - groupSymbol: Builds - groupName: Various builds - symbol: gcc-4.8 - -- task: - metadata: - name: "Linux 32 (opt, gcc-6.1)" - description: "Linux 32 (opt, gcc-6.1)" - - payload: - env: - GCC_VERSION: gcc-6 - GXX_VERSION: g++-6 - BUILD_OPT: 1 - - extra: - treeherder: - build: - platform: linux32 - machine: - platform: linux32 - collection: - opt: true - groupSymbol: Builds - groupName: Various builds - symbol: gcc-6.1 - -- task: - metadata: - name: "Linux 32 (opt, NSS_NO_PKCS11_BYPASS=1)" - description: "Linux 32 (opt, NSS_NO_PKCS11_BYPASS=1)" - - payload: - env: - NSS_NO_PKCS11_BYPASS: 1 - BUILD_OPT: 1 - - extra: - treeherder: - build: - platform: linux32 - machine: - platform: linux32 - collection: - opt: true - groupSymbol: Builds - groupName: Various builds - symbol: noPkcs11Bypass diff --git a/automation/taskcluster/graph/linux/build64-asan.yml b/automation/taskcluster/graph/linux/build64-asan.yml deleted file mode 100644 index d7fca39e2..000000000 --- a/automation/taskcluster/graph/linux/build64-asan.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -- task: - metadata: - name: "Linux 64 (ASan, debug)" - description: "Linux 64 (ASan, debug)" - - payload: - env: - GCC_VERSION: clang - GXX_VERSION: clang++ - NSS_DISABLE_ARENA_FREE_LIST: 1 - NSS_DISABLE_UNLOAD: 1 - USE_ASAN: 1 - USE_64: 1 - - extra: - treeherder: - build: - platform: linux64 - machine: - platform: linux64 - collection: - asan: true - - tests: - - chains - - cipher - - crmf - - db - - ec - - fips - - gtests - - lowhash - - merge - - sdr - - smime - - ssl - - tools diff --git a/automation/taskcluster/graph/linux/build64-debug.yml b/automation/taskcluster/graph/linux/build64-debug.yml deleted file mode 100644 index 197727bb3..000000000 --- a/automation/taskcluster/graph/linux/build64-debug.yml +++ /dev/null @@ -1,146 +0,0 @@ ---- -- task: - metadata: - name: "Linux 64 (debug)" - description: "Linux 64 (debug)" - - payload: - env: - USE_64: 1 - - extra: - treeherder: - build: - platform: linux64 - machine: - platform: linux64 - collection: - debug: true - - tests: - - chains - - cipher - - crmf - - db - - ec - - fips - - gtests - - lowhash - - merge - - sdr - - smime - - ssl - - tools - -- task: - metadata: - name: "Linux 64 (debug, clang-3.8)" - description: "Linux 64 (debug, clang-3.8)" - - payload: - env: - GCC_VERSION: clang - GXX_VERSION: clang++ - USE_64: 1 - - extra: - treeherder: - build: - platform: linux64 - machine: - platform: linux64 - collection: - debug: true - groupSymbol: Builds - groupName: Various builds - symbol: clang-3.8 - -- task: - metadata: - name: "Linux 64 (debug, gcc-4.8)" - description: "Linux 64 (debug, gcc-4.8)" - - payload: - env: - GCC_VERSION: gcc-4.8 - GXX_VERSION: g++-4.8 - USE_64: 1 - - extra: - treeherder: - build: - platform: linux64 - machine: - platform: linux64 - collection: - debug: true - groupSymbol: Builds - groupName: Various builds - symbol: gcc-4.8 - -- task: - metadata: - name: "Linux 64 (debug, gcc-6.1)" - description: "Linux 64 (debug, gcc-6.1)" - - payload: - env: - GCC_VERSION: gcc-6 - GXX_VERSION: g++-6 - USE_64: 1 - - extra: - treeherder: - build: - platform: linux64 - machine: - platform: linux64 - collection: - debug: true - groupSymbol: Builds - groupName: Various builds - symbol: gcc-6.1 - -- task: - metadata: - name: "Linux 64 (debug, NSS_NO_PKCS11_BYPASS=1)" - description: "Linux 64 (debug, NSS_NO_PKCS11_BYPASS=1)" - - payload: - env: - NSS_NO_PKCS11_BYPASS: 1 - USE_64: 1 - - extra: - treeherder: - build: - platform: linux64 - machine: - platform: linux64 - collection: - debug: true - groupSymbol: Builds - groupName: Various builds - symbol: noPkcs11Bypass - -- task: - metadata: - name: "Linux 64 (debug, NSS_DISABLE_LIBPKIX=1)" - description: "Linux 64 (debug, NSS_DISABLE_LIBPKIX=1)" - - payload: - env: - NSS_DISABLE_LIBPKIX: 1 - USE_64: 1 - - extra: - treeherder: - build: - platform: linux64 - machine: - platform: linux64 - collection: - debug: true - groupSymbol: Builds - groupName: Various builds - symbol: noLibpkix diff --git a/automation/taskcluster/graph/linux/build64-opt.yml b/automation/taskcluster/graph/linux/build64-opt.yml deleted file mode 100644 index ff57c5312..000000000 --- a/automation/taskcluster/graph/linux/build64-opt.yml +++ /dev/null @@ -1,129 +0,0 @@ ---- -- task: - metadata: - name: "Linux 64 (opt)" - description: "Linux 64 (opt)" - - payload: - env: - BUILD_OPT: 1 - USE_64: 1 - - extra: - treeherder: - build: - platform: linux64 - machine: - platform: linux64 - collection: - opt: true - - tests: - - chains - - cipher - - crmf - - db - - ec - - fips - - gtests - - lowhash - - merge - - sdr - - smime - - ssl - - tools - -- task: - metadata: - name: "Linux 64 (opt, clang-3.8)" - description: "Linux 64 (opt, clang-3.8)" - - payload: - env: - GCC_VERSION: clang - GXX_VERSION: clang++ - BUILD_OPT: 1 - USE_64: 1 - - extra: - treeherder: - build: - platform: linux64 - machine: - platform: linux64 - collection: - opt: true - groupSymbol: Builds - groupName: Various builds - symbol: clang-3.8 - -- task: - metadata: - name: "Linux 64 (opt, gcc-4.8)" - description: "Linux 64 (opt, gcc-4.8)" - - payload: - env: - GCC_VERSION: gcc-4.8 - GXX_VERSION: g++-4.8 - BUILD_OPT: 1 - USE_64: 1 - - extra: - treeherder: - build: - platform: linux64 - machine: - platform: linux64 - collection: - opt: true - groupSymbol: Builds - groupName: Various builds - symbol: gcc-4.8 - -- task: - metadata: - name: "Linux 64 (opt, gcc-6.1)" - description: "Linux 64 (opt, gcc-6.1)" - - payload: - env: - GCC_VERSION: gcc-6 - GXX_VERSION: g++-6 - BUILD_OPT: 1 - USE_64: 1 - - extra: - treeherder: - build: - platform: linux64 - machine: - platform: linux64 - collection: - opt: true - groupSymbol: Builds - groupName: Various builds - symbol: gcc-6.1 - -- task: - metadata: - name: "Linux 64 (opt, NSS_NO_PKCS11_BYPASS=1)" - description: "Linux 64 (opt, NSS_NO_PKCS11_BYPASS=1)" - - payload: - env: - NSS_NO_PKCS11_BYPASS: 1 - BUILD_OPT: 1 - USE_64: 1 - - extra: - treeherder: - build: - platform: linux64 - machine: - platform: linux64 - collection: - opt: true - groupSymbol: Builds - groupName: Various builds - symbol: noPkcs11Bypass diff --git a/automation/taskcluster/graph/package.json b/automation/taskcluster/graph/package.json index e58ab0b0d..d866adebc 100644 --- a/automation/taskcluster/graph/package.json +++ b/automation/taskcluster/graph/package.json @@ -4,7 +4,15 @@ "private": true, "author": "Tim Taubert <ttaubert@mozilla.com>", "description": "Decision Task for NSS", + "scripts": { + "compile": "babel-compile -p taskcluster src:lib", + "install": "npm run compile" + }, "dependencies": { + "babel-cli": "^6.14.0", + "babel-compile": "^2.0.0", + "babel-preset-taskcluster": "^3.0.0", + "babel-runtime": "^6.11.6", "flatmap": "0.0.3", "intersect": "^1.0.1", "js-yaml": "^3.6.1", diff --git a/automation/taskcluster/graph/src/context_hash.js b/automation/taskcluster/graph/src/context_hash.js new file mode 100644 index 000000000..67bf25784 --- /dev/null +++ b/automation/taskcluster/graph/src/context_hash.js @@ -0,0 +1,40 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import fs from "fs"; +import path from "path"; +import crypto from "crypto"; +import flatmap from "flatmap"; + +// Compute the SHA-256 digest. +function sha256(data) { + let hash = crypto.createHash("sha256"); + hash.update(data); + return hash.digest("hex"); +} + +// Recursively collect a list of all files of a given directory. +function collectFilesInDirectory(dir) { + return flatmap(fs.readdirSync(dir), entry => { + let entry_path = path.join(dir, entry); + + if (fs.lstatSync(entry_path).isDirectory()) { + return collectFilesInDirectory(entry_path); + } + + return [entry_path]; + }); +} + +// Compute a context hash for the given context path. +export default function (context_path) { + let root = path.join(__dirname, "../../../.."); + let dir = path.join(root, context_path); + let files = collectFilesInDirectory(dir).sort(); + let hashes = files.map(file => { + return sha256(file + "|" + fs.readFileSync(file, "utf-8")); + }); + + return sha256(hashes.join(",")); +} diff --git a/automation/taskcluster/graph/src/extend.js b/automation/taskcluster/graph/src/extend.js new file mode 100644 index 000000000..633baa2ac --- /dev/null +++ b/automation/taskcluster/graph/src/extend.js @@ -0,0 +1,362 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import merge from "./merge"; +import * as queue from "./queue"; + +const LINUX_IMAGE = {name: "linux", path: "automation/taskcluster/docker"}; + +const WINDOWS_CHECKOUT_CMD = + "bash -c \"hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || " + + "(sleep 2; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss) || " + + "(sleep 5; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss)\""; + +/*****************************************************************************/ + +queue.filter(task => { + // Remove extra builds on ASan and ARM. + if (task.collection == "asan" || task.collection == "arm-debug") { + return task.group != "Builds"; + } + + return true; +}); + +queue.map(task => { + if (task.collection == "asan") { + // CRMF and FIPS tests still leak, unfortunately. + if (task.tests == "crmf" || task.tests == "fips") { + task.env.ASAN_OPTIONS = "detect_leaks=0"; + } + } + + if (task.collection == "arm-debug") { + // These tests take quite some time on our poor ARM devices. + if (task.tests == "chains" || (task.tests == "ssl" && task.cycle == "standard")) { + task.maxRunTime = 14400; + } + } + + return task; +}); + +/*****************************************************************************/ + +export default async function main() { + await scheduleLinux("Linux 32 (opt)", { + env: {BUILD_OPT: "1"}, + platform: "linux32", + image: LINUX_IMAGE + }); + + await scheduleLinux("Linux 32 (debug)", { + platform: "linux32", + collection: "debug", + image: LINUX_IMAGE + }); + + await scheduleLinux("Linux 64 (opt)", { + env: {USE_64: "1", BUILD_OPT: "1"}, + platform: "linux64", + image: LINUX_IMAGE + }); + + await scheduleLinux("Linux 64 (debug)", { + env: {USE_64: "1"}, + platform: "linux64", + collection: "debug", + image: LINUX_IMAGE + }); + + await scheduleLinux("Linux 64 (ASan, debug)", { + env: { + NSS_DISABLE_ARENA_FREE_LIST: "1", + NSS_DISABLE_UNLOAD: "1", + GCC_VERSION: "clang", + GXX_VERSION: "clang++", + USE_ASAN: "1", + USE_64: "1" + }, + platform: "linux64", + collection: "asan", + image: LINUX_IMAGE + }); + + await scheduleWindows("Windows 2012 64 (opt)", { + env: {BUILD_OPT: "1"} + }); + + await scheduleWindows("Windows 2012 64 (debug)", { + collection: "debug" + }); + + await scheduleTools(); + + await scheduleLinux("Linux 32 (ARM, debug)", { + image: "ttaubert/nss-rpi-ci:0.0.3", + provisioner: "localprovisioner", + collection: "arm-debug", + workerType: "nss-rpi", + platform: "linux32", + maxRunTime: 7200, + tier: 3 + }); +} + +/*****************************************************************************/ + +async function scheduleLinux(name, base) { + // Build base definition. + let build_base = merge(base, { + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh" + ], + artifacts: { + public: { + expires: 24 * 7, + type: "directory", + path: "/home/worker/artifacts" + } + }, + kind: "build", + symbol: "B" + }); + + // The task that builds NSPR+NSS. + let task_build = queue.scheduleTask(merge(build_base, {name})); + + // The task that generates certificates. + let task_cert = queue.scheduleTask(merge(build_base, { + name: "Certificates", + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/gen_certs.sh" + ], + parent: task_build, + symbol: "Certs" + })); + + // Schedule tests. + scheduleTests(task_build, task_cert, merge(base, { + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh" + ] + })); + + // Extra builds. + let extra_base = merge({group: "Builds"}, build_base); + queue.scheduleTask(merge(extra_base, { + name: `${name} w/ clang-3.8`, + env: { + GCC_VERSION: "clang", + GXX_VERSION: "clang++" + }, + symbol: "clang-3.8" + })); + + queue.scheduleTask(merge(extra_base, { + name: `${name} w/ gcc-4.8`, + env: { + GCC_VERSION: "gcc-4.8", + GXX_VERSION: "g++-4.8" + }, + symbol: "gcc-4.8" + })); + + queue.scheduleTask(merge(extra_base, { + name: `${name} w/ gcc-6.1`, + env: { + GCC_VERSION: "gcc-6", + GXX_VERSION: "g++-6" + }, + symbol: "gcc-6.1" + })); + + queue.scheduleTask(merge(extra_base, { + name: `${name} w/ NSS_NO_PKCS11_BYPASS=1`, + env: {NSS_NO_PKCS11_BYPASS: "1"}, + symbol: "noPkcs11Bypass" + })); + + return queue.submit(); +} + +/*****************************************************************************/ + +async function scheduleWindows(name, base) { + base = merge(base, { + workerType: "nss-win2012r2", + platform: "windows2012-64", + env: { + PATH: "c:\\mozilla-build\\python;c:\\mozilla-build\\msys\\local\\bin;" + + "c:\\mozilla-build\\7zip;c:\\mozilla-build\\info-zip;" + + "c:\\mozilla-build\\python\\Scripts;c:\\mozilla-build\\yasm;" + + "c:\\mozilla-build\\msys\\bin;c:\\Windows\\system32;" + + "c:\\mozilla-build\\upx391w;c:\\mozilla-build\\moztools-x64\\bin;" + + "c:\\mozilla-build\\wget", + DOMSUF: "localdomain", + HOST: "localhost", + USE_64: "1" + } + }); + + // Build base definition. + let build_base = merge(base, { + command: [ + WINDOWS_CHECKOUT_CMD, + "bash -c nss/automation/taskcluster/windows/build.sh" + ], + artifacts: [{ + expires: 24 * 7, + type: "directory", + path: "public\\build" + }], + kind: "build", + symbol: "B" + }); + + // The task that builds NSPR+NSS. + let task_build = queue.scheduleTask(merge(build_base, {name})); + + // The task that generates certificates. + let task_cert = queue.scheduleTask(merge(build_base, { + name: "Certificates", + command: [ + WINDOWS_CHECKOUT_CMD, + "bash -c nss/automation/taskcluster/windows/gen_certs.sh" + ], + parent: task_build, + symbol: "Certs" + })); + + // Schedule tests. + scheduleTests(task_build, task_cert, merge(base, { + command: [ + WINDOWS_CHECKOUT_CMD, + "bash -c nss/automation/taskcluster/windows/run_tests.sh" + ] + })); + + // Extra builds. + let extra_base = merge({group: "Builds"}, build_base); + queue.scheduleTask(merge(extra_base, { + name: `${name} w/ NSS_NO_PKCS11_BYPASS=1`, + env: {NSS_NO_PKCS11_BYPASS: "1"}, + symbol: "noPkcs11Bypass" + })); + + return queue.submit(); +} + +/*****************************************************************************/ + +function scheduleTests(task_build, task_cert, test_base) { + test_base = merge({kind: "test"}, test_base); + + // Schedule tests that do NOT need certificates. + let no_cert_base = merge(test_base, {parent: task_build}); + queue.scheduleTask(merge(no_cert_base, { + name: "Gtests", symbol: "Gtest", tests: "ssl_gtests gtests" + })); + queue.scheduleTask(merge(no_cert_base, { + name: "Chains tests", symbol: "Chains", tests: "chains" + })); + queue.scheduleTask(merge(no_cert_base, { + name: "Cipher tests", symbol: "Cipher", tests: "cipher" + })); + queue.scheduleTask(merge(no_cert_base, { + name: "EC tests", symbol: "EC", tests: "ec" + })); + queue.scheduleTask(merge(no_cert_base, { + name: "Lowhash tests", symbol: "Lowhash", tests: "lowhash" + })); + queue.scheduleTask(merge(no_cert_base, { + name: "SDR tests", symbol: "SDR", tests: "sdr" + })); + + // Schedule tests that need certificates. + let cert_base = merge(test_base, {parent: task_cert}); + queue.scheduleTask(merge(cert_base, { + name: "CRMF tests", symbol: "CRMF", tests: "crmf" + })); + queue.scheduleTask(merge(cert_base, { + name: "DB tests", symbol: "DB", tests: "dbtests" + })); + queue.scheduleTask(merge(cert_base, { + name: "FIPS tests", symbol: "FIPS", tests: "fips" + })); + queue.scheduleTask(merge(cert_base, { + name: "Merge tests", symbol: "Merge", tests: "merge" + })); + queue.scheduleTask(merge(cert_base, { + name: "S/MIME tests", symbol: "SMIME", tests: "smime" + })); + queue.scheduleTask(merge(cert_base, { + name: "Tools tests", symbol: "Tools", tests: "tools" + })); + + // SSL tests, need certificates too. + let ssl_base = merge(cert_base, {tests: "ssl"}); + queue.scheduleTask(merge(ssl_base, { + name: "SSL tests (standard)", symbol: "standard", cycle: "standard" + })); + queue.scheduleTask(merge(ssl_base, { + name: "SSL tests (pkix)", symbol: "pkix", cycle: "pkix" + })); + queue.scheduleTask(merge(ssl_base, { + name: "SSL tests (sharedb)", symbol: "sharedb", cycle: "sharedb" + })); + queue.scheduleTask(merge(ssl_base, { + name: "SSL tests (upgradedb)", symbol: "upgradedb", cycle: "upgradedb" + })); +} + +/*****************************************************************************/ + +async function scheduleTools() { + let base = { + image: LINUX_IMAGE, + platform: "nss-tools", + kind: "test" + }; + + queue.scheduleTask(merge(base, { + symbol: "clang-format-3.8", + name: "clang-format-3.8", + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/run_clang_format.sh" + ] + })); + + queue.scheduleTask(merge(base, { + symbol: "scan-build-3.8", + name: "scan-build-3.8", + env: { + USE_64: "1", + GCC_VERSION: "clang", + GXX_VERSION: "clang++" + }, + artifacts: { + public: { + expires: 24, + type: "directory", + path: "/home/worker/artifacts" + } + }, + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/run_scan_build.sh" + ] + })); + + return queue.submit(); +} diff --git a/automation/taskcluster/graph/src/image_builder.js b/automation/taskcluster/graph/src/image_builder.js new file mode 100644 index 000000000..093eacb44 --- /dev/null +++ b/automation/taskcluster/graph/src/image_builder.js @@ -0,0 +1,62 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import * as queue from "./queue"; +import context_hash from "./context_hash"; +import taskcluster from "taskcluster-client"; + +async function taskHasImageArtifact(taskId) { + let queue = new taskcluster.Queue(); + let {artifacts} = await queue.listLatestArtifacts(taskId); + return artifacts.some(artifact => artifact.name == "public/image.tar"); +} + +async function findTaskWithImageArtifact(ns) { + let index = new taskcluster.Index(); + let {taskId} = await index.findTask(ns); + let has_image = await taskHasImageArtifact(taskId); + return has_image ? taskId : null; +} + +export async function findTask({name, path}) { + let hash = await context_hash(path); + let ns = `docker.images.v1.${process.env.TC_PROJECT}.${name}.hash.${hash}`; + return findTaskWithImageArtifact(ns).catch(() => null); +} + +export async function buildTask({name, path}) { + let hash = await context_hash(path); + let ns = `docker.images.v1.${process.env.TC_PROJECT}.${name}.hash.${hash}`; + + return { + name: "Image Builder", + image: "taskcluster/image_builder:0.1.5", + routes: ["index." + ns], + env: { + HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY, + BASE_REPOSITORY: process.env.NSS_HEAD_REPOSITORY, + HEAD_REV: process.env.NSS_HEAD_REVISION, + HEAD_REF: process.env.NSS_HEAD_REVISION, + PROJECT: process.env.TC_PROJECT, + CONTEXT_PATH: path, + HASH: hash + }, + artifacts: { + "public/image.tar": { + type: "file", + expires: 24 * 365, + path: "/artifacts/image.tar" + } + }, + command: [ + "/bin/bash", + "-c", + "/home/worker/bin/build_image.sh" + ], + platform: "nss-decision", + features: ["dind"], + kind: "build", + symbol: "I" + }; +} diff --git a/automation/taskcluster/graph/src/index.js b/automation/taskcluster/graph/src/index.js new file mode 100644 index 000000000..4153e1b18 --- /dev/null +++ b/automation/taskcluster/graph/src/index.js @@ -0,0 +1,14 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import * as try_syntax from "./try_syntax"; +import extend from "./extend"; + +// Init try syntax filter. +if (process.env.TC_PROJECT == "nss-try") { + try_syntax.initFilter(); +} + +// Extend the task graph. +extend().catch(console.error); diff --git a/automation/taskcluster/graph/src/merge.js b/automation/taskcluster/graph/src/merge.js new file mode 100644 index 000000000..17043dd8e --- /dev/null +++ b/automation/taskcluster/graph/src/merge.js @@ -0,0 +1,10 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import {recursive as merge} from "merge"; + +// We always want to clone. +export default function (...args) { + return merge(true, ...args); +} diff --git a/automation/taskcluster/graph/src/queue.js b/automation/taskcluster/graph/src/queue.js new file mode 100644 index 000000000..dbeb2622c --- /dev/null +++ b/automation/taskcluster/graph/src/queue.js @@ -0,0 +1,235 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import {clone} from "merge"; +import merge from "./merge"; +import slugid from "slugid"; +import taskcluster from "taskcluster-client"; +import * as image_builder from "./image_builder"; + +let maps = []; +let filters = []; + +let tasks = new Map(); +let image_tasks = new Map(); + +let queue = new taskcluster.Queue({ + baseUrl: "http://taskcluster/queue/v1/" +}); + +function fromNow(hours) { + let d = new Date(); + d.setHours(d.getHours() + (hours|0)); + return d.toJSON(); +} + +function parseRoutes(routes) { + return [ + `tc-treeherder.v2.${process.env.TC_PROJECT}.${process.env.NSS_HEAD_REVISION}.${process.env.NSS_PUSHLOG_ID}`, + ...routes + ]; +} + +function parseFeatures(list) { + return list.reduce((map, feature) => { + map[feature] = true; + return map; + }, {}); +} + +function parseArtifacts(artifacts) { + let copy = clone(artifacts); + Object.keys(copy).forEach(key => { + copy[key].expires = fromNow(copy[key].expires); + }); + return copy; +} + +function parseCollection(name) { + let collection = {}; + collection[name] = true; + return collection; +} + +function parseTreeherder(def) { + let treeherder = { + build: { + platform: def.platform + }, + machine: { + platform: def.platform + }, + symbol: def.symbol, + jobKind: def.kind + }; + + if (def.group) { + treeherder.groupSymbol = def.group; + } + + if (def.collection) { + treeherder.collection = parseCollection(def.collection); + } + + if (def.tier) { + treeherder.tier = def.tier; + } + + return treeherder; +} + +function convertTask(def) { + let dependencies = def.dependencies || []; + + let env = merge({ + NSS_HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY, + NSS_HEAD_REVISION: process.env.NSS_HEAD_REVISION + }, def.env || {}); + + if (def.parent) { + dependencies.push(def.parent); + env.TC_PARENT_TASK_ID = def.parent; + } + + if (def.tests) { + env.NSS_TESTS = def.tests; + } + + if (def.cycle) { + env.NSS_CYCLES = def.cycle; + } + + let payload = { + env, + command: def.command, + maxRunTime: def.maxRunTime || 3600 + }; + + if (def.image) { + payload.image = def.image; + } + + if (def.features) { + payload.features = parseFeatures(def.features); + } + + if (def.artifacts) { + payload.artifacts = parseArtifacts(def.artifacts); + } + + return { + provisionerId: def.provisioner || "aws-provisioner-v1", + workerType: def.workerType || "hg-worker", + schedulerId: "task-graph-scheduler", + + created: fromNow(0), + deadline: fromNow(24), + + dependencies, + routes: parseRoutes(def.routes || []), + + metadata: { + name: def.name, + description: def.name, + owner: process.env.TC_OWNER, + source: process.env.TC_SOURCE + }, + + payload, + + extra: { + treeherder: parseTreeherder(def) + } + }; +} + +export function map(fun) { + maps.push(fun); +} + +export function filter(fun) { + filters.push(fun); +} + +export function scheduleTask(def) { + let taskId = slugid.v4(); + tasks.set(taskId, merge({}, def)); + return taskId; +} + +export async function submit() { + let promises = []; + + for (let [taskId, task] of tasks) { + // Allow filtering tasks before we schedule them. + if (!filters.every(filter => filter(task))) { + continue; + } + + // Allow changing tasks before we schedule them. + maps.forEach(map => { task = map(merge({}, task)) }); + + let log_id = `${task.name} @ ${task.platform}[${task.collection || "opt"}]`; + console.log(`+ Submitting ${log_id}.`); + + // Convert the task definition. + task = await convertTask(task); + + // Convert the docker image definition. + let image_def = task.payload.image; + if (image_def && image_def.hasOwnProperty("path")) { + let key = `${image_def.name}:${image_def.path}`; + let data = {}; + + // Check the cache first. + if (image_tasks.has(key)) { + data = image_tasks.get(key); + } else { + data.taskId = await image_builder.findTask(image_def); + data.isPending = !data.taskId; + + // No task found. + if (data.isPending) { + let image_task = await image_builder.buildTask(image_def); + + // Schedule a new image builder task immediately. + data.taskId = slugid.v4(); + + try { + await queue.createTask(data.taskId, convertTask(image_task)); + } catch (e) { + console.error("! FAIL: Scheduling image builder task failed."); + continue; /* Skip this task on failure. */ + } + } + + // Store in cache. + image_tasks.set(key, data); + } + + if (data.isPending) { + task.dependencies.push(data.taskId); + } + + task.payload.image = { + path: "public/image.tar", + taskId: data.taskId, + type: "task-image" + }; + } + + // Schedule the task. + promises.push(queue.createTask(taskId, task).catch(err => { + console.error(`! FAIL: Scheduling ${log_id} failed.`, err); + })); + } + + // Wait for all requests to finish. + if (promises.length) { + await Promise.all(promises); + console.log("=== Total:", promises.length, "tasks. ==="); + } + + tasks.clear(); +} diff --git a/automation/taskcluster/graph/try_syntax.js b/automation/taskcluster/graph/src/try_syntax.js index b2fbfbefa..032d4515c 100644 --- a/automation/taskcluster/graph/try_syntax.js +++ b/automation/taskcluster/graph/src/try_syntax.js @@ -2,8 +2,9 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -var intersect = require("intersect"); -var parse_args = require("minimist"); +import * as queue from "./queue"; +import intersect from "intersect"; +import parse_args from "minimist"; function parseOptions(opts) { opts = parse_args(opts.split(/\s+/), { @@ -13,7 +14,7 @@ function parseOptions(opts) { }); // Parse build types (d=debug, o=opt). - var builds = intersect(opts.build.split(""), ["d", "o"]); + let builds = intersect(opts.build.split(""), ["d", "o"]); // If the given value is nonsense default to debug and opt builds. if (builds.length == 0) { @@ -21,8 +22,8 @@ function parseOptions(opts) { } // Parse platforms. - var allPlatforms = ["linux", "linux64", "linux64-asan", "win64", "arm"]; - var platforms = intersect(opts.platform.split(/\s*,\s*/), allPlatforms); + let allPlatforms = ["linux", "linux64", "linux64-asan", "win64", "arm"]; + let platforms = intersect(opts.platform.split(/\s*,\s*/), allPlatforms); // If the given value is nonsense or "none" default to all platforms. if (platforms.length == 0 && opts.platform != "none") { @@ -30,9 +31,9 @@ function parseOptions(opts) { } // Parse unit tests. - var allUnitTests = ["crmf", "chains", "cipher", "db", "ec", "fips", "gtest", + let allUnitTests = ["crmf", "chains", "cipher", "db", "ec", "fips", "gtest", "lowhash", "merge", "sdr", "smime", "tools", "ssl"]; - var unittests = intersect(opts.unittests.split(/\s*,\s*/), allUnitTests); + let unittests = intersect(opts.unittests.split(/\s*,\s*/), allUnitTests); // If the given value is "all" run all tests. // If it's nonsense then don't run any tests. @@ -43,8 +44,8 @@ function parseOptions(opts) { } // Parse tools. - var allTools = ["clang-format", "scan-build"]; - var tools = intersect(opts.tools.split(/\s*,\s*/), allTools); + let allTools = ["clang-format", "scan-build"]; + let tools = intersect(opts.tools.split(/\s*,\s*/), allTools); // If the given value is "all" run all tools. // If it's nonsense then don't run any tools. @@ -63,36 +64,20 @@ function parseOptions(opts) { }; } -function filterTasks(tasks, comment) { - // Check for try syntax in changeset comment. - var match = comment.match(/^\s*try:\s*(.*)\s*$/); - if (!match) { - return tasks; - } - - var opts = parseOptions(match[1]); - - return tasks.filter(function (task) { - var env = task.task.payload.env || {}; - var th = task.task.extra.treeherder; - var machine = th.machine.platform; - var coll = th.collection || {}; - var found; - +function filter(opts) { + return function (task) { // Filter tools. We can immediately return here as those // are not affected by platform or build type selectors. - if (machine == "nss-tools") { - return opts.tools.some(function (tool) { - var symbol = th.symbol.toLowerCase(); - return symbol.startsWith(tool); + if (task.platform == "nss-tools") { + return opts.tools.some(tool => { + return task.symbol.toLowerCase().startsWith(tool); }); } // Filter unit tests. - if (env.NSS_TESTS && env.TC_PARENT_TASK_ID) { - found = opts.unittests.some(function (test) { - var symbol = (th.groupSymbol || th.symbol).toLowerCase(); - return symbol.startsWith(test); + if (task.tests) { + let found = opts.unittests.some(test => { + return (task.group || task.symbol).toLowerCase().startsWith(test); }); if (!found) { @@ -101,13 +86,15 @@ function filterTasks(tasks, comment) { } // Filter extra builds. - if (th.groupSymbol == "Builds" && !opts.extra) { + if (task.group == "Builds" && !opts.extra) { return false; } + let coll = name => name == (task.collection || "opt"); + // Filter by platform. - found = opts.platforms.some(function (platform) { - var aliases = { + let found = opts.platforms.some(platform => { + let aliases = { "linux": "linux32", "linux64-asan": "linux64", "win64": "windows2012-64", @@ -115,15 +102,15 @@ function filterTasks(tasks, comment) { }; // Check the platform name. - var keep = machine == (aliases[platform] || platform); + let keep = (task.platform == (aliases[platform] || platform)); // Additional checks. if (platform == "linux64-asan") { - keep &= coll.asan; + keep &= coll("asan"); } else if (platform == "arm") { - keep &= (coll["arm-opt"] || coll["arm-debug"]); + keep &= coll("arm-opt") || coll("arm-debug"); } else { - keep &= (coll.opt || coll.debug); + keep &= coll("opt") || coll("debug"); } return keep; @@ -134,10 +121,20 @@ function filterTasks(tasks, comment) { } // Finally, filter by build type. - var isDebug = coll.debug || coll.asan || coll["arm-debug"]; - return (isDebug && opts.builds.indexOf("d") > -1) || - (!isDebug && opts.builds.indexOf("o") > -1); - }); + let isDebug = coll("debug") || coll("asan") || coll("arm-debug"); + return (isDebug && opts.builds.includes("d")) || + (!isDebug && opts.builds.includes("o")); + } } -module.exports.filterTasks = filterTasks; +export function initFilter() { + let comment = process.env.TC_COMMENT || ""; + + // Check for try syntax in changeset comment. + let match = comment.match(/^\s*try:\s*(.*)\s*$/); + + // Add try syntax filter. + if (match) { + queue.filter(filter(parseOptions(match[1]))); + } +} diff --git a/automation/taskcluster/graph/tests/chains.yml b/automation/taskcluster/graph/tests/chains.yml deleted file mode 100644 index b553d8d39..000000000 --- a/automation/taskcluster/graph/tests/chains.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- task: - metadata: - name: Chains tests - description: Chains tests - - payload: - maxRunTime: 14400 - env: - NSS_TESTS: chains - - extra: - treeherder: - symbol: Chains diff --git a/automation/taskcluster/graph/tests/cipher.yml b/automation/taskcluster/graph/tests/cipher.yml deleted file mode 100644 index e1d0f9faa..000000000 --- a/automation/taskcluster/graph/tests/cipher.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- task: - metadata: - name: Cipher tests - description: Cipher tests - - payload: - env: - NSS_TESTS: cipher - - extra: - treeherder: - symbol: Cipher diff --git a/automation/taskcluster/graph/tests/crmf.yml b/automation/taskcluster/graph/tests/crmf.yml deleted file mode 100644 index a29a9c5e2..000000000 --- a/automation/taskcluster/graph/tests/crmf.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- task: - metadata: - name: CRMF tests - description: CRMF tests - - payload: - env: - ASAN_OPTIONS: detect_leaks=0 - NSS_TESTS: crmf - - extra: - treeherder: - symbol: CRMF diff --git a/automation/taskcluster/graph/tests/db.yml b/automation/taskcluster/graph/tests/db.yml deleted file mode 100644 index b5ee77653..000000000 --- a/automation/taskcluster/graph/tests/db.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- task: - metadata: - name: DB tests - description: DB tests - - payload: - env: - NSS_TESTS: dbtests - - extra: - treeherder: - symbol: DB diff --git a/automation/taskcluster/graph/tests/ec.yml b/automation/taskcluster/graph/tests/ec.yml deleted file mode 100644 index c6e21917e..000000000 --- a/automation/taskcluster/graph/tests/ec.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- task: - metadata: - name: EC tests - description: EC tests - - payload: - env: - NSS_TESTS: ec - - extra: - treeherder: - symbol: EC diff --git a/automation/taskcluster/graph/tests/fips.yml b/automation/taskcluster/graph/tests/fips.yml deleted file mode 100644 index 58484c7b2..000000000 --- a/automation/taskcluster/graph/tests/fips.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- task: - metadata: - name: FIPS tests - description: FIPS tests - - payload: - env: - ASAN_OPTIONS: detect_leaks=0 - NSS_TESTS: fips - - extra: - treeherder: - symbol: FIPS diff --git a/automation/taskcluster/graph/tests/gtests.yml b/automation/taskcluster/graph/tests/gtests.yml deleted file mode 100644 index d4a355a24..000000000 --- a/automation/taskcluster/graph/tests/gtests.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- task: - metadata: - name: GTests - description: GTests - - payload: - env: - NSS_TESTS: ssl_gtests gtests - - extra: - treeherder: - symbol: GTest diff --git a/automation/taskcluster/graph/tests/lowhash.yml b/automation/taskcluster/graph/tests/lowhash.yml deleted file mode 100644 index 097a92377..000000000 --- a/automation/taskcluster/graph/tests/lowhash.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- task: - metadata: - name: Lowhash tests - description: Lowhash tests - - payload: - env: - NSS_TESTS: lowhash - - extra: - treeherder: - symbol: Lowhash diff --git a/automation/taskcluster/graph/tests/merge.yml b/automation/taskcluster/graph/tests/merge.yml deleted file mode 100644 index 7db008a1d..000000000 --- a/automation/taskcluster/graph/tests/merge.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- task: - metadata: - name: Merge tests - description: Merge tests - - payload: - env: - NSS_TESTS: merge - - extra: - treeherder: - symbol: Merge diff --git a/automation/taskcluster/graph/tests/sdr.yml b/automation/taskcluster/graph/tests/sdr.yml deleted file mode 100644 index 12d881eda..000000000 --- a/automation/taskcluster/graph/tests/sdr.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- task: - metadata: - name: SDR tests - description: SDR tests - - payload: - env: - NSS_TESTS: sdr - - extra: - treeherder: - symbol: SDR diff --git a/automation/taskcluster/graph/tests/smime.yml b/automation/taskcluster/graph/tests/smime.yml deleted file mode 100644 index f5c3ab8fb..000000000 --- a/automation/taskcluster/graph/tests/smime.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- task: - metadata: - name: S/MIME tests - description: S/MIME tests - - payload: - env: - NSS_TESTS: smime - - extra: - treeherder: - symbol: SMIME diff --git a/automation/taskcluster/graph/tests/ssl.yml b/automation/taskcluster/graph/tests/ssl.yml deleted file mode 100644 index da66c4384..000000000 --- a/automation/taskcluster/graph/tests/ssl.yml +++ /dev/null @@ -1,65 +0,0 @@ ---- -- task: - metadata: - name: "SSL tests (standard)" - description: "SSL tests (standard)" - - payload: - maxRunTime: 14400 - env: - NSS_CYCLES: standard - NSS_TESTS: ssl - - extra: - treeherder: - symbol: standard - groupSymbol: SSL - groupName: SSL tests - -- task: - metadata: - name: "SSL tests (pkix)" - description: "SSL tests (pkix)" - - payload: - env: - NSS_CYCLES: pkix - NSS_TESTS: ssl - - extra: - treeherder: - symbol: pkix - groupSymbol: SSL - groupName: SSL tests - -- task: - metadata: - name: "SSL tests (sharedb)" - description: "SSL tests (sharedb)" - - payload: - env: - NSS_CYCLES: sharedb - NSS_TESTS: ssl - - extra: - treeherder: - symbol: sharedb - groupSymbol: SSL - groupName: SSL tests - -- task: - metadata: - name: "SSL tests (upgradedb)" - description: "SSL tests (upgradedb)" - - payload: - env: - NSS_CYCLES: upgradedb - NSS_TESTS: ssl - - extra: - treeherder: - symbol: upgradedb - groupSymbol: SSL - groupName: SSL tests diff --git a/automation/taskcluster/graph/tests/tools.yml b/automation/taskcluster/graph/tests/tools.yml deleted file mode 100644 index 2edcce01a..000000000 --- a/automation/taskcluster/graph/tests/tools.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- task: - metadata: - name: Tools tests - description: Tools tests - - payload: - env: - NSS_TESTS: tools - - extra: - treeherder: - symbol: Tools diff --git a/automation/taskcluster/graph/tools/_build_base.yml b/automation/taskcluster/graph/tools/_build_base.yml deleted file mode 100644 index cd7244b97..000000000 --- a/automation/taskcluster/graph/tools/_build_base.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -reruns: 0 - -task: - created: !from_now 0 - deadline: !from_now 24 - provisionerId: aws-provisioner-v1 - workerType: hg-worker - schedulerId: task-graph-scheduler - - metadata: - owner: !env TC_OWNER - source: !env TC_SOURCE - - payload: - maxRunTime: 3600 - image: automation/taskcluster/docker - - env: - NSS_HEAD_REPOSITORY: !env NSS_HEAD_REPOSITORY - NSS_HEAD_REVISION: !env NSS_HEAD_REVISION - - extra: - treeherder: - build: - platform: nss-tools - machine: - platform: nss-tools - jobKind: test diff --git a/automation/taskcluster/graph/tools/clang-format.yml b/automation/taskcluster/graph/tools/clang-format.yml deleted file mode 100644 index 16c18dd2c..000000000 --- a/automation/taskcluster/graph/tools/clang-format.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -- task: - metadata: - name: clang-format-3.8 - description: clang-format-3.8 - - payload: - command: - - "/bin/bash" - - "-c" - - "bin/checkout.sh && nss/automation/taskcluster/scripts/run_clang_format.sh" - - extra: - treeherder: - symbol: clang-format-3.8 diff --git a/automation/taskcluster/graph/tools/scan-build.yml b/automation/taskcluster/graph/tools/scan-build.yml deleted file mode 100644 index 8733d845d..000000000 --- a/automation/taskcluster/graph/tools/scan-build.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -- task: - metadata: - name: scan-build-3.8 - description: scan-build-3.8 - - payload: - artifacts: - public: - type: directory - path: /home/worker/artifacts - expires: !from_now 24 - - command: - - "/bin/bash" - - "-c" - - "bin/checkout.sh && nss/automation/taskcluster/scripts/run_scan_build.sh" - - env: - GCC_VERSION: clang - GXX_VERSION: clang++ - USE_64: 1 - - extra: - treeherder: - symbol: scan-build-3.8 diff --git a/automation/taskcluster/graph/windows/_build_base.yml b/automation/taskcluster/graph/windows/_build_base.yml deleted file mode 100644 index a75a7a1af..000000000 --- a/automation/taskcluster/graph/windows/_build_base.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -reruns: 2 - -task: - created: !from_now 0 - deadline: !from_now 24 - provisionerId: aws-provisioner-v1 - workerType: nss-win2012r2 - schedulerId: task-graph-scheduler - - metadata: - owner: !env TC_OWNER - source: !env TC_SOURCE - - payload: - maxRunTime: 3600 - - artifacts: - - type: directory - path: "public\\build" - expires: !from_now 24 - - command: - - "bash -c \"hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || (sleep 2; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss) || (sleep 5; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss)\"" - - "bash -c nss/automation/taskcluster/windows/build.sh" - - env: - PATH: "c:\\mozilla-build\\python;c:\\mozilla-build\\msys\\local\\bin;c:\\mozilla-build\\7zip;c:\\mozilla-build\\info-zip;c:\\mozilla-build\\python\\Scripts;c:\\mozilla-build\\yasm;c:\\mozilla-build\\msys\\bin;c:\\Windows\\system32;c:\\mozilla-build\\upx391w;c:\\mozilla-build\\moztools-x64\\bin;c:\\mozilla-build\\wget" - NSS_HEAD_REPOSITORY: !env NSS_HEAD_REPOSITORY - NSS_HEAD_REVISION: !env NSS_HEAD_REVISION - DOMSUF: localdomain - HOST: localhost - - extra: - treeherder: - jobKind: build - symbol: B diff --git a/automation/taskcluster/graph/windows/_test_base.yml b/automation/taskcluster/graph/windows/_test_base.yml deleted file mode 100644 index 0ffaabc75..000000000 --- a/automation/taskcluster/graph/windows/_test_base.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -reruns: 2 - -task: - created: !from_now 0 - deadline: !from_now 24 - provisionerId: aws-provisioner-v1 - workerType: nss-win2012r2 - schedulerId: task-graph-scheduler - - metadata: - owner: !env TC_OWNER - source: !env TC_SOURCE - - payload: - maxRunTime: 3600 - - command: - - "hg clone -r %NSS_HEAD_REVISION% %NSS_HEAD_REPOSITORY% nss" - - "bash -c nss/automation/taskcluster/windows/run_tests.sh" - - extra: - treeherder: - jobKind: test diff --git a/automation/taskcluster/graph/windows/build64-debug.yml b/automation/taskcluster/graph/windows/build64-debug.yml deleted file mode 100644 index ccc47f2d7..000000000 --- a/automation/taskcluster/graph/windows/build64-debug.yml +++ /dev/null @@ -1,55 +0,0 @@ ---- -- task: - metadata: - name: "Windows 2012 64 (debug)" - description: "Windows 2012 64 (debug)" - - payload: - env: - USE_64: 1 - - extra: - treeherder: - build: - platform: windows2012-64 - machine: - platform: windows2012-64 - collection: - debug: true - - tests: - - chains - - cipher - - crmf - - db - - ec - - fips - - gtests - - lowhash - - merge - - sdr - - smime - - ssl - - tools - -- task: - metadata: - name: "Windows 2012 64 (debug, NSS_NO_PKCS11_BYPASS=1)" - description: "Windows 2012 64 (debug, NSS_NO_PKCS11_BYPASS=1)" - - payload: - env: - NSS_NO_PKCS11_BYPASS: 1 - USE_64: 1 - - extra: - treeherder: - build: - platform: windows2012-64 - machine: - platform: windows2012-64 - collection: - debug: true - groupSymbol: Builds - groupName: Various builds - symbol: noPkcs11Bypass diff --git a/automation/taskcluster/graph/windows/build64-opt.yml b/automation/taskcluster/graph/windows/build64-opt.yml deleted file mode 100644 index ad4d46e31..000000000 --- a/automation/taskcluster/graph/windows/build64-opt.yml +++ /dev/null @@ -1,57 +0,0 @@ ---- -- task: - metadata: - name: "Windows 2012 64 (opt)" - description: "Windows 2012 64 (opt)" - - payload: - env: - BUILD_OPT: 1 - USE_64: 1 - - extra: - treeherder: - build: - platform: windows2012-64 - machine: - platform: windows2012-64 - collection: - opt: true - - tests: - - chains - - cipher - - crmf - - db - - ec - - fips - - gtests - - lowhash - - merge - - sdr - - smime - - ssl - - tools - -- task: - metadata: - name: "Windows 2012 64 (opt, NSS_NO_PKCS11_BYPASS=1)" - description: "Windows 2012 64 (opt, NSS_NO_PKCS11_BYPASS=1)" - - payload: - env: - NSS_NO_PKCS11_BYPASS: 1 - BUILD_OPT: 1 - USE_64: 1 - - extra: - treeherder: - build: - platform: windows2012-64 - machine: - platform: windows2012-64 - collection: - opt: true - groupSymbol: Builds - groupName: Various builds - symbol: noPkcs11Bypass diff --git a/automation/taskcluster/graph/yaml.js b/automation/taskcluster/graph/yaml.js deleted file mode 100644 index dd4f6ba47..000000000 --- a/automation/taskcluster/graph/yaml.js +++ /dev/null @@ -1,51 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -var fs = require("fs"); -var yaml = require("js-yaml"); - -// Register custom YAML types. -var YAML_SCHEMA = yaml.Schema.create([ - // Point in time at $now + x hours. - new yaml.Type('!from_now', { - kind: "scalar", - - resolve: function (data) { - return true; - }, - - construct: function (data) { - var d = new Date(); - d.setHours(d.getHours() + (data|0)); - return d.toJSON(); - } - }), - - // Environment variables. - new yaml.Type('!env', { - kind: "scalar", - - resolve: function (data) { - return true; - }, - - construct: function (data) { - return process.env[data] || "{{" + data.toLowerCase() + "}}"; - } - }) -]); - -// Parse a given YAML file. -function parse(file, fallback) { - // Return fallback if the file doesn't exist. - if (!fs.existsSync(file) && fallback) { - return fallback; - } - - // Otherwise, read the file or fail. - var source = fs.readFileSync(file, "utf-8"); - return yaml.load(source, {schema: YAML_SCHEMA}); -} - -module.exports.parse = parse; diff --git a/automation/taskcluster/scripts/build.sh b/automation/taskcluster/scripts/build.sh index a6a202164..c44a321d8 100755 --- a/automation/taskcluster/scripts/build.sh +++ b/automation/taskcluster/scripts/build.sh @@ -18,12 +18,6 @@ hg_clone https://hg.mozilla.org/projects/nspr nspr default # Build. make -C nss nss_build_all -# Generate certificates. -NSS_TESTS=cert NSS_CYCLES="standard pkix sharedb" $(dirname $0)/run_tests.sh - -# Reset test counter so that test runs pick up our certificates. -echo 1 > tests_results/security/localhost - # Package. mkdir artifacts -tar cvfjh artifacts/dist.tar.bz2 dist tests_results +tar cvfjh artifacts/dist.tar.bz2 dist diff --git a/automation/taskcluster/scripts/extend_task_graph.sh b/automation/taskcluster/scripts/extend_task_graph.sh index abbd4ecd7..5a3fb8d98 100755 --- a/automation/taskcluster/scripts/extend_task_graph.sh +++ b/automation/taskcluster/scripts/extend_task_graph.sh @@ -12,5 +12,5 @@ mkdir -p /home/worker/artifacts # Install Node.JS dependencies. cd nss/automation/taskcluster/graph/ && npm install -# Build the task graph definition. -nodejs build.js > /home/worker/artifacts/graph.json +# Extend the task graph. +node lib/index.js diff --git a/automation/taskcluster/scripts/gen_certs.sh b/automation/taskcluster/scripts/gen_certs.sh new file mode 100755 index 000000000..56b690d67 --- /dev/null +++ b/automation/taskcluster/scripts/gen_certs.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -v -e -x + +source $(dirname $0)/tools.sh + +if [ $(id -u) = 0 ]; then + # Set compiler. + switch_compilers + + # Stupid Docker. + echo "127.0.0.1 localhost.localdomain" >> /etc/hosts + + # Drop privileges by re-running this script. + exec su worker $0 +fi + +# Fetch artifact if needed. +fetch_dist + +# Generate certificates. +NSS_TESTS=cert NSS_CYCLES="standard pkix sharedb" $(dirname $0)/run_tests.sh + +# Reset test counter so that test runs pick up our certificates. +echo 1 > tests_results/security/localhost + +# Package. +mkdir artifacts +tar cvfjh artifacts/dist.tar.bz2 dist tests_results diff --git a/automation/taskcluster/scripts/run_tests.sh b/automation/taskcluster/scripts/run_tests.sh index b917a9d48..d8341ee82 100755 --- a/automation/taskcluster/scripts/run_tests.sh +++ b/automation/taskcluster/scripts/run_tests.sh @@ -2,9 +2,9 @@ set -v -e -x -if [ $(id -u) = 0 ]; then - source $(dirname $0)/tools.sh +source $(dirname $0)/tools.sh +if [ $(id -u) = 0 ]; then # Set compiler. switch_compilers @@ -16,10 +16,7 @@ if [ $(id -u) = 0 ]; then fi # Fetch artifact if needed. -if [ ! -d "dist" ]; then - curl --retry 3 -Lo dist.tar.bz2 https://queue.taskcluster.net/v1/task/$TC_PARENT_TASK_ID/artifacts/public/dist.tar.bz2 - tar xvjf dist.tar.bz2 -fi +fetch_dist # Run tests. cd nss/tests && ./all.sh diff --git a/automation/taskcluster/scripts/tools.sh b/automation/taskcluster/scripts/tools.sh index 57f45d6be..632c909c2 100644 --- a/automation/taskcluster/scripts/tools.sh +++ b/automation/taskcluster/scripts/tools.sh @@ -27,3 +27,15 @@ hg_clone() { done exit 1 } + +fetch_dist() { + url=https://queue.taskcluster.net/v1/task/$TC_PARENT_TASK_ID/artifacts/public/dist.tar.bz2 + if [ ! -d "dist" ]; then + for i in 0 2 5; do + sleep $i + curl --retry 3 -Lo dist.tar.bz2 $url && tar xvjf dist.tar.bz2 && return + rm -fr dist.tar.bz2 dist + done + exit 1 + fi +} diff --git a/automation/taskcluster/windows/build.sh b/automation/taskcluster/windows/build.sh index c275df54d..6c8a47470 100644 --- a/automation/taskcluster/windows/build.sh +++ b/automation/taskcluster/windows/build.sh @@ -11,11 +11,5 @@ hg_clone https://hg.mozilla.org/projects/nspr nspr default # Build. make -C nss nss_build_all -# Generate certificates. -NSS_TESTS=cert NSS_CYCLES="standard pkix sharedb" nss/tests/all.sh - -# Reset test counter so that test runs pick up our certificates. -echo 1 > tests_results/security/localhost - # Package. -7z a public/build/dist.7z dist tests_results +7z a public/build/dist.7z dist diff --git a/automation/taskcluster/windows/gen_certs.sh b/automation/taskcluster/windows/gen_certs.sh new file mode 100644 index 000000000..ead16bbc3 --- /dev/null +++ b/automation/taskcluster/windows/gen_certs.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -v -e -x + +# Set up the toolchain. +source $(dirname $0)/setup.sh + +# Fetch artifact. +wget -t 3 --retry-connrefused -w 5 --random-wait https://queue.taskcluster.net/v1/task/$TC_PARENT_TASK_ID/artifacts/public/build/dist.7z -O dist.7z +7z x dist.7z + +# Generate certificates. +NSS_TESTS=cert NSS_CYCLES="standard pkix sharedb" nss/tests/all.sh + +# Reset test counter so that test runs pick up our certificates. +echo 1 > tests_results/security/localhost + +# Package. +7z a public/build/dist.7z dist tests_results |