Bug 1802331 - compress docker image artifact with zstd. r=nss-reviewers,releng-reviewers,jlorenzo,bbeurdouche

When we moved tasks to run on GCP from AWS in bug 1799315, we started
using a newer version of docker-worker including the changes from bug
1637302; as a result, artifacts are compressed with gzip before upload
to s3, and downloads now come with a "content-encoding: gzip" header and
compressed content, regardless of the client's "accept-encoding".
Unfortunately docker-worker doesn't handle that encoding and expects an
artifact called image.tar to be uncompressed.  To work around that
issue, we now compress docker images in image_builder with zstd before
upload.

[Ideally we'd install the zstd package in the nssdev/image_builder
docker image itself instead of doing it in every task, however I'm not
sure who owns that or how it's built so this might be good enough for
right now.]

Differential Revision: https://phabricator.services.mozilla.com/D163306

3 files changed, 8 insertions, 5 deletions

diff --git a/automation/taskcluster/graph/src/image_builder.js b/automation/taskcluster/graph/src/image_builder.js
index b69b31602..7bf632d29 100644
--- a/automation/taskcluster/graph/src/image_builder.js
+++ b/automation/taskcluster/graph/src/image_builder.js
@@ -9,7 +9,7 @@ import taskcluster from "taskcluster-client";
 async function taskHasImageArtifact(taskId) {
   let queue = new taskcluster.Queue(taskcluster.fromEnvVars());
   let {artifacts} = await queue.listLatestArtifacts(taskId);
-  return artifacts.some(artifact => artifact.name == "public/image.tar");
+  return artifacts.some(artifact => artifact.name == "public/image.tar.zst");
 }
 
 async function findTaskWithImageArtifact(ns) {
@@ -41,10 +41,10 @@ export async function buildTask({name, path}) {
       HASH: hash
     },
     artifacts: {
-      "public/image.tar": {
+      "public/image.tar.zst": {
         type: "file",
         expires: 24 * 90,
-        path: "/artifacts/image.tar"
+        path: "/artifacts/image.tar.zst"
       }
     },
     command: [
diff --git a/automation/taskcluster/graph/src/queue.js b/automation/taskcluster/graph/src/queue.js
index 2e62d4bb7..1baa60417 100644
--- a/automation/taskcluster/graph/src/queue.js
+++ b/automation/taskcluster/graph/src/queue.js
@@ -277,7 +277,7 @@ export async function submit() {
       }
 
       task.payload.image = {
-        path: "public/image.tar",
+        path: "public/image.tar.zst",
         taskId: data.taskId,
         type: "task-image"
       };
diff --git a/automation/taskcluster/scripts/build_image.sh b/automation/taskcluster/scripts/build_image.sh
index b8715dbe9..3a469cf22 100755
--- a/automation/taskcluster/scripts/build_image.sh
+++ b/automation/taskcluster/scripts/build_image.sh
@@ -18,7 +18,10 @@ CONTEXT_PATH="/home/worker/nss/$CONTEXT_PATH"
 test -d "$CONTEXT_PATH" || raise_error "Context Path $CONTEXT_PATH does not exist."
 test -f "$CONTEXT_PATH/Dockerfile" || raise_error "Dockerfile must be present in $CONTEXT_PATH."
 
+apt-get update
+apt-get -y install zstd
+
 docker build -t "$PROJECT:$HASH" "$CONTEXT_PATH"
 
 mkdir /artifacts
-docker save "$PROJECT:$HASH" > /artifacts/image.tar
+docker save "$PROJECT:$HASH" | zstd > /artifacts/image.tar.zst