Auto merge of #59926 - pietroalbini:android-sdk-manager, r=alexcrichton

ci: use a custom android sdk manager with pinning and mirroring

Google's own sdkmanager has two issues that make it unsuitable for us:

* Mirroring has to be done manually, which is annoying because we need to figure out on our own all the URLs to copy (I couldn't find any documentation when building this PR, had to use mitmproxy).
* There is no support for pinning, which means an update on Google's side can break our CI, as it happened multiple times.

This PR replaces all our usage of sdkmanager with a custom Python script which mimics its behavior, but with the two issues fixes.

sdkmanager's logic for installing packages is thankfully very simple: the package name (like `system-images;android-18;default;armeabi-v7a`) is the directory where the package should live (with `;` replaced with `/`), so to install a package we only need to extract its contents in the right directory.

r? @alexcrichton
cc @kennytm
fixes https://github.com/rust-lang/rust/issues/59778

4 files changed, 224 insertions, 68 deletions

diff --git a/src/ci/docker/arm-android/Dockerfile b/src/ci/docker/arm-android/Dockerfile
index bbf700ae233..b934d1ce971 100644
--- a/src/ci/docker/arm-android/Dockerfile
+++ b/src/ci/docker/arm-android/Dockerfile
@@ -7,23 +7,21 @@ COPY scripts/android-ndk.sh /scripts/
 RUN . /scripts/android-ndk.sh && \
     download_and_make_toolchain android-ndk-r15c-linux-x86_64.zip arm 14
 
-# Note:
-# Do not upgrade to `openjdk-9-jre-headless`, as it will cause certificate error
-# when installing the Android SDK (see PR #45193). This is unfortunate, but
-# every search result suggested either disabling HTTPS or replacing JDK 9 by
-# JDK 8 as the solution (e.g. https://stackoverflow.com/q/41421340). :|
 RUN dpkg --add-architecture i386 && \
     apt-get update && \
     apt-get install -y --no-install-recommends \
   libgl1-mesa-glx \
   libpulse0 \
   libstdc++6:i386 \
-  openjdk-8-jre-headless \
-  tzdata
+  openjdk-9-jre-headless \
+  tzdata \
+  wget \
+  python3
 
 COPY scripts/android-sdk.sh /scripts/
-RUN . /scripts/android-sdk.sh && \
-    download_and_create_avd 4333796 armeabi-v7a 18 5264690
+COPY scripts/android-sdk-manager.py /scripts/
+COPY arm-android/android-sdk.lock /android/sdk/android-sdk.lock
+RUN /scripts/android-sdk.sh
 
 ENV PATH=$PATH:/android/sdk/emulator
 ENV PATH=$PATH:/android/sdk/tools
diff --git a/src/ci/docker/arm-android/android-sdk.lock b/src/ci/docker/arm-android/android-sdk.lock
new file mode 100644
index 00000000000..a1be8a4346b
--- /dev/null
+++ b/src/ci/docker/arm-android/android-sdk.lock
@@ -0,0 +1,6 @@
+emulator emulator-linux-5264690.zip 48c1cda2bdf3095d9d9d5c010fbfb3d6d673e3ea
+patcher;v4 3534162-studio.sdk-patcher.zip 046699c5e2716ae11d77e0bad814f7f33fab261e
+platform-tools platform-tools_r28.0.2-linux.zip 46a4c02a9b8e4e2121eddf6025da3c979bf02e28
+platforms;android-18 android-18_r03.zip e6b09b3505754cbbeb4a5622008b907262ee91cb
+system-images;android-18;default;armeabi-v7a sys-img/android/armeabi-v7a-18_r05.zip 580b583720f7de671040d5917c8c9db0c7aa03fd
+tools sdk-tools-linux-4333796.zip 8c7c28554a32318461802c1291d76fccfafde054
diff --git a/src/ci/docker/scripts/android-sdk-manager.py b/src/ci/docker/scripts/android-sdk-manager.py
new file mode 100755
index 00000000000..7c9a8b82e92
--- /dev/null
+++ b/src/ci/docker/scripts/android-sdk-manager.py
@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+# Simpler reimplementation of Android's sdkmanager
+# Extra features of this implementation are pinning and mirroring
+
+# These URLs are the Google repositories containing the list of available
+# packages and their versions. The list has been generated by listing the URLs
+# fetched while executing `tools/bin/sdkmanager --list`
+BASE_REPOSITORY = "https://dl.google.com/android/repository/"
+REPOSITORIES = [
+    "sys-img/android/sys-img2-1.xml",
+    "sys-img/android-wear/sys-img2-1.xml",
+    "sys-img/android-wear-cn/sys-img2-1.xml",
+    "sys-img/android-tv/sys-img2-1.xml",
+    "sys-img/google_apis/sys-img2-1.xml",
+    "sys-img/google_apis_playstore/sys-img2-1.xml",
+    "addon2-1.xml",
+    "glass/addon2-1.xml",
+    "extras/intel/addon2-1.xml",
+    "repository2-1.xml",
+]
+
+# Available hosts: linux, macosx and windows
+HOST_OS = "linux"
+
+# Mirroring options
+MIRROR_BUCKET = "rust-lang-ci2"
+MIRROR_BASE_DIR = "rust-ci-mirror/android/"
+
+import argparse
+import hashlib
+import os
+import subprocess
+import sys
+import tempfile
+import urllib.request
+import xml.etree.ElementTree as ET
+
+class Package:
+    def __init__(self, path, url, sha1, deps=None):
+        if deps is None:
+            deps = []
+        self.path = path.strip()
+        self.url = url.strip()
+        self.sha1 = sha1.strip()
+        self.deps = deps
+
+    def download(self, base_url):
+        _, file = tempfile.mkstemp()
+        url = base_url + self.url
+        subprocess.run(["curl", "-o", file, url], check=True)
+        # Ensure there are no hash mismatches
+        with open(file, "rb") as f:
+            sha1 = hashlib.sha1(f.read()).hexdigest()
+            if sha1 != self.sha1:
+                raise RuntimeError(
+                    "hash mismatch for package " + self.path + ": " +
+                    sha1 + " vs " + self.sha1 + " (known good)"
+                )
+        return file
+
+    def __repr__(self):
+        return "<Package "+self.path+" at "+self.url+" (sha1="+self.sha1+")"
+
+def fetch_url(url):
+    page = urllib.request.urlopen(url)
+    return page.read()
+
+def fetch_repository(base, repo_url):
+    packages = {}
+    root = ET.fromstring(fetch_url(base + repo_url))
+    for package in root:
+        if package.tag != "remotePackage":
+            continue
+        path = package.attrib["path"]
+
+        for archive in package.find("archives"):
+            host_os = archive.find("host-os")
+            if host_os is not None and host_os.text != HOST_OS:
+                continue
+            complete = archive.find("complete")
+            url = os.path.join(os.path.dirname(repo_url), complete.find("url").text)
+            sha1 = complete.find("checksum").text
+
+            deps = []
+            dependencies = package.find("dependencies")
+            if dependencies is not None:
+                for dep in dependencies:
+                    deps.append(dep.attrib["path"])
+
+            packages[path] = Package(path, url, sha1, deps)
+            break
+
+    return packages
+
+def fetch_repositories():
+    packages = {}
+    for repo in REPOSITORIES:
+        packages.update(fetch_repository(BASE_REPOSITORY, repo))
+    return packages
+
+class Lockfile:
+    def __init__(self, path):
+        self.path = path
+        self.packages = {}
+        if os.path.exists(path):
+            with open(path) as f:
+                for line in f:
+                    path, url, sha1 = line.split(" ")
+                    self.packages[path] = Package(path, url, sha1)
+
+    def add(self, packages, name, *, update=True):
+        if name not in packages:
+            raise NameError("package not found: " + name)
+        if not update and name in self.packages:
+            return
+        self.packages[name] = packages[name]
+        for dep in packages[name].deps:
+            self.add(packages, dep, update=False)
+
+    def save(self):
+        packages = list(sorted(self.packages.values(), key=lambda p: p.path))
+        with open(self.path, "w") as f:
+            for package in packages:
+                f.write(package.path + " " + package.url + " " + package.sha1 + "\n")
+
+def cli_add_to_lockfile(args):
+    lockfile = Lockfile(args.lockfile)
+    packages = fetch_repositories()
+    for package in args.packages:
+        lockfile.add(packages, package)
+    lockfile.save()
+
+def cli_update_mirror(args):
+    lockfile = Lockfile(args.lockfile)
+    for package in lockfile.packages.values():
+        path = package.download(BASE_REPOSITORY)
+        subprocess.run([
+            "aws", "s3", "mv", path,
+            "s3://" + MIRROR_BUCKET + "/" + MIRROR_BASE_DIR + package.url,
+            "--profile=" + args.awscli_profile,
+        ], check=True)
+
+def cli_install(args):
+    lockfile = Lockfile(args.lockfile)
+    for package in lockfile.packages.values():
+        # Download the file from the mirror into a temp file
+        url = "https://" + MIRROR_BUCKET + ".s3.amazonaws.com/" + MIRROR_BASE_DIR
+        downloaded = package.download(url)
+        # Extract the file in a temporary directory
+        extract_dir = tempfile.mkdtemp()
+        subprocess.run([
+            "unzip", "-q", downloaded, "-d", extract_dir,
+        ], check=True)
+        # Figure out the prefix used in the zip
+        subdirs = [d for d in os.listdir(extract_dir) if not d.startswith(".")]
+        if len(subdirs) != 1:
+            raise RuntimeError("extracted directory contains more than one dir")
+        # Move the extracted files in the proper directory
+        dest = os.path.join(args.dest, package.path.replace(";", "/"))
+        os.makedirs("/".join(dest.split("/")[:-1]), exist_ok=True)
+        os.rename(os.path.join(extract_dir, subdirs[0]), dest)
+        os.unlink(downloaded)
+
+def cli():
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers()
+
+    add_to_lockfile = subparsers.add_parser("add-to-lockfile")
+    add_to_lockfile.add_argument("lockfile")
+    add_to_lockfile.add_argument("packages", nargs="+")
+    add_to_lockfile.set_defaults(func=cli_add_to_lockfile)
+
+    update_mirror = subparsers.add_parser("update-mirror")
+    update_mirror.add_argument("lockfile")
+    update_mirror.add_argument("--awscli-profile", default="default")
+    update_mirror.set_defaults(func=cli_update_mirror)
+
+    install = subparsers.add_parser("install")
+    install.add_argument("lockfile")
+    install.add_argument("dest")
+    install.set_defaults(func=cli_install)
+
+    args = parser.parse_args()
+    if not hasattr(args, "func"):
+        print("error: a subcommand is required (see --help)")
+        exit(1)
+    args.func(args)
+
+if __name__ == "__main__":
+    cli()
diff --git a/src/ci/docker/scripts/android-sdk.sh b/src/ci/docker/scripts/android-sdk.sh
index e78e3795c04..0b86a2f2dff 100644..100755
--- a/src/ci/docker/scripts/android-sdk.sh
+++ b/src/ci/docker/scripts/android-sdk.sh
@@ -2,66 +2,28 @@ set -ex
 
 export ANDROID_HOME=/android/sdk
 PATH=$PATH:"${ANDROID_HOME}/tools/bin"
+LOCKFILE="${ANDROID_HOME}/android-sdk.lock"
 
-download_sdk() {
-    mkdir -p /android
-    curl -fo sdk.zip "https://dl.google.com/android/repository/sdk-tools-linux-$1.zip"
-    unzip -q sdk.zip -d "$ANDROID_HOME"
-    rm -f sdk.zip
-}
-
-download_sysimage() {
-    abi=$1
-    api=$2
-
-    # See https://developer.android.com/studio/command-line/sdkmanager.html for
-    # usage of `sdkmanager`.
-    #
-    # The output from sdkmanager is so noisy that it will occupy all of the 4 MB
-    # log extremely quickly. Thus we must silence all output.
-    yes | sdkmanager --licenses > /dev/null
-    yes | sdkmanager platform-tools \
-        "platforms;android-$api" \
-        "system-images;android-$api;default;$abi" > /dev/null
-}
-
-download_emulator() {
-    # Download a pinned version of the emulator since upgrades can cause issues
-    curl -fo emulator.zip "https://dl.google.com/android/repository/emulator-linux-$1.zip"
-    rm -rf "${ANDROID_HOME}/emulator"
-    unzip -q emulator.zip -d "${ANDROID_HOME}"
-    rm -f emulator.zip
-}
-
-create_avd() {
-    abi=$1
-    api=$2
+# To add a new packages to the SDK or to update an existing one you need to
+# run the command:
+#
+#    android-sdk-manager.py add-to-lockfile $LOCKFILE <package-name>
+#
+# Then, after every lockfile update the mirror has to be synchronized as well:
+#
+#    android-sdk-manager.py update-mirror $LOCKFILE
+#
+/scripts/android-sdk-manager.py install "${LOCKFILE}" "${ANDROID_HOME}"
 
-    # See https://developer.android.com/studio/command-line/avdmanager.html for
-    # usage of `avdmanager`.
-    echo no | avdmanager create avd \
-        -n "$abi-$api" \
-        -k "system-images;android-$api;default;$abi"
-}
+details=$(cat "${LOCKFILE}" \
+    | grep system-images \
+    | sed 's/^system-images;android-\([0-9]\+\);default;\([a-z0-9-]\+\) /\1 \2 /g')
+api="$(echo "${details}" | awk '{print($1)}')"
+abi="$(echo "${details}" | awk '{print($2)}')"
 
-download_and_create_avd() {
-    download_sdk $1
-    download_sysimage $2 $3
-    create_avd $2 $3
-    download_emulator $4
-}
+# See https://developer.android.com/studio/command-line/avdmanager.html for
+# usage of `avdmanager`.
+echo no | avdmanager create avd \
+    -n "$abi-$api" \
+    -k "system-images;android-$api;default;$abi"
 
-# Usage:
-#
-#       download_and_create_avd 4333796 armeabi-v7a 18 5264690
-#
-# 4333796 =>
-#   SDK tool version.
-#   Copy from https://developer.android.com/studio/index.html#command-tools
-# armeabi-v7a =>
-#   System image ABI
-# 18 =>
-#   Android API Level (18 = Android 4.3 = Jelly Bean MR2)
-# 5264690 =>
-#   Android Emulator version.
-#   Copy from the "build_id" in the `/android/sdk/emulator/emulator -version` output