diff options
author | bors <bors@rust-lang.org> | 2023-04-30 20:02:00 +0000 |
---|---|---|
committer | bors <bors@rust-lang.org> | 2023-04-30 20:02:00 +0000 |
commit | 9ecda8de85ce893cc3fc748ab06be0b8250147a7 (patch) | |
tree | 07079ca29bfd36b9248499c68c043cf7ef0d929d | |
parent | f2eb9f85b9b52e6538c3c7fc160725963272d471 (diff) | |
parent | a98968ee0e03b15ce8656d77f75ae664a0b2d6b5 (diff) | |
download | rust-9ecda8de85ce893cc3fc748ab06be0b8250147a7.tar.gz |
Auto merge of #110427 - Nilstrieb:parallel-bootstrap-startup, r=albertlarsan68
Parallelize initial Rust download in bootstrap
Parallelize the initial download of Rust in `bootstrap.py`
`time ./x.py --help` after `rm -r build`
Before: 33s
After: 27s
-rw-r--r-- | src/bootstrap/bootstrap.py | 101 |
1 files changed, 86 insertions, 15 deletions
diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index bb3e6dd91d9..dd0b091a2a3 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -13,6 +13,7 @@ import tarfile import tempfile from time import time +from multiprocessing import Pool, cpu_count try: import lzma @@ -406,6 +407,48 @@ class Stage0Toolchain: return self.version + "-" + self.date +class DownloadInfo: + """A helper class that can be pickled into a parallel subprocess""" + + def __init__( + self, + base_download_url, + download_path, + bin_root, + tarball_path, + tarball_suffix, + checksums_sha256, + pattern, + verbose, + ): + self.base_download_url = base_download_url + self.download_path = download_path + self.bin_root = bin_root + self.tarball_path = tarball_path + self.tarball_suffix = tarball_suffix + self.checksums_sha256 = checksums_sha256 + self.pattern = pattern + self.verbose = verbose + +def download_component(download_info): + if not os.path.exists(download_info.tarball_path): + get( + download_info.base_download_url, + download_info.download_path, + download_info.tarball_path, + download_info.checksums_sha256, + verbose=download_info.verbose, + ) + +def unpack_component(download_info): + unpack( + download_info.tarball_path, + download_info.tarball_suffix, + download_info.bin_root, + match=download_info.pattern, + verbose=download_info.verbose, + ) + class RustBuild(object): """Provide all the methods required to build Rust""" def __init__(self): @@ -460,17 +503,49 @@ class RustBuild(object): ) run_powershell([script]) shutil.rmtree(bin_root) + + key = self.stage0_compiler.date + cache_dst = os.path.join(self.build_dir, "cache") + rustc_cache = os.path.join(cache_dst, key) + if not os.path.exists(rustc_cache): + os.makedirs(rustc_cache) + tarball_suffix = '.tar.gz' if lzma is None else '.tar.xz' - filename = "rust-std-{}-{}{}".format( - rustc_channel, self.build, tarball_suffix) - pattern = "rust-std-{}".format(self.build) - self._download_component_helper(filename, pattern, tarball_suffix) - filename = "rustc-{}-{}{}".format(rustc_channel, self.build, - tarball_suffix) - self._download_component_helper(filename, "rustc", tarball_suffix) - filename = "cargo-{}-{}{}".format(rustc_channel, self.build, - tarball_suffix) - self._download_component_helper(filename, "cargo", tarball_suffix) + + toolchain_suffix = "{}-{}{}".format(rustc_channel, self.build, tarball_suffix) + + tarballs_to_download = [ + ("rust-std-{}".format(toolchain_suffix), "rust-std-{}".format(self.build)), + ("rustc-{}".format(toolchain_suffix), "rustc"), + ("cargo-{}".format(toolchain_suffix), "cargo"), + ] + + tarballs_download_info = [ + DownloadInfo( + base_download_url=self.download_url, + download_path="dist/{}/{}".format(self.stage0_compiler.date, filename), + bin_root=self.bin_root(), + tarball_path=os.path.join(rustc_cache, filename), + tarball_suffix=tarball_suffix, + checksums_sha256=self.checksums_sha256, + pattern=pattern, + verbose=self.verbose, + ) + for filename, pattern in tarballs_to_download + ] + + # Download the components serially to show the progress bars properly. + for download_info in tarballs_download_info: + download_component(download_info) + + # Unpack the tarballs in parallle. + # In Python 2.7, Pool cannot be used as a context manager. + p = Pool(min(len(tarballs_download_info), cpu_count())) + try: + p.map(unpack_component, tarballs_download_info) + finally: + p.close() + if self.should_fix_bins_and_dylibs(): self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root)) @@ -486,13 +561,9 @@ class RustBuild(object): rust_stamp.write(key) def _download_component_helper( - self, filename, pattern, tarball_suffix, + self, filename, pattern, tarball_suffix, rustc_cache, ): key = self.stage0_compiler.date - cache_dst = os.path.join(self.build_dir, "cache") - rustc_cache = os.path.join(cache_dst, key) - if not os.path.exists(rustc_cache): - os.makedirs(rustc_cache) tarball = os.path.join(rustc_cache, filename) if not os.path.exists(tarball): |