diff options
Diffstat (limited to 'hadrian/bootstrap/bootstrap.py')
| -rwxr-xr-x | hadrian/bootstrap/bootstrap.py | 234 |
1 files changed, 170 insertions, 64 deletions
diff --git a/hadrian/bootstrap/bootstrap.py b/hadrian/bootstrap/bootstrap.py index 8ed4d5588c..d5580ab484 100755 --- a/hadrian/bootstrap/bootstrap.py +++ b/hadrian/bootstrap/bootstrap.py @@ -21,6 +21,8 @@ from pathlib import Path import platform import shutil import subprocess +import tempfile +import sys from textwrap import dedent from typing import Set, Optional, Dict, List, Tuple, \ NewType, BinaryIO, NamedTuple, TypeVar @@ -31,7 +33,7 @@ BUILDDIR = Path('_build') BINDIR = BUILDDIR / 'bin' # binaries go there (--bindir) DISTDIR = BUILDDIR / 'dists' # --builddir -UNPACKED = BUILDDIR / 'unpacked' # where we unpack tarballs +UNPACKED = BUILDDIR / 'unpacked' # where we unpack final package tarballs TARBALLS = BUILDDIR / 'tarballs' # where we download tarballks PSEUDOSTORE = BUILDDIR / 'pseudostore' # where we install packages ARTIFACTS = BUILDDIR / 'artifacts' # Where we put the archive @@ -46,6 +48,8 @@ class PackageSource(Enum): HACKAGE = 'hackage' LOCAL = 'local' +url = str + BuiltinDep = NamedTuple('BuiltinDep', [ ('package', PackageName), ('version', Version), @@ -68,10 +72,18 @@ BootstrapInfo = NamedTuple('BootstrapInfo', [ ('dependencies', List[BootstrapDep]), ]) +FetchInfo = NamedTuple('FetchInfo', [ + ('url', url), + ('sha256', SHA256Hash) +]) + +FetchPlan = Dict[Path, FetchInfo] + class Compiler: def __init__(self, ghc_path: Path): if not ghc_path.is_file(): - raise TypeError(f'GHC {ghc_path} is not a file') + print(f'GHC {ghc_path} is not a file') + sys.exit(1) self.ghc_path = ghc_path.resolve() @@ -111,40 +123,11 @@ def package_cabal_url(package: PackageName, version: Version, revision: int) -> return f'http://hackage.haskell.org/package/{package}-{version}/revision/{revision}.cabal' def verify_sha256(expected_hash: SHA256Hash, f: Path): + print(f"Verifying {f}...") h = hash_file(hashlib.sha256(), f.open('rb')) if h != expected_hash: raise BadTarball(f, expected_hash, h) -def fetch_package(package: PackageName, - version: Version, - src_sha256: SHA256Hash, - revision: Optional[int], - cabal_sha256: Optional[SHA256Hash], - ) -> (Path, Path): - import urllib.request - - # Download source distribution - tarball = TARBALLS / f'{package}-{version}.tar.gz' - if not tarball.exists(): - print(f'Fetching {package}-{version}...') - tarball.parent.mkdir(parents=True, exist_ok=True) - url = package_url(package, version) - with urllib.request.urlopen(url) as resp: - shutil.copyfileobj(resp, tarball.open('wb')) - - verify_sha256(src_sha256, tarball) - - # Download revised cabal file - cabal_file = TARBALLS / f'{package}.cabal' - if revision is not None and not cabal_file.exists(): - assert cabal_sha256 is not None - url = package_cabal_url(package, version, revision) - with urllib.request.urlopen(url) as resp: - shutil.copyfileobj(resp, cabal_file.open('wb')) - verify_sha256(cabal_sha256, cabal_file) - - return (tarball, cabal_file) - def read_bootstrap_info(path: Path) -> BootstrapInfo: obj = json.load(path.open()) @@ -166,15 +149,18 @@ def check_builtin(dep: BuiltinDep, ghc: Compiler) -> None: print(f'Using {dep.package}-{dep.version} from GHC...') return -def install_dep(dep: BootstrapDep, ghc: Compiler) -> None: - dist_dir = (DISTDIR / f'{dep.package}-{dep.version}').resolve() - +def resolve_dep(dep : BootstrapDep) -> Path: if dep.source == PackageSource.HACKAGE: - assert dep.src_sha256 is not None - (tarball, cabal_file) = fetch_package(dep.package, dep.version, dep.src_sha256, - dep.revision, dep.cabal_sha256) + + tarball = TARBALLS / f'{dep.package}-{dep.version}.tar.gz' + verify_sha256(dep.src_sha256, tarball) + + cabal_file = TARBALLS / f'{dep.package}.cabal' + verify_sha256(dep.cabal_sha256, cabal_file) + UNPACKED.mkdir(parents=True, exist_ok=True) shutil.unpack_archive(tarball.resolve(), UNPACKED, 'gztar') + sdist_dir = UNPACKED / f'{dep.package}-{dep.version}' # Update cabal file with revision @@ -183,9 +169,16 @@ def install_dep(dep: BootstrapDep, ghc: Compiler) -> None: elif dep.source == PackageSource.LOCAL: if dep.package == 'hadrian': - sdist_dir = Path('../').resolve() + sdist_dir = Path(sys.path[0]).parent.resolve() else: raise ValueError(f'Unknown local package {dep.package}') + return sdist_dir + + +def install_dep(dep: BootstrapDep, ghc: Compiler) -> None: + dist_dir = (DISTDIR / f'{dep.package}-{dep.version}').resolve() + + sdist_dir = resolve_dep(dep) install_sdist(dist_dir, sdist_dir, ghc, dep.flags) @@ -298,7 +291,6 @@ def archive_name(version): return f'hadrian-{version}-{machine}-{version}' def make_archive(hadrian_path): - import tempfile print(f'Creating distribution tarball') @@ -324,28 +316,88 @@ def make_archive(hadrian_path): return archivename +def fetch_from_plan(plan : FetchPlan, output_dir : Path): + import urllib.request + + output_dir.resolve() + output_dir.mkdir(parents=True, exist_ok=True) + + for path in plan: + output_path = output_dir / path + url = plan[path].url + sha = plan[path].sha256 + if not output_path.exists(): + print(f'Fetching {url}...') + with urllib.request.urlopen(url) as resp: + shutil.copyfileobj(resp, output_path.open('wb')) + verify_sha256(sha, output_path) + +def gen_fetch_plan(info : BootstrapInfo) -> FetchPlan : + sources_dict = {} + for dep in info.dependencies: + if dep.package != 'hadrian': + sources_dict[f"{dep.package}-{dep.version}.tar.gz"] = FetchInfo(package_url(dep.package, dep.version), dep.src_sha256) + if dep.revision is not None: + sources_dict[f"{dep.package}.cabal"] = FetchInfo(package_cabal_url(dep.package, dep.version, dep.revision), dep.cabal_sha256) + return sources_dict + +def find_ghc(compiler) -> Compiler: + # Find compiler + if compiler is None: + path = shutil.which('ghc') + if path is None: + raise ValueError("Couldn't find ghc in PATH") + ghc = Compiler(Path(path)) + else: + ghc = Compiler(compiler) + return ghc + + def main() -> None: import argparse parser = argparse.ArgumentParser( description="bootstrapping utility for hadrian.", epilog = USAGE, formatter_class = argparse.RawDescriptionHelpFormatter) - parser.add_argument('-d', '--deps', type=Path, default='bootstrap-deps.json', - help='bootstrap dependency file') + parser.add_argument('-d', '--deps', type=Path, help='bootstrap dependency file (plan-bootstrap.json)') parser.add_argument('-w', '--with-compiler', type=Path, help='path to GHC') - args = parser.parse_args() + parser.add_argument('-s', '--bootstrap-sources', type=Path, + help='Path to prefetched bootstrap sources tarball') - # Find compiler - if args.with_compiler is None: - path = shutil.which('ghc') - if path is None: - raise ValueError("Couldn't find ghc in PATH") - ghc = Compiler(Path(path)) - else: - ghc = Compiler(args.with_compiler) + subparsers = parser.add_subparsers(dest="command") + + parser_list = subparsers.add_parser('list-sources', help='list all sources required to download') + parser_list.add_argument('-o','--output', type=Path, default='fetch_plan.json') + + parser_fetch = subparsers.add_parser('fetch', help='fetch all required sources from hackage (for offline builds)') + parser_fetch.add_argument('-o','--output', type=Path, default='bootstrap-sources') + parser_fetch.add_argument('-p','--fetch-plan', type=Path, default=None, help="A json document that lists the urls required for download (optional)") + + args = parser.parse_args() - print(f'Bootstrapping hadrian with GHC {ghc.version} at {ghc.ghc_path}...') + ghc = None + + if args.deps is None: + if args.bootstrap_sources is None: + # find appropriate plan in the same directory as the script + ghc = find_ghc(args.with_compiler) + args.deps = Path(sys.path[0]) / f"plan-bootstrap-{ghc.version.replace('.','_')}.json" + print(f"defaulting bootstrap plan to {args.deps}") + # We have a tarball with all the required information, unpack it and use for further + elif args.bootstrap_sources is not None and args.command != 'list-sources': + print(f'Unpacking {args.bootstrap_sources} to {TARBALLS}') + shutil.unpack_archive(args.bootstrap_sources.resolve(), TARBALLS, 'gztar') + args.deps = TARBALLS / 'plan-bootstrap.json' + print(f"using plan-bootstrap.json ({args.deps}) from {args.bootstrap_sources}") + else: + print("We need a bootstrap plan (plan-bootstrap.json) or a tarball containing bootstrap information") + print("Perhaps pick an appropriate one from: ") + for child in Path(sys.path[0]).iterdir: + if child.match('plan-bootstrap-*.json'): + print(child) + sys.exit(1) + info = read_bootstrap_info(args.deps) print(dedent(""" DO NOT use this script if you have another recent cabal-install available. @@ -353,25 +405,79 @@ def main() -> None: architectures. """)) - info = read_bootstrap_info(args.deps) - bootstrap(info, ghc) - hadrian_path = (BINDIR / 'hadrian').resolve() - archive = make_archive(hadrian_path) + if(args.command == 'fetch'): + if args.fetch_plan is not None: + plan = { path : FetchInfo(p["url"],p["sha256"]) for path, p in json.load(args.fetch_plan.open()).items() } + else: + plan = gen_fetch_plan(info) + + # In temporary directory, create a directory which we will archive + tmpdir = TMPDIR.resolve() + tmpdir.mkdir(parents=True, exist_ok=True) + + rootdir = Path(tempfile.mkdtemp(dir=tmpdir)) + + fetch_from_plan(plan, rootdir) + + shutil.copyfile(args.deps, rootdir / 'plan-bootstrap.json') - print(dedent(f''' - Bootstrapping finished! + fmt = 'gztar' + if platform.system() == 'Windows': fmt = 'zip' + + archivename = shutil.make_archive(args.output, fmt, root_dir=rootdir) - The resulting hadrian executable can be found at + print(f'Bootstrap sources saved to {archivename}') + print(f'Use `bootstrap.py -d {args.deps} -s {archivename}` to continue') - {hadrian_path} + elif(args.command == 'list-sources'): + plan = gen_fetch_plan(info) + with open(args.output, 'w') as out: + json.dump({path : val._asdict() for path,val in plan.items()}, out) + print(f"Required hackage sources saved to {args.output}") + tarfmt= "\n./" + print(f""" +Download the files listed in {args.output} and save them to a tarball ($TARBALL), along with {args.deps} +The contents of $TARBALL should look like: - It have been archived for distribution in +./ +./plan-bootstrap.json +./{tarfmt.join(path for path in plan)} - {archive} +Then use `bootstrap.py -s $TARBALL` to continue +Alternatively, you could use `bootstrap.py -d {args.deps} fetch -o $TARBALL` to download and generate the tarball, skipping this step +""") - You can use this executable to build GHC. - ''')) + elif(args.command == None): + if ghc is None: + ghc = find_ghc(args.with_compiler) + + print(f'Bootstrapping hadrian with GHC {ghc.version} at {ghc.ghc_path}...') + + if args.bootstrap_sources is None: + plan = gen_fetch_plan(info) + fetch_from_plan(plan, TARBALLS) + + bootstrap(info, ghc) + hadrian_path = (BINDIR / 'hadrian').resolve() + + archive = make_archive(hadrian_path) + + print(dedent(f''' + Bootstrapping finished! + + The resulting hadrian executable can be found at + + {hadrian_path} + + It have been archived for distribution in + + {archive} + + You can use this executable to build GHC. + ''')) + else: + print(f"No such command: {args.command}") def subprocess_run(args, **kwargs): "Like subprocess.run, but also print what we run" |