path: root/.gitlab/upload_ghc_libs.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
A tool for uploading GHC's core libraries to Hackage.

This is a utility for preparing and uploading source distributions and
documentation of GHC's core libraries to Hackage. This should be run in
a GHC tree of the release commit after having run ./configure.

There are two modes, preparation and upload.

* The `prepare` mode takes a link to a bindist and creates a folder containing the
  source and doc tarballs ready to upload to hackage.
* The `upload` mode takes the folder created by prepare and performs the upload to
  hackage.
"""

from subprocess import run, check_call
from getpass import getpass
import shutil
from pathlib import Path
from typing import NamedTuple, Callable, List, Dict, Optional
import tempfile
import re
import pickle
import os


WORK_DIR = Path('.upload-libs')
WORK_DIR.mkdir(exist_ok=True)
OUT_DIR = WORK_DIR / 'docs'
OUT_DIR.mkdir(exist_ok=True)

class Package(NamedTuple):
    name: str
    path: Path
    prepare_sdist: Callable[[], None]

class Credentials(NamedTuple):
    username: str
    password: str

def no_prep():
    pass

def prep_base():
    shutil.copy('config.guess', 'libraries/base')
    shutil.copy('config.sub', 'libraries/base')

def build_copy_file(pkg: Package, f: Path):
    target = Path('_build') / 'stage1' / pkg.path / 'build' / f
    dest = pkg.path / f
    build_file_hadrian(target)
    print(f'Copying {target} to {dest}...')
    dest.parent.mkdir(exist_ok=True, parents=True)
    shutil.copyfile(target, dest)


def build_file_hadrian(target: Path):
    build_cabal = Path('hadrian') / 'build-cabal'
    if not build_cabal.is_file():
        build_cabal = Path('hadrian') / 'build.cabal.sh'

    print(f'Building {target}...')
    run([build_cabal, target], check=True)

def modify_file(pkg: Package, fname: Path, f: Callable[[str], str]):
    target = pkg.path / fname
    s = target.read_text()
    target.write_text(f(s))

def prep_ghc_prim():
    build_copy_file(PACKAGES['ghc-prim'], Path('GHC/PrimopWrappers.hs'))

def prep_ghc_bignum():
    shutil.copy('config.guess', 'libraries/base')
    shutil.copy('config.sub', 'libraries/base')

def prep_ghc_boot():
    build_copy_file(PACKAGES['ghc-boot'], Path('GHC/Platform/Host.hs'))
    build_copy_file(PACKAGES['ghc-boot'], Path('GHC/Version.hs'))

def prep_ghc():
    # Drop RTS includes from `include-dirs` as Hackage rejects this
    modify_file(PACKAGES['ghc'], 'ghc.cabal',
                lambda s: s.replace('../rts/dist/build', ''))
    build_copy_file(PACKAGES['ghc'], 'GHC/Platform/Constants.hs')
    build_copy_file(PACKAGES['ghc'], 'GHC/Settings/Config.hs')

PACKAGES = {
    pkg.name: pkg
    for pkg in [
        Package('base', Path("libraries/base"), prep_base),
        Package('ghc-prim', Path("libraries/ghc-prim"), prep_ghc_prim),
        Package('integer-gmp', Path("libraries/integer-gmp"), no_prep),
        Package('ghc-bignum', Path("libraries/ghc-bignum"), prep_ghc_bignum),
        Package('template-haskell', Path("libraries/template-haskell"), no_prep),
        Package('ghc-heap', Path("libraries/ghc-heap"), no_prep),
        Package('ghc-boot', Path("libraries/ghc-boot"), prep_ghc_boot),
        Package('ghc-boot-th', Path("libraries/ghc-boot-th"), no_prep),
        Package('ghc-compact', Path("libraries/ghc-compact"), no_prep),
        Package('libiserv', Path("libraries/libiserv"), no_prep),
        Package('ghc', Path("compiler"), prep_ghc),
    ]
}
# Dict[str, Package]

def cabal_upload(tarball: Path, creds: Credentials, publish: bool=False, extra_args=[]):
    if publish:
        extra_args += ['--publish']

    creds_args = [
        f'--username={creds.username}',
        f'--password={creds.password}'
    ]
    run(['cabal', 'upload'] + extra_args + [tarball] + creds_args, check=True)

def prepare_sdist(pkg: Package):

    print(f'Preparing package {pkg.name}...')
    shutil.rmtree(pkg.path / 'dist-newstyle', ignore_errors=True)
    build_file_hadrian(pkg.path / '{}.cabal'.format(pkg.name))
    pkg.prepare_sdist()

    # Upload source tarball
    run(['cabal', 'sdist'], cwd=pkg.path, check=True)
    sdist = list((pkg.path / 'dist-newstyle' / 'sdist').glob('*.tar.gz'))[0]
    res_path = shutil.copy(sdist, OUT_DIR)
    return os.path.relpath(res_path, OUT_DIR)

def upload_pkg_sdist(sdist : Path, pkg: Package, publish: bool, creds: Credentials):
    publish_tag = '-publish' if publish else ''
    stamp = WORK_DIR / f'{pkg.name}-sdist{publish_tag}'
    if stamp.is_file():
        return
    print(f'Uploading package {pkg.name}...')
    cabal_upload(sdist, publish=publish, creds=creds)
    stamp.write_text('')

def get_version(cabal_file: Path) -> Optional[str]:
    m = re.search(r'^version:\s*(\d+(\.\d+)*)', cabal_file.read_text(), re.I | re.M)
    return None if m is None else m.group(1)


def prepare_docs(bindist: Path, pkg: Package):
    """
    Prepare Haddock documentation for a package. bindist
    is the path to an extract binary distribution produced by
    hadrian.
    """
    cabal_file = pkg.path / f'{pkg.name}.cabal'
    version = get_version(cabal_file)
    assert version is not None
    docdir = bindist / 'doc' / 'html' / 'libraries' / (pkg.name + "-" + version)

    # Build the documentation tarball from the bindist documentation
    stem = f'{pkg.name}-{version}-docs'
    tmp = tempfile.TemporaryDirectory(stem)
    shutil.copytree(docdir, Path(tmp.name) / stem)
    tarball = Path(f'{stem}.tar.gz')
    run(['tar', '-czf', OUT_DIR / tarball, '-H', 'ustar', '-C', tmp.name, stem])
    return tarball

def upload_docs(tarball : Path, pkg : Package, publish : bool, creds: Credentials):
    publish_tag = '-publish' if publish else ''
    stamp = WORK_DIR / f'{pkg.name}-docs{publish_tag}'
    if stamp.is_file():
        return
    # Upload the documentation tarball
    print(f'Uploading documentation for {pkg.name}...')
    cabal_upload(tarball, publish=publish, extra_args=['--documentation'], creds=creds)
    stamp.write_text('')

def upload_pkg(pkg: Package, d : Path, meta, publish : bool, creds: Credentials):
    print(f'Uploading {pkg.name}...')
    upload_pkg_sdist(d / meta['sdist'], pkg, publish=publish, creds=creds)
    upload_docs(d / meta['docs'], pkg, publish=publish, creds=creds)

def prepare_pkg(bindist : Path, pkg : Package):
    if pkg.path.exists():
        print(f'Processing {pkg.name}...')
        p1 = prepare_sdist(pkg)
        p2 = prepare_docs(bindist, pkg)
        return { 'sdist' : p1, 'docs': p2 }
    else:
        print(f"Package {pkg.name} doesn't exist... skipping")

def main() -> None:
    import argparse

    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('pkg', type=str, nargs='*', help='package to upload')
    subparsers = parser.add_subparsers(dest="command")

    parser_prepare = subparsers.add_parser('prepare')
    parser_prepare.add_argument('--bindist', required=True, type=Path, help='extracted binary distribution')

    parser_upload = subparsers.add_parser('upload')
    parser_upload.add_argument('--docs', required = True, type=Path, help='folder created by --prepare')
    parser_upload.add_argument('--publish', action='store_true', help='Publish Hackage packages instead of just uploading candidates')
    args = parser.parse_args()

    pkgs = args.pkg
    for pkg_name in pkgs:
        assert pkg_name in PACKAGES

    if pkgs == []:
        pkgs = PACKAGES.keys()

    if args.command == "prepare":

        manifest = {}
        for pkg_name in pkgs:
            print(pkg_name)
            pkg = PACKAGES[pkg_name]
            pkg_meta = prepare_pkg(args.bindist, pkg)
            manifest[pkg] = pkg_meta
        manifest_path = WORK_DIR / 'docs' / 'manifest.pickle'
        with open(WORK_DIR / 'docs' / 'manifest.pickle', 'wb') as fout:
            pickle.dump(manifest, fout)

    elif args.command == "upload":
        username = input('Hackage username: ')
        password = getpass('Hackage password: ')
        creds = Credentials(username, password)
        manifest_path = args.docs
        with open(manifest_path / 'manifest.pickle', 'rb') as fin:
            manifest = pickle.load(fin)
        for pkg, item in manifest.items():
            if pkg.name in pkgs:
                print(pkg, item)
                upload_pkg(pkg, manifest_path, item, publish=args.publish, creds=creds)

if __name__ == '__main__':
    main()