diff options
author | Ben Gamari <ben@smart-cactus.org> | 2023-01-27 14:08:46 -0500 |
---|---|---|
committer | Ben Gamari <ben@smart-cactus.org> | 2023-02-14 15:05:25 -0500 |
commit | f6426b6f33926fada1a82a50b89d736f390f7666 (patch) | |
tree | 716dcd2c7f7b03b63260c69ee6c55b2ab7ecbd15 | |
parent | 2ea1a6bc7d7c2946b4a3d1c2c19083e09401f9f1 (diff) | |
download | haskell-wip/T22686.tar.gz |
gitlab: Collect metadata about binary distributionswip/T22686
Fixes #22686.
-rwxr-xr-x | .gitlab/bindist_metadata.py | 155 | ||||
-rwxr-xr-x | .gitlab/ci.sh | 4 | ||||
-rwxr-xr-x | .gitlab/gen_ci.hs | 1 | ||||
-rw-r--r-- | .gitlab/jobs.yaml | 69 |
4 files changed, 229 insertions, 0 deletions
diff --git a/.gitlab/bindist_metadata.py b/.gitlab/bindist_metadata.py new file mode 100755 index 0000000000..5542f6e3f3 --- /dev/null +++ b/.gitlab/bindist_metadata.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 + +import sys +import os +import shutil +import re +import ast +from pathlib import Path +import subprocess +import json +from typing import Dict, List, Set, Optional, NamedTuple + +def run(args: List[str]) -> str: + return subprocess.check_output(args, encoding='UTF-8') + +def parse_hadrian_cfg(cfg: str) -> Dict[str,str]: + res = {} + for l in cfg.split('\n'): + if l.startswith('#'): + continue + elif '=' in l: + i = l.find('=') + k = l[:i].strip() + v = l[i+1:].strip() + res[k] = v + + return res + +def get_ghc_info(ghc: Path) -> Dict[str,str]: + import ast + out = run([ghc, '--info']) + pairs = ast.literal_eval(out.strip()) + res = {} + for k,v in pairs: + if v == 'YES': + v = True + elif v == 'NO': + v = False + res[k] = v + + return res + +def get_dynamic_deps(objfile: Path) -> Set[Path]: + out = run(['ldd', objfile]) + return { Path(m.group(1)) for m in re.finditer('=> *([^ ]+)', out) } + +def get_configure_cmdline() -> str: + r = Path('config.log').read_text() + m = re.search(r' $ .+', r) + return m + +class Package(NamedTuple): + name: str + version: str + +def find_providing_package(f: Path) -> Optional[Package]: + if shutil.which('dpkg'): + out = run(['dpkg-query', '--search', f]).strip() + pkg,_file = out.split(':') + + out = run(['dpkg-query', '--show', pkg]).strip() + _pkg,version = out.split() + return Package(pkg, version) + elif shutil.which('rpm'): + out = run(['rpm', '-qf', f, '--queryformat=%{NAME} %{VERSION}\n']).strip() + pkg,version = out.split() + return Package(pkg, version) + elif shutil.which('apk'): + out = run(['apk', 'info', '--who-owns', f]).strip() + pkg = re.find('is owned by ([.+])', out) + + # Determining the version of an installed package is far too difficult; + # some day perhaps upstream will address + # https://gitlab.alpinelinux.org/alpine/apk-tools/-/issues/10704 + db = Path('/lib/apk/db/installed').read_text() + m = re.find(f'P:{pkg}\nV:([.+])\n', db) + version = m.group(1) + return Package(pkg, version) + else: + return None + +def main() -> None: + ghc = Path('_build/stage1/bin/ghc') + ghc_pkg = Path('_build/stage1/bin/ghc-pkg') + + metadata = {} + + system_config = Path('.') / 'hadrian' / 'cfg' / 'system.config' + cfg = parse_hadrian_cfg(system_config.read_text()) + + ###### + # GHC build configuration + ###### + metadata['ghc_version'] = cfg['project-version'] + metadata['git_commit_id'] = cfg['project-git-commit-id'] + metadata['tables_next_to_code'] = cfg['tables-next-to-code'] + metadata['unregisterised'] = cfg['ghc-unregisterised'] + metadata['build_triple'] = cfg['build-platform'] + metadata['host_triple'] = cfg['host-platform'] + metadata['target_triple'] = cfg['target-platform'] + metadata['build_flavour'] = os.environ.get('BUILD_FLAVOUR') + metadata['configure_cmdline'] = get_configure_cmdline() + + opsys = cfg['build-os'] + + ###### + # Information about the bootstrapping environment + ###### + lsb_release = None + if opsys == 'linux': + lsb_release = run(['lsb_release']) + + metadata['bootstrap_environment'] = { + 'ghc': run([cfg['system-ghc'], '--version']).split('\n')[0], + 'cc': run([cfg['system-cc'], '--version']).split('\n')[0], + 'lsb_release': lsb_release, + } + + ###### + # Information about the bootstrapping environment's packages + ###### + metadata['dynamic_deps'] = None + if opsys != 'darwin': + dyn_deps = get_dynamic_deps(ghc) + print(dyn_deps) + deps = { + dep.name: find_providing_package(dep) + for dep in dyn_deps + if not dep.is_relative_to(Path('.').resolve()) + } + metadata['dynamic_deps'] = deps + + ###### + # The contents of the compiler's global package database + ###### + def call_ghc_pkg(args: List[str]) -> str: + return run([ghc_pkg, '--simple-output'] + args).strip() + + metadata['global_packages'] = { + pkg: { + 'version': call_ghc_pkg(['field', pkg, 'version']), + 'extra-libraries': call_ghc_pkg(['field', pkg, 'extra-libraries']).split(), + } + for pkg in call_ghc_pkg(['list', '--names-only']).split() + } + + ###### + # Information about the resulting compiler + ###### + metadata['inplace_ghc_info'] = get_ghc_info(ghc) + + json.dump(metadata, sys.stdout, indent=2) + +if __name__ == '__main__': + main() diff --git a/.gitlab/ci.sh b/.gitlab/ci.sh index c447696ae6..2b31acb34e 100755 --- a/.gitlab/ci.sh +++ b/.gitlab/ci.sh @@ -478,6 +478,7 @@ function check_msys2_deps() { # Ensure that GHC on Windows doesn't have any dynamic dependencies on msys2 case "$(uname)" in MSYS_*|MINGW*) + info "Checking for unwanted msys2 dependencies..." sysroot="$(cygpath "$SYSTEMROOT")" PATH="$sysroot/System32:$sysroot;$sysroot/Wbem" $@ \ || fail "'$@' failed; there may be unwanted dynamic dependencies." @@ -584,6 +585,9 @@ function test_hadrian() { check_msys2_deps _build/stage1/bin/ghc --version check_release_build + info "Collecting binary distribution metadata..." + run "$TOP/.gitlab/bindist_metadata.py" > metadata.json + # Ensure that statically-linked builds are actually static if [[ "${BUILD_FLAVOUR}" = *static* ]]; then bad_execs="" diff --git a/.gitlab/gen_ci.hs b/.gitlab/gen_ci.hs index c5adc90662..7e0528b62d 100755 --- a/.gitlab/gen_ci.hs +++ b/.gitlab/gen_ci.hs @@ -680,6 +680,7 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} } { junitReport = "junit.xml" , expireIn = "2 weeks" , artifactPaths = [binDistName arch opsys buildConfig ++ ".tar.xz" + ,"metadata.json" ,"junit.xml"] , artifactsWhen = ArtifactsAlways } diff --git a/.gitlab/jobs.yaml b/.gitlab/jobs.yaml index 6db398f22b..3c34d6a63d 100644 --- a/.gitlab/jobs.yaml +++ b/.gitlab/jobs.yaml @@ -11,6 +11,7 @@ "expire_in": "2 weeks", "paths": [ "ghc-aarch64-darwin-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -73,6 +74,7 @@ "expire_in": "2 weeks", "paths": [ "ghc-aarch64-linux-deb10-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -131,6 +133,7 @@ "expire_in": "2 weeks", "paths": [ "ghc-i386-linux-deb9-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -189,6 +192,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-aarch64-darwin-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -252,6 +256,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-aarch64-linux-deb10-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -311,6 +316,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-aarch64-linux-deb10-validate+llvm.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -370,6 +376,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-i386-linux-deb9-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -429,6 +436,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-darwin-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -494,6 +502,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-freebsd13-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -555,6 +564,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-alpine3_12-cross_wasm32-wasi-release+fully_static.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -616,6 +626,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-alpine3_12-int_native-cross_wasm32-wasi-release+fully_static.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -677,6 +688,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-alpine3_12-int_native-validate+fully_static.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -740,6 +752,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-alpine3_12-unreg-cross_wasm32-wasi-release+fully_static.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -801,6 +814,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-alpine3_12-validate+fully_static.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -864,6 +878,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-centos7-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -924,6 +939,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-deb10-int_native-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -983,6 +999,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-deb10-no_tntc-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -1042,6 +1059,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-deb10-numa-slow-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -1102,6 +1120,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-deb10-unreg-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -1161,6 +1180,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-deb10-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -1220,6 +1240,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-deb10-validate+debug_info.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -1279,6 +1300,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-deb10-validate+llvm.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -1338,6 +1360,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-deb10-validate+thread_sanitizer.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -1399,6 +1422,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-deb11-cross_aarch64-linux-gnu-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -1460,6 +1484,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-deb11-int_native-cross_javascript-unknown-ghcjs-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -1522,6 +1547,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-deb11-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -1581,6 +1607,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-deb9-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -1640,6 +1667,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-fedora33-release.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -1701,6 +1729,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-fedora33-release.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -1763,6 +1792,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-fedora33-validate+debug_info.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -1824,6 +1854,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-rocky8-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -1884,6 +1915,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-ubuntu18_04-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -1943,6 +1975,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-linux-ubuntu20_04-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -2001,6 +2034,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-windows-int_native-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -2060,6 +2094,7 @@ "expire_in": "8 weeks", "paths": [ "ghc-x86_64-windows-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -2120,6 +2155,7 @@ "expire_in": "1 year", "paths": [ "ghc-aarch64-darwin-release.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -2184,6 +2220,7 @@ "expire_in": "1 year", "paths": [ "ghc-aarch64-linux-deb10-release+no_split_sections.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -2244,6 +2281,7 @@ "expire_in": "1 year", "paths": [ "ghc-i386-linux-deb9-release+no_split_sections.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -2304,6 +2342,7 @@ "expire_in": "1 year", "paths": [ "ghc-x86_64-darwin-release.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -2370,6 +2409,7 @@ "expire_in": "1 year", "paths": [ "ghc-x86_64-linux-alpine3_12-int_native-release+fully_static.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -2434,6 +2474,7 @@ "expire_in": "1 year", "paths": [ "ghc-x86_64-linux-alpine3_12-release+fully_static+no_split_sections.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -2498,6 +2539,7 @@ "expire_in": "1 year", "paths": [ "ghc-x86_64-linux-centos7-release+no_split_sections.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -2559,6 +2601,7 @@ "expire_in": "1 year", "paths": [ "ghc-x86_64-linux-deb10-release.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -2619,6 +2662,7 @@ "expire_in": "1 year", "paths": [ "ghc-x86_64-linux-deb10-release+debug_info.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -2679,6 +2723,7 @@ "expire_in": "1 year", "paths": [ "ghc-x86_64-linux-deb11-release.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -2739,6 +2784,7 @@ "expire_in": "1 year", "paths": [ "ghc-x86_64-linux-deb9-release+no_split_sections.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -2799,6 +2845,7 @@ "expire_in": "1 year", "paths": [ "ghc-x86_64-linux-fedora33-release.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -2861,6 +2908,7 @@ "expire_in": "1 year", "paths": [ "ghc-x86_64-linux-fedora33-release+debug_info.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -2923,6 +2971,7 @@ "expire_in": "1 year", "paths": [ "ghc-x86_64-linux-fedora33-release.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -2986,6 +3035,7 @@ "expire_in": "1 year", "paths": [ "ghc-x86_64-linux-rocky8-release.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -3047,6 +3097,7 @@ "expire_in": "1 year", "paths": [ "ghc-x86_64-linux-ubuntu18_04-release.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -3107,6 +3158,7 @@ "expire_in": "1 year", "paths": [ "ghc-x86_64-linux-ubuntu20_04-release.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -3166,6 +3218,7 @@ "expire_in": "1 year", "paths": [ "ghc-x86_64-windows-int_native-release+no_split_sections.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -3226,6 +3279,7 @@ "expire_in": "1 year", "paths": [ "ghc-x86_64-windows-release+no_split_sections.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -3287,6 +3341,7 @@ "expire_in": "2 weeks", "paths": [ "ghc-x86_64-darwin-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -3351,6 +3406,7 @@ "expire_in": "2 weeks", "paths": [ "ghc-x86_64-freebsd13-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -3411,6 +3467,7 @@ "expire_in": "2 weeks", "paths": [ "ghc-x86_64-linux-alpine3_12-cross_wasm32-wasi-release+fully_static.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -3471,6 +3528,7 @@ "expire_in": "2 weeks", "paths": [ "ghc-x86_64-linux-alpine3_12-validate+fully_static.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -3533,6 +3591,7 @@ "expire_in": "2 weeks", "paths": [ "ghc-x86_64-linux-deb10-int_native-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -3591,6 +3650,7 @@ "expire_in": "2 weeks", "paths": [ "ghc-x86_64-linux-deb10-no_tntc-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -3650,6 +3710,7 @@ "expire_in": "2 weeks", "paths": [ "ghc-x86_64-linux-deb10-numa-slow-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -3709,6 +3770,7 @@ "expire_in": "2 weeks", "paths": [ "ghc-x86_64-linux-deb10-unreg-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -3767,6 +3829,7 @@ "expire_in": "2 weeks", "paths": [ "ghc-x86_64-linux-deb10-validate+debug_info.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -3825,6 +3888,7 @@ "expire_in": "2 weeks", "paths": [ "ghc-x86_64-linux-deb10-validate+llvm.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -3883,6 +3947,7 @@ "expire_in": "2 weeks", "paths": [ "ghc-x86_64-linux-deb10-validate+thread_sanitizer.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -3944,6 +4009,7 @@ "expire_in": "2 weeks", "paths": [ "ghc-x86_64-linux-deb11-cross_aarch64-linux-gnu-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -4004,6 +4070,7 @@ "expire_in": "2 weeks", "paths": [ "ghc-x86_64-linux-deb11-int_native-cross_javascript-unknown-ghcjs-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -4065,6 +4132,7 @@ "expire_in": "2 weeks", "paths": [ "ghc-x86_64-linux-fedora33-release.tar.xz", + "metadata.json", "junit.xml" ], "reports": { @@ -4124,6 +4192,7 @@ "expire_in": "2 weeks", "paths": [ "ghc-x86_64-windows-validate.tar.xz", + "metadata.json", "junit.xml" ], "reports": { |