diff options
| -rw-r--r-- | .gitlab-ci.yml | 135 | ||||
| -rwxr-xr-x | .gitlab/gen_ci.hs | 135 | ||||
| -rwxr-xr-x | .gitlab/generate_job_metadata | 5 | ||||
| -rwxr-xr-x | .gitlab/generate_jobs | 2 | ||||
| -rw-r--r-- | .gitlab/rel_eng/default.nix | 2 | ||||
| -rw-r--r-- | .gitlab/rel_eng/mk-ghcup-metadata/.gitignore | 3 | ||||
| -rw-r--r-- | .gitlab/rel_eng/mk-ghcup-metadata/README.mkd | 56 | ||||
| -rw-r--r-- | .gitlab/rel_eng/mk-ghcup-metadata/default.nix | 13 | ||||
| -rwxr-xr-x | .gitlab/rel_eng/mk-ghcup-metadata/mk_ghcup_metadata.py | 274 | ||||
| -rw-r--r-- | .gitlab/rel_eng/mk-ghcup-metadata/setup.py | 14 |
10 files changed, 614 insertions, 25 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 059c36ca07..1304e4aa11 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -944,3 +944,138 @@ pages: artifacts: paths: - public + +############################################################# +# Generation of GHCUp metadata +############################################################# + + +# TODO: MP: This way of determining the project version is sadly very slow. +# It seems overkill to have to setup a complete environment, and build hadrian to get +# it to generate a single file containing the version information. +project-version: + stage: packaging + image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV" + tags: + - x86_64-linux + variables: + BUILD_FLAVOUR: default + script: + # Calculate the project version + - sudo chown ghc:ghc -R . + - .gitlab/ci.sh setup + - .gitlab/ci.sh configure + - .gitlab/ci.sh run_hadrian VERSION + - echo "ProjectVersion=$(cat VERSION)" > version.sh + + needs: [] + dependencies: [] + artifacts: + paths: + - version.sh + rules: + - if: '$NIGHTLY' + - if: '$RELEASE_JOB == "yes"' + +.ghcup-metadata: + stage: deploy + image: "nixos/nix:2.12.0" + dependencies: null + tags: + - x86_64-linux + variables: + BUILD_FLAVOUR: default + GIT_SUBMODULE_STRATEGY: "none" + before_script: + - echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf + - nix-channel --update + - cat version.sh + # Calculate the project version + - . ./version.sh + + # Download existing ghcup metadata + - nix shell --extra-experimental-features nix-command --extra-experimental-features flakes nixpkgs#wget -c wget "https://raw.githubusercontent.com/haskell/ghcup-metadata/develop/ghcup-0.0.7.yaml" + + - .gitlab/generate_job_metadata + + artifacts: + paths: + - metadata_test.yaml + - version.sh + +ghcup-metadata-nightly: + extends: .ghcup-metadata + # Explicit needs for validate pipeline because we only need certain bindists + needs: + - job: nightly-x86_64-linux-fedora33-release + artifacts: false + - job: nightly-x86_64-linux-centos7-validate + artifacts: false + - job: nightly-x86_64-darwin-validate + artifacts: false + - job: nightly-aarch64-darwin-validate + artifacts: false + - job: nightly-x86_64-windows-validate + artifacts: false + - job: nightly-x86_64-linux-alpine3_12-int_native-validate+fully_static + artifacts: false + - job: nightly-x86_64-linux-deb9-validate + artifacts: false + - job: nightly-i386-linux-deb9-validate + artifacts: false + - job: nightly-x86_64-linux-deb10-validate + artifacts: false + - job: nightly-aarch64-linux-deb10-validate + artifacts: false + - job: nightly-x86_64-linux-deb11-validate + artifacts: false + - job: source-tarball + artifacts: false + - job: project-version + script: + - nix shell --extra-experimental-features nix-command -f .gitlab/rel_eng -c ghcup-metadata --metadata ghcup-0.0.7.yaml --pipeline-id="$CI_PIPELINE_ID" --version="$ProjectVersion" > "metadata_test.yaml" + rules: + - if: $NIGHTLY + +ghcup-metadata-release: + # No explicit needs for release pipeline as we assume we need everything and everything will pass. + extends: .ghcup-metadata + script: + - nix shell --extra-experimental-features nix-command -f .gitlab/rel_eng -c ghcup-metadata --release-mode --metadata ghcup-0.0.7.yaml --pipeline-id="$CI_PIPELINE_ID" --version="$ProjectVersion" > "metadata_test.yaml" + rules: + - if: '$RELEASE_JOB == "yes"' + +.ghcup-metadata-testing: + stage: deploy + variables: + UPSTREAM_PROJECT_PATH: "$CI_PROJECT_PATH" + UPSTREAM_PROJECT_ID: "$CI_PROJECT_ID" + UPSTREAM_PIPELINE_ID: "$CI_PIPELINE_ID" + RELEASE_JOB: "$RELEASE_JOB" + trigger: + project: "ghc/ghcup-ci" + branch: "upstream-testing" + strategy: "depend" + +ghcup-metadata-testing-nightly: + needs: + - job: ghcup-metadata-nightly + artifacts: false + extends: .ghcup-metadata-testing + variables: + NIGHTLY: "$NIGHTLY" + UPSTREAM_JOB_NAME: "ghcup-metadata-nightly" + rules: + - if: '$NIGHTLY == "1"' + +ghcup-metadata-testing-release: + needs: + - job: ghcup-metadata-release + artifacts: false + extends: .ghcup-metadata-testing + variables: + UPSTREAM_JOB_NAME: "ghcup-metadata-release" + rules: + - if: '$RELEASE_JOB == "yes"' + when: manual + diff --git a/.gitlab/gen_ci.hs b/.gitlab/gen_ci.hs index 9e8130657f..95458e097c 100755 --- a/.gitlab/gen_ci.hs +++ b/.gitlab/gen_ci.hs @@ -17,6 +17,7 @@ import Data.List (intercalate) import Data.Set (Set) import qualified Data.Set as S import System.Environment +import Data.Maybe {- Note [Generating the CI pipeline] @@ -84,6 +85,16 @@ names of jobs to update these other places. 3. The ghc-head-from script downloads release artifacts based on a pipeline change. 4. Some subsequent CI jobs have explicit dependencies (for example docs-tarball, perf, perf-nofib) +Note [Generation Modes] +~~~~~~~~~~~~~~~~~~~~~~~ + +There are two different modes this script can operate in: + +* `gitlab`: Generates a job.yaml which defines all the pipelines for the platforms +* `metadata`: Generates a file which maps a platform the the "default" validate and + nightly pipeline. This file is intended to be used when generating + ghcup metadata. + -} ----------------------------------------------------------------------------- @@ -337,6 +348,9 @@ instance (Ord k, Semigroup v) => Monoid (MonoidalMap k v) where mminsertWith :: Ord k => (a -> a -> a) -> k -> a -> MonoidalMap k a -> MonoidalMap k a mminsertWith f k v (MonoidalMap m) = MonoidalMap (Map.insertWith f k v m) +mmlookup :: Ord k => k -> MonoidalMap k a -> Maybe a +mmlookup k (MonoidalMap m) = Map.lookup k m + type Variables = MonoidalMap String [String] (=:) :: String -> String -> Variables @@ -567,6 +581,7 @@ data Job , jobArtifacts :: Artifacts , jobCache :: Cache , jobRules :: OnOffRules + , jobPlatform :: (Arch, Opsys) } instance ToJSON Job where @@ -590,9 +605,11 @@ instance ToJSON Job where ] -- | Build a job description from the system description and 'BuildConfig' -job :: Arch -> Opsys -> BuildConfig -> (String, Job) -job arch opsys buildConfig = (jobName, Job {..}) +job :: Arch -> Opsys -> BuildConfig -> NamedJob Job +job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} } where + jobPlatform = (arch, opsys) + jobRules = emptyRules jobName = testEnv arch opsys buildConfig @@ -702,20 +719,20 @@ delVariable k j = j { jobVariables = MonoidalMap $ Map.delete k $ unMonoidalMap -- Building the standard jobs -- -- | Make a normal validate CI job -validate :: Arch -> Opsys -> BuildConfig -> (String, Job) +validate :: Arch -> Opsys -> BuildConfig -> NamedJob Job validate = job -- | Make a normal nightly CI job -nightly :: Arch -> Opsys -> BuildConfig -> ([Char], Job) +nightly :: Arch -> Opsys -> BuildConfig -> NamedJob Job nightly arch opsys bc = - let (n, j) = job arch opsys bc - in ("nightly-" ++ n, addJobRule Nightly . keepArtifacts "8 weeks" . highCompression $ j) + let NamedJob n j = job arch opsys bc + in NamedJob { name = "nightly-" ++ n, jobInfo = addJobRule Nightly . keepArtifacts "8 weeks" . highCompression $ j} -- | Make a normal release CI job -release :: Arch -> Opsys -> BuildConfig -> ([Char], Job) +release :: Arch -> Opsys -> BuildConfig -> NamedJob Job release arch opsys bc = - let (n, j) = job arch opsys (bc { buildFlavour = Release }) - in ("release-" ++ n, addJobRule ReleaseOnly . keepArtifacts "1 year" . ignorePerfFailures . highCompression $ j) + let NamedJob n j = job arch opsys (bc { buildFlavour = Release }) + in NamedJob { name = "release-" ++ n, jobInfo = addJobRule ReleaseOnly . keepArtifacts "1 year" . ignorePerfFailures . highCompression $ j} -- Specific job modification functions @@ -758,17 +775,33 @@ addValidateRule t = modifyValidateJobs (addJobRule t) disableValidate :: JobGroup Job -> JobGroup Job disableValidate = addValidateRule Disable +data NamedJob a = NamedJob { name :: String, jobInfo :: a } deriving Functor + +renameJob :: (String -> String) -> NamedJob a -> NamedJob a +renameJob f (NamedJob n i) = NamedJob (f n) i + +instance ToJSON a => ToJSON (NamedJob a) where + toJSON nj = object + [ "name" A..= name nj + , "jobInfo" A..= jobInfo nj ] + -- Jobs are grouped into either triples or pairs depending on whether the -- job is just validate and nightly, or also release. -data JobGroup a = StandardTriple { v :: (String, a) - , n :: (String, a) - , r :: (String, a) } - | ValidateOnly { v :: (String, a) - , n :: (String, a) } deriving Functor +data JobGroup a = StandardTriple { v :: NamedJob a + , n :: NamedJob a + , r :: NamedJob a } + | ValidateOnly { v :: NamedJob a + , n :: NamedJob a } deriving Functor + +instance ToJSON a => ToJSON (JobGroup a) where + toJSON jg = object + [ "n" A..= n jg + , "r" A..= r jg + ] rename :: (String -> String) -> JobGroup a -> JobGroup a -rename f (StandardTriple (nv, v) (nn, n) (nr, r)) = StandardTriple (f nv, v) (f nn, n) (f nr, r) -rename f (ValidateOnly (nv, v) (nn, n)) = ValidateOnly (f nv, v) (f nn, n) +rename f (StandardTriple nv nn nr) = StandardTriple (renameJob f nv) (renameJob f nn) (renameJob f nr) +rename f (ValidateOnly nv nn) = ValidateOnly (renameJob f nv) (renameJob f nn) -- | Construct a 'JobGroup' which consists of a validate, nightly and release build with -- a specific config. @@ -789,13 +822,21 @@ validateBuilds :: Arch -> Opsys -> BuildConfig -> JobGroup Job validateBuilds a op bc = ValidateOnly (validate a op bc) (nightly a op bc) flattenJobGroup :: JobGroup a -> [(String, a)] -flattenJobGroup (StandardTriple a b c) = [a,b,c] -flattenJobGroup (ValidateOnly a b) = [a, b] +flattenJobGroup (StandardTriple a b c) = map flattenNamedJob [a,b,c] +flattenJobGroup (ValidateOnly a b) = map flattenNamedJob [a, b] + +flattenNamedJob :: NamedJob a -> (String, a) +flattenNamedJob (NamedJob n i) = (n, i) -- | Specification for all the jobs we want to build. jobs :: Map String Job -jobs = Map.fromList $ concatMap (filter is_enabled_job . flattenJobGroup) +jobs = Map.fromList $ concatMap (filter is_enabled_job . flattenJobGroup) job_groups + where + is_enabled_job (_, Job {jobRules = OnOffRules {..}}) = not $ Disable `S.member` rule_set + +job_groups :: [JobGroup Job] +job_groups = [ disableValidate (standardBuilds Amd64 (Linux Debian10)) , standardBuildsWithConfig Amd64 (Linux Debian10) dwarf , validateBuilds Amd64 (Linux Debian10) nativeInt @@ -838,10 +879,7 @@ jobs = Map.fromList $ concatMap (filter is_enabled_job . flattenJobGroup) ] where - is_enabled_job (_, Job {jobRules = OnOffRules {..}}) = not $ Disable `S.member` rule_set - hackage_doc_job = rename (<> "-hackage") . modifyJobs (addVariable "HADRIAN_ARGS" "--haddock-base-url") - tsan_jobs = modifyJobs ( addVariable "TSAN_OPTIONS" "suppressions=$CI_PROJECT_DIR/rts/.tsan-suppressions" @@ -865,10 +903,59 @@ jobs = Map.fromList $ concatMap (filter is_enabled_job . flattenJobGroup) , buildFlavour = Release -- TODO: This needs to be validate but wasm backend doesn't pass yet } + +mkPlatform :: Arch -> Opsys -> String +mkPlatform arch opsys = archName arch <> "-" <> opsysName opsys + +-- | This map tells us for a specific arch/opsys combo what the job name for +-- nightly/release pipelines is. This is used by the ghcup metadata generation so that +-- things like bindist names etc are kept in-sync. +-- +-- For cases where there are just +-- +-- Otherwise: +-- * Prefer jobs which have a corresponding release pipeline +-- * Explicitly require tie-breaking for other cases. +platform_mapping :: Map String (JobGroup BindistInfo) +platform_mapping = Map.map go $ + Map.fromListWith combine [ (uncurry mkPlatform (jobPlatform (jobInfo $ v j)), j) | j <- job_groups ] + where + whitelist = [ "x86_64-linux-alpine3_12-int_native-validate+fully_static" + , "x86_64-linux-deb10-validate" + , "x86_64-linux-fedora33-release" + , "x86_64-windows-validate" + ] + + combine a b + | name (v a) `elem` whitelist = a -- Explicitly selected + | name (v b) `elem` whitelist = b + | hasReleaseBuild a, not (hasReleaseBuild b) = a -- Has release build, but other doesn't + | hasReleaseBuild b, not (hasReleaseBuild a) = b + | otherwise = error (show (name (v a)) ++ show (name (v b))) + + go = fmap (BindistInfo . unwords . fromJust . mmlookup "BIN_DIST_NAME" . jobVariables) + + hasReleaseBuild (StandardTriple{}) = True + hasReleaseBuild (ValidateOnly{}) = False + +data BindistInfo = BindistInfo { bindistName :: String } + +instance ToJSON BindistInfo where + toJSON (BindistInfo n) = object [ "bindistName" A..= n ] + + main :: IO () main = do - as <- getArgs + ass <- getArgs + case ass of + -- See Note [Generation Modes] + ("gitlab":as) -> write_result as jobs + ("metadata":as) -> write_result as platform_mapping + _ -> error "gen_ci.hs <gitlab|metadata> [file.json]" + +write_result as obj = (case as of [] -> B.putStrLn (fp:_) -> B.writeFile fp) - (A.encode jobs) + (A.encode obj) + diff --git a/.gitlab/generate_job_metadata b/.gitlab/generate_job_metadata new file mode 100755 index 0000000000..017f578f51 --- /dev/null +++ b/.gitlab/generate_job_metadata @@ -0,0 +1,5 @@ +#! /usr/bin/env nix-shell +#!nix-shell -i bash -p cabal-install "haskell.packages.ghc924.ghcWithPackages (pkgs: with pkgs; [aeson])" git jq + +cd "$(dirname "${BASH_SOURCE[0]}")" +cabal run gen_ci -- metadata jobs-metadata.json diff --git a/.gitlab/generate_jobs b/.gitlab/generate_jobs index 049157e8c3..0df674cceb 100755 --- a/.gitlab/generate_jobs +++ b/.gitlab/generate_jobs @@ -7,7 +7,7 @@ set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")" tmp=$(mktemp) -cabal run gen_ci -- $tmp +cabal run gen_ci -- gitlab $tmp rm -f jobs.yaml echo "### THIS IS A GENERATED FILE, DO NOT MODIFY DIRECTLY" > jobs.yaml cat $tmp | jq | tee -a jobs.yaml diff --git a/.gitlab/rel_eng/default.nix b/.gitlab/rel_eng/default.nix index 42435ba476..4cd6e98499 100644 --- a/.gitlab/rel_eng/default.nix +++ b/.gitlab/rel_eng/default.nix @@ -5,6 +5,7 @@ let sources = import ./nix/sources.nix; in with nixpkgs; let fetch-gitlab-artifacts = nixpkgs.callPackage ./fetch-gitlab-artifacts {}; + mk-ghcup-metadata = nixpkgs.callPackage ./mk-ghcup-metadata { fetch-gitlab=fetch-gitlab-artifacts;}; bindistPrepEnv = pkgs.buildFHSUserEnv { @@ -50,5 +51,6 @@ in paths = [ scripts fetch-gitlab-artifacts + mk-ghcup-metadata ]; } diff --git a/.gitlab/rel_eng/mk-ghcup-metadata/.gitignore b/.gitlab/rel_eng/mk-ghcup-metadata/.gitignore new file mode 100644 index 0000000000..1b01e3c7e9 --- /dev/null +++ b/.gitlab/rel_eng/mk-ghcup-metadata/.gitignore @@ -0,0 +1,3 @@ +result +fetch-gitlab +out diff --git a/.gitlab/rel_eng/mk-ghcup-metadata/README.mkd b/.gitlab/rel_eng/mk-ghcup-metadata/README.mkd new file mode 100644 index 0000000000..fe16439b61 --- /dev/null +++ b/.gitlab/rel_eng/mk-ghcup-metadata/README.mkd @@ -0,0 +1,56 @@ +# mk-ghcup-metadata + +This script is used to automatically generate metadata suitable for consumption by +GHCUp. + +# Usage + +``` +nix run -f .gitlab/rel_eng/ -c ghcup-metadata +``` + +``` +options: + -h, --help show this help message and exit + --metadata METADATA Path to GHCUp metadata + --pipeline-id PIPELINE_ID + Which pipeline to generate metadata for + --release-mode Generate metadata which points to downloads folder + --fragment Output the generated fragment rather than whole modified file + --version VERSION Version of the GHC compiler +``` + +The script also requires the `.gitlab/jobs-metadata.yaml` file which can be generated +by running `.gitlab/generate_jobs_metadata` script if you want to run it locally. + + +## CI Pipelines + +The metadata is generated by the nightly and release pipelines. + +* Nightly pipelines generate metadata where the bindist URLs point immediatley to + nightly artifacts. +* Release jobs can pass the `--release-mode` flag which downloads the artifacts from + the pipeline but the final download URLs for users point into the downloads folder. + +The mapping from platform to bindist is not clever, it is just what the GHCUp developers +tell us to use. + +## Testing Pipelines + +The metadata is tested by the `ghcup-ci` repo which is triggered by the +`ghcup-metadata-testing-nightly` job. + +This job sets the following variables which are then used by the downstream job +to collect the metadata from the correct place: + +* `UPSTREAM_PIPELINE_ID` - The pipeline ID which the generated metadata lives in +* `UPSTREAM_PROJECT_ID` - The project ID for the upstream project (almost always `1` (for ghc/ghc)) +* `UPSTREAM_JOB_NAME` - The job which the metadata belongs to (ie `ghcup-metadata-nightly`) +* `UPSTREAM_PROJECT_PATH` - The path of the upstream project (almost always ghc/ghc) + +Nightly pipelines are tested automaticaly but release pipelines are manually triggered +as the testing requires the bindists to be uploaded into the final release folder. + + + diff --git a/.gitlab/rel_eng/mk-ghcup-metadata/default.nix b/.gitlab/rel_eng/mk-ghcup-metadata/default.nix new file mode 100644 index 0000000000..61498503a4 --- /dev/null +++ b/.gitlab/rel_eng/mk-ghcup-metadata/default.nix @@ -0,0 +1,13 @@ +{ nix-gitignore, python3Packages, fetch-gitlab }: + +let + ghcup-metadata = { buildPythonPackage, python-gitlab, pyyaml }: + buildPythonPackage { + pname = "ghcup-metadata"; + version = "0.0.1"; + src = nix-gitignore.gitignoreSource [] ./.; + propagatedBuildInputs = [fetch-gitlab python-gitlab pyyaml ]; + preferLocalBuild = true; + }; +in +python3Packages.callPackage ghcup-metadata { } diff --git a/.gitlab/rel_eng/mk-ghcup-metadata/mk_ghcup_metadata.py b/.gitlab/rel_eng/mk-ghcup-metadata/mk_ghcup_metadata.py new file mode 100755 index 0000000000..394fb4e298 --- /dev/null +++ b/.gitlab/rel_eng/mk-ghcup-metadata/mk_ghcup_metadata.py @@ -0,0 +1,274 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i python3 -p curl "python3.withPackages (ps:[ps.pyyaml ps.python-gitlab ])" + +""" +A tool for generating metadata suitable for GHCUp + +There are two ways to prepare metadata: + +* From a nightly pipeline. +* From a release pipeline. + +In any case the script takes the same arguments: + + +* --metadata: The path to existing GHCup metadata to which we want to add the new entry. +* --version: GHC version of the pipeline +* --pipeline-id: The pipeline to generate metadata for +* --release-mode: Download from a release pipeline but generate URLs to point to downloads folder. +* --fragment: Only print out the updated fragment rather than the modified file + +The script will then download the relevant bindists to compute the hashes. The +generated metadata is printed to stdout. + +The metadata can then be used by passing the `--url-source` flag to ghcup. +""" + +from subprocess import run, check_call +from getpass import getpass +import shutil +from pathlib import Path +from typing import NamedTuple, Callable, List, Dict, Optional +import tempfile +import re +import pickle +import os +import yaml +import gitlab +from urllib.request import urlopen +import hashlib +import sys +import json +import urllib.parse +import fetch_gitlab + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +gl = gitlab.Gitlab('https://gitlab.haskell.org', per_page=100) + +# TODO: Take this file as an argument +metadata_file = ".gitlab/jobs-metadata.json" + +release_base = "https://downloads.haskell.org/~ghc/{version}/ghc-{version}-{bindistName}" + +eprint(f"Reading job metadata from {metadata_file}.") +with open(metadata_file, 'r') as f: + job_mapping = json.load(f) + +eprint(f"Supported platforms: {job_mapping.keys()}") + + +# Artifact precisely specifies a job what the bindist to download is called. +class Artifact(NamedTuple): + job_name: str + name: str + subdir: str + +# Platform spec provides a specification which is agnostic to Job +# PlatformSpecs are converted into Artifacts by looking in the jobs-metadata.json file. +class PlatformSpec(NamedTuple): + name: str + subdir: str + +source_artifact = Artifact('source-tarball', 'ghc-{version}-src.tar.xz', 'ghc-{version}' ) + +def debian(arch, n): + return linux_platform(arch, "{arch}-linux-deb{n}".format(arch=arch, n=n)) + +def darwin(arch): + return PlatformSpec ( '{arch}-darwin'.format(arch=arch) + , 'ghc-{version}-{arch}-apple-darwin'.format(arch=arch, version="{version}") ) + +windowsArtifact = PlatformSpec ( 'x86_64-windows' + , 'ghc-{version}-x86_64-unknown-mingw' ) + +def centos(n): + return linux_platform("x86_64", "x86_64-linux-centos{n}".format(n=n)) + +def fedora(n): + return linux_platform("x86_64", "x86_64-linux-fedora{n}".format(n=n)) + +def alpine(n): + return linux_platform("x86_64", "x86_64-linux-alpine{n}".format(n=n)) + +def linux_platform(arch, opsys): + return PlatformSpec( opsys, 'ghc-{version}-{arch}-unknown-linux'.format(version="{version}", arch=arch) ) + + +base_url = 'https://gitlab.haskell.org/ghc/ghc/-/jobs/{job_id}/artifacts/raw/{artifact_name}' + + +hash_cache = {} + +# Download a URL and return its hash +def download_and_hash(url): + if url in hash_cache: return hash_cache[url] + eprint ("Opening {}".format(url)) + response = urlopen(url) + sz = response.headers['content-length'] + hasher = hashlib.sha256() + CHUNK = 2**22 + for n,text in enumerate(iter(lambda: response.read(CHUNK), '')): + if not text: break + eprint("{:.2f}% {} / {} of {}".format (((n + 1) * CHUNK) / int(sz) * 100, (n + 1) * CHUNK, sz, url)) + hasher.update(text) + digest = hasher.hexdigest() + hash_cache[url] = digest + return digest + +# Make the metadata for one platform. +def mk_one_metadata(release_mode, version, job_map, artifact): + job_id = job_map[artifact.job_name].id + + url = base_url.format(job_id=job_id, artifact_name=urllib.parse.quote_plus(artifact.name.format(version=version))) + + # In --release-mode, the URL in the metadata needs to point into the downloads folder + # rather then the pipeline. + if release_mode: + final_url = release_base.format( version=version + , bindistName=urllib.parse.quote_plus(f"{fetch_gitlab.job_triple(artifact.job_name)}.tar.xz")) + else: + final_url = url + + eprint(f"Making metadata for: {artifact}") + eprint(f"Bindist URL: {url}") + eprint(f"Download URL: {final_url}") + + # Download and hash from the release pipeline, this must not change anyway during upload. + h = download_and_hash(url) + + res = { "dlUri": final_url, "dlSubdir": artifact.subdir.format(version=version), "dlHash" : h } + eprint(res) + return res + +# Turns a platform into an Artifact respecting pipeline_type +# Looks up the right job to use from the .gitlab/jobs-metadata.json file +def mk_from_platform(pipeline_type, platform): + info = job_mapping[platform.name][pipeline_type] + eprint(f"From {platform.name} / {pipeline_type} selecting {info['name']}") + return Artifact(info['name'] , f"{info['jobInfo']['bindistName']}.tar.xz", platform.subdir) + +# Generate the new metadata for a specific GHC mode etc +def mk_new_yaml(release_mode, version, pipeline_type, job_map): + def mk(platform): + eprint("\n=== " + platform.name + " " + ('=' * (75 - len(platform.name)))) + return mk_one_metadata(release_mode, version, job_map, mk_from_platform(pipeline_type, platform)) + + # Here are all the bindists we can distribute + centos7 = mk(centos(7)) + fedora33 = mk(fedora(33)) + darwin_x86 = mk(darwin("x86_64")) + darwin_arm64 = mk(darwin("aarch64")) + windows = mk(windowsArtifact) + alpine3_12 = mk(alpine("3_12")) + deb9 = mk(debian("x86_64", 9)) + deb10 = mk(debian("x86_64", 10)) + deb11 = mk(debian("x86_64", 11)) + deb10_arm64 = mk(debian("aarch64", 10)) + deb9_i386 = mk(debian("i386", 9)) + + source = mk_one_metadata(release_mode, version, job_map, source_artifact) + + # The actual metadata, this is not a precise science, but just what the ghcup + # developers want. + + a64 = { "Linux_Debian": { "< 10": deb9 + , "(>= 10 && < 11)": deb10 + , ">= 11": deb11 + , "unknown_versioning": deb11 } + , "Linux_Ubuntu" : { "unknown_versioning": deb10 + , "( >= 16 && < 19 )": deb9 + } + , "Linux_Mint" : { "< 20": deb9 + , ">= 20": deb10 } + , "Linux_CentOS" : { "( >= 7 && < 8 )" : centos7 + , "unknown_versioning" : centos7 } + , "Linux_Fedora" : { ">= 33": fedora33 + , "unknown_versioning": centos7 } + , "Linux_RedHat" : { "unknown_versioning": centos7 } + #MP: Replace here with Rocky8 when that job is in the pipeline + , "Linux_UnknownLinux" : { "unknown_versioning": fedora33 } + , "Darwin" : { "unknown_versioning" : darwin_x86 } + , "Windows" : { "unknown_versioning" : windows } + , "Linux_Alpine" : { "unknown_versioning": alpine3_12 } + + } + + a32 = { "Linux_Debian": { "<10": deb9_i386, "unknown_versioning": deb9_i386 } + , "Linux_Ubuntu": { "unknown_versioning": deb9_i386 } + , "Linux_Mint" : { "unknown_versioning": deb9_i386 } + , "Linux_UnknownLinux" : { "unknown_versioning": deb9_i386 } + } + + arm64 = { "Linux_UnknownLinux": { "unknown_versioning": deb10_arm64 } + , "Darwin": { "unknown_versioning": darwin_arm64 } + } + + if release_mode: + version_parts = version.split('.') + if len(version_parts) == 3: + final_version = version + elif len(version_parts) == 4: + final_version = '.'.join(version_parts[:2] + [str(int(version_parts[2]) + 1)]) + change_log = f"https://downloads.haskell.org/~ghc/{version}/docs/users_guide/{final_version}-notes.html" + else: + change_log = "https://gitlab.haskell.org" + + return { "viTags": ["Latest", "TODO_base_version"] + # Check that this link exists + , "viChangeLog": change_log + , "viSourceDL": source + , "viPostRemove": "*ghc-post-remove" + , "viArch": { "A_64": a64 + , "A_32": a32 + , "A_ARM64": arm64 + } + } + + +def main() -> None: + import argparse + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--metadata', required=True, type=Path, help='Path to GHCUp metadata') + parser.add_argument('--pipeline-id', required=True, type=int, help='Which pipeline to generate metadata for') + parser.add_argument('--release-mode', action='store_true', help='Generate metadata which points to downloads folder') + parser.add_argument('--fragment', action='store_true', help='Output the generated fragment rather than whole modified file') + # TODO: We could work out the --version from the project-version CI job. + parser.add_argument('--version', required=True, type=str, help='Version of the GHC compiler') + args = parser.parse_args() + + project = gl.projects.get(1, lazy=True) + pipeline = project.pipelines.get(args.pipeline_id) + jobs = pipeline.jobs.list() + job_map = { job.name: job for job in jobs } + # Bit of a hacky way to determine what pipeline we are dealing with but + # the aarch64-darwin job should stay stable for a long time. + if 'nightly-aarch64-darwin-validate' in job_map: + pipeline_type = 'n' + if args.release_mode: + raise Exception("Incompatible arguments: nightly pipeline but using --release-mode") + + elif 'release-aarch64-darwin-release' in job_map: + pipeline_type = 'r' + else: + raise Exception("Not a nightly nor release pipeline") + eprint(f"Pipeline Type: {pipeline_type}") + + + new_yaml = mk_new_yaml(args.release_mode, args.version, pipeline_type, job_map) + if args.fragment: + print(yaml.dump({ args.version : new_yaml })) + + else: + with open(args.metadata, 'r') as file: + ghcup_metadata = yaml.safe_load(file) + ghcup_metadata['ghcupDownloads']['GHC'][args.version] = new_yaml + print(yaml.dump(ghcup_metadata)) + + +if __name__ == '__main__': + main() + diff --git a/.gitlab/rel_eng/mk-ghcup-metadata/setup.py b/.gitlab/rel_eng/mk-ghcup-metadata/setup.py new file mode 100644 index 0000000000..d4f4efe5e4 --- /dev/null +++ b/.gitlab/rel_eng/mk-ghcup-metadata/setup.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python + +from distutils.core import setup + +setup(name='ghcup-metadata', + author='Matthew Pickering', + author_email='matthew@well-typed.com', + py_modules=['mk_ghcup_metadata'], + entry_points={ + 'console_scripts': [ + 'ghcup-metadata=mk_ghcup_metadata:main', + ] + } + ) |