Add scripts to generate ghcup metadata on nightly and release pipelines

1. A python script in .gitlab/rel_eng/mk-ghcup-metadata which generates
   suitable metadata for consumption by GHCUp for the relevant
   pipelines.

  - The script generates the metadata just as the ghcup maintainers
    want, without taking into account platform/library combinations. It
    is updated manually when the mapping changes.

  - The script downloads the bindists which ghcup wants to distribute,
    calculates the hash and generates the yaml in the correct structure.

  - The script is documented in the .gitlab/rel_eng/mk-ghcup-metadata/README.mk file

1a. The script requires us to understand the mapping from platform ->
    job. To choose the preferred bindist for each platform the
    .gitlab/gen_ci.hs script is modified to allow outputting a metadata
    file which answers the question about which job produces the
    bindist which we want to distribute to users for a specific
    platform.

2. Pipelines to run on nightly and release jobs to generate metadata

  - ghcup-metadata-nightly: Generates metadata which points directly to
    artifacts in the nightly job.

  - ghcup-metadata-release: Generates metadata suitable for inclusion
    directly in ghcup by pointing to the downloads folder where the
    bindist will be uploaded to.

2a. Trigger jobs which test the generated metadata in the downstream
    `ghccup-ci` repo. See that repo for documentation about what is
    tested and how but essentially we test in a variety of clean images
    that ghcup can download and install the bindists we say exist in our
    metadata.

10 files changed, 614 insertions, 25 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 059c36ca07..1304e4aa11 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -944,3 +944,138 @@ pages:
   artifacts:
     paths:
       - public
+
+#############################################################
+# Generation of GHCUp metadata
+#############################################################
+
+
+# TODO: MP: This way of determining the project version is sadly very slow.
+# It seems overkill to have to setup a complete environment, and build hadrian to get
+# it to generate a single file containing the version information.
+project-version:
+  stage: packaging
+  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb10:$DOCKER_REV"
+  tags:
+    - x86_64-linux
+  variables:
+    BUILD_FLAVOUR: default
+  script:
+    # Calculate the project version
+    - sudo chown ghc:ghc -R .
+    - .gitlab/ci.sh setup
+    - .gitlab/ci.sh configure
+    - .gitlab/ci.sh run_hadrian VERSION
+    - echo "ProjectVersion=$(cat VERSION)" > version.sh
+
+  needs: []
+  dependencies: []
+  artifacts:
+    paths:
+      - version.sh
+  rules:
+    - if: '$NIGHTLY'
+    - if: '$RELEASE_JOB == "yes"'
+
+.ghcup-metadata:
+  stage: deploy
+  image: "nixos/nix:2.12.0"
+  dependencies: null
+  tags:
+    - x86_64-linux
+  variables:
+    BUILD_FLAVOUR: default
+    GIT_SUBMODULE_STRATEGY: "none"
+  before_script:
+    - echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf
+    - nix-channel --update
+    - cat version.sh
+    # Calculate the project version
+    - . ./version.sh
+
+    # Download existing ghcup metadata
+    - nix shell --extra-experimental-features nix-command --extra-experimental-features flakes nixpkgs#wget -c wget "https://raw.githubusercontent.com/haskell/ghcup-metadata/develop/ghcup-0.0.7.yaml"
+
+    - .gitlab/generate_job_metadata
+
+  artifacts:
+    paths:
+      - metadata_test.yaml
+      - version.sh
+
+ghcup-metadata-nightly:
+  extends: .ghcup-metadata
+  # Explicit needs for validate pipeline because we only need certain bindists
+  needs:
+    - job: nightly-x86_64-linux-fedora33-release
+      artifacts: false
+    - job: nightly-x86_64-linux-centos7-validate
+      artifacts: false
+    - job: nightly-x86_64-darwin-validate
+      artifacts: false
+    - job: nightly-aarch64-darwin-validate
+      artifacts: false
+    - job: nightly-x86_64-windows-validate
+      artifacts: false
+    - job: nightly-x86_64-linux-alpine3_12-int_native-validate+fully_static
+      artifacts: false
+    - job: nightly-x86_64-linux-deb9-validate
+      artifacts: false
+    - job: nightly-i386-linux-deb9-validate
+      artifacts: false
+    - job: nightly-x86_64-linux-deb10-validate
+      artifacts: false
+    - job: nightly-aarch64-linux-deb10-validate
+      artifacts: false
+    - job: nightly-x86_64-linux-deb11-validate
+      artifacts: false
+    - job: source-tarball
+      artifacts: false
+    - job: project-version
+  script:
+    - nix shell --extra-experimental-features nix-command -f .gitlab/rel_eng -c ghcup-metadata --metadata ghcup-0.0.7.yaml --pipeline-id="$CI_PIPELINE_ID" --version="$ProjectVersion" > "metadata_test.yaml"
+  rules:
+    - if: $NIGHTLY
+
+ghcup-metadata-release:
+  # No explicit needs for release pipeline as we assume we need everything and everything will pass.
+  extends: .ghcup-metadata
+  script:
+    - nix shell --extra-experimental-features nix-command -f .gitlab/rel_eng -c ghcup-metadata --release-mode --metadata ghcup-0.0.7.yaml --pipeline-id="$CI_PIPELINE_ID" --version="$ProjectVersion" > "metadata_test.yaml"
+  rules:
+    - if: '$RELEASE_JOB == "yes"'
+
+.ghcup-metadata-testing:
+  stage: deploy
+  variables:
+    UPSTREAM_PROJECT_PATH: "$CI_PROJECT_PATH"
+    UPSTREAM_PROJECT_ID: "$CI_PROJECT_ID"
+    UPSTREAM_PIPELINE_ID: "$CI_PIPELINE_ID"
+    RELEASE_JOB: "$RELEASE_JOB"
+  trigger:
+    project: "ghc/ghcup-ci"
+    branch: "upstream-testing"
+    strategy: "depend"
+
+ghcup-metadata-testing-nightly:
+  needs:
+    - job: ghcup-metadata-nightly
+      artifacts: false
+  extends: .ghcup-metadata-testing
+  variables:
+      NIGHTLY: "$NIGHTLY"
+      UPSTREAM_JOB_NAME: "ghcup-metadata-nightly"
+  rules:
+    - if: '$NIGHTLY == "1"'
+
+ghcup-metadata-testing-release:
+  needs:
+    - job: ghcup-metadata-release
+      artifacts: false
+  extends: .ghcup-metadata-testing
+  variables:
+      UPSTREAM_JOB_NAME: "ghcup-metadata-release"
+  rules:
+    - if: '$RELEASE_JOB == "yes"'
+  when: manual
+
diff --git a/.gitlab/gen_ci.hs b/.gitlab/gen_ci.hs
index 9e8130657f..95458e097c 100755
--- a/.gitlab/gen_ci.hs
+++ b/.gitlab/gen_ci.hs
@@ -17,6 +17,7 @@ import Data.List (intercalate)
 import Data.Set (Set)
 import qualified Data.Set as S
 import System.Environment
+import Data.Maybe
 
 {-
 Note [Generating the CI pipeline]
@@ -84,6 +85,16 @@ names of jobs to update these other places.
 3. The ghc-head-from script downloads release artifacts based on a pipeline change.
 4. Some subsequent CI jobs have explicit dependencies (for example docs-tarball, perf, perf-nofib)
 
+Note [Generation Modes]
+~~~~~~~~~~~~~~~~~~~~~~~
+
+There are two different modes this  script can operate in:
+
+* `gitlab`: Generates a job.yaml which defines all the pipelines for the platforms
+* `metadata`: Generates a file which maps a platform the the "default" validate and
+              nightly pipeline. This file is intended to be used when generating
+              ghcup metadata.
+
 -}
 
 -----------------------------------------------------------------------------
@@ -337,6 +348,9 @@ instance (Ord k, Semigroup v) => Monoid (MonoidalMap k v) where
 mminsertWith :: Ord k => (a -> a -> a) -> k -> a -> MonoidalMap k a -> MonoidalMap k a
 mminsertWith f k v (MonoidalMap m) = MonoidalMap (Map.insertWith f k v m)
 
+mmlookup :: Ord k => k -> MonoidalMap k a -> Maybe a
+mmlookup k (MonoidalMap m) = Map.lookup k m
+
 type Variables = MonoidalMap String [String]
 
 (=:) :: String -> String -> Variables
@@ -567,6 +581,7 @@ data Job
         , jobArtifacts :: Artifacts
         , jobCache :: Cache
         , jobRules :: OnOffRules
+        , jobPlatform  :: (Arch, Opsys)
         }
 
 instance ToJSON Job where
@@ -590,9 +605,11 @@ instance ToJSON Job where
     ]
 
 -- | Build a job description from the system description and 'BuildConfig'
-job :: Arch -> Opsys -> BuildConfig -> (String, Job)
-job arch opsys buildConfig = (jobName, Job {..})
+job :: Arch -> Opsys -> BuildConfig -> NamedJob Job
+job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} }
   where
+    jobPlatform = (arch, opsys)
+
     jobRules = emptyRules
 
     jobName = testEnv arch opsys buildConfig
@@ -702,20 +719,20 @@ delVariable k j = j { jobVariables = MonoidalMap $ Map.delete k $ unMonoidalMap
 -- Building the standard jobs
 --
 -- | Make a normal validate CI job
-validate :: Arch -> Opsys -> BuildConfig -> (String, Job)
+validate :: Arch -> Opsys -> BuildConfig -> NamedJob Job
 validate = job
 
 -- | Make a normal nightly CI job
-nightly :: Arch -> Opsys -> BuildConfig -> ([Char], Job)
+nightly :: Arch -> Opsys -> BuildConfig -> NamedJob Job
 nightly arch opsys bc =
-  let (n, j) = job arch opsys bc
-  in ("nightly-" ++ n, addJobRule Nightly . keepArtifacts "8 weeks" . highCompression $ j)
+  let NamedJob n j = job arch opsys bc
+  in NamedJob { name = "nightly-" ++ n, jobInfo = addJobRule Nightly . keepArtifacts "8 weeks" . highCompression $ j}
 
 -- | Make a normal release CI job
-release :: Arch -> Opsys -> BuildConfig -> ([Char], Job)
+release :: Arch -> Opsys -> BuildConfig -> NamedJob Job
 release arch opsys bc =
-  let (n, j) = job arch opsys (bc { buildFlavour = Release })
-  in ("release-" ++ n, addJobRule ReleaseOnly . keepArtifacts "1 year" . ignorePerfFailures . highCompression $ j)
+  let NamedJob n j = job arch opsys (bc { buildFlavour = Release })
+  in NamedJob { name = "release-" ++ n, jobInfo = addJobRule ReleaseOnly . keepArtifacts "1 year" . ignorePerfFailures . highCompression $ j}
 
 -- Specific job modification functions
 
@@ -758,17 +775,33 @@ addValidateRule t = modifyValidateJobs (addJobRule t)
 disableValidate :: JobGroup Job -> JobGroup Job
 disableValidate = addValidateRule Disable
 
+data NamedJob a = NamedJob { name :: String, jobInfo :: a } deriving Functor
+
+renameJob :: (String -> String) -> NamedJob a -> NamedJob a
+renameJob f (NamedJob n i) = NamedJob (f n) i
+
+instance ToJSON a => ToJSON (NamedJob a) where
+  toJSON nj = object
+    [ "name" A..= name nj
+    , "jobInfo" A..= jobInfo nj ]
+
 -- Jobs are grouped into either triples or pairs depending on whether the
 -- job is just validate and nightly, or also release.
-data JobGroup a = StandardTriple { v :: (String, a)
-                                 , n :: (String, a)
-                                 , r :: (String, a) }
-                | ValidateOnly   { v :: (String, a)
-                                 , n :: (String, a) } deriving Functor
+data JobGroup a = StandardTriple { v :: NamedJob a
+                                 , n :: NamedJob a
+                                 , r :: NamedJob a }
+                | ValidateOnly   { v :: NamedJob a
+                                 , n :: NamedJob a } deriving Functor
+
+instance ToJSON a => ToJSON (JobGroup a) where
+  toJSON jg = object
+    [ "n" A..= n jg
+    , "r" A..= r jg
+    ]
 
 rename :: (String -> String) -> JobGroup a -> JobGroup a
-rename f (StandardTriple (nv, v) (nn, n) (nr, r)) = StandardTriple (f nv, v) (f nn, n) (f nr, r)
-rename f (ValidateOnly (nv, v) (nn, n)) = ValidateOnly (f nv, v) (f nn, n)
+rename f (StandardTriple nv nn nr) = StandardTriple (renameJob f nv) (renameJob f nn) (renameJob f nr)
+rename f (ValidateOnly nv nn) = ValidateOnly (renameJob f nv) (renameJob f nn)
 
 -- | Construct a 'JobGroup' which consists of a validate, nightly and release build with
 -- a specific config.
@@ -789,13 +822,21 @@ validateBuilds :: Arch -> Opsys -> BuildConfig -> JobGroup Job
 validateBuilds a op bc = ValidateOnly (validate a op bc) (nightly a op bc)
 
 flattenJobGroup :: JobGroup a -> [(String, a)]
-flattenJobGroup (StandardTriple a b c) = [a,b,c]
-flattenJobGroup (ValidateOnly a b) = [a, b]
+flattenJobGroup (StandardTriple a b c) = map flattenNamedJob [a,b,c]
+flattenJobGroup (ValidateOnly a b) = map flattenNamedJob [a, b]
+
+flattenNamedJob :: NamedJob a -> (String, a)
+flattenNamedJob (NamedJob n i) = (n, i)
 
 
 -- | Specification for all the jobs we want to build.
 jobs :: Map String Job
-jobs = Map.fromList $ concatMap (filter is_enabled_job . flattenJobGroup)
+jobs = Map.fromList $ concatMap (filter is_enabled_job . flattenJobGroup) job_groups
+  where
+    is_enabled_job (_, Job {jobRules = OnOffRules {..}}) = not $ Disable `S.member` rule_set
+
+job_groups :: [JobGroup Job]
+job_groups =
      [ disableValidate (standardBuilds Amd64 (Linux Debian10))
      , standardBuildsWithConfig Amd64 (Linux Debian10) dwarf
      , validateBuilds Amd64 (Linux Debian10) nativeInt
@@ -838,10 +879,7 @@ jobs = Map.fromList $ concatMap (filter is_enabled_job . flattenJobGroup)
      ]
 
   where
-    is_enabled_job (_, Job {jobRules = OnOffRules {..}}) = not $ Disable `S.member` rule_set
-
     hackage_doc_job = rename (<> "-hackage") . modifyJobs (addVariable "HADRIAN_ARGS" "--haddock-base-url")
-
     tsan_jobs =
       modifyJobs
         ( addVariable "TSAN_OPTIONS" "suppressions=$CI_PROJECT_DIR/rts/.tsan-suppressions"
@@ -865,10 +903,59 @@ jobs = Map.fromList $ concatMap (filter is_enabled_job . flattenJobGroup)
           , buildFlavour     = Release -- TODO: This needs to be validate but wasm backend doesn't pass yet
         }
 
+
+mkPlatform :: Arch -> Opsys -> String
+mkPlatform arch opsys = archName arch <> "-" <> opsysName opsys
+
+-- | This map tells us for a specific arch/opsys combo what the job name for
+-- nightly/release pipelines is. This is used by the ghcup metadata generation so that
+-- things like bindist names etc are kept in-sync.
+--
+-- For cases where there are just
+--
+-- Otherwise:
+--  * Prefer jobs which have a corresponding release pipeline
+--  * Explicitly require tie-breaking for other cases.
+platform_mapping :: Map String (JobGroup BindistInfo)
+platform_mapping = Map.map go $
+  Map.fromListWith combine [ (uncurry mkPlatform (jobPlatform (jobInfo $ v j)), j) | j <- job_groups ]
+  where
+    whitelist = [ "x86_64-linux-alpine3_12-int_native-validate+fully_static"
+                , "x86_64-linux-deb10-validate"
+                , "x86_64-linux-fedora33-release"
+                , "x86_64-windows-validate"
+                ]
+
+    combine a b
+      | name (v a) `elem` whitelist = a -- Explicitly selected
+      | name (v b) `elem` whitelist = b
+      | hasReleaseBuild a, not (hasReleaseBuild b) = a -- Has release build, but other doesn't
+      | hasReleaseBuild b, not (hasReleaseBuild a) = b
+      | otherwise = error (show (name (v a)) ++ show (name (v b)))
+
+    go = fmap (BindistInfo . unwords . fromJust . mmlookup "BIN_DIST_NAME" . jobVariables)
+
+    hasReleaseBuild (StandardTriple{}) = True
+    hasReleaseBuild (ValidateOnly{}) = False
+
+data BindistInfo = BindistInfo { bindistName :: String }
+
+instance ToJSON BindistInfo where
+  toJSON (BindistInfo n) = object [ "bindistName" A..= n ]
+
+
 main :: IO ()
 main = do
-  as <- getArgs
+  ass <- getArgs
+  case ass of
+    -- See Note [Generation Modes]
+    ("gitlab":as) -> write_result as jobs
+    ("metadata":as) -> write_result as platform_mapping
+    _ -> error "gen_ci.hs <gitlab|metadata> [file.json]"
+
+write_result as obj =
   (case as of
     [] -> B.putStrLn
     (fp:_) -> B.writeFile fp)
-    (A.encode jobs)
+    (A.encode obj)
+
diff --git a/.gitlab/generate_job_metadata b/.gitlab/generate_job_metadata
new file mode 100755
index 0000000000..017f578f51
--- /dev/null
+++ b/.gitlab/generate_job_metadata
@@ -0,0 +1,5 @@
+#! /usr/bin/env nix-shell
+#!nix-shell -i bash -p cabal-install "haskell.packages.ghc924.ghcWithPackages (pkgs: with pkgs; [aeson])" git jq
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+cabal run gen_ci -- metadata jobs-metadata.json
diff --git a/.gitlab/generate_jobs b/.gitlab/generate_jobs
index 049157e8c3..0df674cceb 100755
--- a/.gitlab/generate_jobs
+++ b/.gitlab/generate_jobs
@@ -7,7 +7,7 @@ set -euo pipefail
 
 cd "$(dirname "${BASH_SOURCE[0]}")"
 tmp=$(mktemp)
-cabal run gen_ci -- $tmp
+cabal run gen_ci -- gitlab $tmp
 rm -f jobs.yaml
 echo "### THIS IS A GENERATED FILE, DO NOT MODIFY DIRECTLY" > jobs.yaml
 cat $tmp | jq | tee -a jobs.yaml
diff --git a/.gitlab/rel_eng/default.nix b/.gitlab/rel_eng/default.nix
index 42435ba476..4cd6e98499 100644
--- a/.gitlab/rel_eng/default.nix
+++ b/.gitlab/rel_eng/default.nix
@@ -5,6 +5,7 @@ let sources = import ./nix/sources.nix; in
 with nixpkgs;
 let
   fetch-gitlab-artifacts = nixpkgs.callPackage ./fetch-gitlab-artifacts {};
+  mk-ghcup-metadata = nixpkgs.callPackage ./mk-ghcup-metadata { fetch-gitlab=fetch-gitlab-artifacts;};
 
 
   bindistPrepEnv = pkgs.buildFHSUserEnv {
@@ -50,5 +51,6 @@ in
     paths = [
       scripts
       fetch-gitlab-artifacts
+      mk-ghcup-metadata
     ];
   }
diff --git a/.gitlab/rel_eng/mk-ghcup-metadata/.gitignore b/.gitlab/rel_eng/mk-ghcup-metadata/.gitignore
new file mode 100644
index 0000000000..1b01e3c7e9
--- /dev/null
+++ b/.gitlab/rel_eng/mk-ghcup-metadata/.gitignore
@@ -0,0 +1,3 @@
+result
+fetch-gitlab
+out
diff --git a/.gitlab/rel_eng/mk-ghcup-metadata/README.mkd b/.gitlab/rel_eng/mk-ghcup-metadata/README.mkd
new file mode 100644
index 0000000000..fe16439b61
--- /dev/null
+++ b/.gitlab/rel_eng/mk-ghcup-metadata/README.mkd
@@ -0,0 +1,56 @@
+# mk-ghcup-metadata
+
+This script is used to automatically generate metadata suitable for consumption by
+GHCUp.
+
+# Usage
+
+```
+nix run -f .gitlab/rel_eng/ -c ghcup-metadata
+```
+
+```
+options:
+  -h, --help            show this help message and exit
+  --metadata METADATA   Path to GHCUp metadata
+  --pipeline-id PIPELINE_ID
+                        Which pipeline to generate metadata for
+  --release-mode        Generate metadata which points to downloads folder
+  --fragment            Output the generated fragment rather than whole modified file
+  --version VERSION     Version of the GHC compiler
+```
+
+The script also requires the `.gitlab/jobs-metadata.yaml` file which can be generated
+by running `.gitlab/generate_jobs_metadata` script if you want to run it locally.
+
+
+## CI Pipelines
+
+The metadata is generated by the nightly and release pipelines.
+
+* Nightly pipelines generate metadata where the bindist URLs point immediatley to
+  nightly artifacts.
+* Release jobs can pass the `--release-mode` flag which downloads the artifacts from
+  the pipeline but the final download URLs for users point into the downloads folder.
+
+The mapping from platform to bindist is not clever, it is just what the GHCUp developers
+tell us to use.
+
+## Testing Pipelines
+
+The metadata is tested by the `ghcup-ci` repo which is triggered by the
+`ghcup-metadata-testing-nightly` job.
+
+This job sets the following variables which are then used by the downstream job
+to collect the metadata from the correct place:
+
+* `UPSTREAM_PIPELINE_ID`  - The pipeline ID which the generated metadata lives in
+* `UPSTREAM_PROJECT_ID`   - The project ID for the upstream project (almost always `1` (for ghc/ghc))
+* `UPSTREAM_JOB_NAME`     - The job which the metadata belongs to (ie `ghcup-metadata-nightly`)
+* `UPSTREAM_PROJECT_PATH` - The path of the upstream project (almost always ghc/ghc)
+
+Nightly pipelines are tested automaticaly but release pipelines are manually triggered
+as the testing requires the bindists to be uploaded into the final release folder.
+
+
+
diff --git a/.gitlab/rel_eng/mk-ghcup-metadata/default.nix b/.gitlab/rel_eng/mk-ghcup-metadata/default.nix
new file mode 100644
index 0000000000..61498503a4
--- /dev/null
+++ b/.gitlab/rel_eng/mk-ghcup-metadata/default.nix
@@ -0,0 +1,13 @@
+{ nix-gitignore, python3Packages, fetch-gitlab }:
+
+let
+  ghcup-metadata = { buildPythonPackage, python-gitlab, pyyaml }:
+    buildPythonPackage {
+      pname = "ghcup-metadata";
+      version = "0.0.1";
+      src = nix-gitignore.gitignoreSource [] ./.;
+      propagatedBuildInputs = [fetch-gitlab python-gitlab pyyaml ];
+      preferLocalBuild = true;
+    };
+in
+python3Packages.callPackage ghcup-metadata { }
diff --git a/.gitlab/rel_eng/mk-ghcup-metadata/mk_ghcup_metadata.py b/.gitlab/rel_eng/mk-ghcup-metadata/mk_ghcup_metadata.py
new file mode 100755
index 0000000000..394fb4e298
--- /dev/null
+++ b/.gitlab/rel_eng/mk-ghcup-metadata/mk_ghcup_metadata.py
@@ -0,0 +1,274 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i python3 -p curl  "python3.withPackages (ps:[ps.pyyaml ps.python-gitlab ])"
+
+"""
+A tool for generating metadata suitable for GHCUp
+
+There are two ways to prepare metadata:
+
+* From a nightly pipeline.
+* From a release pipeline.
+
+In any case the script takes the same arguments:
+
+
+* --metadata: The path to existing GHCup metadata to which we want to add the new entry.
+* --version: GHC version of the pipeline
+* --pipeline-id: The pipeline to generate metadata for
+* --release-mode: Download from a release pipeline but generate URLs to point to downloads folder.
+* --fragment: Only print out the updated fragment rather than the modified file
+
+The script will then download the relevant bindists to compute the hashes. The
+generated metadata is printed to stdout.
+
+The metadata can then be used by passing the `--url-source` flag to ghcup.
+"""
+
+from subprocess import run, check_call
+from getpass import getpass
+import shutil
+from pathlib import Path
+from typing import NamedTuple, Callable, List, Dict, Optional
+import tempfile
+import re
+import pickle
+import os
+import yaml
+import gitlab
+from urllib.request import urlopen
+import hashlib
+import sys
+import json
+import urllib.parse
+import fetch_gitlab
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
+gl = gitlab.Gitlab('https://gitlab.haskell.org', per_page=100)
+
+# TODO: Take this file as an argument
+metadata_file = ".gitlab/jobs-metadata.json"
+
+release_base = "https://downloads.haskell.org/~ghc/{version}/ghc-{version}-{bindistName}"
+
+eprint(f"Reading job metadata from {metadata_file}.")
+with open(metadata_file, 'r') as f:
+  job_mapping = json.load(f)
+
+eprint(f"Supported platforms: {job_mapping.keys()}")
+
+
+# Artifact precisely specifies a job what the bindist to download is called.
+class Artifact(NamedTuple):
+    job_name: str
+    name: str
+    subdir: str
+
+# Platform spec provides a specification which is agnostic to Job
+# PlatformSpecs are converted into Artifacts by looking in the jobs-metadata.json file.
+class PlatformSpec(NamedTuple):
+    name: str
+    subdir: str
+
+source_artifact = Artifact('source-tarball', 'ghc-{version}-src.tar.xz', 'ghc-{version}' )
+
+def debian(arch, n):
+    return linux_platform(arch, "{arch}-linux-deb{n}".format(arch=arch, n=n))
+
+def darwin(arch):
+    return PlatformSpec ( '{arch}-darwin'.format(arch=arch)
+                        , 'ghc-{version}-{arch}-apple-darwin'.format(arch=arch, version="{version}") )
+
+windowsArtifact = PlatformSpec ( 'x86_64-windows'
+                               , 'ghc-{version}-x86_64-unknown-mingw' )
+
+def centos(n):
+    return linux_platform("x86_64", "x86_64-linux-centos{n}".format(n=n))
+
+def fedora(n):
+    return linux_platform("x86_64", "x86_64-linux-fedora{n}".format(n=n))
+
+def alpine(n):
+    return linux_platform("x86_64", "x86_64-linux-alpine{n}".format(n=n))
+
+def linux_platform(arch, opsys):
+    return PlatformSpec( opsys, 'ghc-{version}-{arch}-unknown-linux'.format(version="{version}", arch=arch) )
+
+
+base_url = 'https://gitlab.haskell.org/ghc/ghc/-/jobs/{job_id}/artifacts/raw/{artifact_name}'
+
+
+hash_cache = {}
+
+# Download a URL and return its hash
+def download_and_hash(url):
+    if url in hash_cache: return hash_cache[url]
+    eprint ("Opening {}".format(url))
+    response = urlopen(url)
+    sz = response.headers['content-length']
+    hasher = hashlib.sha256()
+    CHUNK = 2**22
+    for n,text in enumerate(iter(lambda: response.read(CHUNK), '')):
+        if not text: break
+        eprint("{:.2f}% {} / {} of {}".format (((n + 1) * CHUNK) / int(sz) * 100, (n + 1) * CHUNK, sz, url))
+        hasher.update(text)
+    digest = hasher.hexdigest()
+    hash_cache[url] = digest
+    return digest
+
+# Make the metadata for one platform.
+def mk_one_metadata(release_mode, version, job_map, artifact):
+    job_id = job_map[artifact.job_name].id
+
+    url = base_url.format(job_id=job_id, artifact_name=urllib.parse.quote_plus(artifact.name.format(version=version)))
+
+    # In --release-mode, the URL in the metadata needs to point into the downloads folder
+    # rather then the pipeline.
+    if release_mode:
+        final_url = release_base.format( version=version
+                                       , bindistName=urllib.parse.quote_plus(f"{fetch_gitlab.job_triple(artifact.job_name)}.tar.xz"))
+    else:
+        final_url = url
+
+    eprint(f"Making metadata for: {artifact}")
+    eprint(f"Bindist URL: {url}")
+    eprint(f"Download URL: {final_url}")
+
+    # Download and hash from the release pipeline, this must not change anyway during upload.
+    h = download_and_hash(url)
+
+    res = { "dlUri": final_url, "dlSubdir": artifact.subdir.format(version=version), "dlHash" : h }
+    eprint(res)
+    return res
+
+# Turns a platform into an Artifact respecting pipeline_type
+# Looks up the right job to use from the .gitlab/jobs-metadata.json file
+def mk_from_platform(pipeline_type, platform):
+    info = job_mapping[platform.name][pipeline_type]
+    eprint(f"From {platform.name} / {pipeline_type} selecting {info['name']}")
+    return Artifact(info['name'] , f"{info['jobInfo']['bindistName']}.tar.xz", platform.subdir)
+
+# Generate the new metadata for a specific GHC mode etc
+def mk_new_yaml(release_mode, version, pipeline_type, job_map):
+    def mk(platform):
+        eprint("\n=== " + platform.name + " " + ('=' * (75 - len(platform.name))))
+        return mk_one_metadata(release_mode, version, job_map, mk_from_platform(pipeline_type, platform))
+
+    # Here are all the bindists we can distribute
+    centos7 = mk(centos(7))
+    fedora33 = mk(fedora(33))
+    darwin_x86 = mk(darwin("x86_64"))
+    darwin_arm64 = mk(darwin("aarch64"))
+    windows = mk(windowsArtifact)
+    alpine3_12 = mk(alpine("3_12"))
+    deb9 = mk(debian("x86_64", 9))
+    deb10 = mk(debian("x86_64", 10))
+    deb11 = mk(debian("x86_64", 11))
+    deb10_arm64 = mk(debian("aarch64", 10))
+    deb9_i386 = mk(debian("i386", 9))
+
+    source = mk_one_metadata(release_mode, version, job_map, source_artifact)
+
+    # The actual metadata, this is not a precise science, but just what the ghcup
+    # developers want.
+
+    a64 = { "Linux_Debian": { "< 10": deb9
+                           , "(>= 10 && < 11)": deb10
+                           , ">= 11": deb11
+                           , "unknown_versioning": deb11 }
+          , "Linux_Ubuntu" : { "unknown_versioning": deb10
+                             , "( >= 16 && < 19 )": deb9
+                             }
+          , "Linux_Mint"   : { "< 20": deb9
+                             , ">= 20": deb10 }
+          , "Linux_CentOS"  : { "( >= 7 && < 8 )" : centos7
+                              , "unknown_versioning" : centos7  }
+          , "Linux_Fedora"  : { ">= 33": fedora33
+                              , "unknown_versioning": centos7 }
+          , "Linux_RedHat"  : { "unknown_versioning": centos7 }
+          #MP: Replace here with Rocky8 when that job is in the pipeline
+          , "Linux_UnknownLinux" : { "unknown_versioning": fedora33 }
+          , "Darwin" : { "unknown_versioning" : darwin_x86 }
+          , "Windows" : { "unknown_versioning" :  windows }
+          , "Linux_Alpine" : { "unknown_versioning": alpine3_12 }
+
+          }
+
+    a32 = { "Linux_Debian": { "<10": deb9_i386, "unknown_versioning": deb9_i386 }
+          , "Linux_Ubuntu": { "unknown_versioning": deb9_i386 }
+          , "Linux_Mint" : { "unknown_versioning": deb9_i386 }
+          , "Linux_UnknownLinux" : { "unknown_versioning": deb9_i386 }
+          }
+
+    arm64 = { "Linux_UnknownLinux": { "unknown_versioning": deb10_arm64 }
+            , "Darwin": { "unknown_versioning": darwin_arm64 }
+            }
+
+    if release_mode:
+        version_parts = version.split('.')
+        if len(version_parts) == 3:
+            final_version = version
+        elif len(version_parts) == 4:
+            final_version = '.'.join(version_parts[:2] + [str(int(version_parts[2]) + 1)])
+        change_log = f"https://downloads.haskell.org/~ghc/{version}/docs/users_guide/{final_version}-notes.html"
+    else:
+        change_log =  "https://gitlab.haskell.org"
+
+    return { "viTags": ["Latest", "TODO_base_version"]
+        # Check that this link exists
+        , "viChangeLog": change_log
+        , "viSourceDL": source
+        , "viPostRemove": "*ghc-post-remove"
+        , "viArch": { "A_64": a64
+                    , "A_32": a32
+                    , "A_ARM64": arm64
+                    }
+        }
+
+
+def main() -> None:
+    import argparse
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--metadata', required=True, type=Path, help='Path to GHCUp metadata')
+    parser.add_argument('--pipeline-id', required=True, type=int, help='Which pipeline to generate metadata for')
+    parser.add_argument('--release-mode', action='store_true', help='Generate metadata which points to downloads folder')
+    parser.add_argument('--fragment', action='store_true', help='Output the generated fragment rather than whole modified file')
+    # TODO: We could work out the --version from the project-version CI job.
+    parser.add_argument('--version', required=True, type=str, help='Version of the GHC compiler')
+    args = parser.parse_args()
+
+    project = gl.projects.get(1, lazy=True)
+    pipeline = project.pipelines.get(args.pipeline_id)
+    jobs = pipeline.jobs.list()
+    job_map = { job.name: job for job in jobs }
+    # Bit of a hacky way to determine what pipeline we are dealing with but
+    # the aarch64-darwin job should stay stable for a long time.
+    if 'nightly-aarch64-darwin-validate' in job_map:
+        pipeline_type = 'n'
+        if args.release_mode:
+            raise Exception("Incompatible arguments: nightly pipeline but using --release-mode")
+
+    elif 'release-aarch64-darwin-release' in job_map:
+        pipeline_type = 'r'
+    else:
+        raise Exception("Not a nightly nor release pipeline")
+    eprint(f"Pipeline Type: {pipeline_type}")
+
+
+    new_yaml = mk_new_yaml(args.release_mode, args.version, pipeline_type, job_map)
+    if args.fragment:
+        print(yaml.dump({ args.version : new_yaml }))
+
+    else:
+        with open(args.metadata, 'r') as file:
+            ghcup_metadata = yaml.safe_load(file)
+            ghcup_metadata['ghcupDownloads']['GHC'][args.version] = new_yaml
+            print(yaml.dump(ghcup_metadata))
+
+
+if __name__ == '__main__':
+    main()
+
diff --git a/.gitlab/rel_eng/mk-ghcup-metadata/setup.py b/.gitlab/rel_eng/mk-ghcup-metadata/setup.py
new file mode 100644
index 0000000000..d4f4efe5e4
--- /dev/null
+++ b/.gitlab/rel_eng/mk-ghcup-metadata/setup.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+
+from distutils.core import setup
+
+setup(name='ghcup-metadata',
+      author='Matthew Pickering',
+      author_email='matthew@well-typed.com',
+      py_modules=['mk_ghcup_metadata'],
+      entry_points={
+          'console_scripts': [
+              'ghcup-metadata=mk_ghcup_metadata:main',
+          ]
+      }
+     )