From 6a65b8c2f5b4bc7abdb0ca3b5876df694acb8194 Mon Sep 17 00:00:00 2001 From: Alp Mestanogullari Date: Wed, 26 Feb 2020 17:03:31 +0100 Subject: hadrian: improve dependency tracking for the check-* programs The code in Rules.Register responsible for finding all the build artifacts that Cabal installs when registering a library (static/shared libs, .hi files, ...) was looking in the wrong place. This patch fixes that logic and makes sure we gather all those artifacts in a list to declare that the rule for a given `.conf` file, our proxy for "Hadrian, please install this package in the package db for this stage", also produces those artifacts under the said package database. We also were completely missing some logic to declare that the check-* programs have dependencies besides their source code, at least when testing an in-tree compiler. Finally, this patch also removes redundant packages from 'testsuitePackages', since they should already be covered by the stagePackages lists from Settings.Default. With this patch, after a complete build and freezing stage 1, a change to `compiler/parser/Parser.y` results in rebuilding the ghc lib, reinstalling it, and rebuilding the few programs that depend on it, _including_ `check-ppr` and `check-api-annotations` (therefore fixing #17273). --- hadrian/src/Rules/Register.hs | 29 ++++++++++++++--------------- hadrian/src/Rules/Test.hs | 28 +++++++++++++++++++++------- hadrian/src/Settings/Default.hs | 13 +------------ 3 files changed, 36 insertions(+), 34 deletions(-) diff --git a/hadrian/src/Rules/Register.hs b/hadrian/src/Rules/Register.hs index fdb3202afe..51df69f116 100644 --- a/hadrian/src/Rules/Register.hs +++ b/hadrian/src/Rules/Register.hs @@ -9,7 +9,6 @@ import Expression ( getContextData ) import Hadrian.BuildPath import Hadrian.Expression import Hadrian.Haskell.Cabal -import Oracles.Setting import Packages import Rules.Rts import {-# SOURCE #-} Rules.Library (needLibrary) @@ -109,7 +108,7 @@ registerPackageRules rs stage = do _ -> buildConf rs ctx conf buildConf :: [(Resource, Int)] -> Context -> FilePath -> Action () -buildConf _ context@Context {..} conf = do +buildConf _ context@Context {..} _conf = do depPkgIds <- cabalDependencies context ensureConfigured context need =<< mapM (\pkgId -> packageDbPath stage <&> (-/- pkgId <.> "conf")) depPkgIds @@ -137,19 +136,19 @@ buildConf _ context@Context {..} conf = do Cabal.copyPackage context Cabal.registerPackage context - -- The above two steps produce an entry in the package database, with copies - -- of many of the files we have build, e.g. Haskell interface files. We need - -- to record this side effect so that Shake can cache these files too. - -- See why we need 'fixWindows': https://gitlab.haskell.org/ghc/ghc/issues/16073 - let fixWindows path = do - version <- setting GhcVersion - hostOs <- cabalOsString <$> setting BuildOs - hostArch <- cabalArchString <$> setting BuildArch - let dir = hostArch ++ "-" ++ hostOs ++ "-ghc-" ++ version - return $ if windowsHost then path -/- "../.." -/- dir else path - pkgDbPath <- fixWindows =<< packageDbPath stage - let dir = pkgDbPath -/- takeBaseName conf - files <- liftIO $ getDirectoryFilesIO "." [dir -/- "**"] + -- We declare that this rule also produces files matching: + -- - /stage/lib/--ghc-/*libHS* + -- (for .so files, Cabal's registration mechanism places them there) + -- - /stage/lib/--ghc-//** + -- (for interface files, static libs, ghci libs, includes, ...) + -- + -- so that if any change ends up modifying a library (but not its .conf + -- file), we still rebuild things that depend on it. + dir <- (-/-) <$> libPath context <*> distDir stage + pkgid <- pkgIdentifier package + files <- liftIO $ + (++) <$> getDirectoryFilesIO "." [dir -/- "*libHS"++pkgid++"*"] + <*> getDirectoryFilesIO "." [dir -/- pkgid -/- "**"] produces files copyConf :: [(Resource, Int)] -> Context -> FilePath -> Action () diff --git a/hadrian/src/Rules/Test.hs b/hadrian/src/Rules/Test.hs index d4a528445b..82a37ebe41 100644 --- a/hadrian/src/Rules/Test.hs +++ b/hadrian/src/Rules/Test.hs @@ -6,6 +6,8 @@ import Base import CommandLine import Expression import Flavour +import Hadrian.Haskell.Cabal.Type (packageDependencies) +import Hadrian.Oracles.Cabal (readPackageData) import Oracles.Setting import Oracles.TestSettings import Packages @@ -30,10 +32,10 @@ checkApiAnnotationsProgPath, checkApiAnnotationsSourcePath :: FilePath checkApiAnnotationsProgPath = "test/bin/check-api-annotations" <.> exe checkApiAnnotationsSourcePath = "utils/check-api-annotations/Main.hs" -checkPrograms :: [(FilePath, FilePath)] +checkPrograms :: [(FilePath, FilePath, Package)] checkPrograms = - [ (checkPprProgPath, checkPprSourcePath) - , (checkApiAnnotationsProgPath, checkApiAnnotationsSourcePath) + [ (checkPprProgPath, checkPprSourcePath, checkPpr) + , (checkApiAnnotationsProgPath, checkApiAnnotationsSourcePath, checkApiAnnotations) ] ghcConfigPath :: FilePath @@ -53,16 +55,28 @@ testRules = do -- Rules for building check-ppr and check-ppr-annotations with the compiler -- we are going to test (in-tree or out-of-tree). - forM_ checkPrograms $ \(progPath, sourcePath) -> + forM_ checkPrograms $ \(progPath, sourcePath, progPkg) -> root -/- progPath %> \path -> do + need [ sourcePath ] testGhc <- testCompiler <$> userSetting defaultTestArgs top <- topDirectory + depsPkgs <- packageDependencies <$> readPackageData progPkg + + -- when we're about to test an in-tree compiler, we make sure that + -- we have the corresponding GHC binary available, along with the + -- necessary libraries to build the check-* programs when (testGhc `elem` ["stage1", "stage2", "stage3"]) $ do let stg = stageOf testGhc - need . (:[]) =<< programPath (Context stg ghc vanilla) + ghcPath <- programPath (Context stg ghc vanilla) + depsLibs <- traverse + (\p -> pkgRegisteredLibraryFile (vanillaContext stg p)) + depsPkgs + need (ghcPath : depsLibs) + bindir <- getBinaryDirectory testGhc - cmd [bindir "ghc" <.> exe] - ["-package", "ghc", "-o", top -/- path, top -/- sourcePath] + cmd [bindir "ghc" <.> exe] $ + concatMap (\p -> ["-package", pkgName p]) depsPkgs ++ + ["-o", top -/- path, top -/- sourcePath] root -/- ghcConfigPath %> \_ -> do args <- userSetting defaultTestArgs diff --git a/hadrian/src/Settings/Default.hs b/hadrian/src/Settings/Default.hs index b0d0f04ab3..78d46b46be 100644 --- a/hadrian/src/Settings/Default.hs +++ b/hadrian/src/Settings/Default.hs @@ -125,18 +125,7 @@ stage2Packages = stage1Packages -- | Packages that are built only for the testsuite. testsuitePackages :: Action [Package] -testsuitePackages = do - return $ [ checkApiAnnotations - , checkPpr - , ghci - , ghcCompact - , ghcPkg - , hpcBin - , hsc2hs - , iserv - , runGhc - , unlit ] ++ - [ timeout | windowsHost ] +testsuitePackages = return [ timeout | windowsHost ] -- | Default build ways for library packages: -- * We always build 'vanilla' way. -- cgit v1.2.1