From a5e76a073afc8ffdde274a4cb3d09847f2d35be9 Mon Sep 17 00:00:00 2001 From: Alp Mestanogullari Date: Tue, 11 Dec 2018 13:11:32 -0500 Subject: Hadrian: ability to run a subset of the testsuite This was supposed to be working already but didn't work when we specified several tests with --only. This patch not only fixes this but also makes it possible to specify a subset of tests to run with the TEST environment variable, like the make build system. Here are some examples: hadrian/build.sh test --only=plugins01 hadrian/build.sh test --only="plugins01 plugins02" TEST="plugins01 plugins02" hadrian/build.sh test TEST=plugins03 hadrian/build.sh test --only="plugins01 plugins02" When both the TEST environment variable and the --only flag are used, we simply concatenate the list of tests from both sources and ask the testsuite driver to run them all. This patch addresses #16026. Test Plan: hadrian/build.sh test --only="plugins01 plugins02" Reviewers: bgamari, snowleopard Reviewed By: bgamari, snowleopard Subscribers: rwbarton, carter GHC Trac Issues: #16026 Differential Revision: https://phabricator.haskell.org/D5431 --- hadrian/README.md | 14 ++-- hadrian/doc/testsuite.md | 130 +++++++++++++++++++++++++++++++ hadrian/src/CommandLine.hs | 9 ++- hadrian/src/Settings/Builders/RunTest.hs | 6 +- 4 files changed, 144 insertions(+), 15 deletions(-) create mode 100644 hadrian/doc/testsuite.md (limited to 'hadrian') diff --git a/hadrian/README.md b/hadrian/README.md index 7bd5fa8a34..c88d98e99d 100644 --- a/hadrian/README.md +++ b/hadrian/README.md @@ -170,16 +170,12 @@ workflow, for now. #### Testing -* `build validate` runs GHC tests by simply executing `make fast` in `testsuite/tests` -directory. This can be used instead of `sh validate --fast --no-clean` in the existing -build system. Note: this will rebuild Stage2 GHC, `ghc-pkg` and `hpc` if they are out of date. +To run GHC's testsuite, use `build test`. See +[doc/testsuite.md](doc/testsuite.md) to learn about all the options +you can use to mimic what the Make build system offers. -* `build test` runs GHC tests by calling the `testsuite/driver/runtests.py` python -script with appropriate flags. The current implementation is limited and cannot -replace the `validate` script (see [#187][validation-issue]). - -* `build selftest` runs tests of the build system. Current test coverage is close to -zero (see [#197][test-issue]). +`build selftest` runs tests of the build system. Current test coverage +is close to zero (see [#197][test-issue]). Troubleshooting --------------- diff --git a/hadrian/doc/testsuite.md b/hadrian/doc/testsuite.md new file mode 100644 index 0000000000..bdc7186b54 --- /dev/null +++ b/hadrian/doc/testsuite.md @@ -0,0 +1,130 @@ +# Running the testsuite + +## Quickstart + +The simplest way to run the testsuite is + +``` sh +# assuming 'build' is the build script you're using +# (hadrian/build.sh, hadrian/build.bat, ...) +build test +``` + +This is the equivalent of running `make test` with the +Make build system: it will run the entire testsuite in +normal mode (as opposed to slow or fast). If you have not +built GHC before, this will also build a stage 2 GHC in +the default flavour along with many libraries and programs +needed by the tests. + +## Running only a subset of the testsuite + +You can use the `TEST` environment variable, like with the +Make build system, or the `--only=...` command line argument. +This is best illustrated with examples: + +``` sh +# only run the test named 'sometest' +build test --only=sometest + +# only run 'test1' and 'test2' +build test --only="test1 test2" + +# only run 'sometest' +TEST=sometest build test + +# only run 'test1' and 'test2' +TEST="test1 test2" build test + +# only run 'test1', 'test2', 'test3' and 'test4' +TEST="test1 test2" build test --only="test3 test4" +``` + +## Performance tests + +You can use the `--only-perf` and `--skip-perf` flags to +respectively run only the performance tests or everything +but the performance tests. + +``` sh +# just performance tests, equivalent to: +# make test ONLY_PERF_TESTS=YES +build test --only-perf + +# skip performance tests, equivalent to: +# make test SKIP_PERF_TESTS=YES +build test --skip-perf +``` + +## Test speed + +You can run the tests in `slow`, `normal` (default) or `fast` +mode using `--test-speed={slow, normal, fast}`. + +``` sh +# equivalent to: make slowtest +build test --test-speed=slow + +# equivalent to: make test +build test --test-speed=normal + +# equivalent to: make fasttest +build test --test-speed=fast +``` + +## Test ways + +You can specify which test ways to use using `--test-way=`, +once per way that you want to run. + +``` sh +# equivalent to: make test WAY=ghci +build test --test-way=ghci + +# equivalent to: make test WAY=ghci WAY=hpc +build test --test-way=ghci --test-way=hpc +``` + +## Misc. + +``` + --summary[=TEST_SUMMARY] +``` +Where to output the test summary file. + +--- + +``` + --test-verbose[=TEST_VERBOSE] +``` +A verbosity value between 0 and 5. 0 is silent, 4 and higher +activates extra output. + +--- + +``` + --test-compiler[=TEST_COMPILER] +``` +Use given compiler [Default=stage2]. + +--- + +``` + --test-config-file[=CONFIG_FILE] +``` +Configuration file for testsuite. Default is +`testsuite/config/ghc` + +--- + +``` + --config[=EXTRA_TEST_CONFIG] +``` +Configurations to run test, in `key=value` format. + +--- + +``` + --summary-junit[=TEST_SUMMARY_JUNIT] +``` +Output testsuite summary in JUnit format. diff --git a/hadrian/src/CommandLine.hs b/hadrian/src/CommandLine.hs index 36f3818d1d..1f940f2152 100644 --- a/hadrian/src/CommandLine.hs +++ b/hadrian/src/CommandLine.hs @@ -46,7 +46,7 @@ data TestArgs = TestArgs , testConfigFile :: String , testConfigs :: [String] , testJUnit :: Maybe FilePath - , testOnly :: Maybe String + , testOnly :: [String] , testOnlyPerf :: Bool , testSkipPerf :: Bool , testSpeed :: TestSpeed @@ -62,7 +62,7 @@ defaultTestArgs = TestArgs , testConfigFile = "testsuite/config/ghc" , testConfigs = [] , testJUnit = Nothing - , testOnly = Nothing + , testOnly = [] , testOnlyPerf = False , testSkipPerf = False , testSpeed = Fast @@ -142,7 +142,10 @@ readTestJUnit :: Maybe String -> Either String (CommandLineArgs -> CommandLineAr readTestJUnit filepath = Right $ \flags -> flags { testArgs = (testArgs flags) { testJUnit = filepath } } readTestOnly :: Maybe String -> Either String (CommandLineArgs -> CommandLineArgs) -readTestOnly tests = Right $ \flags -> flags { testArgs = (testArgs flags) { testOnly = tests } } +readTestOnly tests = Right $ \flags -> + flags { testArgs = (testArgs flags) { testOnly = tests' } } + + where tests' = maybe [] words tests readTestOnlyPerf :: Either String (CommandLineArgs -> CommandLineArgs) readTestOnlyPerf = Right $ \flags -> flags { testArgs = (testArgs flags) { testOnlyPerf = True } } diff --git a/hadrian/src/Settings/Builders/RunTest.hs b/hadrian/src/Settings/Builders/RunTest.hs index 734fecdb49..6188b644ea 100644 --- a/hadrian/src/Settings/Builders/RunTest.hs +++ b/hadrian/src/Settings/Builders/RunTest.hs @@ -129,14 +129,14 @@ runTestBuilderArgs = builder RunTest ? do -- | Command line arguments for running GHC's test script. getTestArgs :: Args getTestArgs = do + -- targets specified in the TEST env var + testEnvTargets <- maybe [] words <$> expr (liftIO $ lookupEnv "TEST") args <- expr $ userSetting defaultTestArgs bindir <- expr $ setBinaryDirectory (testCompiler args) compiler <- expr $ setCompiler (testCompiler args) globalVerbosity <- shakeVerbosity <$> expr getShakeOptions let configFileArg= ["--config-file=" ++ (testConfigFile args)] - testOnlyArg = case testOnly args of - Just cases -> map ("--only=" ++) (words cases) - Nothing -> [] + testOnlyArg = map ("--only=" ++) (testOnly args ++ testEnvTargets) onlyPerfArg = if testOnlyPerf args then Just "--only-perf-tests" else Nothing -- cgit v1.2.1