From 099e622dc5559cced775e9694deec6c798f590f3 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Mon, 4 Nov 2013 09:02:01 +1300 Subject: * ``run`` now accepts ``--isolated`` as a parameter, which will cause each selected test to be run independently. This can be useful to both workaround isolation bugs and detect tests that can not be run independently. (Robert Collins) --- NEWS | 8 ++++++++ doc/MANUAL.txt | 15 +++++++++++++++ testrepository/commands/run.py | 23 +++++++++++++++++++++-- testrepository/tests/commands/test_run.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 663d563..d079ac6 100644 --- a/NEWS +++ b/NEWS @@ -5,6 +5,14 @@ testrepository release notes NEXT (In development) +++++++++++++++++++++ +CHANGES +------- + +* ``run`` now accepts ``--isolated`` as a parameter, which will cause each + selected test to be run independently. This can be useful to both workaround + isolation bugs and detect tests that can not be run independently. + (Robert Collins) + INTERNALS --------- diff --git a/doc/MANUAL.txt b/doc/MANUAL.txt index 28b286b..d0b298b 100644 --- a/doc/MANUAL.txt +++ b/doc/MANUAL.txt @@ -366,6 +366,21 @@ will perform that analysis for you. (This requires that your test runner is then either the isolation issue is racy, or it is a 3-or-more test isolation issue. Neither of those cases are automated today. +Forcing isolation +~~~~~~~~~~~~~~~~~ + +Sometimes it is useful to force a separate test runner instance for each test +executed. The ``--isolated`` flag will cause testr to execute a separate runner +per test:: + + $ testr run --isolated + +In this mode testr first determines tests to run (either automatically listed, +using the failing set, or a user supplied load-list), and then spawns one test +runner per test it runs. To avoid cross-test-runner interactions concurrency +is disabled in this mode. ``--analyze-isolation`` supercedes ``--isolated`` if +they are both supplied. + Repositories ~~~~~~~~~~~~ diff --git a/testrepository/commands/run.py b/testrepository/commands/run.py index a18c235..7b6cd40 100644 --- a/testrepository/commands/run.py +++ b/testrepository/commands/run.py @@ -143,6 +143,9 @@ class run(Command): optparse.Option("--analyze-isolation", action="store_true", default=False, help="Search the last test run for 2-test test isolation interactions."), + optparse.Option("--isolated", action="store_true", + default=False, + help="Run each test id in a separate test runner."), ] args = [StringArgument('testfilters', 0, None), DoubledashArgument(), StringArgument('testargs', 0, None)] @@ -192,7 +195,22 @@ class run(Command): if not self.ui.options.analyze_isolation: cmd = testcommand.get_run_command(ids, self.ui.arguments['testargs'], test_filters = filters) - return self._run_tests(cmd) + if self.ui.options.isolated: + result = 0 + cmd.setUp() + try: + ids = cmd.list_tests() + finally: + cmd.cleanUp() + for test_id in ids: + cmd = testcommand.get_run_command([test_id], + self.ui.arguments['testargs'], test_filters=filters) + run_result = self._run_tests(cmd) + if run_result > result: + result = run_result + return result + else: + return self._run_tests(cmd) else: # Where do we source data about the cause of conflicts. # XXX: Should instead capture the run id in with the failing test @@ -329,7 +347,8 @@ class run(Command): def run_tests(): run_procs = [('subunit', ReturnCodeToSubunit(proc)) for proc in cmd.run_tests()] options = {} - if self.ui.options.failing or self.ui.options.analyze_isolation: + if (self.ui.options.failing or self.ui.options.analyze_isolation + or self.ui.options.isolated): options['partial'] = True load_ui = decorator.UI(input_streams=run_procs, options=options, decorated=self.ui) diff --git a/testrepository/tests/commands/test_run.py b/testrepository/tests/commands/test_run.py index 3168e94..da94ca2 100644 --- a/testrepository/tests/commands/test_run.py +++ b/testrepository/tests/commands/test_run.py @@ -446,6 +446,35 @@ class TestCommand(ResourcedTestCase): ], ui.outputs) self.assertEqual(0, result) + def test_isolated_runs_multiple_processes(self): + ui, cmd = self.get_test_ui_and_cmd(options=[('isolated', True)]) + cmd.repository_factory = memory.RepositoryFactory() + self.setup_repo(cmd, ui) + self.set_config( + '[DEFAULT]\ntest_command=foo $IDLIST $LISTOPT\n' + 'test_id_option=--load-list $IDFILE\n' + 'test_list_option=--list\n') + params, capture_ids = self.capture_ids(list_result=['ab', 'cd', 'ef']) + self.useFixture(MonkeyPatch( + 'testrepository.testcommand.TestCommand.get_run_command', + capture_ids)) + cmd_result = cmd.execute() + self.assertEqual([ + ('results', Wildcard), + ('summary', True, 0, -3, None, None, [('id', 1, None)]), + ('results', Wildcard), + ('summary', True, 0, 0, None, None, [('id', 2, None)]), + ('results', Wildcard), + ('summary', True, 0, 0, None, None, [('id', 3, None)]), + ], ui.outputs) + self.assertEqual(0, cmd_result) + # once to list, then 3 each executing one test. + self.assertThat(params, HasLength(4)) + self.assertThat(params[0][1], Equals(None)) + self.assertThat(params[1][1], Equals(['ab'])) + self.assertThat(params[2][1], Equals(['cd'])) + self.assertThat(params[3][1], Equals(['ef'])) + def read_all(stream): return stream.read() -- cgit v1.2.1