diff options
| -rwxr-xr-x | yarn | 151 | ||||
| -rw-r--r-- | yarnlib/__init__.py | 1 | ||||
| -rw-r--r-- | yarnlib/scenario_runner.py | 202 | ||||
| -rw-r--r-- | yarnlib/scenario_runner_tests.py | 129 |
4 files changed, 377 insertions, 106 deletions
@@ -118,6 +118,13 @@ class YarnRunner(cliapp.Application): '%String(scenario_name): ' '%String(step_name)') + if self.settings['tempdir']: + self.tempdir = os.path.abspath(self.settings['tempdir']) + if not os.path.exists(self.tempdir): + os.mkdir(self.tempdir) + else: + self.tempdir = tempfile.mkdtemp() + scenarios, implementations = self.parse_scenarios(args) sv = yarnlib.ScenarioValidator(scenarios) sv.validate_all() @@ -136,13 +143,21 @@ class YarnRunner(cliapp.Application): self.steps_run = 0 self.timings = [] + scenario_runner = yarnlib.ScenarioRunner(shell_prelude, os.getcwd(), + self.parse_env(), + pre_step_cb=self.pre_step, + post_step_cb=self.post_step) + start_time = time.time() failed_scenarios = [] for scenario in self.select_scenarios(scenarios): - if not self.run_scenario(scenario, shell_prelude): + if not self.run_scenario(scenario_runner, scenario): failed_scenarios.append(scenario) duration = time.time() - start_time + if not self.settings['snapshot']: + shutil.rmtree(self.tempdir) + if not self.settings['quiet']: self.ts.clear() self.ts.finish() @@ -219,7 +234,7 @@ class YarnRunner(cliapp.Application): return scenarios - def run_scenario(self, scenario, shell_prelude): + def run_scenario(self, scenario_runner, scenario): self.start_scenario_timing(scenario.name) started = time.time() @@ -234,62 +249,15 @@ class YarnRunner(cliapp.Application): self.remember_scenario_timing(time.time() - started) return True - if self.settings['tempdir']: - tempdir = os.path.abspath(self.settings['tempdir']) - if not os.path.exists(tempdir): - os.mkdir(tempdir) - else: - tempdir = tempfile.mkdtemp() - - os.mkdir(self.scenario_dir(tempdir, scenario)) - datadir = self.datadir(tempdir, scenario) + os.mkdir(self.scenario_dir(self.tempdir, scenario)) + datadir = self.datadir(self.tempdir, scenario) os.mkdir(datadir) self.info('DATADIR is %s' % datadir) homedir = self.homedir(datadir) os.mkdir(homedir) self.info('HOME for tests is %s' % homedir) - assuming = [s for s in scenario.steps if s.what == 'ASSUMING'] - cleanup = [s for s in scenario.steps if s.what == 'FINALLY'] - normal = [s for s in scenario.steps if s not in assuming + cleanup] - - ok = True - step_number = 0 - - for step in assuming: - exit = self.run_step(datadir, scenario, step, shell_prelude, False) - step_number += 1 - self.snapshot_datadir( - tempdir, datadir, scenario, step_number, step) - if exit != 0: - self.ts.notify( - 'Skipping "%s" because "%s %s" failed' % - (scenario.name, step.what, step.text)) - self.skipped_for_assuming += 1 - break - else: - for step in normal: - exit = self.run_step( - datadir, scenario, step, shell_prelude, True) - step_number += 1 - self.snapshot_datadir( - tempdir, datadir, scenario, step_number, step) - if exit != 0: - ok = False - break - - for step in cleanup: - exit = self.run_step( - datadir, scenario, step, shell_prelude, True) - step_number += 1 - self.snapshot_datadir( - tempdir, datadir, scenario, step_number, step) - if exit != 0: - ok = False - break - - if not self.settings['snapshot']: - shutil.rmtree(tempdir) + ok = scenario_runner.run_scenario(scenario, datadir, homedir) self.remember_scenario_timing(time.time() - started) return ok @@ -297,42 +265,16 @@ class YarnRunner(cliapp.Application): def homedir(self, datadir): return os.path.join(datadir, 'HOME') - def clean_env(self): - '''Return a clean environment for running tests.''' - - whitelisted = [ - 'PATH', - ] - - hardcoded = { - 'TERM': 'dumb', - 'SHELL': '/bin/sh', - 'LC_ALL': 'C', - 'USER': 'tomjon', - 'USERNAME': 'tomjon', - 'LOGNAME': 'tomjon', - } - - env = {} - - for key in whitelisted: - if key in os.environ: - env[key] = os.environ[key] - - for key in hardcoded: - env[key] = hardcoded[key] - + def parse_env(self): for option_arg in self.settings['env']: if '=' not in option_arg: raise cliapp.AppException( '--env argument must contain "=" ' 'to separate environment variable name and value') key, value = option_arg.split('=', 1) - env[key] = value + yield key, value - return env - - def run_step(self, datadir, scenario, step, shell_prelude, report_error): + def pre_step(self, step, **ignored): started = time.time() self.info('Running step "%s %s"' % (step.what, step.text)) @@ -340,19 +282,12 @@ class YarnRunner(cliapp.Application): self.ts['step_name'] = '%s %s' % (step.what, step.text) self.steps_run += 1 - m = yarnlib.implements_matches_step(step.implementation, step) - assert m is not None - env = self.clean_env() - env['DATADIR'] = datadir - env['SRCDIR'] = os.getcwd() - env['HOME'] = self.homedir(datadir) - for i, match in enumerate(m.groups('')): - env['MATCH_%d' % (i+1)] = match + return (started,) - shell_script = '%s\n\n%s\n' % ( - shell_prelude, step.implementation.shell) - exit, stdout, stderr = cliapp.runcmd_unchecked( - ['sh', '-xeuc', shell_script], env=env) + def post_step(self, scenario, step, step_number, step_env, + exit, stdout, stderr, pre_step_userdata): + stopped = time.time() + (started,) = pre_step_userdata logging.debug('Exit code: %d' % exit) if stdout: @@ -363,20 +298,24 @@ class YarnRunner(cliapp.Application): logging.debug('Standard error:\n%s' % self.indent(stderr)) else: logging.debug('Standard error: empty') - - if exit != 0 and report_error: - self.error( - 'ERROR: In scenario "%s"\nstep "%s %s" failed,\n' - 'with exit code %d:\n' - 'Standard output from shell command:\n%s' - 'Standard error from shell command:\n%s' % - (scenario.name, step.what, step.text, exit, - self.indent(stdout), self.indent(stderr))) - + if exit != 0: + if step.what == 'ASSUMING': + self.ts.notify( + 'Skipping "%s" because "%s %s" failed' % + (scenario.name, step.what, step.text)) + self.skipped_for_assuming += 1 + else: + self.error( + 'ERROR: In scenario "%s"\nstep "%s %s" failed,\n' + 'with exit code %d:\n' + 'Standard output from shell command:\n%s' + 'Standard error from shell command:\n%s' % + (scenario.name, step.what, step.text, exit, + self.indent(stdout), self.indent(stderr))) self.remember_step_timing( - '%s %s' % (step.what, step.text), time.time() - started) - - return exit + '%s %s' % (step.what, step.text), stopped - started) + self.snapshot_datadir(self.tempdir, step_env['DATADIR'], + scenario, step_number, step) def scenario_dir(self, tempdir, scenario): return os.path.join(tempdir, self.nice(scenario.name)) diff --git a/yarnlib/__init__.py b/yarnlib/__init__.py index e17223a..515082e 100644 --- a/yarnlib/__init__.py +++ b/yarnlib/__init__.py @@ -25,6 +25,7 @@ from scenario_step_connector import (implements_matches_step, ScenarioStepConnector, StepNotImplementedError, StepMultipleImplementationsError) +from scenario_runner import ScenarioRunner from shell_libraries import load_shell_libraries diff --git a/yarnlib/scenario_runner.py b/yarnlib/scenario_runner.py new file mode 100644 index 0000000..e0cdf62 --- /dev/null +++ b/yarnlib/scenario_runner.py @@ -0,0 +1,202 @@ +# Copyright 2014 Lars Wirzenius and Codethink Limited +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# =*= License: GPL-3+ =*= + + +import cliapp +import os + +import yarnlib + + +def default_pre_step(scenario, step, step_number, step_env): + '''Default callback run before each step in ScenarioRunner. + + Parameters: + + * `scenario`: The yarnlib.Scenario that is about to be run. + * `step`: The yarnlib.ScenarioStep that is about to be run. + * `step_number`: Per-scenario counter for the step that is about to be run, + starting from 1. + * `step_env`: Environment that will be used for this step. + * `return`: Any returned values are passed to `post_step_cb`. + + All parameters are passed as keyword-arguments, so the order of + parameters is not important, and unused arguments can be ignored by + putting **kwargs in the parameter definition. + + ''' + pass + + +def default_post_step(scenario, step, step_number, step_env, + exit, stdout, stderr, pre_step_userdata): + '''Default callback run after each step in ScenarioRunner. + + Parameters: + + * `scenario`: The yarnlib.Scenario that has just been run. + * `step`: The yarnlib.ScenarioStep that has just been run. + * `step_number`: Per-scenario counter for the step that is about to be run, + starting from 1. + * `step_env`: Environment that was used for this step. + * `exit`: Return code of the step that was run. + * `stdout`: Standard output of the step that was just run. + * `stderr`: Standard error of the step that was just run. + * `pre_step_userdata`: Return value from `pre_step_cb`. + + All parameters are passed as keyword-arguments, so the order of + parameters is not important, and unused arguments can be ignored by + putting **kwargs in the parameter definition. + + ''' + pass + + +# Arguably this could just be the run_scenario method with all the +# constructor's parameters passed in, but reeks of poor design with that +# many parameters. +# However, decoupling the ScenarioRunner from the scenarios allows the +# same scenarios to be run against different projects that aim to satisfy +# the same requirements, or multiple yarn suites on the same project, +# if the project aims to satisfy multiple sets of requirements. +# While this use is far less likely than having 1 yarn suite per project, +# it's a useful metric for deciding how to split the arguments between the +# constructor and the run_scenario method. +class ScenarioRunner(object): + '''Sets up an environment to run scenarios on a Project. + + Parameters: + + * `shell_prelude`: Prefix to all shell scripts run, e.g. shell libraries. + * `srcdir`: Path to the root of the source tree. + This is available as the $SRCDIR environment variable + and is the directory the scenarios are run from. + * `extra_env`: dict, or iterable of pairs of variables to add to + the clean environment provided for every command. + * `pre_step_cb`: Callback run before each step in a scenario. + See default_pre_step for an explanation of its signature. + Its return value is passed to `post_step_cb`. + * `post_step_cb`: Callback run after each step in a scenario. + See default_post_step for an explanation of its signature. + * `cmdrunner`: Function used to run step commands in place of + `cliapp.runcmd_unchecked`, intended to be replaceable + by unit tests. + + ''' + + + def __init__(self, shell_prelude, srcdir, extra_env=(), + pre_step_cb=default_pre_step, post_step_cb=default_post_step, + cmdrunner=cliapp.runcmd_unchecked): + self.shell_prelude = shell_prelude + self.srcdir = srcdir + self.env = self.clean_env(extra_env, SRCDIR=srcdir) + self.pre_step_cb = pre_step_cb + self.post_step_cb = post_step_cb + self.cmdrunner = cmdrunner + + def run_scenario(self, scenario, datadir, homedir): + assuming = [s for s in scenario.steps if s.what == 'ASSUMING'] + cleanup = [s for s in scenario.steps if s.what == 'FINALLY'] + normal = [s for s in scenario.steps if s not in assuming + cleanup] + + ok = True + step_number = 1 + + scenario_env = dict(self.env) + scenario_env['HOME'] = homedir + scenario_env['DATADIR'] = datadir + + for step in assuming: + exit = self.run_step(scenario, step, scenario_env, step_number) + step_number += 1 + if exit != 0: + break + else: + for step in normal: + exit = self.run_step(scenario, step, scenario_env, step_number) + step_number += 1 + if exit != 0: + ok = False + break + + for step in cleanup: + exit = self.run_step(scenario, step, scenario_env, step_number) + step_number += 1 + if exit != 0: + ok = False + break + + return ok + + def run_step(self, scenario, step, scenario_env, step_number): + m = yarnlib.implements_matches_step(step.implementation, step) + assert m is not None + step_env = dict(scenario_env) + for i, match in enumerate(m.groups('')): + step_env['MATCH_%d' % (i+1)] = match + + # All parameters passed as keyword-arguments, so that the callback + # may declare parameters in any order, and ignore any parameters + # by specifying **kwargs + pre_step_userdata = self.pre_step_cb(scenario=scenario, step=step, + step_number=step_number, + step_env=step_env) + + shell_script = '%s\n\n%s\n' % ( + self.shell_prelude, step.implementation.shell) + exit, stdout, stderr = self.cmdrunner( + ['sh', '-xeuc', shell_script], env=step_env, cwd=self.srcdir) + + # All parameters passed as keyword-arguments, so that the callback + # may declare parameters in any order, and ignore any parameters + # by specifying **kwargs + self.post_step_cb(scenario=scenario, step=step, + step_number=step_number, step_env=step_env, + exit=exit, stdout=stdout, stderr=stderr, + pre_step_userdata=pre_step_userdata) + + return exit + + @staticmethod + def clean_env(extra_env, **kwarg_env): + '''Return a clean environment for running tests.''' + + whitelisted = [ + 'PATH', + ] + + hardcoded = { + 'TERM': 'dumb', + 'SHELL': '/bin/sh', + 'LC_ALL': 'C', + 'USER': 'tomjon', + 'USERNAME': 'tomjon', + 'LOGNAME': 'tomjon', + } + + env = {} + + for key in whitelisted: + if key in os.environ: + env[key] = os.environ[key] + + env.update(hardcoded) + env.update(extra_env) + env.update(kwarg_env) + + return env diff --git a/yarnlib/scenario_runner_tests.py b/yarnlib/scenario_runner_tests.py new file mode 100644 index 0000000..cda6397 --- /dev/null +++ b/yarnlib/scenario_runner_tests.py @@ -0,0 +1,129 @@ +# Copyright 2014 Codethink Limited +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# =*= License: GPL-3+ =*= + + +import itertools +import unittest + +import yarnlib + + +class ScenarioRunnerEnvironmentTests(unittest.TestCase): + + def setUp(self): + self.cmdlog = [] + self.scenario = yarnlib.Scenario('foo') + step = yarnlib.ScenarioStep('THEN', 'foo bar') + step.implementation = yarnlib.Implementation('THEN', r'foo (\S+)', + 'echo foo $MATCH_1') + self.scenario.steps = [step] + + def fake_cmdrunner(self, cmd, env=None, cwd=None): + self.cmdlog.append((cmd, env, cwd)) + return 0, '', '' + + def test_command_env(self): + sr = yarnlib.ScenarioRunner('', '/tmp', {'FOO': 'bar'}, + cmdrunner=self.fake_cmdrunner) + sr.run_scenario(self.scenario, '/tmp/datadir', '/tmp/home') + cmd, cmd_env, cmd_cwd = self.cmdlog[0] + self.assertIn('FOO', cmd_env) + self.assertEqual(cmd_env['FOO'], 'bar') + self.assertEqual(cmd_env['SRCDIR'], '/tmp') + self.assertEqual(cmd_env['DATADIR'], '/tmp/datadir') + self.assertEqual(cmd_env['HOME'], '/tmp/home') + self.assertEqual(cmd_env['MATCH_1'], 'bar') + self.assertEqual('/tmp', cmd_cwd) + + def test_command_args(self): + prelude = 'TESTVAR=foo\n' + impl_shell = self.scenario.steps[0].implementation.shell + sr = yarnlib.ScenarioRunner(prelude, '/tmp', + cmdrunner=self.fake_cmdrunner) + sr.run_scenario(self.scenario, '/tmp/datadir', '/tmp/home') + shell, opts, command = self.cmdlog[0][0] + self.assertTrue(shell.endswith('sh')) + self.assertTrue(command.startswith(prelude)) + self.assertIn(impl_shell, command) + + +class ScenarioRunnerFlowTests(unittest.TestCase): + + def setUp(self): + self.cmdlog = [] + self.scenario_runner = ( + yarnlib.ScenarioRunner('', 'srcdir', + cmdrunner=self.fake_cmdrunner)) + for verb, cmd in itertools.product(('ASSUMING', 'THEN', 'FINALLY'), + ('true', 'false')): + step = yarnlib.ScenarioStep(verb, cmd) + step.implementation = yarnlib.Implementation(verb, cmd, cmd) + setattr(self, '%s_%s' % (verb.lower(), cmd), step) + + def fake_cmdrunner(self, cmd, *args, **kwargs): + self.cmdlog.append(cmd) + if 'false' in cmd[-1]: + return 1, '', '' + return 0, '', '' + + def run_scenario(self, *steps): + scenario = yarnlib.Scenario('foo') + scenario.steps = steps + return self.scenario_runner.run_scenario(scenario, 'data', 'home') + + def test_assuming_skips_remaining(self): + ok = self.run_scenario( + self.assuming_false, + self.then_true, + self.finally_true, + ) + self.assertTrue(ok) + self.assertEqual(len(self.cmdlog), 1) + + def test_cleanup_run_on_success(self): + ok = self.run_scenario( + self.assuming_true, + self.then_true, + self.finally_true, + ) + self.assertTrue(ok) + self.assertEqual(len(self.cmdlog), 3) + + def test_cleanup_run_on_failure(self): + ok = self.run_scenario( + self.then_false, + self.finally_true, + ) + self.assertFalse(ok) + self.assertEqual(len(self.cmdlog), 2) + + def test_skip_steps_after_failure(self): + ok = self.run_scenario( + self.then_false, + self.then_true, + ) + self.assertFalse(ok) + self.assertEqual(len(self.cmdlog), 1) + + def test_cleanup_failure(self): + ok = self.run_scenario( + self.then_true, + self.finally_false, + self.finally_true, + ) + self.assertFalse(ok) + self.assertEqual(len(self.cmdlog), 2) |