diff options
Diffstat (limited to 'yarn')
| -rwxr-xr-x | yarn | 149 |
1 files changed, 141 insertions, 8 deletions
@@ -18,6 +18,7 @@ import cliapp +import collections import logging import os import re @@ -63,12 +64,21 @@ class YarnRunner(cliapp.Application): 'it should be empty or not exist', metavar='DIR') + self.settings.string_list( + ['env'], + 'add NAME=VALUE to the environment when tests are run', + metavar='NAME=VALUE') + self.settings.boolean( ['snapshot'], 'make snapshots of test working directory ' 'after each scenario step; you probably ' 'want to use this with --tempdir') + self.settings.boolean( + ['timings'], + 'report wall clock time for each scenario and step') + def info(self, msg): if self.settings['verbose']: logging.info(msg) @@ -104,6 +114,9 @@ class YarnRunner(cliapp.Application): 'step %Index(step,steps): %String(step_name)') scenarios, implementations = self.parse_scenarios(args) + self.check_there_are_scenarios(scenarios) + self.check_for_duplicate_scenario_names(scenarios) + self.check_for_thens(scenarios) self.connect_implementations(scenarios, implementations) shell_prelude = self.load_shell_libraries() @@ -111,6 +124,10 @@ class YarnRunner(cliapp.Application): self.ts['num_scenarios'] = len(scenarios) self.info('Found %d scenarios' % len(scenarios)) + self.scenarios_run = 0 + self.steps_run = 0 + self.timings = [] + start_time = time.time() failed_scenarios = [] for scenario in self.select_scenarios(scenarios): @@ -128,9 +145,13 @@ class YarnRunner(cliapp.Application): if not self.settings['quiet']: print ( - 'Scenario test suite PASS, with %d scenarios, ' + 'Scenario test suite PASS, with %d scenarios ' + '(%d total steps), ' 'in %.1f seconds' % - (len(scenarios), duration)) + (self.scenarios_run, self.steps_run, duration)) + + if self.settings['timings']: + self.report_timings() def parse_scenarios(self, filenames): mdparser = yarnlib.MarkdownParser() @@ -145,6 +166,36 @@ class YarnRunner(cliapp.Application): return block_parser.scenarios, block_parser.implementations + def check_there_are_scenarios(self, scenarios): + if not scenarios: + raise cliapp.AppException( + 'There are no scenarios; must have at least one.') + + def check_for_duplicate_scenario_names(self, scenarios): + counts = collections.Counter() + for s in scenarios: + counts[s.name] += 1 + + duplicates = [name for name in counts if counts[name] > 1] + if duplicates: + duplist = ''.join(' %s\n' % name for name in duplicates) + raise cliapp.AppException( + 'There are scenarios with duplicate names:\n%s' % duplist) + + def check_for_thens(self, scenarios): + no_thens = [] + for scenario in scenarios: + for step in scenario.steps: + if step.what == 'THEN': + break + else: + no_thens.append(scenario) + + if no_thens: + raise cliapp.AppException( + 'Some scenarios have no THENs:\n%s' % + ''.join(' "%s"\n' % s.name for s in scenarios)) + def connect_implementations(self, scenarios, implementations): for scenario in scenarios: for step in scenario.steps: @@ -153,11 +204,11 @@ class YarnRunner(cliapp.Application): def connect_implementation(self, scenario, step, implementations): matching = [i for i in implementations if step.what == i.what and - re.match('(%s)$' % i.regexp, step.text, re.I)] + self.implements_matches_step(i, step)] if len(matching) == 0: raise cliapp.AppException( - 'Scenario %s, step "%s %s" has no matching ' + 'Scenario "%s", step "%s %s" has no matching ' 'implementation' % (scenario.name, step.what, step.text)) if len(matching) > 1: @@ -206,17 +257,22 @@ class YarnRunner(cliapp.Application): return scenarios def run_scenario(self, scenario, shell_prelude): + self.start_scenario_timing(scenario.name) + started = time.time() + self.info('Running scenario %s' % scenario.name) self.ts['scenario'] = scenario self.ts['scenario_name'] = scenario.name self.ts['steps'] = scenario.steps + self.scenarios_run += 1 if self.settings['no-act']: self.info('Pretending everything went OK') + self.remember_scenario_timing(time.time() - started) return True if self.settings['tempdir']: - tempdir = self.settings['tempdir'] + tempdir = os.path.abspath(self.settings['tempdir']) if not os.path.exists(tempdir): os.mkdir(tempdir) else: @@ -240,6 +296,9 @@ class YarnRunner(cliapp.Application): self.snapshot_datadir( tempdir, datadir, scenario, step_number, step) if exit != 0: + self.ts.notify( + 'Skipping "%s" because "%s %s" failed' % + (scenario.name, step.what, step.text)) break else: for step in normal: @@ -265,24 +324,65 @@ class YarnRunner(cliapp.Application): if not self.settings['snapshot']: shutil.rmtree(tempdir) + self.remember_scenario_timing(time.time() - started) return ok + def clean_env(self): + '''Return a clean environment for running tests.''' + + whitelisted = [ + 'PATH', + ] + + hardcoded = { + 'TERM': 'dumb', + 'SHELL': '/bin/sh', + 'LC_ALL': 'C', + 'USER': 'tomjon', + 'USERNAME': 'tomjon', + 'LOGNAME': 'tomjon', + 'HOME': '/this/path/does/not/exist', + } + + env = {} + + for key in whitelisted: + if key in os.environ: + env[key] = os.environ[key] + + for key in hardcoded: + env[key] = hardcoded[key] + + for option_arg in self.settings['env']: + if '=' not in option_arg: + raise cliapp.AppException( + '--env argument must contain "=" ' + 'to separate environment variable name and value') + key, value = option_arg.split('=', 1) + env[key] = value + + return env + def run_step(self, datadir, scenario, step, shell_prelude, report_error): + started = time.time() + self.info('Running step "%s %s"' % (step.what, step.text)) self.ts['step'] = step self.ts['step_name'] = '%s %s' % (step.what, step.text) + self.steps_run += 1 - m = re.match(step.implementation.regexp, step.text) + m = self.implements_matches_step(step.implementation, step) assert m is not None - env = os.environ.copy() + env = self.clean_env() env['DATADIR'] = datadir + env['SRCDIR'] = os.getcwd() for i, match in enumerate(m.groups('')): env['MATCH_%d' % (i+1)] = match shell_script = '%s\n\n%s\n' % ( shell_prelude, step.implementation.shell) exit, stdout, stderr = cliapp.runcmd_unchecked( - ['sh', '-euc', shell_script], env=env) + ['sh', '-xeuc', shell_script], env=env) logging.debug('Exit code: %d' % exit) if stdout: @@ -303,6 +403,9 @@ class YarnRunner(cliapp.Application): (scenario.name, step.what, step.text, exit, self.indent(stdout), self.indent(stderr))) + self.remember_step_timing( + '%s %s' % (step.what, step.text), time.time() - started) + return exit def scenario_dir(self, tempdir, scenario): @@ -336,8 +439,38 @@ class YarnRunner(cliapp.Application): nice = ''.join(nice) return nice + def implements_matches_step(self, implements, step): + '''Return re.Match if implements matches the step. + + Otherwise, return None. + + ''' + + m = re.match(implements.regexp, step.text, re.I) + if m and m.end() != len(step.text): + return None + return m + def indent(self, s): return ''.join(' %s\n' % line for line in s.splitlines()) + def start_scenario_timing(self, scenario_name): + self.timings.append((scenario_name, None, [])) + + def remember_scenario_timing(self, duration): + scenario_name, _, step_tuples = self.timings[-1] + self.timings[-1] = (scenario_name, duration, step_tuples) + + def remember_step_timing(self, step_name, step_duration): + scenario_name, scenario_duration, step_tuples = self.timings[-1] + step_tuples = step_tuples + [(step_name, step_duration)] + self.timings[-1] = (scenario_name, scenario_duration, step_tuples) + + def report_timings(self): + for scenario_name, scenario_duration, step_tuples in self.timings: + print '%5.1f %s' % (scenario_duration, scenario_name) + for step_name, step_duration in step_tuples: + print ' %5.1f %s' % (step_duration, step_name) + YarnRunner(version=cmdtestlib.__version__).run() |