From 1e6e96f65dccbe6be94edf588184349fc9be4bd4 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 23 Jul 2013 21:56:35 +0100 Subject: Prepare release 0.9 --- NEWS | 2 +- cmdtestlib.py | 2 +- debian/changelog | 6 ++++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index cb4584d..ca4d551 100644 --- a/NEWS +++ b/NEWS @@ -3,7 +3,7 @@ NEWS for cmdtest This file summarizes changes between releases of cmdtest. -Version 0.X, released UNRELEASED +Version 0.9, released 2013-07-23 -------------------------------- * Yarn now warns if an input file has no code blocks. diff --git a/cmdtestlib.py b/cmdtestlib.py index e6ca6bf..dee2c00 100644 --- a/cmdtestlib.py +++ b/cmdtestlib.py @@ -14,7 +14,7 @@ # along with this program. If not, see . -__version__ = '0.8.3' +__version__ = '0.9' import os diff --git a/debian/changelog b/debian/changelog index 46eb065..7bbbe5c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +cmdtest (0.9-1) unstable; urgency=low + + * New upstream release. + + -- Lars Wirzenius Tue, 23 Jul 2013 21:56:29 +0100 + cmdtest (0.8.3-1) unstable; urgency=low * New upstream. -- cgit v1.2.1 From 3fb60d86c01ef3bc615d39e69ee0011aef906685 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 30 Jul 2013 21:36:45 +0100 Subject: Add quotes around scenario name in error message --- yarn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn b/yarn index 842f308..288e23b 100755 --- a/yarn +++ b/yarn @@ -157,7 +157,7 @@ class YarnRunner(cliapp.Application): if len(matching) == 0: raise cliapp.AppException( - 'Scenario %s, step "%s %s" has no matching ' + 'Scenario "%s", step "%s %s" has no matching ' 'implementation' % (scenario.name, step.what, step.text)) if len(matching) > 1: -- cgit v1.2.1 From 8443dc250bb58091c435db2b29e2029c434d1750 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 31 Jul 2013 18:37:33 +0100 Subject: Set SRCDIR and clean environment --- yarn | 28 +++++++++++++++++++++++++++- yarn.tests/env.script | 18 ++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100755 yarn.tests/env.script diff --git a/yarn b/yarn index 288e23b..6fbd63a 100755 --- a/yarn +++ b/yarn @@ -267,6 +267,31 @@ class YarnRunner(cliapp.Application): return ok + def clean_env(self): + '''Return a clean environment for running tests.''' + + whitelisted = [ + 'TERM', + 'USER', + 'USERNAME', + 'PATH', + 'HOME', + 'LOGNAME', + ] + + hardcoded = { + 'SHELL': '/bin/sh', + 'LC_ALL': 'C', + } + + env = {} + for key in whitelisted: + if key in os.environ: + env[key] = os.environ[key] + for key in hardcoded: + env[key] = hardcoded[key] + return env + def run_step(self, datadir, scenario, step, shell_prelude, report_error): self.info('Running step "%s %s"' % (step.what, step.text)) self.ts['step'] = step @@ -274,8 +299,9 @@ class YarnRunner(cliapp.Application): m = re.match(step.implementation.regexp, step.text) assert m is not None - env = os.environ.copy() + env = self.clean_env() env['DATADIR'] = datadir + env['SRCDIR'] = os.getcwd() for i, match in enumerate(m.groups('')): env['MATCH_%d' % (i+1)] = match diff --git a/yarn.tests/env.script b/yarn.tests/env.script new file mode 100755 index 0000000..2ef35c6 --- /dev/null +++ b/yarn.tests/env.script @@ -0,0 +1,18 @@ +#!/bin/sh + +set -eu + +cat << 'EOF' > "$DATADIR/env.yarn" + SCENARIO check environment + THEN DATADIR is set + AND SRCDIR is set + AND NOTSET is not set + + IMPLEMENTS THEN (\S+) is set + env | grep "^$MATCH_1=" + + IMPLEMENTS THEN (\S+) is not set + ! env | grep "^$MATCH_1=" +EOF + +NOTSET=foo ./run-yarn "$DATADIR/env.yarn" -- cgit v1.2.1 From 2399b141f96604818c3a05b1e7e2cc1a8505f3b8 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 31 Jul 2013 18:38:48 +0100 Subject: Update NEWS about SRCDIR, clean env --- NEWS | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/NEWS b/NEWS index ca4d551..4951f4c 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,14 @@ NEWS for cmdtest This file summarizes changes between releases of cmdtest. +Version 0.10, released UNRELEASED +--------------------------------- + +* Yarn now cleans the environment when it runs shell commands for the + implementation steps. It also sets the `SRCDIR` environment variable + to point at the root of the source tree (the directory where yarn + was invoked from). + Version 0.9, released 2013-07-23 -------------------------------- -- cgit v1.2.1 From 8e8b0139119f102b08283c528ddbae70728bc188 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 31 Jul 2013 18:40:03 +0100 Subject: Add SRCDIR to the README --- README.yarn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.yarn b/README.yarn index 76d57b5..7006a66 100644 --- a/README.yarn +++ b/README.yarn @@ -136,6 +136,10 @@ The following keywords are defined. The test runner creates a temporary directory, whose name is given to the shell code in the `DATADIR` environment variable. + The test runner sets the `SRCDIR` environment variable to the + path to the directory it was invoked from (by convention, the + root of the source tree of the project). + The shell commands get invoked with `/bin/sh -eu`, and need to be written accordingly. Be careful about commands that return a non-zero exit code. There will eventually be a library of shell -- cgit v1.2.1 From 02fe4bfce7b93c54e7e0e1a68a822dcd7764b30f Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 31 Jul 2013 18:41:45 +0100 Subject: Add to README mention of environment cleanup --- README.yarn | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.yarn b/README.yarn index 7006a66..16f7522 100644 --- a/README.yarn +++ b/README.yarn @@ -140,6 +140,11 @@ The following keywords are defined. path to the directory it was invoked from (by convention, the root of the source tree of the project). + The test runner removes all other environment variables, except + `TERM`, `USER`, `USERNAME`, `LOGNAME`, `HOME`, and `PATH`. It also + forces `SHELL` set to `/bin/sh`, and `LC_ALL` set to `C`, in order + to have as clean an environment as possible for tests to run in. + The shell commands get invoked with `/bin/sh -eu`, and need to be written accordingly. Be careful about commands that return a non-zero exit code. There will eventually be a library of shell -- cgit v1.2.1 From 6fb702dcc45f991b7bf3106c994a83a302d7ec1c Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 31 Jul 2013 18:47:21 +0100 Subject: Make --tempdir absolute --- yarn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn b/yarn index 6fbd63a..fbf928d 100755 --- a/yarn +++ b/yarn @@ -216,7 +216,7 @@ class YarnRunner(cliapp.Application): return True if self.settings['tempdir']: - tempdir = self.settings['tempdir'] + tempdir = os.path.abspath(self.settings['tempdir']) if not os.path.exists(tempdir): os.mkdir(tempdir) else: -- cgit v1.2.1 From 79a6709f4cc49e028fffaead40192a8bbe79f002 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 31 Jul 2013 18:48:48 +0100 Subject: Make yarn run shell scripts with -x This makes debugging easier. --- yarn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn b/yarn index fbf928d..202d726 100755 --- a/yarn +++ b/yarn @@ -308,7 +308,7 @@ class YarnRunner(cliapp.Application): shell_script = '%s\n\n%s\n' % ( shell_prelude, step.implementation.shell) exit, stdout, stderr = cliapp.runcmd_unchecked( - ['sh', '-euc', shell_script], env=env) + ['sh', '-xeuc', shell_script], env=env) logging.debug('Exit code: %d' % exit) if stdout: -- cgit v1.2.1 From 0e875126d927ffd17fb714abb914c857b638ce74 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 2 Aug 2013 19:21:07 +0100 Subject: Report number of scenarios, steps actually run --- yarn | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/yarn b/yarn index 202d726..2d9f52b 100755 --- a/yarn +++ b/yarn @@ -111,6 +111,9 @@ class YarnRunner(cliapp.Application): self.ts['num_scenarios'] = len(scenarios) self.info('Found %d scenarios' % len(scenarios)) + self.scenarios_run = 0 + self.steps_run = 0 + start_time = time.time() failed_scenarios = [] for scenario in self.select_scenarios(scenarios): @@ -128,9 +131,10 @@ class YarnRunner(cliapp.Application): if not self.settings['quiet']: print ( - 'Scenario test suite PASS, with %d scenarios, ' + 'Scenario test suite PASS, with %d scenarios ' + '(%d total steps), ' 'in %.1f seconds' % - (len(scenarios), duration)) + (self.scenarios_run, self.steps_run, duration)) def parse_scenarios(self, filenames): mdparser = yarnlib.MarkdownParser() @@ -210,6 +214,7 @@ class YarnRunner(cliapp.Application): self.ts['scenario'] = scenario self.ts['scenario_name'] = scenario.name self.ts['steps'] = scenario.steps + self.scenarios_run += 1 if self.settings['no-act']: self.info('Pretending everything went OK') @@ -296,6 +301,7 @@ class YarnRunner(cliapp.Application): self.info('Running step "%s %s"' % (step.what, step.text)) self.ts['step'] = step self.ts['step_name'] = '%s %s' % (step.what, step.text) + self.steps_run += 1 m = re.match(step.implementation.regexp, step.text) assert m is not None -- cgit v1.2.1 From f1b64b8535c73381d819d8c74c16dca067caf69c Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 2 Aug 2013 19:30:58 +0100 Subject: Add yarn --timings option --- yarn | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/yarn b/yarn index 2d9f52b..ce0b9cb 100755 --- a/yarn +++ b/yarn @@ -69,6 +69,10 @@ class YarnRunner(cliapp.Application): 'after each scenario step; you probably ' 'want to use this with --tempdir') + self.settings.boolean( + ['timings'], + 'report wall clock time for each scenario and step') + def info(self, msg): if self.settings['verbose']: logging.info(msg) @@ -113,6 +117,7 @@ class YarnRunner(cliapp.Application): self.scenarios_run = 0 self.steps_run = 0 + self.timings = [] start_time = time.time() failed_scenarios = [] @@ -136,6 +141,9 @@ class YarnRunner(cliapp.Application): 'in %.1f seconds' % (self.scenarios_run, self.steps_run, duration)) + if self.settings['timings']: + self.report_timings() + def parse_scenarios(self, filenames): mdparser = yarnlib.MarkdownParser() for filename in filenames: @@ -210,6 +218,9 @@ class YarnRunner(cliapp.Application): return scenarios def run_scenario(self, scenario, shell_prelude): + self.start_scenario_timing(scenario.name) + started = time.time() + self.info('Running scenario %s' % scenario.name) self.ts['scenario'] = scenario self.ts['scenario_name'] = scenario.name @@ -218,6 +229,7 @@ class YarnRunner(cliapp.Application): if self.settings['no-act']: self.info('Pretending everything went OK') + self.remember_scenario_timing(time.time() - started) return True if self.settings['tempdir']: @@ -270,6 +282,7 @@ class YarnRunner(cliapp.Application): if not self.settings['snapshot']: shutil.rmtree(tempdir) + self.remember_scenario_timing(time.time() - started) return ok def clean_env(self): @@ -298,6 +311,8 @@ class YarnRunner(cliapp.Application): return env def run_step(self, datadir, scenario, step, shell_prelude, report_error): + started = time.time() + self.info('Running step "%s %s"' % (step.what, step.text)) self.ts['step'] = step self.ts['step_name'] = '%s %s' % (step.what, step.text) @@ -335,6 +350,9 @@ class YarnRunner(cliapp.Application): (scenario.name, step.what, step.text, exit, self.indent(stdout), self.indent(stderr))) + self.remember_step_timing( + '%s %s' % (step.what, step.text), time.time() - started) + return exit def scenario_dir(self, tempdir, scenario): @@ -371,5 +389,23 @@ class YarnRunner(cliapp.Application): def indent(self, s): return ''.join(' %s\n' % line for line in s.splitlines()) + def start_scenario_timing(self, scenario_name): + self.timings.append((scenario_name, None, [])) + + def remember_scenario_timing(self, duration): + scenario_name, _, step_tuples = self.timings[-1] + self.timings[-1] = (scenario_name, duration, step_tuples) + + def remember_step_timing(self, step_name, step_duration): + scenario_name, scenario_duration, step_tuples = self.timings[-1] + step_tuples = step_tuples + [(step_name, step_duration)] + self.timings[-1] = (scenario_name, scenario_duration, step_tuples) + + def report_timings(self): + for scenario_name, scenario_duration, step_tuples in self.timings: + print '%5.1f %s' % (scenario_duration, scenario_name) + for step_name, step_duration in step_tuples: + print ' %5.1f %s' % (step_duration, step_name) + YarnRunner(version=cmdtestlib.__version__).run() -- cgit v1.2.1 From 3ee1542566627fb33eae7edcb67b97e28de7f975 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 2 Aug 2013 19:31:38 +0100 Subject: Update NEWS about yarn --timings --- NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS b/NEWS index 4951f4c..35bbc38 100644 --- a/NEWS +++ b/NEWS @@ -11,6 +11,9 @@ Version 0.10, released UNRELEASED to point at the root of the source tree (the directory where yarn was invoked from). +* A new option, `--timings`, has been added to yarn to report how long + each scenario and each step took. + Version 0.9, released 2013-07-23 -------------------------------- -- cgit v1.2.1 From a3bfd4de13e415c3c8267762311340579967cbbb Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 19 Aug 2013 18:53:54 +0100 Subject: Complain if a scenario has no THENs --- NEWS | 3 +++ yarn | 15 +++++++++++++++ yarn.tests/no-then.exit | 1 + yarn.tests/no-then.script | 14 ++++++++++++++ yarn.tests/no-then.stderr | 3 +++ 5 files changed, 36 insertions(+) create mode 100644 yarn.tests/no-then.exit create mode 100755 yarn.tests/no-then.script create mode 100644 yarn.tests/no-then.stderr diff --git a/NEWS b/NEWS index 35bbc38..92c1e37 100644 --- a/NEWS +++ b/NEWS @@ -14,6 +14,9 @@ Version 0.10, released UNRELEASED * A new option, `--timings`, has been added to yarn to report how long each scenario and each step took. +* Yarn now complains if a scenario has no THEN steps. Suggested by + Richard Maw. + Version 0.9, released 2013-07-23 -------------------------------- diff --git a/yarn b/yarn index ce0b9cb..3bb26e8 100755 --- a/yarn +++ b/yarn @@ -108,6 +108,7 @@ class YarnRunner(cliapp.Application): 'step %Index(step,steps): %String(step_name)') scenarios, implementations = self.parse_scenarios(args) + self.check_for_thens(scenarios) self.connect_implementations(scenarios, implementations) shell_prelude = self.load_shell_libraries() @@ -157,6 +158,20 @@ class YarnRunner(cliapp.Application): return block_parser.scenarios, block_parser.implementations + def check_for_thens(self, scenarios): + no_thens = [] + for scenario in scenarios: + for step in scenario.steps: + if step.what == 'THEN': + break + else: + no_thens.append(scenario) + + if no_thens: + raise cliapp.AppException( + 'Some scenarios have no THENs:\n%s' % + ''.join(' "%s"\n' % s.name for s in scenarios)) + def connect_implementations(self, scenarios, implementations): for scenario in scenarios: for step in scenario.steps: diff --git a/yarn.tests/no-then.exit b/yarn.tests/no-then.exit new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/yarn.tests/no-then.exit @@ -0,0 +1 @@ +1 diff --git a/yarn.tests/no-then.script b/yarn.tests/no-then.script new file mode 100755 index 0000000..491853f --- /dev/null +++ b/yarn.tests/no-then.script @@ -0,0 +1,14 @@ +#!/bin/sh + +set -eu + + +cat < "$DATADIR/1.yarn" + SCENARIO foo + WHEN doing ok + + IMPLEMENTS WHEN doing ok + true +EOF + +./run-yarn "$DATADIR/1.yarn" diff --git a/yarn.tests/no-then.stderr b/yarn.tests/no-then.stderr new file mode 100644 index 0000000..b018036 --- /dev/null +++ b/yarn.tests/no-then.stderr @@ -0,0 +1,3 @@ +ERROR: Some scenarios have no THENs: + "foo" + -- cgit v1.2.1 From 0f7cfd5fc4f94446c54bb2712eb0aaf5043f7cd5 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 25 Aug 2013 13:27:01 +0100 Subject: Add test for duplicate scenario names failing --- yarn.tests/duplicate-scenario-names.exit | 1 + yarn.tests/duplicate-scenario-names.script | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 yarn.tests/duplicate-scenario-names.exit create mode 100755 yarn.tests/duplicate-scenario-names.script diff --git a/yarn.tests/duplicate-scenario-names.exit b/yarn.tests/duplicate-scenario-names.exit new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/yarn.tests/duplicate-scenario-names.exit @@ -0,0 +1 @@ +1 diff --git a/yarn.tests/duplicate-scenario-names.script b/yarn.tests/duplicate-scenario-names.script new file mode 100755 index 0000000..770f00a --- /dev/null +++ b/yarn.tests/duplicate-scenario-names.script @@ -0,0 +1,18 @@ +#!/bin/sh + +set -eu + + +cat < "$DATADIR/test.yarn" + SCENARIO foo + THEN nop + + SCENARIO foo + THEN nop + + IMPLEMENTS THEN nop + true +EOF + +./run-yarn "$DATADIR/test.yarn" + -- cgit v1.2.1 From 99eb7132499d14b604d67d8d8104243c3f9a35e6 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 25 Aug 2013 13:38:14 +0100 Subject: Implement check for duplicate scenario names --- yarn | 13 +++++++++++++ yarn.tests/duplicate-scenario-names.stderr | 3 +++ 2 files changed, 16 insertions(+) create mode 100644 yarn.tests/duplicate-scenario-names.stderr diff --git a/yarn b/yarn index 3bb26e8..c13681b 100755 --- a/yarn +++ b/yarn @@ -18,6 +18,7 @@ import cliapp +import collections import logging import os import re @@ -108,6 +109,7 @@ class YarnRunner(cliapp.Application): 'step %Index(step,steps): %String(step_name)') scenarios, implementations = self.parse_scenarios(args) + self.check_for_duplicate_scenario_names(scenarios) self.check_for_thens(scenarios) self.connect_implementations(scenarios, implementations) shell_prelude = self.load_shell_libraries() @@ -158,6 +160,17 @@ class YarnRunner(cliapp.Application): return block_parser.scenarios, block_parser.implementations + def check_for_duplicate_scenario_names(self, scenarios): + counts = collections.Counter() + for s in scenarios: + counts[s.name] += 1 + + duplicates = [name for name in counts if counts[name] > 1] + if duplicates: + duplist = ''.join(' %s\n' % name for name in duplicates) + raise cliapp.AppException( + 'There are scenarios with duplicate names:\n%s' % duplist) + def check_for_thens(self, scenarios): no_thens = [] for scenario in scenarios: diff --git a/yarn.tests/duplicate-scenario-names.stderr b/yarn.tests/duplicate-scenario-names.stderr new file mode 100644 index 0000000..d3fbc95 --- /dev/null +++ b/yarn.tests/duplicate-scenario-names.stderr @@ -0,0 +1,3 @@ +ERROR: There are scenarios with duplicate names: + foo + -- cgit v1.2.1 From 736c408ac2b8768150867b89f4a497edcffd2edd Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 25 Aug 2013 13:38:54 +0100 Subject: Update NEWS --- NEWS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS b/NEWS index 92c1e37..2682b85 100644 --- a/NEWS +++ b/NEWS @@ -14,9 +14,13 @@ Version 0.10, released UNRELEASED * A new option, `--timings`, has been added to yarn to report how long each scenario and each step took. +Bug fixes: + * Yarn now complains if a scenario has no THEN steps. Suggested by Richard Maw. +* Yarn now checks for duplicate scenario names. + Version 0.9, released 2013-07-23 -------------------------------- -- cgit v1.2.1 From ba82e75356b37ff7e248b984ca8484c9cd6d6141 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 25 Aug 2013 13:51:08 +0100 Subject: Clean environmnet harder --- NEWS | 8 +++++--- yarn | 10 +++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/NEWS b/NEWS index 2682b85..67d870f 100644 --- a/NEWS +++ b/NEWS @@ -7,9 +7,11 @@ Version 0.10, released UNRELEASED --------------------------------- * Yarn now cleans the environment when it runs shell commands for the - implementation steps. It also sets the `SRCDIR` environment variable - to point at the root of the source tree (the directory where yarn - was invoked from). + implementation steps. The PATH variable is kept from the user's + environment, every other variable is either removed or hardcoded to + a specific value. Additionally yarn sets the `SRCDIR` environment + variable to point at the root of the source tree (the directory + where yarn was invoked from). * A new option, `--timings`, has been added to yarn to report how long each scenario and each step took. diff --git a/yarn b/yarn index c13681b..3e9be0d 100755 --- a/yarn +++ b/yarn @@ -317,17 +317,17 @@ class YarnRunner(cliapp.Application): '''Return a clean environment for running tests.''' whitelisted = [ - 'TERM', - 'USER', - 'USERNAME', 'PATH', - 'HOME', - 'LOGNAME', ] hardcoded = { + 'TERM': 'dumb', 'SHELL': '/bin/sh', 'LC_ALL': 'C', + 'USER': 'tomjon', + 'USERNAME': 'tomjon', + 'LOGNAME': 'tomjon', + 'HOME': '/this/path/does/not/exist', } env = {} -- cgit v1.2.1 From 7cba7b30e615e627984047b00cdd0de0b201efb5 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 25 Aug 2013 13:54:49 +0100 Subject: Check there are scenarios --- NEWS | 3 +++ yarn | 6 ++++++ yarn.tests/no-scenarios.exit | 1 + yarn.tests/no-scenarios.script | 9 +++++++++ yarn.tests/no-scenarios.stderr | 1 + 5 files changed, 20 insertions(+) create mode 100644 yarn.tests/no-scenarios.exit create mode 100755 yarn.tests/no-scenarios.script create mode 100644 yarn.tests/no-scenarios.stderr diff --git a/NEWS b/NEWS index 67d870f..a57a186 100644 --- a/NEWS +++ b/NEWS @@ -21,6 +21,9 @@ Bug fixes: * Yarn now complains if a scenario has no THEN steps. Suggested by Richard Maw. +* Yarn now gives an error if there are no scenarios. Suggested by + Daniel Silverstone and others. + * Yarn now checks for duplicate scenario names. Version 0.9, released 2013-07-23 diff --git a/yarn b/yarn index 3e9be0d..a84fcd4 100755 --- a/yarn +++ b/yarn @@ -109,6 +109,7 @@ class YarnRunner(cliapp.Application): 'step %Index(step,steps): %String(step_name)') scenarios, implementations = self.parse_scenarios(args) + self.check_there_are_scenarios(scenarios) self.check_for_duplicate_scenario_names(scenarios) self.check_for_thens(scenarios) self.connect_implementations(scenarios, implementations) @@ -160,6 +161,11 @@ class YarnRunner(cliapp.Application): return block_parser.scenarios, block_parser.implementations + def check_there_are_scenarios(self, scenarios): + if not scenarios: + raise cliapp.AppException( + 'There are no scenarios; must have at least one.') + def check_for_duplicate_scenario_names(self, scenarios): counts = collections.Counter() for s in scenarios: diff --git a/yarn.tests/no-scenarios.exit b/yarn.tests/no-scenarios.exit new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/yarn.tests/no-scenarios.exit @@ -0,0 +1 @@ +1 diff --git a/yarn.tests/no-scenarios.script b/yarn.tests/no-scenarios.script new file mode 100755 index 0000000..ffd8e96 --- /dev/null +++ b/yarn.tests/no-scenarios.script @@ -0,0 +1,9 @@ +#!/bin/sh + +set -eu + + +cat < "$DATADIR/test.yarn" +EOF + +./run-yarn "$DATADIR/test.yarn" diff --git a/yarn.tests/no-scenarios.stderr b/yarn.tests/no-scenarios.stderr new file mode 100644 index 0000000..66311c1 --- /dev/null +++ b/yarn.tests/no-scenarios.stderr @@ -0,0 +1 @@ +ERROR: There are no scenarios; must have at least one. -- cgit v1.2.1 From c4038a5a4b2efc6a7654d4ca81b2ae8b1b28a667 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 25 Aug 2013 14:02:29 +0100 Subject: Report scenaries skipped due to ASSUMING --- NEWS | 2 ++ yarn | 3 +++ 2 files changed, 5 insertions(+) diff --git a/NEWS b/NEWS index a57a186..8f5f5df 100644 --- a/NEWS +++ b/NEWS @@ -16,6 +16,8 @@ Version 0.10, released UNRELEASED * A new option, `--timings`, has been added to yarn to report how long each scenario and each step took. +* Yarn now reports scenarios skipped because of ASSUMING failing. + Bug fixes: * Yarn now complains if a scenario has no THEN steps. Suggested by diff --git a/yarn b/yarn index a84fcd4..2e02eb3 100755 --- a/yarn +++ b/yarn @@ -291,6 +291,9 @@ class YarnRunner(cliapp.Application): self.snapshot_datadir( tempdir, datadir, scenario, step_number, step) if exit != 0: + self.ts.notify( + 'Skipping "%s" because "%s %s" failed' % + (scenario.name, step.what, step.text)) break else: for step in normal: -- cgit v1.2.1 From 1dbacde0bfa4463434abe3ecbccd2c03a29a3c16 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 25 Aug 2013 14:05:05 +0100 Subject: Fix expected output of test --- yarn.tests/assuming-failure.stdout | 1 + 1 file changed, 1 insertion(+) create mode 100644 yarn.tests/assuming-failure.stdout diff --git a/yarn.tests/assuming-failure.stdout b/yarn.tests/assuming-failure.stdout new file mode 100644 index 0000000..d7494ec --- /dev/null +++ b/yarn.tests/assuming-failure.stdout @@ -0,0 +1 @@ +Skipping "foo" because "ASSUMING something" failed -- cgit v1.2.1 From 3573f2213988fdc477d3cbf123981f7397f42cdb Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 25 Aug 2013 14:11:50 +0100 Subject: Add --env option Suggested by Daniel Silverstone --- NEWS | 7 ++++--- yarn | 16 ++++++++++++++++ yarn.tests/env-option.script | 15 +++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) create mode 100755 yarn.tests/env-option.script diff --git a/NEWS b/NEWS index 8f5f5df..15b29a1 100644 --- a/NEWS +++ b/NEWS @@ -9,9 +9,10 @@ Version 0.10, released UNRELEASED * Yarn now cleans the environment when it runs shell commands for the implementation steps. The PATH variable is kept from the user's environment, every other variable is either removed or hardcoded to - a specific value. Additionally yarn sets the `SRCDIR` environment - variable to point at the root of the source tree (the directory - where yarn was invoked from). + a specific value. More variables can be added explicitly to the test + environment with the new `--env NAME=VALUE` option. Additionally + yarn sets the `SRCDIR` environment variable to point at the root of + the source tree (the directory where yarn was invoked from). * A new option, `--timings`, has been added to yarn to report how long each scenario and each step took. diff --git a/yarn b/yarn index 2e02eb3..501d10e 100755 --- a/yarn +++ b/yarn @@ -64,6 +64,11 @@ class YarnRunner(cliapp.Application): 'it should be empty or not exist', metavar='DIR') + self.settings.string_list( + ['env'], + 'add NAME=VALUE to the environment when tests are run', + metavar='NAME=VALUE') + self.settings.boolean( ['snapshot'], 'make snapshots of test working directory ' @@ -340,11 +345,22 @@ class YarnRunner(cliapp.Application): } env = {} + for key in whitelisted: if key in os.environ: env[key] = os.environ[key] + for key in hardcoded: env[key] = hardcoded[key] + + for option_arg in self.settings['env']: + if '=' not in option_arg: + raise cliapp.AppException( + '--env argument must contain "=" ' + 'to separate environment variable name and value') + key, value = option_arg.split('=', 1) + env[key] = value + return env def run_step(self, datadir, scenario, step, shell_prelude, report_error): diff --git a/yarn.tests/env-option.script b/yarn.tests/env-option.script new file mode 100755 index 0000000..9724ffc --- /dev/null +++ b/yarn.tests/env-option.script @@ -0,0 +1,15 @@ +#!/bin/sh + +set -eu + + +cat < "$DATADIR/test.yarn" + SCENARIO foo + THEN yoyo is set + + IMPLEMENTS THEN yoyo is set + env | grep '^yoyo=' +EOF + +./run-yarn --env yoyo=something "$DATADIR/test.yarn" + -- cgit v1.2.1 From f16588d5c07ec4e7fb08d2de2575258c36210fea Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 2 Sep 2013 19:20:29 +0100 Subject: Document DATADIR and SRCDIR in the manpage --- NEWS | 3 +++ yarn.1.in | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/NEWS b/NEWS index 15b29a1..797291e 100644 --- a/NEWS +++ b/NEWS @@ -19,6 +19,9 @@ Version 0.10, released UNRELEASED * Yarn now reports scenarios skipped because of ASSUMING failing. +* Yarn manual page now documents DATADIR and SRCDIR environment + variables. + Bug fixes: * Yarn now complains if a scenario has no THEN steps. Suggested by diff --git a/yarn.1.in b/yarn.1.in index 85d2d8e..f1708d9 100644 --- a/yarn.1.in +++ b/yarn.1.in @@ -109,7 +109,26 @@ by a programmer, but given a well-designed set of steps, with enough flexibility in their implementation, that quite a good test suite can be written. +.PP +The shell commands in an IMPLEMENTS section are run in the directory +in which the user ran +.BR yarn . +The environment variable +.B SRCDIR +is set to the fully qualified path to that directory. .SH OPTIONS +.SH ENVIRONMENT +.TP +.B DATADIR +Fully qualified pathname to a temporary directory, +in which the tests can use files. +The temporary directory is removed at the end of the test execution, +unless the user specifies otherwise with \-\-snapshot. +.TP +.B SRCDIR +Fully qualitifed pathname to the directory in which the user ran +.BR yarn . +This is useful when the tests want to change the directory. .SH EXAMPLE To run .B yarn -- cgit v1.2.1 From d9f0a876a0ec1c5320d77b0b4314bbd4a5b11bd3 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 19 Sep 2013 19:09:00 +0100 Subject: Match IMPLEMENTS always case-insensitively Reported-by: Jannis Pohlmann --- NEWS | 3 +++ yarn | 16 ++++++++++++++-- yarn.tests/env.script | 1 + 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 797291e..9aad0a2 100644 --- a/NEWS +++ b/NEWS @@ -32,6 +32,9 @@ Bug fixes: * Yarn now checks for duplicate scenario names. +* Yarn now always checks for IMPLEMENTS sections with case-insensitive + matching. Reported, with test case, by Jannis Pohlmann. + Version 0.9, released 2013-07-23 -------------------------------- diff --git a/yarn b/yarn index 501d10e..f4abcdc 100755 --- a/yarn +++ b/yarn @@ -204,7 +204,7 @@ class YarnRunner(cliapp.Application): def connect_implementation(self, scenario, step, implementations): matching = [i for i in implementations if step.what == i.what and - re.match('(%s)$' % i.regexp, step.text, re.I)] + self.implements_matches_step(i, step)] if len(matching) == 0: raise cliapp.AppException( @@ -371,7 +371,7 @@ class YarnRunner(cliapp.Application): self.ts['step_name'] = '%s %s' % (step.what, step.text) self.steps_run += 1 - m = re.match(step.implementation.regexp, step.text) + m = self.implements_matches_step(step.implementation, step) assert m is not None env = self.clean_env() env['DATADIR'] = datadir @@ -439,6 +439,18 @@ class YarnRunner(cliapp.Application): nice = ''.join(nice) return nice + def implements_matches_step(self, implements, step): + '''Return re.Match if implements matches the step. + + Otherwise, return None. + + ''' + + m = re.match(implements.regexp, step.text, re.I) + if m and m.end() != len(step.text): + return None + return m + def indent(self, s): return ''.join(' %s\n' % line for line in s.splitlines()) diff --git a/yarn.tests/env.script b/yarn.tests/env.script index 2ef35c6..d14a21a 100755 --- a/yarn.tests/env.script +++ b/yarn.tests/env.script @@ -9,6 +9,7 @@ cat << 'EOF' > "$DATADIR/env.yarn" AND NOTSET is not set IMPLEMENTS THEN (\S+) is set + env env | grep "^$MATCH_1=" IMPLEMENTS THEN (\S+) is not set -- cgit v1.2.1 From b15657651e587bf66399bc1a234603fe2bd3889b Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sat, 5 Oct 2013 15:09:30 +0100 Subject: Release version 0.10 --- NEWS | 2 +- cmdtestlib.py | 2 +- debian/changelog | 6 ++++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 9aad0a2..47fbd88 100644 --- a/NEWS +++ b/NEWS @@ -3,7 +3,7 @@ NEWS for cmdtest This file summarizes changes between releases of cmdtest. -Version 0.10, released UNRELEASED +Version 0.10, released 2013-10-05 --------------------------------- * Yarn now cleans the environment when it runs shell commands for the diff --git a/cmdtestlib.py b/cmdtestlib.py index dee2c00..66cb9e9 100644 --- a/cmdtestlib.py +++ b/cmdtestlib.py @@ -14,7 +14,7 @@ # along with this program. If not, see . -__version__ = '0.9' +__version__ = '0.10' import os diff --git a/debian/changelog b/debian/changelog index 7bbbe5c..2bd069f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +cmdtest (0.10-1) unstable; urgency=low + + * New upstream release. + + -- Lars Wirzenius Sat, 05 Oct 2013 15:07:43 +0100 + cmdtest (0.9-1) unstable; urgency=low * New upstream release. -- cgit v1.2.1