diff options
Diffstat (limited to 'test/coveragetest.py')
| -rw-r--r-- | test/coveragetest.py | 304 |
1 files changed, 198 insertions, 106 deletions
diff --git a/test/coveragetest.py b/test/coveragetest.py index d1631a3d..9b85b034 100644 --- a/test/coveragetest.py +++ b/test/coveragetest.py @@ -1,11 +1,11 @@ """Base test case class for coverage testing.""" -import imp, os, random, re, shutil, sys, tempfile, textwrap, unittest +import imp, os, random, shlex, shutil, sys, tempfile, textwrap import coverage -from coverage.backward import set, sorted, StringIO # pylint: disable-msg=W0622 +from coverage.backward import sorted, StringIO # pylint: disable-msg=W0622 from backtest import run_command - +from backunittest import TestCase class Tee(object): """A file-like that writes to all the file-likes it has.""" @@ -13,18 +13,21 @@ class Tee(object): def __init__(self, *files): """Make a Tee that writes to all the files in `files.`""" self.files = files - + def write(self, data): """Write `data` to all the files.""" for f in self.files: f.write(data) -class CoverageTest(unittest.TestCase): +# Status returns for the command line. +OK, ERR = 0, 1 + +class CoverageTest(TestCase): """A base class for Coverage test cases.""" run_in_temp_dir = True - + def setUp(self): if self.run_in_temp_dir: # Create a temporary directory. @@ -34,49 +37,106 @@ class CoverageTest(unittest.TestCase): os.makedirs(self.temp_dir) self.old_dir = os.getcwd() os.chdir(self.temp_dir) - - # Preserve changes to PYTHONPATH. - self.old_pypath = os.environ.get('PYTHONPATH', '') - + + # Modules should be importable from this temp directory. self.old_syspath = sys.path[:] sys.path.insert(0, '') - + # Keep a counter to make every call to check_coverage unique. self.n = 0 - # Use a Tee to capture stdout. + # Record environment variables that we changed with set_environ. + self.environ_undos = {} + + # Capture stdout and stderr so we can examine them in tests. + # nose keeps stdout from littering the screen, so we can safely Tee it, + # but it doesn't capture stderr, so we don't want to Tee stderr to the + # real stderr, since it will interfere with our nice field of dots. self.old_stdout = sys.stdout self.captured_stdout = StringIO() sys.stdout = Tee(sys.stdout, self.captured_stdout) - + self.old_stderr = sys.stderr + self.captured_stderr = StringIO() + sys.stderr = self.captured_stderr + + # Record sys.modules here so we can restore it in tearDown. + self.old_modules = dict(sys.modules) + def tearDown(self): if self.run_in_temp_dir: - # Restore the original sys.path and PYTHONPATH + # Restore the original sys.path. sys.path = self.old_syspath - os.environ['PYTHONPATH'] = self.old_pypath - + # Get rid of the temporary directory. os.chdir(self.old_dir) shutil.rmtree(self.temp_root) - - # Restore stdout. + + # Restore the environment. + self.undo_environ() + + # Restore stdout and stderr sys.stdout = self.old_stdout + sys.stderr = self.old_stderr + + # Remove any new modules imported during the test run. This lets us + # import the same source files for more than one test. + for m in [m for m in sys.modules if m not in self.old_modules]: + del sys.modules[m] + + def set_environ(self, name, value): + """Set an environment variable `name` to be `value`. + + The environment variable is set, and record is kept that it was set, + so that `tearDown` can restore its original value. + + """ + if name not in self.environ_undos: + self.environ_undos[name] = os.environ.get(name) + os.environ[name] = value + + def original_environ(self, name, if_missing=None): + """The environment variable `name` from when the test started.""" + if name in self.environ_undos: + ret = self.environ_undos[name] + else: + ret = os.environ.get(name) + if ret is None: + ret = if_missing + return ret + + def undo_environ(self): + """Undo all the changes made by `set_environ`.""" + for name, value in self.environ_undos.items(): + if value is None: + del os.environ[name] + else: + os.environ[name] = value def stdout(self): """Return the data written to stdout during the test.""" return self.captured_stdout.getvalue() + def stderr(self): + """Return the data written to stderr during the test.""" + return self.captured_stderr.getvalue() + def make_file(self, filename, text): """Create a temp file. - - `filename` is the file name, and `text` is the content. - + + `filename` is the path to the file, including directories if desired, + and `text` is the content. + """ # Tests that call `make_file` should be run in a temp environment. assert self.run_in_temp_dir text = textwrap.dedent(text) - + + # Make sure the directories are available. + dirs, _ = os.path.split(filename) + if dirs and not os.path.exists(dirs): + os.makedirs(dirs) + # Create the file. f = open(filename, 'w') f.write(text) @@ -86,7 +146,7 @@ class CoverageTest(unittest.TestCase): """Import the module named modname, and return the module object.""" modfile = modname + '.py' f = open(modfile, 'r') - + for suff in imp.get_suffixes(): if suff[0] == '.py': break @@ -107,49 +167,73 @@ class CoverageTest(unittest.TestCase): modname = 'coverage_test_' + self.noise + str(self.n) self.n += 1 return modname - + # Map chars to numbers for arcz_to_arcs _arcz_map = {'.': -1} _arcz_map.update(dict([(c, ord(c)-ord('0')) for c in '123456789'])) _arcz_map.update(dict( [(c, 10+ord(c)-ord('A')) for c in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'] )) - + def arcz_to_arcs(self, arcz): """Convert a compact textual representation of arcs to a list of pairs. - + The text has space-separated pairs of letters. Period is -1, 1-9 are 1-9, A-Z are 10 through 36. The resulting list is sorted regardless of the order of the input pairs. - + ".1 12 2." --> [(-1,1), (1,2), (2,-1)] - + + Minus signs can be included in the pairs: + + "-11, 12, 2-5" --> [(-1,1), (1,2), (2,-5)] + """ arcs = [] - for a,b in arcz.split(): - arcs.append((self._arcz_map[a], self._arcz_map[b])) + for pair in arcz.split(): + asgn = bsgn = 1 + if len(pair) == 2: + a,b = pair + else: + assert len(pair) == 3 + if pair[0] == '-': + _,a,b = pair + asgn = -1 + else: + assert pair[1] == '-' + a,_,b = pair + bsgn = -1 + arcs.append((asgn*self._arcz_map[a], bsgn*self._arcz_map[b])) return sorted(arcs) + def assertEqualArcs(self, a1, a2, msg=None): + """Assert that the arc lists `a1` and `a2` are equal.""" + # Make them into multi-line strings so we can see what's going wrong. + s1 = "\n".join([repr(a) for a in a1]) + "\n" + s2 = "\n".join([repr(a) for a in a2]) + "\n" + self.assertMultiLineEqual(s1, s2, msg) + def check_coverage(self, text, lines=None, missing="", excludes=None, report="", arcz=None, arcz_missing="", arcz_unpredicted=""): """Check the coverage measurement of `text`. - + The source `text` is run and measured. `lines` are the line numbers - that are executable, `missing` are the lines not executed, `excludes` - are regexes to match against for excluding lines, and `report` is - the text of the measurement report. - + that are executable, or a list of possible line numbers, any of which + could match. `missing` are the lines not executed, `excludes` are + regexes to match against for excluding lines, and `report` is the text + of the measurement report. + For arc measurement, `arcz` is a string that can be decoded into arcs in the code (see `arcz_to_arcs` for the encoding scheme), `arcz_missing` are the arcs that are not executed, and `arcs_unpredicted` are the arcs executed in the code, but not deducible from the code. - + """ # We write the code into a file so that we can import it. # Coverage wants to deal with things as modules with file names. modname = self.get_module_name() - + self.make_file(modname+".py", text) arcs = arcs_missing = arcs_unpredicted = None @@ -157,7 +241,7 @@ class CoverageTest(unittest.TestCase): arcs = self.arcz_to_arcs(arcz) arcs_missing = self.arcz_to_arcs(arcz_missing or "") arcs_unpredicted = self.arcz_to_arcs(arcz_unpredicted or "") - + # Start up Coverage. cov = coverage.coverage(branch=(arcs_missing is not None)) cov.erase() @@ -165,11 +249,12 @@ class CoverageTest(unittest.TestCase): cov.exclude(exc) cov.start() - # Import the python file, executing it. - mod = self.import_module(modname) - - # Stop Coverage. - cov.stop() + try: # pragma: recursive coverage + # Import the python file, executing it. + mod = self.import_module(modname) + finally: # pragma: recursive coverage + # Stop Coverage. + cov.stop() # Clean up our side effects del sys.modules[modname] @@ -178,8 +263,12 @@ class CoverageTest(unittest.TestCase): analysis = cov._analyze(mod) if lines is not None: if type(lines[0]) == type(1): + # lines is just a list of numbers, it must match the statements + # found in the code. self.assertEqual(analysis.statements, lines) else: + # lines is a list of possible line number lists, one of them + # must match. for line_list in lines: if analysis.statements == line_list: break @@ -188,26 +277,33 @@ class CoverageTest(unittest.TestCase): analysis.statements ) - if missing is not None: - if type(missing) == type(""): - self.assertEqual(analysis.missing_formatted(), missing) + if type(missing) == type(""): + self.assertEqual(analysis.missing_formatted(), missing) + else: + for missing_list in missing: + if analysis.missing_formatted() == missing_list: + break else: - for missing_list in missing: - if analysis.missing == missing_list: - break - else: - self.fail("None of the missing choices matched %r" % - analysis.missing_formatted() - ) + self.fail("None of the missing choices matched %r" % + analysis.missing_formatted() + ) if arcs is not None: - self.assertEqual(analysis.arc_possibilities(), arcs) + self.assertEqualArcs( + analysis.arc_possibilities(), arcs, "Possible arcs differ" + ) if arcs_missing is not None: - self.assertEqual(analysis.arcs_missing(), arcs_missing) + self.assertEqualArcs( + analysis.arcs_missing(), arcs_missing, + "Missing arcs differ" + ) if arcs_unpredicted is not None: - self.assertEqual(analysis.arcs_unpredicted(), arcs_unpredicted) + self.assertEqualArcs( + analysis.arcs_unpredicted(), arcs_unpredicted, + "Unpredicted arcs differ" + ) if report: frep = StringIO() @@ -215,68 +311,64 @@ class CoverageTest(unittest.TestCase): rep = " ".join(frep.getvalue().split("\n")[2].split()[1:]) self.assertEqual(report, rep) - def assert_raises_msg(self, excClass, msg, callableObj, *args, **kwargs): - """ Just like unittest.TestCase.assertRaises, - but checks that the message is right too. - """ - try: - callableObj(*args, **kwargs) - except excClass: - _, exc, _ = sys.exc_info() - excMsg = str(exc) - if not msg: - # No message provided: it passes. - return #pragma: no cover - elif excMsg == msg: - # Message provided, and we got the right message: it passes. - return - else: #pragma: no cover - # Message provided, and it didn't match: fail! - raise self.failureException( - "Right exception, wrong message: got '%s' expected '%s'" % - (excMsg, msg) - ) - # No need to catch other exceptions: They'll fail the test all by - # themselves! - else: #pragma: no cover - if hasattr(excClass,'__name__'): - excName = excClass.__name__ - else: - excName = str(excClass) - raise self.failureException( - "Expected to raise %s, didn't get an exception at all" % - excName - ) - def nice_file(self, *fparts): """Canonicalize the filename composed of the parts in `fparts`.""" fname = os.path.join(*fparts) return os.path.normcase(os.path.abspath(os.path.realpath(fname))) - + + def command_line(self, args, ret=OK, _covpkg=None): + """Run `args` through the command line. + + Use this when you want to run the full coverage machinery, but in the + current process. Exceptions may be thrown from deep in the code. + Asserts that `ret` is returned by `CoverageScript.command_line`. + + Compare with `run_command`. + + Returns None. + + """ + script = coverage.CoverageScript(_covpkg=_covpkg) + ret_actual = script.command_line(shlex.split(args)) + self.assertEqual(ret_actual, ret) + def run_command(self, cmd): - """ Run the command-line `cmd`, print its output. + """Run the command-line `cmd` in a subprocess, and print its output. + + Use this when you need to test the process behavior of coverage. + + Compare with `command_line`. + + Returns the process' stdout text. + + """ + _, output = self.run_command_status(cmd) + return output + + def run_command_status(self, cmd, status=0): + """Run the command-line `cmd` in a subprocess, and print its output. + + Use this when you need to test the process behavior of coverage. + + Compare with `command_line`. + + Returns a pair: the process' exit status and stdout text. + + The `status` argument is returned as the status on older Pythons where + we can't get the actual exit status of the process. + """ # Add our test modules directory to PYTHONPATH. I'm sure there's too # much path munging here, but... here = os.path.dirname(self.nice_file(coverage.__file__, "..")) testmods = self.nice_file(here, 'test/modules') zipfile = self.nice_file(here, 'test/zipmods.zip') - pypath = self.old_pypath + pypath = self.original_environ('PYTHONPATH', "") if pypath: pypath += os.pathsep pypath += testmods + os.pathsep + zipfile - os.environ['PYTHONPATH'] = pypath - - _, output = run_command(cmd) - print(output) - return output + self.set_environ('PYTHONPATH', pypath) - def assert_equal_sets(self, s1, s2): - """Assert that the two arguments are equal as sets.""" - self.assertEqual(set(s1), set(s2)) - - def assert_matches(self, s, regex): - """Assert that `s` matches `regex`.""" - m = re.search(regex, s) - if not m: - raise self.failureException("%r doesn't match %r" % (s, regex)) + status, output = run_command(cmd, status=status) + print(output) + return status, output |