Merge pull request #920 from asottile/coverage

Instrument code coverage

5 files changed, 82 insertions, 76 deletions

diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 0000000..57fc88e
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,40 @@
+[run]
+source = .
+branch = true
+parallel = true
+omit =
+    */.tox/*
+    */__main__.py
+    */setup.py
+    */venv*/*
+    # TODO: separate the tests from the test data
+    testsuite/E*.py
+    testsuite/W*.py
+    testsuite/latin-1.py
+    testsuite/noqa.py
+    testsuite/python*.py
+    testsuite/utf-8-bom.py
+
+[report]
+show_missing = True
+skip_covered = True
+# TODO: increase this
+fail_under = 90
+exclude_lines =
+    # a more strict default pragma
+    \# pragma: no cover\b
+
+    # allow defensive code
+    ^\s*raise AssertionError\b
+    ^\s*raise NotImplementedError\b
+    ^\s*return NotImplemented\b
+    ^\s*raise$
+
+    # typing-related code
+    ^if (False|TYPE_CHECKING):
+    : \.\.\.$
+    ^ +\.\.\.$
+    -> ['"]?NoReturn['"]?:
+
+    # non-runnable code
+    if __name__ == ['"]__main__['"]:$
diff --git a/.gitignore b/.gitignore
index 2a40a53..a284817 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1,9 @@
 *.egg
 *.egg-info
 *.pyc
-.tox
-dist
-docs/_build
-build/
+/.coverage
+/.tox
+/build/
+/dist
 /venv*/
+docs/_build
diff --git a/testsuite/test_all.py b/testsuite/test_all.py
index 0e4bc7d..3508674 100644
--- a/testsuite/test_all.py
+++ b/testsuite/test_all.py
@@ -45,31 +45,3 @@ class PycodestyleTestCase(unittest.TestCase):
         report = self._style.check_files(files)
         self.assertEqual(list(report.messages.keys()), ['W504'],
                          msg='Failures: %s' % report.messages)
-
-
-def suite():
-    from testsuite import (
-        test_api,
-        test_blank_lines,
-        test_parser,
-        test_shell,
-        test_util,
-    )
-
-    suite = unittest.TestSuite()
-    suite.addTest(unittest.makeSuite(PycodestyleTestCase))
-    suite.addTest(unittest.makeSuite(test_api.APITestCase))
-    suite.addTest(unittest.makeSuite(test_blank_lines.TestBlankLinesDefault))
-    suite.addTest(unittest.makeSuite(test_blank_lines.TestBlankLinesTwisted))
-    suite.addTest(unittest.makeSuite(test_parser.ParserTestCase))
-    suite.addTest(unittest.makeSuite(test_shell.ShellTestCase))
-    suite.addTest(unittest.makeSuite(test_util.UtilTestCase))
-    return suite
-
-
-def _main():
-    return unittest.TextTestRunner(verbosity=2).run(suite())
-
-
-if __name__ == '__main__':
-    sys.exit(not _main())
diff --git a/testsuite/test_api.py b/testsuite/test_api.py
index c69e54d..ad96074 100644
--- a/testsuite/test_api.py
+++ b/testsuite/test_api.py
@@ -43,8 +43,7 @@ class APITestCase(unittest.TestCase):
 
     def test_register_physical_check(self):
         def check_dummy(physical_line, line_number):
-            if False:
-                yield
+            raise NotImplementedError
         pycodestyle.register_check(check_dummy, ['Z001'])
 
         self.assertTrue(check_dummy in pycodestyle._checks['physical_line'])
@@ -53,13 +52,12 @@ class APITestCase(unittest.TestCase):
         self.assertEqual(args, ['physical_line', 'line_number'])
 
         options = pycodestyle.StyleGuide().options
-        self.assertTrue(any(func == check_dummy
-                            for name, func, args in options.physical_checks))
+        functions = [func for _, func, _ in options.physical_checks]
+        self.assertIn(check_dummy, functions)
 
     def test_register_logical_check(self):
         def check_dummy(logical_line, tokens):
-            if False:
-                yield
+            raise NotImplementedError
         pycodestyle.register_check(check_dummy, ['Z401'])
 
         self.assertTrue(check_dummy in pycodestyle._checks['logical_line'])
@@ -74,8 +72,8 @@ class APITestCase(unittest.TestCase):
         self.assertEqual(args, ['logical_line', 'tokens'])
 
         options = pycodestyle.StyleGuide().options
-        self.assertTrue(any(func == check_dummy
-                            for name, func, args in options.logical_checks))
+        functions = [func for _, func, _ in options.logical_checks]
+        self.assertIn(check_dummy, functions)
 
     def test_register_ast_check(self):
         pycodestyle.register_check(DummyChecker, ['Z701'])
@@ -86,17 +84,17 @@ class APITestCase(unittest.TestCase):
         self.assertTrue(args is None)
 
         options = pycodestyle.StyleGuide().options
-        self.assertTrue(any(cls == DummyChecker
-                            for name, cls, args in options.ast_checks))
+        classes = [cls for _, cls, _ in options.ast_checks]
+        self.assertIn(DummyChecker, classes)
 
     def test_register_invalid_check(self):
         class InvalidChecker(DummyChecker):
             def __init__(self, filename):
-                pass
+                raise NotImplementedError
 
         def check_dummy(logical, tokens):
-            if False:
-                yield
+            raise NotImplementedError
+
         pycodestyle.register_check(InvalidChecker, ['Z741'])
         pycodestyle.register_check(check_dummy, ['Z441'])
 
@@ -272,28 +270,28 @@ class APITestCase(unittest.TestCase):
 
         # Do run E11 checks
         options = pycodestyle.StyleGuide().options
-        self.assertTrue(any(func == pycodestyle.indentation
-                            for name, func, args in options.logical_checks))
+        functions = [func for _, func, _ in options.logical_checks]
+        self.assertIn(pycodestyle.indentation, functions)
         options = pycodestyle.StyleGuide(select=['E']).options
-        self.assertTrue(any(func == pycodestyle.indentation
-                            for name, func, args in options.logical_checks))
+        functions = [func for _, func, _ in options.logical_checks]
+        self.assertIn(pycodestyle.indentation, functions)
         options = pycodestyle.StyleGuide(ignore=['W']).options
-        self.assertTrue(any(func == pycodestyle.indentation
-                            for name, func, args in options.logical_checks))
+        functions = [func for _, func, _ in options.logical_checks]
+        self.assertIn(pycodestyle.indentation, functions)
         options = pycodestyle.StyleGuide(ignore=['E12']).options
-        self.assertTrue(any(func == pycodestyle.indentation
-                            for name, func, args in options.logical_checks))
+        functions = [func for _, func, _ in options.logical_checks]
+        self.assertIn(pycodestyle.indentation, functions)
 
         # Do not run E11 checks
         options = pycodestyle.StyleGuide(select=['W']).options
-        self.assertFalse(any(func == pycodestyle.indentation
-                             for name, func, args in options.logical_checks))
+        functions = [func for _, func, _ in options.logical_checks]
+        self.assertNotIn(pycodestyle.indentation, functions)
         options = pycodestyle.StyleGuide(ignore=['E']).options
-        self.assertFalse(any(func == pycodestyle.indentation
-                             for name, func, args in options.logical_checks))
+        functions = [func for _, func, _ in options.logical_checks]
+        self.assertNotIn(pycodestyle.indentation, functions)
         options = pycodestyle.StyleGuide(ignore=['E11']).options
-        self.assertFalse(any(func == pycodestyle.indentation
-                             for name, func, args in options.logical_checks))
+        functions = [func for _, func, _ in options.logical_checks]
+        self.assertNotIn(pycodestyle.indentation, functions)
 
     def test_styleguide_init_report(self):
         style = pycodestyle.StyleGuide(paths=[E11])
@@ -327,9 +325,7 @@ class APITestCase(unittest.TestCase):
     def test_check_unicode(self):
         # Do not crash if lines are Unicode (Python 2.x)
         pycodestyle.register_check(DummyChecker, ['Z701'])
-        source = '#\n'
-        if hasattr(source, 'decode'):
-            source = source.decode('ascii')
+        source = u'#\n'
 
         pep8style = pycodestyle.StyleGuide()
         count_errors = pep8style.input_file('stdin', lines=[source])
@@ -345,13 +341,9 @@ class APITestCase(unittest.TestCase):
         count_errors = pep8style.input_file('stdin', lines=['\x00\n'])
 
         stdout = sys.stdout.getvalue()
-        if 'SyntaxError' in stdout:
-            # PyPy 2.2 returns a SyntaxError
-            expected = "stdin:1:2: E901 SyntaxError"
-        elif 'ValueError' in stdout:
-            # Python 3.5.
+        if 'ValueError' in stdout:  # pragma: no cover (python 3.5+)
             expected = "stdin:1:1: E901 ValueError"
-        else:
+        else:  # pragma: no cover (< python3.5)
             expected = "stdin:1:1: E901 TypeError"
         self.assertTrue(stdout.startswith(expected),
                         msg='Output %r does not start with %r' %
diff --git a/tox.ini b/tox.ini
index fc42fc1..f4356a5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,19 +5,20 @@
 
 [tox]
 envlist = py27, py34, py35, py36, py37, py38, pypy, pypy3, jython
-skipsdist = True
 skip_missing_interpreters = True
 
 [testenv]
+deps = coverage
 commands =
-	{envpython} setup.py install
-	{envpython} pycodestyle.py --max-doc-length=72 --testsuite testsuite
-	{envpython} pycodestyle.py --statistics pycodestyle.py
-	{envpython} pycodestyle.py --max-doc-length=72 --doctest
-	{envpython} setup.py test
+    python -m pycodestyle --statistics pycodestyle.py
+    coverage run -m pycodestyle --max-doc-length=72 --testsuite testsuite
+    coverage run -m pycodestyle --max-doc-length=72 --doctest
+    coverage run -m unittest discover testsuite -vv
+    coverage combine
+    coverage report
 
 [testenv:flake8]
-deps =
-    flake8
+skip_install = true
+deps = flake8
 commands =
     flake8 pycodestyle.py