Added a new test suite for functional tests.

Main changes:
 - version restrictions are not encoded in file names any more, but in option files
 - expected messages are annotated in the test files directly

17 files changed, 305 insertions, 31 deletions

diff --git a/MANIFEST.in b/MANIFEST.in
index 8c28721..6810ec5 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -7,7 +7,7 @@ include examples/*.py examples/pylintrc examples/pylintrc_camelcase
 include elisp/startup elisp/*.el
 include man/*.1
 recursive-include doc *.rst *.jpeg Makefile *.html *.py
-recursive-include test *.py *.txt *.dot *.sh
+recursive-include test *.py *.txt *.dot *.sh *.args
 include test/input/similar* 
 include test/input/noext
 include test/data/ascript
diff --git a/test/messages/func_newstyle___slots___py30.txt b/test/functional/__init__.py
index e69de29..e69de29 100644
--- a/test/messages/func_newstyle___slots___py30.txt
+++ b/test/functional/__init__.py
diff --git a/test/functional/bad_inline_option.args b/test/functional/bad_inline_option.args
new file mode 100644
index 0000000..b211d72
--- /dev/null
+++ b/test/functional/bad_inline_option.args
@@ -0,0 +1,2 @@
+[Messages Control]
+enable=I
diff --git a/test/input/func_i0010.py b/test/functional/bad_inline_option.py
index d56762f..be6e240 100644
--- a/test/input/func_i0010.py
+++ b/test/functional/bad_inline_option.py
@@ -1,5 +1,5 @@
-# pylint: errors-only
 """errors-only is not usable as an inline option"""
-__revision__ = None
+# +1: [bad-inline-option]
+# pylint: errors-only
 
 CONST = "This is not a pylint: inline option."
diff --git a/test/functional/bad_inline_option.txt b/test/functional/bad_inline_option.txt
new file mode 100644
index 0000000..91ac1af
--- /dev/null
+++ b/test/functional/bad_inline_option.txt
@@ -0,0 +1 @@
+bad-inline-option:3::Unable to consider inline option 'errors-only'
diff --git a/test/input/func_noerror___future___import.py b/test/functional/future_import.py
index 5c77516..e4d3785 100644
--- a/test/input/func_noerror___future___import.py
+++ b/test/functional/future_import.py
@@ -1,5 +1,3 @@
 """a docstring"""
 
 from __future__ import generators
-
-__revision__ = 1
diff --git a/test/input/func_newstyle___slots__.py b/test/functional/newstyle__slots__.py
index f01c40e..306d6a1 100644
--- a/test/input/func_newstyle___slots__.py
+++ b/test/functional/newstyle__slots__.py
@@ -1,17 +1,17 @@
 # pylint: disable=R0903
 """test __slots__ on old style class"""
 
-__revision__ = 1
 
-class OkOk(object):
+class NewStyleClass(object):
     """correct usage"""
     __slots__ = ('a', 'b')
 
-class HaNonNonNon:
+
+class OldStyleClass:  # <3.0:[old-style-class,slots-on-old-class]
     """bad usage"""
     __slots__ = ('a', 'b')
 
     def __init__(self):
         pass
 
-__slots__ = 'hop' # pfff
+__slots__ = 'hop'
diff --git a/test/functional/newstyle__slots__.txt b/test/functional/newstyle__slots__.txt
new file mode 100644
index 0000000..afc9117
--- /dev/null
+++ b/test/functional/newstyle__slots__.txt
@@ -0,0 +1,2 @@
+old-style-class:10:OldStyleClass:Old-style class defined.
+slots-on-old-class:10:OldStyleClass:Use of __slots__ on an old style class
diff --git a/test/functional/unpacked_exceptions.args b/test/functional/unpacked_exceptions.args
new file mode 100644
index 0000000..a650233
--- /dev/null
+++ b/test/functional/unpacked_exceptions.args
@@ -0,0 +1,2 @@
+[testoptions]
+max_pyver=3.0
diff --git a/test/functional/unpacked_exceptions.py b/test/functional/unpacked_exceptions.py
new file mode 100644
index 0000000..f3c633f
--- /dev/null
+++ b/test/functional/unpacked_exceptions.py
@@ -0,0 +1,11 @@
+"""Test for redefine-in-handler, overwriting names in exception handlers."""
+
+def new_style():
+    """Some exceptions can be unpacked."""
+    try:
+        pass
+    except IOError, (errno, message):  # [unpacking-in-except]
+        print errno, message
+    # +1: [redefine-in-handler,redefine-in-handler,unpacking-in-except]
+    except IOError, (new_style, tuple):
+        print new_style, tuple
diff --git a/test/functional/unpacked_exceptions.txt b/test/functional/unpacked_exceptions.txt
new file mode 100644
index 0000000..ec51f20
--- /dev/null
+++ b/test/functional/unpacked_exceptions.txt
@@ -0,0 +1,4 @@
+unpacking-in-except:7:new_style:Implicit unpacking of exceptions is not supported in Python 3
+unpacking-in-except:10:new_style:Implicit unpacking of exceptions is not supported in Python 3
+redefine-in-handler:10:new_style:Redefining name 'new_style' from outer scope (line 3) in exception handler
+redefine-in-handler:10:new_style:Redefining name 'tuple' from builtins in exception handler
diff --git a/test/input/func_unpack_exception_py_30.py b/test/input/func_unpack_exception_py_30.py
deleted file mode 100644
index 356915e..0000000
--- a/test/input/func_unpack_exception_py_30.py
+++ /dev/null
@@ -1,13 +0,0 @@
-"""Test for W0623, overwriting names in exception handlers."""
-
-__revision__ = ''
-
-def new_style():
-    """Some exceptions can be unpacked."""
-    try:
-        pass
-    except IOError, (errno, message): # this is fine
-        print errno, message
-    except IOError, (new_style, tuple): # W0623 twice
-        print new_style, tuple
-
diff --git a/test/messages/func_i0010.txt b/test/messages/func_i0010.txt
deleted file mode 100644
index 2bc4372..0000000
--- a/test/messages/func_i0010.txt
+++ /dev/null
@@ -1 +0,0 @@
-I:  1: Unable to consider inline option 'errors-only'
diff --git a/test/messages/func_newstyle___slots__.txt b/test/messages/func_newstyle___slots__.txt
deleted file mode 100644
index cb69e14..0000000
--- a/test/messages/func_newstyle___slots__.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-C: 10:HaNonNonNon: Old-style class defined.
-E: 10:HaNonNonNon: Use of __slots__ on an old style class
diff --git a/test/messages/func_unpack_exception_py_30.txt b/test/messages/func_unpack_exception_py_30.txt
deleted file mode 100644
index aaff92f..0000000
--- a/test/messages/func_unpack_exception_py_30.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-W:  9:new_style: Implicit unpacking of exceptions is not supported in Python 3
-W: 11:new_style: Implicit unpacking of exceptions is not supported in Python 3
-W: 11:new_style: Redefining name 'new_style' from outer scope (line 5) in exception handler
-W: 11:new_style: Redefining name 'tuple' from builtins in exception handler
-
diff --git a/test/test_func.py b/test/test_func.py
index 63c585f..d3fb9be 100644
--- a/test/test_func.py
+++ b/test/test_func.py
@@ -44,6 +44,8 @@ class LintTestNonExistentModuleTC(LintTestUsingModule):
 
 class TestTests(testlib.TestCase):
     """check that all testable messages have been checked"""
+    PORTED = set(['I0001', 'I0010', 'W0712', 'E1001'])
+    
     @testlib.tag('coverage')
     def test_exhaustivity(self):
         # skip fatal messages
@@ -54,7 +56,7 @@ class TestTests(testlib.TestCase):
                 not_tested.remove(msgid)
             except KeyError:
                 continue
-        not_tested -= set(('I0001',))
+        not_tested -= self.PORTED
         if PY3K:
             not_tested.remove('W0403') # relative-import
         self.assertFalse(not_tested)
diff --git a/test/test_functional.py b/test/test_functional.py
new file mode 100644
index 0000000..ae12360
--- /dev/null
+++ b/test/test_functional.py
@@ -0,0 +1,273 @@
+"""Functional full-module tests for PyLint."""
+import ConfigParser
+import cStringIO
+import operator
+import os
+import re
+import sys
+
+from logilab.common import testlib
+
+from pylint import lint
+from pylint import reporters
+from pylint import checkers
+
+class NoFileError(Exception):
+    pass
+
+
+# If message files should be updated instead of checked.
+UPDATE = False
+
+# Common sub-expressions.
+_MESSAGE = {'msg': r'[a-z][a-z\-]+'}
+# Matches a #, 
+#  - followed by a comparison operator and a Python version (optional),
+#  - followed by an line number with a +/- (optional),
+#  - followed by a list of bracketed message symbols.
+# Used to extract expected messages from testdata files.
+_EXPECTED_RE = re.compile(
+    r'\s*#\s*(?:(?P<line>[+-]?[0-9]+):)?'
+    r'(?:(?P<op>[><=]+) *(?P<version>[0-9.]+):)?'
+    r'\s*\[(?P<msgs>%(msg)s(?:,\s*%(msg)s)*)\]' % _MESSAGE)
+
+
+def parse_python_version(str):
+    return tuple(int(digit) for digit in str.split('.'))
+
+
+class TestReporter(reporters.BaseReporter):
+    def add_message(self, msg_id, location, msg):
+        self.messages.append(reporters.Message(self, msg_id, location, msg))
+    
+    def on_set_current_module(self, module, filepath):
+        self.messages = []
+
+    def display_results(self, layout):
+        """Ignore layouts."""
+
+
+class TestFile(object):
+    """A single functional test case file with options."""
+
+    _CONVERTERS = {
+        'min_pyver': parse_python_version,
+        'max_pyver': parse_python_version,
+    }
+
+
+    def __init__(self, directory, filename):
+        self._directory = directory
+        self.base = filename.replace('.py', '')
+        self.options = {
+            'min_pyver': (2, 5),
+            'max_pyver': (4, 0),
+            }
+        self._parse_options()
+
+    def _parse_options(self):
+        cp = ConfigParser.ConfigParser()
+        cp.add_section('testoptions')
+        try:
+            cp.read(self.option_file)
+        except NoFileError:
+            pass
+
+        for name, value in cp.items('testoptions'):
+            conv = self._CONVERTERS.get(name, lambda v: v)
+            self.options[name] = conv(value)
+
+    @property
+    def option_file(self):
+        return self._file_type('.args')
+
+    @property
+    def module(self):
+        package = os.path.basename(self._directory)
+        return '.'.join([package, self.base])
+
+    @property
+    def expected_output(self):
+        return self._file_type('.txt')
+
+    @property
+    def source(self):
+        return self._file_type('.py')
+
+    def _file_type(self, ext):
+        name = os.path.join(self._directory, self.base + ext)
+        if os.path.exists(name):
+            return name
+        else:
+            raise NoFileError
+
+
+_OPERATORS = {
+    '>': operator.gt,
+    '<': operator.lt,
+    '>=': operator.ge,
+    '<=': operator.le,
+}
+
+def get_expected_messages(stream):
+    """Parses a file and get expected messages.
+
+    :param stream: File-like input stream.
+    :returns: A dict mapping line,msg-symbol tuples to the count on this line.
+    """
+    messages = {}
+    for i, line in enumerate(stream):
+        match = _EXPECTED_RE.search(line)
+        if match is None:
+            continue
+        line = match.group('line')
+        if line is None:
+            line = i + 1
+        elif line.startswith('+') or line.startswith('-'):
+            line = i + 1 + int(line)
+        else:
+            line = int(line)
+
+        version = match.group('version')
+        op = match.group('op')
+        if version:
+            required = parse_python_version(version)
+            if not _OPERATORS[op](sys.version_info, required):
+                continue
+
+        for msg_id in match.group('msgs').split(','):
+            messages.setdefault((line, msg_id.strip()), 0)
+            messages[line, msg_id.strip()] += 1
+    return messages
+
+
+def multiset_difference(left_op, right_op):
+    """Takes two multisets and compares them.
+
+    A multiset is a dict with the cardinality of the key as the value.
+
+    :param left_op: The expected entries.
+    :param right_op: Actual entries.
+
+    :returns: The two multisets of missing and unexpected messages.
+    """
+    missing = left_op.copy()
+    unexpected = {}
+    for key, value in right_op.iteritems():
+        missing.setdefault(key, 0)
+        missing[key] -= value
+        if missing[key] == 0:
+            del missing[key]
+        elif missing[key] < 0:
+            unexpected.setdefault(key, 0)
+            unexpected[key] = -missing.pop(key)
+    return missing, unexpected
+
+
+class LintModuleTest(testlib.TestCase):
+    def __init__(self, test_file):
+        super(LintModuleTest, self).__init__()
+        test_reporter = TestReporter()
+        self._linter = lint.PyLinter()
+        self._linter.set_reporter(test_reporter)
+        self._linter.config.persistent = 0
+        self._linter.disable('I')
+        try:
+            self._linter.load_file_configuration(test_file.option_file)
+        except NoFileError:
+            pass
+        checkers.initialize(self._linter)
+        self._test_file = test_file
+
+    def shortDescription(self):
+        return self._test_file.base
+
+    def _produces_output(self):
+        return True
+
+    def _get_expected(self):
+        with open(self._test_file.source) as fobj:
+            expected = get_expected_messages(fobj)
+        
+        lines = []
+        if self._produces_output() and expected:
+            with open(self._test_file.expected_output, 'U') as fobj:
+                for line in fobj:
+                    parts = line.split(':')
+                    linenum = int(parts[1])
+                    if (linenum, parts[0]) in expected:
+                        lines.append(line)
+        return expected, ''.join(lines)
+
+    def _get_received(self):
+        messages = self._linter.reporter.messages       
+        messages.sort(key=lambda m: (m.line, m.C, m.msg))
+        text_result = cStringIO.StringIO()
+        received = {}
+        for msg in messages:
+            received.setdefault((msg.line, msg.symbol), 0)
+            received[msg.line, msg.symbol] += 1
+            text_result.write(msg.format('{symbol}:{line}:{obj}:{msg}'))
+            text_result.write('\n')
+        return received, text_result.getvalue()
+
+    def runTest(self):
+        self._linter.check([self._test_file.module])
+
+        expected_messages, expected_text = self._get_expected()
+        received_messages, received_text = self._get_received()
+
+        if expected_messages != received_messages:
+            msg = ['Wrong results for file "%s":' % (self._test_file.base)]
+            missing, unexpected = multiset_difference(expected_messages, 
+                                                      received_messages)
+            if missing:
+                msg.append('\nExpected in testdata:')
+                msg.extend(' %3d: %s' % msg for msg in sorted(missing))
+            if unexpected:
+                msg.append('\nUnexpected in testdata:')
+                msg.extend(' %3d: %s' % msg for msg in sorted(unexpected))
+            self.fail('\n'.join(msg))
+        self._check_output_text(expected_messages, expected_text, received_text)
+
+    def _check_output_text(self, expected_messages, expected_text, received_text):
+        self.assertMultiLineEqual(expected_text, received_text)
+
+
+class LintModuleOutputUpdate(LintModuleTest):
+    def _produces_output(self):
+        return False
+
+    def _check_output_text(self, expected_messages, expected_text, received_text):
+        if expected_messages:
+            with open(self._test_file.expected_output, 'w') as fobj:
+                fobj.write(received_text)
+
+
+def active_in_running_python_version(options):
+    return options['min_pyver'] < sys.version_info <= options['max_pyver']
+
+
+def suite():
+    input_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 
+                             'functional')
+    suite = testlib.TestSuite()
+    for fname in os.listdir(input_dir):
+        if fname != '__init__.py' and fname.endswith('.py'):
+            test_file = TestFile(input_dir, fname)
+            if active_in_running_python_version(test_file.options):
+                if UPDATE:
+                    suite.addTest(LintModuleOutputUpdate(test_file))
+                else:
+                    suite.addTest(LintModuleTest(test_file))
+    return suite
+
+
+# TODO(tmarek): Port exhaustivity test from test_func once all tests have been added.
+
+
+if __name__=='__main__':
+    if '-u' in sys.argv:
+        UPDATE = True
+        sys.argv.remove('-u')
+    testlib.unittest_main(defaultTest='suite')