Merged in myint/pylint (pull request #237)

Add "duplicate-except" checker. Patch by Steven Myint.

10 files changed, 82 insertions, 29 deletions

diff --git a/ChangeLog b/ChangeLog
index 6b68696..a760b6e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,12 +1,24 @@
 ChangeLog for Pylint
 --------------------
 
---
+2015-03-14 -- 1.4.3
 
     * Remove three warnings: star-args, abstract-class-little-used,
       abstract-class-not-used. These warnings don't add any real value
       and they don't imply errors or problems in the code.
 
+    * Added a new option for controlling the peephole optimizer in astroid.
+      The option ``--optimize-ast`` will control the peephole optimizer,
+      which is used to optimize a couple of AST subtrees. The current problem
+      solved by the peephole optimizer is when multiple joined strings,
+      with the addition operator, are encountered. If the numbers of such
+      strings is high enough, Pylint will then fail with a maximum recursion
+      depth exceeded error, due to its visitor architecture. The peephole
+      just transforms such calls, if it can, into the final resulting string
+      and this exhibit a problem, because the visit_binop method stops being
+      called (in the optimized AST it will be a Const node).           
+
+
 2015-03-11 -- 1.4.2
 
     * Don't require a docstring for empty modules. Closes issue #261.
diff --git a/DEPENDS b/DEPENDS
index 3a33b85..653dfcb 100644
--- a/DEPENDS
+++ b/DEPENDS
@@ -1,3 +1,3 @@
 python-logilab-common (>= 0.19.0)
-python-astroid (>= 1.3.5)
+python-astroid (>= 1.3.6)
 python-tk
diff --git a/doc/faq.rst b/doc/faq.rst
index a42080e..1ce9a3e 100644
--- a/doc/faq.rst
+++ b/doc/faq.rst
@@ -261,12 +261,25 @@ mangled sys.path.  Pylint doesn't import any of the candidate modules and
 thus doesn't include any of import's side effects (good and bad).  It
 traverses an AST representation of the code.
 
-6.3 I think I found a bug in Pylint. What should I do?
+6.3 Pylint keeps crashing with `Maximum recursion depth exceeded`
+-----------------------------------------------------------------
+
+Pylint can crash with this error if you have a string in your analyzed
+program, created by joining a lot of strings with the addition operator.
+Due to how Pylint works, visiting nodes on a AST tree and due to how
+the BinOp node is represented (the node which represents the string '1+1'
+for instance), the same visit method will be called over and over again, leading
+to a maximum recursion error. You can alleviate this problem by passing
+the flag `--optimize-ast=y` to Pylint. This will activate an optimization
+which will transform such AST subtrees into the final resulting string.
+This flag is off by default. If this is not the case, please report a bug!
+
+6.4 I think I found a bug in Pylint. What should I do?
 -------------------------------------------------------
 
 Read http://docs.pylint.org/contribute#bug-reports-feedback
 
-6.4 I have a question about Pylint that isn't answered here.
+6.5 I have a question about Pylint that isn't answered here.
 ------------------------------------------------------------
 
 Read http://docs.pylint.org/contribute#mailing-lists
diff --git a/pylint/__pkginfo__.py b/pylint/__pkginfo__.py
index 89c50a3..78f6bf2 100644
--- a/pylint/__pkginfo__.py
+++ b/pylint/__pkginfo__.py
@@ -19,10 +19,10 @@ from __future__ import absolute_import
 
 modname = distname = 'pylint'
 
-numversion = (1, 4, 2)
+numversion = (1, 4, 3)
 version = '.'.join([str(num) for num in numversion])
 
-install_requires = ['logilab-common >= 0.53.0', 'astroid >= 1.3.5', 'six']
+install_requires = ['logilab-common >= 0.53.0', 'astroid >= 1.3.6', 'six']
 
 license = 'GPL'
 description = "python code static checker"
diff --git a/pylint/checkers/format.py b/pylint/checkers/format.py
index 8c496ac..97ae142 100644
--- a/pylint/checkers/format.py
+++ b/pylint/checkers/format.py
@@ -81,7 +81,7 @@ MSGS = {
               'bad-indentation',
               'Used when an unexpected number of indentation\'s tabulations or '
               'spaces has been found.'),
-    'C0330': ('Wrong %s indentation%s.\n%s%s',
+    'C0330': ('Wrong %s indentation%s%s.\n%s%s',
               'bad-continuation',
               'TODO'),
     'W0312': ('Found indentation with %ss instead of %ss',
@@ -165,14 +165,22 @@ def _get_indent_length(line):
 def _get_indent_hint_line(bar_positions, bad_position):
     """Return a line with |s for each of the positions in the given lists."""
     if not bar_positions:
-        return ''
+        return ('', '')
+    delta_message = ''
     markers = [(pos, '|') for pos in bar_positions]
+    if len(markers) == 1:
+        # if we have only one marker we'll provide an extra hint on how to fix
+        expected_position = markers[0][0]
+        delta = abs(expected_position - bad_position)
+        direction = 'add' if expected_position > bad_position else 'remove'
+        delta_message = _CONTINUATION_HINT_MESSAGE % (
+            direction, delta, 's' if delta > 1 else '')
     markers.append((bad_position, '^'))
     markers.sort()
     line = [' '] * (markers[-1][0] + 1)
     for position, marker in markers:
         line[position] = marker
-    return ''.join(line)
+    return (''.join(line), delta_message)
 
 
 class _ContinuedIndent(object):
@@ -218,6 +226,7 @@ _CONTINUATION_MSG_PARTS = {
     CONTINUED_BLOCK: ('continued', ' before block'),
 }
 
+_CONTINUATION_HINT_MESSAGE = ' (%s %d space%s)'  # Ex: (remove 2 spaces)
 
 def _Offsets(*args):
     """Valid indentation offsets for a continued line."""
@@ -846,11 +855,12 @@ class FormatChecker(BaseTokenChecker):
 
     def _add_continuation_message(self, state, offsets, tokens, position):
         readable_type, readable_position = _CONTINUATION_MSG_PARTS[state.context_type]
-        hint_line = _get_indent_hint_line(offsets, tokens.start_col(position))
+        hint_line, delta_message = _get_indent_hint_line(offsets, tokens.start_col(position))
         self.add_message(
             'bad-continuation',
             line=tokens.start_line(position),
-            args=(readable_type, readable_position, tokens.line(position), hint_line))
+            args=(readable_type, readable_position, delta_message,
+                  tokens.line(position), hint_line))
 
     @check_messages('multiple-statements')
     def visit_default(self, node):
diff --git a/pylint/lint.py b/pylint/lint.py
index eb8f706..4dfb87c 100644
--- a/pylint/lint.py
+++ b/pylint/lint.py
@@ -406,6 +406,19 @@ class PyLinter(configuration.OptionsManagerMixIn,
                             ' loading into the active Python interpreter and may run'
                             ' arbitrary code')}
                   ),
+
+                ('optimize-ast',
+                  {'type': 'yn', 'metavar': '<yn>', 'default': False,
+                   'help': ('Allow optimization of some AST trees. This will '
+                            'activate a peephole AST optimizer, which will '
+                            'apply various small optimizations. For instance, '
+                            'it can be used to obtain the result of joining '
+                            'multiple strings with the addition operator. '
+                            'Joining a lot of strings can lead to a maximum '
+                            'recursion error in Pylint and this flag can prevent '
+                            'that. It has one side effect, the resulting AST '
+                            'will be different than the one from reality.')}
+                ),
                )
 
     option_groups = (
@@ -895,10 +908,14 @@ class PyLinter(configuration.OptionsManagerMixIn,
         """return a ast(roid) representation for a module"""
         try:
             return MANAGER.ast_from_file(filepath, modname, source=True)
-        except SyntaxError as ex:
-            self.add_message('syntax-error', line=ex.lineno, args=ex.msg)
         except astroid.AstroidBuildingException as ex:
-            self.add_message('parse-error', args=ex)
+            if isinstance(ex.args[0], SyntaxError):
+                ex = ex.args[0]
+                self.add_message('syntax-error',
+                                 line=ex.lineno,
+                                 args=ex.msg)
+            else:
+                self.add_message('parse-error', args=ex)
         except Exception as ex: # pylint: disable=broad-except
             import traceback
             traceback.print_exc()
@@ -940,6 +957,7 @@ class PyLinter(configuration.OptionsManagerMixIn,
         self.stats = {'by_module' : {},
                       'by_msg' : {},
                      }
+        MANAGER.optimize_ast = self.config.optimize_ast
         MANAGER.always_load_extensions = self.config.unsafe_load_any_extension
         MANAGER.extension_package_whitelist.update(
             self.config.extension_pkg_whitelist)
diff --git a/pylint/test/functional/bad_continuation.txt b/pylint/test/functional/bad_continuation.txt
index f970780..23a5733 100644
--- a/pylint/test/functional/bad_continuation.txt
+++ b/pylint/test/functional/bad_continuation.txt
@@ -1,25 +1,25 @@
 bad-continuation:12::"Wrong hanging indentation.
    ]  # [bad-continuation]
 |  ^|"
-bad-continuation:17::"Wrong continued indentation.
+bad-continuation:17::"Wrong continued indentation (remove 3 spaces).
          7,  # [bad-continuation]
       |  ^"
-bad-continuation:25::"Wrong hanging indentation.
+bad-continuation:25::"Wrong hanging indentation (add 1 space).
    'b': 2,  # [bad-continuation]
    ^|"
-bad-continuation:31::"Wrong hanging indentation.
+bad-continuation:31::"Wrong hanging indentation (add 1 space).
    'b': 2,  # [bad-continuation]
    ^|"
 bad-continuation:39::"Wrong continued indentation.
                              # [bad-continuation] is not accepted
       |                   |  ^"
-bad-continuation:40::"Wrong continued indentation.
+bad-continuation:40::"Wrong continued indentation (remove 20 spaces).
                           'contents', # [bad-continuation] nor this.
       |                   ^"
 bad-continuation:49::"Wrong hanging indentation in dict value.
       'value2',  # [bad-continuation]
     | ^ |"
-bad-continuation:59::"Wrong continued indentation.
+bad-continuation:59::"Wrong continued indentation (add 4 spaces).
         'wrong',  # [bad-continuation]
         ^   |"
 bad-continuation:83::"Wrong hanging indentation in dict value.
@@ -28,22 +28,22 @@ bad-continuation:83::"Wrong hanging indentation in dict value.
 bad-continuation:87::"Wrong hanging indentation in dict value.
     'value1',  # [bad-continuation]
     ^ |   |"
-bad-continuation:104::"Wrong hanging indentation before block.
+bad-continuation:104::"Wrong hanging indentation before block (add 4 spaces).
     some_arg,         # [bad-continuation]
     ^   |"
-bad-continuation:105::"Wrong hanging indentation before block.
+bad-continuation:105::"Wrong hanging indentation before block (add 4 spaces).
     some_other_arg):  # [bad-continuation]
     ^   |"
-bad-continuation:125::"Wrong continued indentation.
+bad-continuation:125::"Wrong continued indentation (add 3 spaces).
     ""b"")  # [bad-continuation]
     ^  |"
 bad-continuation:139::"Wrong hanging indentation before block.
    ): pass  # [bad-continuation]
 |  ^|"
-bad-continuation:142::"Wrong continued indentation before block.
+bad-continuation:142::"Wrong continued indentation before block (add 4 spaces).
     2):  # [bad-continuation]
     ^   |"
-bad-continuation:150::"Wrong continued indentation.
+bad-continuation:150::"Wrong continued indentation (remove 2 spaces).
          2 and  # [bad-continuation]
        | ^"
 bad-continuation:155::"Wrong hanging indentation before block.
@@ -58,6 +58,6 @@ bad-continuation:166::"Wrong continued indentation before block.
 bad-continuation:172::"Wrong hanging indentation before block.
   2):  # [bad-continuation]
   ^ |   |"
-bad-continuation:183::"Wrong continued indentation.
+bad-continuation:183::"Wrong continued indentation (add 4 spaces).
     2):  # [bad-continuation]
     ^   |"
diff --git a/pylint/test/messages/func_bad_cont_dictcomp_py27.txt b/pylint/test/messages/func_bad_cont_dictcomp_py27.txt
index 7443d55..51bde77 100644
--- a/pylint/test/messages/func_bad_cont_dictcomp_py27.txt
+++ b/pylint/test/messages/func_bad_cont_dictcomp_py27.txt
@@ -1,6 +1,6 @@
-C: 35: Wrong continued indentation.
+C: 35: Wrong continued indentation (add 2 spaces).
     for x in range(3)}  # [bad-continuation]
     ^ |
-C: 38: Wrong continued indentation.
+C: 38: Wrong continued indentation (remove 4 spaces).
           for x in range(3)}  # [bad-continuation]
       |   ^
diff --git a/pylint/test/messages/func_import_syntax_error.txt b/pylint/test/messages/func_import_syntax_error.txt
index 885378b..e0da926 100644
--- a/pylint/test/messages/func_import_syntax_error.txt
+++ b/pylint/test/messages/func_import_syntax_error.txt
@@ -1,4 +1,4 @@
 C:  1: Missing module docstring
 C:  1: Missing required attribute "__revision__"
-F:  2: Unable to import 'syntax_error' (expected an indented block (<string>, line 2))
+F:  2: Unable to import 'syntax_error'
 W:  2: Unused import syntax_error
diff --git a/pylintrc b/pylintrc
index bd60a2a..828f88b 100644
--- a/pylintrc
+++ b/pylintrc
@@ -28,7 +28,7 @@ include-ids=no
 symbols=no
 
 # Use multiple processes to speed up Pylint.
-jobs=2
+jobs=1
 
 # Allow loading of arbitrary C extensions. Extensions are imported into the
 # active Python interpreter and may run arbitrary code.