Add a hybrid checker for continuation line indentation

PEP8 has some specific recommendations early on which relate to
indenting continuation lines.  Specifically, it distinguishes between
'visual indenting' and 'hanging indents'.  This change implements these
rules, adding several new errors (for now):

 * E120 - traps on a visual indent with hanging indent lines following
 * E121 - traps overly–indented lines inside visual indent regions
 * E122 - catches the evil and wrong indenting default that emacs'
          python-mode applies to 'loose fingernails' (not canonical)
 * E123 - catches continuation lines missing extra indentation when the
          previous line started a hanging indent
 * E124 - catches continuation lines not aligned on 4 spaces; note this
          required a fix to the E23.py test
 * E125 - catches continuation lines with 8 or more space indents
          relative to the previous line without good reason
 * E126 - ensures that indentation leaves blocks visually distinct

All of these errors are directly justifiable from text and examples in
the PEP8 document, and some examples are directly copied from PEP8.  The
exception to this is E122, which the PEP8 standard appears neutral on
but nonetheless I feel is intrinsically related with the rules which are
defined here; to not enforce it would permit multiple valid indent
levels for closing brackets which start lines, leading to ambiguity.
All the PEP8 examples leave the closing bracket on the previous line,
avoiding this question.  This E122 rule agrees with K&R, KNF, Linux,
this VIM auto–indent script:

  http://www.vim.org/scripts/script.php?script_id=974

and the fix to python-mode achieved by the elisp fragment on:

  http://stackoverflow.com/revisions/5361478/2

See testsuite/E12.py for many examples.

3 files changed, 274 insertions, 2 deletions

diff --git a/pep8.py b/pep8.py
index 644953b..d5a699a 100755
--- a/pep8.py
+++ b/pep8.py
@@ -415,6 +415,158 @@ def indentation(logical_line, previous_logical, indent_char,
         return 0, "E113 unexpected indentation"
 
 
+def continuation_line_indentation(logical_line, tokens, indent_level):
+    """
+    Continuation lines should align wrapped elements either vertically using
+    Python's implicit line joining inside parentheses, brackets and braces, or
+    using a hanging indent.
+
+    When using a hanging indent the following considerations should be applied;
+
+    - there should be no arguments on the first line, and
+
+    - further indentation should be used to clearly distinguish itself as a
+      continuation line.
+
+    """
+    first_line = tokens[0][2][0]
+    last_line = tokens[-1][3][0]
+    if first_line == last_line:
+        return
+
+    # indent_next tells us whether the next block is indented; assuming
+    # that it is indented by 4 spaces, then we should not allow 4-space
+    # indents on the final continuation line; in turn, some other
+    # indents are allowed to have an extra 4 spaces.
+    indent_next = logical_line.endswith(':')
+
+    depth = 0
+
+    parens = []  # for looking back to see where a bracket was opened
+    max_physical_line = 0
+    visual_min = [None]  # visual indent columns by indent depth
+    rel_indent = [[0,indent_level]]  # relative indents of physical lines
+    last_indent = None
+    if options.verbose >= 3:
+        print ">>> " + tokens[0][4],
+
+    for token in tokens:
+        token_type, text, start, end, orig_line = token
+        line = start[0] - first_line
+        while len(parens) - 1 < line:
+            parens.append([])
+
+        if line > 0:
+            if max_physical_line < line:
+                # this is the beginning of a continuation line.
+                max_physical_line = line
+                last_indent = start
+                if options.verbose >= 3:
+                    print "... " + orig_line,
+
+                # record the initial indent.  if lines were missing (eg,
+                # multi-line strings) then pad the list out
+                while len(rel_indent) - 1 < line:
+                    rel_indent.append(None)
+                rel_indent[line] = [depth, start[1] - indent_level]
+
+                if depth > 0:
+                    # a bracket expression in a continuation line.
+
+                    # first, find the line that it was opened on
+                    open_line = None
+                    for open_line in range(line - 1, -1, -1):
+                        if len(parens[open_line]) > 0:
+                            break
+                    if open_line is None:
+                        import pdb; pdb.set_trace()
+
+                    # check to see if visual indenting is active
+                    min_indent = visual_min[depth]
+                    if min_indent is not None:
+                        if start[1] < min_indent:
+                            return(start, 'E120 continuation line '
+                                   'insufficiently indentated for visual '
+                                   'indent; hanging indents should have '
+                                   'no arguments on the first line')
+
+                    # for E122
+                    eff_depth = depth
+                    if token_type == tokenize.OP and text in ']})':
+                        eff_depth -= 1
+
+                    # check that this line is either visually indented vs
+                    # the opening parens, or a hanging indent.
+                    if len(parens[open_line]) == 0:
+                        import pdb; pdb.set_trace()
+                    start_col, end_col = parens[open_line][-1]
+                    if start[1] == start_col:
+                        # visual.  fine.
+                        pass
+                    elif min_indent is not None:
+                        over_indent = start[1] - end_col
+                        if start[1] > end_col and not (
+                                over_indent == 4 and indent_next):
+                            return(start, "E121 continuation line over-"
+                                   "indented for visual indent")
+                    else:
+                        # hanging indent.
+                        x = rel_indent[line][1] - rel_indent[open_line][1]
+                        if eff_depth != depth:
+                            if x != 0:
+                                return(
+                                    start, 'E122 lines starting with a '
+                                    'closing bracket should be indented '
+                                    "to match that of the opening "
+                                    "bracket's line"
+                                )
+                        elif (x <= 0):
+                            return(start, 'E123 continuation line '
+                                   'missing indent or outdented')
+                        elif (x % 4):
+                            return(start, 'E124 continuation line not '
+                                   'indented on a 4-space boundary')
+                        elif (x > 4) and not (x == 8 and indent_next):
+                            return(start, "E125 continuation line over-"
+                                   "indented")
+
+        # keep track of bracket depth and look for visual indenting
+        if token_type == tokenize.OP and text in '([{':
+            visual_min.append(None)
+            depth += 1
+            assert(len(visual_min) == depth + 1)
+            if depth > 0:
+                visual_min[depth] = visual_min[depth - 1]
+            parens[line].append([start[1], end[1]])
+            if options.verbose >= 4:
+                print "bracket depth {d} seen, col {c}, visual min = {m}".format(
+                    d=depth,
+                    c=start[1],
+                    m=visual_min[depth],
+                )
+        elif len(parens[line]) > 0 and token_type != tokenize.NL:
+            # text after an open parens starts visual indenting
+            if visual_min[depth] is None:
+                visual_min[depth] = start[1]
+            if options.verbose >= 4:
+                print "bracket depth {d} indent to {c}".format(
+                    d=depth,
+                    c=start[1],
+                )
+
+        if token_type == tokenize.OP and text in ')]}':
+            depth -= 1
+            visual_min.pop()
+            for open_line in range(line, -1, -1):
+                if len(parens[open_line]) > 0:
+                    parens[open_line].pop()
+                    break
+
+    if indent_next and rel_indent[-1][1] == 4:
+        return(last_indent, "E126 statement with indented block ends "
+               "with continuation line indented by 4 spaces")
+
+
 def whitespace_before_parameters(logical_line, tokens):
     """
     Avoid extraneous whitespace in the following situations:
diff --git a/testsuite/E12.py b/testsuite/E12.py
new file mode 100644
index 0000000..0d98b1c
--- /dev/null
+++ b/testsuite/E12.py
@@ -0,0 +1,120 @@
+#: E120
+print "E120", ("visual",
+    "hanging")
+#: E120
+print "E120", ("under-",
+              "under-indent")
+#: E121
+print "E121", ("over-",
+                  "over-indent")
+#: E123
+print "E123", (
+"dent")
+#: E124
+print "E124", (
+  "dent")
+#: E125
+print "E125", (
+            "dent")
+#: E125
+print "E125", (
+        "dent")
+#: Okay
+if (foo == bar and
+        baz == frop):
+    pass
+#: Okay
+if (
+    foo == bar and
+    baz == frop
+):
+    pass
+#: Okay
+if start[1] > end_col and not (
+        over_indent == 4 and indent_next):
+    return(0, "E115 continuation line over-"
+           "indented for visual indent")
+#: Okay
+print "OK", ("visual",
+             "indent")
+#: E120
+# Arguments on first line forbidden when not using vertical alignment
+foo = long_function_name(var_one, var_two,
+    var_three, var_four)
+#: Okay
+# Aligned with opening delimiter
+foo = long_function_name(var_one, var_two,
+                         var_three, var_four)
+#: Okay
+# Extra indentation is not necessary.
+foo = long_function_name(
+    var_one, var_two,
+    var_three, var_four)
+#: E120
+print "Okay", ("visual",
+               "indent_two"
+              )
+#: Okay
+print "Okay", ("visual",
+               "indent_three"
+               )
+#: E120
+print "E120", ("visual",
+               "indent_five"
+)
+#: E122 W291
+print "E122", (   
+    "bad", "hanging", "close"
+    )
+#: Okay
+print "a-ok", (
+    "there",
+    "dude",
+)
+#: Okay
+print "hello", (
+    "there",
+    "dude")
+#: Okay
+print "hello", (
+    "there", "dude")
+#: Okay
+print "hello", (
+    "there", "dude",
+)
+#: E126
+# Further indentation required as indentation is not distinguishable
+
+
+def long_function_name(
+    var_one, var_two, var_three,
+    var_four):
+    print(var_one)
+#: Okay
+
+
+def long_function_name(
+        var_one, var_two, var_three,
+        var_four):
+    print(var_one)
+#: E124
+result = {
+   'key1': 'value',
+   'key2': 'value',
+}
+#: E122
+result = {
+    'foo': [
+        'bar', {
+            'baz': 'frop',
+            }
+        ]
+    }
+#: Okay
+result = {
+    'foo': [
+        'bar', {
+            'baz': 'frop',
+        }
+    ]
+}
diff --git a/testsuite/E23.py b/testsuite/E23.py
index 0f44f0c..db1696e 100644
--- a/testsuite/E23.py
+++ b/testsuite/E23.py
@@ -7,6 +7,6 @@ a = (4,)
 b = (5, )
 
 result = {
-   'key1': 'value',
-   'key2': 'value',
+    'key1': 'value',
+    'key2': 'value',
 }