Fix CFamilyLexer preprocessor tokenization errors (#1830)

CFamilyLexer failed to tokenize preprocessor macros when they were
preceded by line break surrounded by spaces. This was the case because
prerpocessor regex rule expected to start at the beginning of the line,
but the space regex rule matched also the whitespace after the line
break. Now the space rule has been refined not to match the line break.
Because of this, the preprocessor regex rule correctly matches
prerpocessor tokens even when they are preceded by white spaces, at the
cost of adding some more tokens in the token stream in some cases. This
change preserves the behavior of invalid preprocessor usage failing to
tokenize.

18 files changed, 912 insertions, 225 deletions

diff --git a/tests/examplefiles/arduino/Blink.ino.output b/tests/examplefiles/arduino/Blink.ino.output
index 417ac143..862f75a0 100644
--- a/tests/examplefiles/arduino/Blink.ino.output
+++ b/tests/examplefiles/arduino/Blink.ino.output
@@ -1,7 +1,8 @@
 '/*\n  Blink\n  Turns on an LED on for one second, then off for one second, repeatedly.\n \n  This example code is in the public domain.\n */' Comment.Multiline
 '\n'          Text
 
-' \n'         Text
+' '           Text
+'\n'          Text
 
 '// Pin 13 has an LED connected on most Arduino boards.\n' Comment.Single
 
@@ -42,7 +43,8 @@
 'OUTPUT'      Keyword.Reserved
 ')'           Punctuation
 ';'           Punctuation
-'     \n'     Text
+'     '       Text
+'\n'          Text
 
 '}'           Punctuation
 '\n'          Text
diff --git a/tests/examplefiles/c/example.c.output b/tests/examplefiles/c/example.c.output
index de43c8ba..3471ad06 100644
--- a/tests/examplefiles/c/example.c.output
+++ b/tests/examplefiles/c/example.c.output
@@ -313,7 +313,10 @@
 '"'           Literal.String
 ')'           Punctuation
 ';'           Punctuation
-' \n\t'       Text
+' '           Text
+'\n'          Text
+
+'\t'          Text
 'sprintf'     Name
 '('           Punctuation
 'tempString'  Name
@@ -378,7 +381,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'\t\n\t'      Text
+'\t'          Text
+'\n'          Text
+
+'\t'          Text
 'stringBufferAppend' Name
 '('           Punctuation
 'finishedMethodsBuffer' Name
@@ -1155,7 +1161,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'\t\n\t'      Text
+'\t'          Text
+'\n'          Text
+
+'\t'          Text
 'if'          Keyword
 ' '           Text
 '('           Punctuation
@@ -1757,7 +1766,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'\t\n\t'      Text
+'\t'          Text
+'\n'          Text
+
+'\t'          Text
 'stringBufferConcatenate' Name
 '('           Punctuation
 'mainBuffer'  Name
@@ -2981,7 +2993,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'    \n    '  Text
+'    '        Text
+'\n'          Text
+
+'    '        Text
 'ary'         Name
 ' '           Text
 '='           Operator
@@ -7146,7 +7161,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'    \n    '  Text
+'    '        Text
+'\n'          Text
+
+'    '        Text
 'return'      Keyword
 ' '           Text
 'ary'         Name
@@ -11983,7 +12001,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'    \n    '  Text
+'    '        Text
+'\n'          Text
+
+'    '        Text
 'return'      Keyword
 ' '           Text
 'rb_ary_join' Name
@@ -12055,7 +12076,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'    \n    '  Text
+'    '        Text
+'\n'          Text
+
+'    '        Text
 'return'      Keyword
 ' '           Text
 'rb_ary_join' Name
diff --git a/tests/examplefiles/charmci/Charmci.ci.output b/tests/examplefiles/charmci/Charmci.ci.output
index 586cb86b..e5674e18 100644
--- a/tests/examplefiles/charmci/Charmci.ci.output
+++ b/tests/examplefiles/charmci/Charmci.ci.output
@@ -58,7 +58,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'\t\n\t'      Text
+'\t'          Text
+'\n'          Text
+
+'\t'          Text
 'mainchare'   Keyword
 ' '           Text
 'ckcallback_main' Name
diff --git a/tests/examplefiles/cpp/example.cpp.output b/tests/examplefiles/cpp/example.cpp.output
index aea47bec..0e88b682 100644
--- a/tests/examplefiles/cpp/example.cpp.output
+++ b/tests/examplefiles/cpp/example.cpp.output
@@ -11295,7 +11295,10 @@
 '              ' Text
 '// BEGIN Content of ASBeautifier.cpp.BITFIELD.patch:\n' Comment.Single
 
-'              \n                ' Text
+'              ' Text
+'\n'          Text
+
+'                ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'int'         Keyword.Type
@@ -11477,7 +11480,10 @@
 
 '\t\t\t\t\t'  Text
 '}'           Punctuation
-'\t\t\t\t\t\t\t\t\t\n\t\t\t\t' Text
+'\t\t\t\t\t\t\t\t\t' Text
+'\n'          Text
+
+'\t\t\t\t'    Text
 '}'           Punctuation
 '\n'          Text
 
@@ -11528,7 +11534,10 @@
 '              ' Text
 '// END Content of ASBeautifier.cpp.BITFIELD.patch:\n' Comment.Single
 
-'                \n                ' Text
+'                ' Text
+'\n'          Text
+
+'                ' Text
 '-'           Operator
 '-'           Operator
 'tabCount'    Name
@@ -11722,7 +11731,10 @@
 
 '                  ' Text
 '}'           Punctuation
-' \n              ' Text
+' '           Text
+'\n'          Text
+
+'              ' Text
 '// BEGIN content of ASBeautifier.cpp.BITFIELD.patch.bz2\n' Comment.Single
 
 '              ' Text
@@ -11925,7 +11937,10 @@
 
 '\t\t\t\t\t'  Text
 '}'           Punctuation
-'\t\t\t\t\t\t\t\t\t\n\t\t\t\t' Text
+'\t\t\t\t\t\t\t\t\t' Text
+'\n'          Text
+
+'\t\t\t\t'    Text
 '}'           Punctuation
 '\n'          Text
 
diff --git a/tests/examplefiles/cpp/namespace.cpp.output b/tests/examplefiles/cpp/namespace.cpp.output
index d929cb03..fbe050d5 100644
--- a/tests/examplefiles/cpp/namespace.cpp.output
+++ b/tests/examplefiles/cpp/namespace.cpp.output
@@ -164,7 +164,8 @@
 
 '}'           Punctuation
 ';'           Punctuation
-'  \n'        Text
+'  '          Text
+'\n'          Text
 
 '}'           Punctuation
 '\n'          Text
diff --git a/tests/examplefiles/cuda/test.cu.output b/tests/examplefiles/cuda/test.cu.output
index 522c1a2c..2e125b69 100644
--- a/tests/examplefiles/cuda/test.cu.output
+++ b/tests/examplefiles/cuda/test.cu.output
@@ -146,7 +146,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'    \n    '  Text
+'    '        Text
+'\n'          Text
+
+'    '        Text
 'if'          Keyword
 ' '           Text
 '('           Punctuation
diff --git a/tests/examplefiles/ec/test.ec.output b/tests/examplefiles/ec/test.ec.output
index 7091bbf7..be84cddb 100644
--- a/tests/examplefiles/ec/test.ec.output
+++ b/tests/examplefiles/ec/test.ec.output
@@ -52,7 +52,10 @@
 ' '           Text
 'percent'     Name
 ';'           Punctuation
-'      \n   ' Text
+'      '      Text
+'\n'          Text
+
+'   '         Text
 '}'           Punctuation
 ';'           Punctuation
 '\n'          Text
@@ -338,7 +341,10 @@
 '0'           Literal.String.Char
 "'"           Literal.String.Char
 ')'           Punctuation
-' \n               ' Text
+' '           Text
+'\n'          Text
+
+'               ' Text
 'last'        Name
 ' '           Text
 '='           Operator
@@ -553,7 +559,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'         \n         ' Text
+'         '   Text
+'\n'          Text
+
+'         '   Text
 'if'          Keyword
 '('           Punctuation
 'end'         Name
@@ -776,7 +785,10 @@
 ' '           Text
 'percent'     Name
 ';'           Punctuation
-'      \n   ' Text
+'      '      Text
+'\n'          Text
+
+'   '         Text
 '}'           Punctuation
 ';'           Punctuation
 '\n'          Text
@@ -1024,7 +1036,10 @@
 '0'           Literal.String.Char
 "'"           Literal.String.Char
 ')'           Punctuation
-' \n               ' Text
+' '           Text
+'\n'          Text
+
+'               ' Text
 'last'        Name
 ' '           Text
 '='           Operator
@@ -1429,7 +1444,12 @@
 ' '           Text
 'bottom'      Name
 ';'           Punctuation
-'      \n\n   ' Text
+'      '      Text
+'\n'          Text
+
+'\n'          Text
+
+'   '         Text
 'char'        Keyword.Type
 ' '           Text
 '*'           Operator
@@ -1578,7 +1598,10 @@
 '}'           Punctuation
 '\n'          Text
 
-'      \n      ' Text
+'      '      Text
+'\n'          Text
+
+'      '      Text
 '//if(((!left.type && !right.type) && horz.distance) || horz.type == middleRelative)\n' Comment.Single
 
 '      '      Text
@@ -1718,7 +1741,10 @@
 '}'           Punctuation
 '\n'          Text
 
-'      \n      ' Text
+'      '      Text
+'\n'          Text
+
+'      '      Text
 'tempString'  Name
 '['           Punctuation
 '0'           Literal.Number.Integer
@@ -1806,7 +1832,10 @@
 '}'           Punctuation
 '\n'          Text
 
-'      \n      ' Text
+'      '      Text
+'\n'          Text
+
+'      '      Text
 'tempString'  Name
 '['           Punctuation
 '0'           Literal.Number.Integer
@@ -2036,7 +2065,10 @@
 '}'           Punctuation
 '\n'          Text
 
-'      \n      ' Text
+'      '      Text
+'\n'          Text
+
+'      '      Text
 'tempString'  Name
 '['           Punctuation
 '0'           Literal.Number.Integer
@@ -2124,7 +2156,10 @@
 '}'           Punctuation
 '\n'          Text
 
-'      \n      ' Text
+'      '      Text
+'\n'          Text
+
+'      '      Text
 'return'      Keyword
 ' '           Text
 'stringOutput' Name
@@ -2392,7 +2427,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'      \n      ' Text
+'      '      Text
+'\n'          Text
+
+'      '      Text
 'comboBox'    Name
 '.'           Punctuation
 'Create'      Name
@@ -4170,7 +4208,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'      \n      ' Text
+'      '      Text
+'\n'          Text
+
+'      '      Text
 'if'          Keyword
 '('           Punctuation
 'checked'     Name
@@ -5642,7 +5683,10 @@
 '.'           Punctuation
 'type'        Name
 ')'           Punctuation
-' \n         ' Text
+' '           Text
+'\n'          Text
+
+'         '   Text
 '{'           Punctuation
 '\n'          Text
 
@@ -6184,9 +6228,15 @@
 '='           Operator
 ' '           Text
 'AnchorButton' Name
-' \n         ' Text
+' '           Text
+'\n'          Text
+
+'         '   Text
 '{'           Punctuation
-' \n            ' Text
+' '           Text
+'\n'          Text
+
+'            ' Text
 'anchorEditor' Name
 ','           Punctuation
 ' '           Text
@@ -6367,7 +6417,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'               \n               ' Text
+'               ' Text
+'\n'          Text
+
+'               ' Text
 'button'      Name
 '.'           Punctuation
 'anchor'      Name
@@ -6562,7 +6615,10 @@
 ' '           Text
 '2'           Literal.Number.Integer
 ':'           Operator
-' \n               ' Text
+' '           Text
+'\n'          Text
+
+'               ' Text
 'if'          Keyword
 '('           Punctuation
 'anchorValue' Name
@@ -6635,7 +6691,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'               \n               ' Text
+'               ' Text
+'\n'          Text
+
+'               ' Text
 'button'      Name
 '.'           Punctuation
 'anchor'      Name
@@ -6694,7 +6753,10 @@
 ' '           Text
 '3'           Literal.Number.Integer
 ':'           Operator
-' \n               ' Text
+' '           Text
+'\n'          Text
+
+'               ' Text
 'if'          Keyword
 '('           Punctuation
 'anchorValue' Name
@@ -6851,7 +6913,10 @@
 '}'           Punctuation
 '\n'          Text
 
-'      \n   ' Text
+'      '      Text
+'\n'          Text
+
+'   '         Text
 'void'        Keyword.Type
 ' '           Text
 'OnCloseDropDown' Name.Function
diff --git a/tests/examplefiles/ec/test.eh.output b/tests/examplefiles/ec/test.eh.output
index 9f018f41..4b1b09b6 100644
--- a/tests/examplefiles/ec/test.eh.output
+++ b/tests/examplefiles/ec/test.eh.output
@@ -1677,7 +1677,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'   \n   '    Text
+'   '         Text
+'\n'          Text
+
+'   '         Text
 'DBTableEntry' Name
 ' '           Text
 'dbtableEntry' Name
diff --git a/tests/examplefiles/mql/example.mq4.output b/tests/examplefiles/mql/example.mq4.output
index 46291a0e..5fa57e7c 100644
--- a/tests/examplefiles/mql/example.mq4.output
+++ b/tests/examplefiles/mql/example.mq4.output
@@ -940,7 +940,10 @@
 
 '     '       Text
 '}'           Punctuation
-' \n   '      Text
+' '           Text
+'\n'          Text
+
+'   '         Text
 'FileFlush'   Name.Function
 '('           Punctuation
 'ExtHandle'   Name
@@ -1086,7 +1089,10 @@
 '0'           Literal.Number.Integer
 ']'           Punctuation
 ';'           Punctuation
-' \n            ' Text
+' '           Text
+'\n'          Text
+
+'            ' Text
 'if'          Keyword
 '('           Punctuation
 'rate'        Name
@@ -1496,9 +1502,13 @@
 '50'          Literal.Number.Integer
 ')'           Punctuation
 ';'           Punctuation
-' \n     '    Text
+' '           Text
+'\n'          Text
+
+'     '       Text
 '}'           Punctuation
-'      \n'    Text
+'      '      Text
+'\n'          Text
 
 '//---\n'     Comment.Single
 
diff --git a/tests/examplefiles/nesc/IPDispatchC.nc.output b/tests/examplefiles/nesc/IPDispatchC.nc.output
index 251d0b99..9ebcd6ae 100644
--- a/tests/examplefiles/nesc/IPDispatchC.nc.output
+++ b/tests/examplefiles/nesc/IPDispatchC.nc.output
@@ -68,7 +68,10 @@
 '{'           Punctuation
 '\n'          Text
 
-'  \n  '      Text
+'  '          Text
+'\n'          Text
+
+'  '          Text
 'components'  Keyword
 ' '           Text
 'MainC'       Name
@@ -415,7 +418,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'  \n  '      Text
+'  '          Text
+'\n'          Text
+
+'  '          Text
 'IPDispatchP' Name
 '.'           Punctuation
 'FragPool'    Name
diff --git a/tests/examplefiles/nesc/IPDispatchP.nc.output b/tests/examplefiles/nesc/IPDispatchP.nc.output
index 8f93a7cb..9e08dc03 100644
--- a/tests/examplefiles/nesc/IPDispatchP.nc.output
+++ b/tests/examplefiles/nesc/IPDispatchP.nc.output
@@ -1297,7 +1297,10 @@
 
 '  '          Text
 '/*\n   *  Receive-side code.\n   */' Comment.Multiline
-' \n  '       Text
+' '           Text
+'\n'          Text
+
+'  '          Text
 'void'        Keyword.Type
 ' '           Text
 'deliver'     Name
@@ -1459,7 +1462,10 @@
 
 '  '          Text
 "/*\n   * Bulletproof recovery logic is very important to make sure we\n   * don't get wedged with no free buffers.\n   * \n   * The table is managed as follows:\n   *  - unused entries are marked T_UNUSED\n   *  - entries which \n   *     o have a buffer allocated\n   *     o have had a fragment reception before we fired\n   *     are marked T_ACTIVE\n   *  - entries which have not had a fragment reception during the last timer period\n   *     and were active are marked T_ZOMBIE\n   *  - zombie receptions are deleted: their buffer is freed and table entry marked unused.\n   *  - when a fragment is dropped, it is entered into the table as T_FAILED1.\n   *     no buffer is allocated\n   *  - when the timer fires, T_FAILED1 entries are aged to T_FAILED2.\n   * - T_FAILED2 entries are deleted.  Incomming fragments with tags\n   *     that are marked either FAILED1 or FAILED2 are dropped; this\n   *     prevents us from allocating a buffer for a packet which we\n   *     have already dropped fragments from.\n   *\n   */" Comment.Multiline
-' \n  '       Text
+' '           Text
+'\n'          Text
+
+'  '          Text
 'void'        Keyword.Type
 ' '           Text
 'reconstruct_age' Name
@@ -1508,7 +1514,10 @@
 ' '           Text
 'T_UNUSED'    Name
 ')'           Punctuation
-' \n      '   Text
+' '           Text
+'\n'          Text
+
+'      '      Text
 'printf'      Name
 '('           Punctuation
 '"'           Literal.String
@@ -1516,7 +1525,10 @@
 '\\n'         Literal.String.Escape
 '"'           Literal.String
 ','           Punctuation
-' \n                 ' Text
+' '           Text
+'\n'          Text
+
+'                 ' Text
 'recon'       Name
 '-'           Operator
 '>'           Operator
@@ -1534,7 +1546,10 @@
 '>'           Operator
 'r_buf'       Name
 ','           Punctuation
-' \n                 ' Text
+' '           Text
+'\n'          Text
+
+'                 ' Text
 'recon'       Name
 '-'           Operator
 '>'           Operator
@@ -1802,7 +1817,10 @@
 '\\n'         Literal.String.Escape
 '"'           Literal.String
 ','           Punctuation
-' \n                  ' Text
+' '           Text
+'\n'          Text
+
+'                  ' Text
 'cur'         Name
 ','           Punctuation
 ' '           Text
@@ -1911,7 +1929,10 @@
 
 '\n'          Text
 
-'    \n    '  Text
+'    '        Text
+'\n'          Text
+
+'    '        Text
 'printf'      Name
 '('           Punctuation
 '"'           Literal.String
@@ -2169,7 +2190,10 @@
 ')'           Punctuation
 ' '           Text
 '{'           Punctuation
-'          \n          ' Text
+'          '  Text
+'\n'          Text
+
+'          '  Text
 'recon'       Name
 '-'           Operator
 '>'           Operator
@@ -2264,7 +2288,10 @@
 ' '           Text
 'T_UNUSED'    Name
 ')'           Punctuation
-' \n        ' Text
+' '           Text
+'\n'          Text
+
+'        '    Text
 'ret'         Name
 ' '           Text
 '='           Operator
@@ -2774,7 +2801,10 @@
 '}'           Punctuation
 '\n'          Text
 
-'        \n      ' Text
+'        '    Text
+'\n'          Text
+
+'      '      Text
 'if'          Keyword
 ' '           Text
 '('           Punctuation
@@ -2945,7 +2975,10 @@
 '.'           Punctuation
 'ieee_src'    Name
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'sizeof'      Keyword
 '('           Punctuation
 'ieee154_addr_t' Name
@@ -3531,7 +3564,12 @@
 '}'           Punctuation
 '\n'          Text
 
-'  \n\n  '    Text
+'  '          Text
+'\n'          Text
+
+'\n'          Text
+
+'  '          Text
 '/*\n   *  it will pack the message into the fragment pool and enqueue\n   *  those fragments for sending\n   *\n   * it will set\n   *  - payload length\n   *  - version, traffic class and flow label\n   *\n   * the source and destination IP addresses must be set by higher\n   * layers.\n   */' Comment.Multiline
 '\n'          Text
 
@@ -4364,7 +4402,10 @@
 '}'           Punctuation
 '\n'          Text
 
-'       \n    ' Text
+'       '     Text
+'\n'          Text
+
+'    '        Text
 '// printf("got %i frags\\n", s_info->link_fragments);\n' Comment.Single
 
 '  '          Text
@@ -4524,7 +4565,10 @@
 '}'           Punctuation
 '\n'          Text
 
-'    \n    '  Text
+'    '        Text
+'\n'          Text
+
+'    '        Text
 's_entry'     Name
 '-'           Operator
 '>'           Operator
@@ -4590,7 +4634,10 @@
 '\\n'         Literal.String.Escape
 '"'           Literal.String
 ','           Punctuation
-' \n                 ' Text
+' '           Text
+'\n'          Text
+
+'                 ' Text
 'call'        Keyword
 ' '           Text
 'PacketLink'  Name
@@ -4663,7 +4710,10 @@
 ' '           Text
 '='           Operator
 '='           Operator
-' \n               ' Text
+' '           Text
+'\n'          Text
+
+'               ' Text
 's_entry'     Name
 '-'           Operator
 '>'           Operator
diff --git a/tests/examplefiles/objective-c/objc_example.m.output b/tests/examplefiles/objective-c/objc_example.m.output
index 309d901e..8d25c353 100644
--- a/tests/examplefiles/objective-c/objc_example.m.output
+++ b/tests/examplefiles/objective-c/objc_example.m.output
@@ -725,7 +725,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'  \n  '      Text
+'  '          Text
+'\n'          Text
+
+'  '          Text
 '// Array literals\n' Comment.Single
 
 '  '          Text
@@ -811,7 +814,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'  \n  '      Text
+'  '          Text
+'\n'          Text
+
+'  '          Text
 '// Dictionary literals\n' Comment.Single
 
 '  '          Text
@@ -985,7 +991,10 @@
 ';'           Punctuation
 '\n'          Text
 
-'  \n  '      Text
+'  '          Text
+'\n'          Text
+
+'  '          Text
 'NSString'    Name.Builtin.Pseudo
 ' '           Text
 '*'           Operator
diff --git a/tests/examplefiles/ragel-cpp/ragel-cpp_rlscan.output b/tests/examplefiles/ragel-cpp/ragel-cpp_rlscan.output
index c867eccf..1d41e967 100644
--- a/tests/examplefiles/ragel-cpp/ragel-cpp_rlscan.output
+++ b/tests/examplefiles/ragel-cpp/ragel-cpp_rlscan.output
@@ -692,7 +692,9 @@
 'emit'        Name.Variable
 ' '           Text.Whitespace
 '{'           Punctuation
-'\n\t\t'      Text
+'\n'          Text
+
+'\t\t'        Text
 'escapeXML'   Name
 '('           Punctuation
 ' '           Text
@@ -769,7 +771,9 @@
 "'/*'"        Literal.String.Single
 ' '           Text.Whitespace
 '{'           Punctuation
-'\n\t\t\t'    Text
+'\n'          Text
+
+'\t\t\t'      Text
 'write'       Name
 '('           Punctuation
 ' '           Text
@@ -808,7 +812,9 @@
 "'{'"         Literal.String.Single
 ' '           Text.Whitespace
 '{'           Punctuation
-'\n\t\t\t'    Text
+'\n'          Text
+
+'\t\t\t'      Text
 'write'       Name
 '('           Punctuation
 ' '           Text
@@ -828,14 +834,19 @@
 ' '           Text
 '1'           Literal.Number.Integer
 ';'           Punctuation
-' \n\t\t'     Text
+' '           Text
+'\n'          Text
+
+'\t\t'        Text
 '}'           Punctuation
 ';'           Punctuation
 '\n\n\t\t'    Text.Whitespace
 "'}'"         Literal.String.Single
 ' '           Text.Whitespace
 '{'           Punctuation
-'\n\t\t\t'    Text
+'\n'          Text
+
+'\t\t\t'      Text
 'write'       Name
 '('           Punctuation
 ' '           Text
@@ -868,7 +879,9 @@
 ')'           Punctuation
 ' '           Text
 '{'           Punctuation
-'\n\t\t\t\t'  Text
+'\n'          Text
+
+'\t\t\t\t'    Text
 'write'       Name
 '('           Punctuation
 ' '           Text
@@ -890,7 +903,9 @@
 
 '\t\t\t'      Text
 '}'           Punctuation
-'\n\t\t'      Text
+'\n'          Text
+
+'\t\t'        Text
 '}'           Punctuation
 ';'           Punctuation
 '\n\n\t\t'    Text.Whitespace
@@ -934,7 +949,9 @@
 "'}%%'"       Literal.String.Single
 ' '           Text.Whitespace
 '{'           Punctuation
-'\n\t\t\t'    Text
+'\n'          Text
+
+'\t\t\t'      Text
 'if'          Keyword
 ' '           Text
 '('           Punctuation
@@ -945,7 +962,9 @@
 ')'           Punctuation
 ' '           Text
 '{'           Punctuation
-'\n\t\t\t\t'  Text
+'\n'          Text
+
+'\t\t\t\t'    Text
 'write'       Name
 '('           Punctuation
 ' '           Text
@@ -967,14 +986,18 @@
 
 '\t\t\t'      Text
 '}'           Punctuation
-'\n\t\t'      Text
+'\n'          Text
+
+'\t\t'        Text
 '}'           Punctuation
 ';'           Punctuation
 '\n\n\t\t'    Text.Whitespace
 "'\\n'"       Literal.String.Single
 ' '           Text.Whitespace
 '{'           Punctuation
-'\n\t\t\t'    Text
+'\n'          Text
+
+'\t\t\t'      Text
 'if'          Keyword
 ' '           Text
 '('           Punctuation
@@ -984,7 +1007,9 @@
 ')'           Punctuation
 ' '           Text
 '{'           Punctuation
-'\n\t\t\t\t'  Text
+'\n'          Text
+
+'\t\t\t\t'    Text
 'write'       Name
 '('           Punctuation
 ' '           Text
@@ -1006,7 +1031,9 @@
 
 '\t\t\t'      Text
 '}'           Punctuation
-'\n\t\t'      Text
+'\n'          Text
+
+'\t\t'        Text
 '}'           Punctuation
 ';'           Punctuation
 '\n\n\t\t'    Text.Whitespace
@@ -1015,7 +1042,9 @@
 'word'        Name.Variable
 ' '           Text.Whitespace
 '{'           Punctuation
-'\n\t\t\t'    Text
+'\n'          Text
+
+'\t\t\t'      Text
 'write'       Name
 '('           Punctuation
 ' '           Text
@@ -1064,7 +1093,9 @@
 'integer'     Name.Variable
 ' '           Text.Whitespace
 '{'           Punctuation
-'\n\t\t\t'    Text
+'\n'          Text
+
+'\t\t\t'      Text
 'write'       Name
 '('           Punctuation
 ' '           Text
@@ -1113,7 +1144,9 @@
 'hex'         Name.Variable
 ' '           Text.Whitespace
 '{'           Punctuation
-'\n\t\t\t'    Text
+'\n'          Text
+
+'\t\t\t'      Text
 'write'       Name
 '('           Punctuation
 ' '           Text
@@ -1185,7 +1218,9 @@
 '"\'"'        Literal.String.Double
 ' '           Text.Whitespace
 '{'           Punctuation
-'\n\t\t\t'    Text
+'\n'          Text
+
+'\t\t\t'      Text
 'write'       Name
 '('           Punctuation
 ' '           Text
@@ -1247,7 +1282,9 @@
 '\'"\''       Literal.String.Single
 ' '           Text.Whitespace
 '{'           Punctuation
-'\n\t\t\t'    Text
+'\n'          Text
+
+'\t\t\t'      Text
 'write'       Name
 '('           Punctuation
 ' '           Text
@@ -1309,7 +1346,9 @@
 "']'"         Literal.String.Single
 ' '           Text.Whitespace
 '{'           Punctuation
-'\n\t\t\t'    Text
+'\n'          Text
+
+'\t\t\t'      Text
 'write'       Name
 '('           Punctuation
 ' '           Text
@@ -1372,7 +1411,9 @@
 "'/'"         Literal.String.Single
 ' '           Text.Whitespace
 '{'           Punctuation
-'\n\t\t\t'    Text
+'\n'          Text
+
+'\t\t\t'      Text
 'write'       Name
 '('           Punctuation
 ' '           Text
@@ -1421,7 +1462,9 @@
 "'{'"         Literal.String.Single
 ' '           Text.Whitespace
 '{'           Punctuation
-'\n\t\t\t'    Text
+'\n'          Text
+
+'\t\t\t'      Text
 'inline_depth' Name
 ' '           Text
 '='           Operator
@@ -1456,7 +1499,9 @@
 'punct'       Keyword
 ' '           Text.Whitespace
 '{'           Punctuation
-'\n\t\t\t'    Text
+'\n'          Text
+
+'\t\t\t'      Text
 'write'       Name
 '('           Punctuation
 ' '           Text
@@ -1561,7 +1606,9 @@
 "'/*'"        Literal.String.Single
 ' '           Text.Whitespace
 '{'           Punctuation
-'\n\t\t\t'    Text
+'\n'          Text
+
+'\t\t\t'      Text
 'escapeXML'   Name
 '('           Punctuation
 ' '           Text
@@ -1603,7 +1650,10 @@
 "'%%{'"       Literal.String.Single
 ' '           Text.Whitespace
 '{'           Punctuation
-' \n\t\t\t'   Text
+' '           Text
+'\n'          Text
+
+'\t\t\t'      Text
 'write'       Name
 '('           Punctuation
 ' '           Text
@@ -1639,7 +1689,9 @@
 "'%%'"        Literal.String.Single
 ' '           Text.Whitespace
 '{'           Punctuation
-'\n\t\t\t'    Text
+'\n'          Text
+
+'\t\t\t'      Text
 'write'       Name
 '('           Punctuation
 ' '           Text
@@ -1650,14 +1702,20 @@
 ' '           Text
 ')'           Punctuation
 ';'           Punctuation
-' \n\t\t\t'   Text
+' '           Text
+'\n'          Text
+
+'\t\t\t'      Text
 'single_line' Name
 ' '           Text
 '='           Operator
 ' '           Text
 'true'        Name.Builtin
 ';'           Punctuation
-' \n\t\t\t'   Text
+' '           Text
+'\n'          Text
+
+'\t\t\t'      Text
 'fgoto'       Name
 ' '           Text
 'rlscan'      Name
@@ -1671,7 +1729,10 @@
 'default'     Name.Variable
 ' '           Text.Whitespace
 '{'           Punctuation
-' \n\t\t\t'   Text
+' '           Text
+'\n'          Text
+
+'\t\t\t'      Text
 'escapeXML'   Name
 '('           Punctuation
 ' '           Text
@@ -1697,7 +1758,9 @@
 '\n'          Text.Whitespace
 
 '}%%'         Punctuation
-'\n\n'        Text
+'\n'          Text
+
+'\n'          Text
 
 '%%'          Punctuation
 ' '           Text.Whitespace
@@ -1817,7 +1880,9 @@
 ';'           Punctuation
 '\n'          Text
 
-'\n\t'        Text
+'\n'          Text
+
+'\t'          Text
 '/* Read in a block. */' Comment.Multiline
 '\n'          Text
 
@@ -1881,7 +1946,9 @@
 ';'           Punctuation
 '\n'          Text
 
-'\n\t'        Text
+'\n'          Text
+
+'\t'          Text
 'if'          Keyword
 ' '           Text
 '('           Punctuation
diff --git a/tests/examplefiles/swig/swig_java.swg.output b/tests/examplefiles/swig/swig_java.swg.output
index 75d49d77..c5c20b65 100644
--- a/tests/examplefiles/swig/swig_java.swg.output
+++ b/tests/examplefiles/swig/swig_java.swg.output
@@ -506,7 +506,10 @@
 
 '    '        Text
 'else'        Keyword
-' \n      '   Text
+' '           Text
+'\n'          Text
+
+'      '      Text
 'return'      Keyword
 ' '           Text
 '('           Punctuation
@@ -638,7 +641,10 @@
 
 '    '        Text
 'else'        Keyword
-' \n      '   Text
+' '           Text
+'\n'          Text
+
+'      '      Text
 'return'      Keyword
 ' '           Text
 '('           Punctuation
@@ -2750,49 +2756,82 @@
 ' '           Text
 'char'        Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'signed'      Keyword.Type
 ' '           Text
 'char'        Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'char'        Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'short'       Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'short'       Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'int'         Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'int'         Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'long'        Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'long'        Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'long'        Keyword.Type
 ' '           Text
 'long'        Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'float'       Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'double'      Keyword.Type
 '\n'          Text
 
@@ -2822,49 +2861,82 @@
 ' '           Text
 'char'        Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'signed'      Keyword.Type
 ' '           Text
 'char'        Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'char'        Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'short'       Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'short'       Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'int'         Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'int'         Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'long'        Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'long'        Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'long'        Keyword.Type
 ' '           Text
 'long'        Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'float'       Keyword.Type
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'double'      Keyword.Type
 '\n'          Text
 
@@ -3155,49 +3227,82 @@
 ' '           Text
 'char'        Keyword.Type
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'signed'      Keyword.Type
 ' '           Text
 'char'        Keyword.Type
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'char'        Keyword.Type
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'short'       Keyword.Type
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'short'       Keyword.Type
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'int'         Keyword.Type
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'int'         Keyword.Type
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'long'        Keyword.Type
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'long'        Keyword.Type
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'long'        Keyword.Type
 ' '           Text
 'long'        Keyword.Type
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'float'       Keyword.Type
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'double'      Keyword.Type
 '\n'          Text
 
@@ -3216,49 +3321,82 @@
 ' '           Text
 'char'        Keyword.Type
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'signed'      Keyword.Type
 ' '           Text
 'char'        Keyword.Type
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'char'        Keyword.Type
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'short'       Keyword.Type
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'short'       Keyword.Type
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'int'         Keyword.Type
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'int'         Keyword.Type
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'long'        Keyword.Type
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'long'        Keyword.Type
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'long'        Keyword.Type
 ' '           Text
 'long'        Keyword.Type
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'float'       Keyword.Type
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'double'      Keyword.Type
 '\n'          Text
 
@@ -3614,7 +3752,10 @@
 'long'        Keyword.Type
 ' '           Text
 '{'           Punctuation
-' \n  '       Text
+' '           Text
+'\n'          Text
+
+'  '          Text
 'jclass'      Name
 ' '           Text
 'clazz'       Name
@@ -3908,7 +4049,10 @@
 'long'        Keyword.Type
 ' '           Text
 '{'           Punctuation
-' \n  '       Text
+' '           Text
+'\n'          Text
+
+'  '          Text
 'jclass'      Name
 ' '           Text
 'clazz'       Name
@@ -4207,7 +4351,10 @@
 'long'        Keyword.Type
 ' '           Text
 '{'           Punctuation
-' \n  '       Text
+' '           Text
+'\n'          Text
+
+'  '          Text
 'jbyteArray'  Name
 ' '           Text
 'ba'          Name
@@ -5444,7 +5591,10 @@
 ' '           Text
 'false'       Name.Builtin
 ';'           Punctuation
-' \n   '      Text
+' '           Text
+'\n'          Text
+
+'   '         Text
 '$1'          Name
 ' '           Text
 '='           Operator
@@ -5501,7 +5651,10 @@
 ' '           Text
 'false'       Name.Builtin
 ';'           Punctuation
-' \n   '      Text
+' '           Text
+'\n'          Text
+
+'   '         Text
 '$result'     Name
 ' '           Text
 '='           Operator
@@ -5567,7 +5720,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'const'       Keyword
 ' '           Text
 'signed'      Keyword.Type
@@ -5582,7 +5738,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'const'       Keyword
 ' '           Text
 'unsigned'    Keyword.Type
@@ -5597,7 +5756,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'const'       Keyword
 ' '           Text
 'short'       Keyword.Type
@@ -5610,7 +5772,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'const'       Keyword
 ' '           Text
 'unsigned'    Keyword.Type
@@ -5625,7 +5790,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'const'       Keyword
 ' '           Text
 'int'         Keyword.Type
@@ -5638,7 +5806,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'const'       Keyword
 ' '           Text
 'unsigned'    Keyword.Type
@@ -5653,7 +5824,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'const'       Keyword
 ' '           Text
 'long'        Keyword.Type
@@ -5666,7 +5840,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'const'       Keyword
 ' '           Text
 'unsigned'    Keyword.Type
@@ -5681,7 +5858,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'const'       Keyword
 ' '           Text
 'long'        Keyword.Type
@@ -5696,7 +5876,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'const'       Keyword
 ' '           Text
 'float'       Keyword.Type
@@ -5709,7 +5892,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n             ' Text
+' '           Text
+'\n'          Text
+
+'             ' Text
 'const'       Keyword
 ' '           Text
 'double'      Keyword.Type
@@ -5735,7 +5921,10 @@
 ')'           Punctuation
 '$input'      Name
 ';'           Punctuation
-' \n   '      Text
+' '           Text
+'\n'          Text
+
+'   '         Text
 '$1'          Name
 ' '           Text
 '='           Operator
@@ -5898,7 +6087,10 @@
 ')'           Punctuation
 '$input'      Name
 ';'           Punctuation
-' \n   '      Text
+' '           Text
+'\n'          Text
+
+'   '         Text
 '$result'     Name
 ' '           Text
 '='           Operator
@@ -6243,7 +6435,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'const'       Keyword
 ' '           Text
 'signed'      Keyword.Type
@@ -6258,7 +6453,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'const'       Keyword
 ' '           Text
 'unsigned'    Keyword.Type
@@ -6273,7 +6471,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'const'       Keyword
 ' '           Text
 'short'       Keyword.Type
@@ -6286,7 +6487,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'const'       Keyword
 ' '           Text
 'unsigned'    Keyword.Type
@@ -6301,7 +6505,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'const'       Keyword
 ' '           Text
 'int'         Keyword.Type
@@ -6314,7 +6521,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'const'       Keyword
 ' '           Text
 'unsigned'    Keyword.Type
@@ -6329,7 +6539,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'const'       Keyword
 ' '           Text
 'long'        Keyword.Type
@@ -6342,7 +6555,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'const'       Keyword
 ' '           Text
 'unsigned'    Keyword.Type
@@ -6357,7 +6573,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'const'       Keyword
 ' '           Text
 'long'        Keyword.Type
@@ -6372,7 +6591,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'const'       Keyword
 ' '           Text
 'float'       Keyword.Type
@@ -6385,7 +6607,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                         ' Text
+' '           Text
+'\n'          Text
+
+'                         ' Text
 'const'       Keyword
 ' '           Text
 'double'      Keyword.Type
@@ -6424,7 +6649,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'const'       Keyword
 ' '           Text
 'signed'      Keyword.Type
@@ -6439,7 +6667,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'const'       Keyword
 ' '           Text
 'unsigned'    Keyword.Type
@@ -6454,7 +6685,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'const'       Keyword
 ' '           Text
 'short'       Keyword.Type
@@ -6467,7 +6701,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'const'       Keyword
 ' '           Text
 'unsigned'    Keyword.Type
@@ -6482,7 +6719,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'const'       Keyword
 ' '           Text
 'int'         Keyword.Type
@@ -6495,7 +6735,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'const'       Keyword
 ' '           Text
 'unsigned'    Keyword.Type
@@ -6510,7 +6753,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'const'       Keyword
 ' '           Text
 'long'        Keyword.Type
@@ -6523,7 +6769,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'const'       Keyword
 ' '           Text
 'unsigned'    Keyword.Type
@@ -6538,7 +6787,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'const'       Keyword
 ' '           Text
 'long'        Keyword.Type
@@ -6553,7 +6805,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'const'       Keyword
 ' '           Text
 'float'       Keyword.Type
@@ -6566,7 +6821,10 @@
 'temp'        Name
 ')'           Punctuation
 ','           Punctuation
-' \n                          ' Text
+' '           Text
+'\n'          Text
+
+'                          ' Text
 'const'       Keyword
 ' '           Text
 'double'      Keyword.Type
@@ -7009,7 +7267,10 @@
 ')'           Punctuation
 ' '           Text
 '{'           Punctuation
-' \n  '       Text
+' '           Text
+'\n'          Text
+
+'  '          Text
 'jclass'      Name
 ' '           Text
 'clazz'       Name
@@ -7321,7 +7582,10 @@
 '&'           Operator
 ' '           Text
 '{'           Punctuation
-' \n  '       Text
+' '           Text
+'\n'          Text
+
+'  '          Text
 'static'      Keyword
 ' '           Text
 '$*1_ltype'   Name
@@ -7638,7 +7902,10 @@
 '&'           Operator
 ' '           Text
 '{'           Punctuation
-' \n  '       Text
+' '           Text
+'\n'          Text
+
+'  '          Text
 'jbyteArray'  Name
 ' '           Text
 'ba'          Name
@@ -7949,7 +8216,10 @@
 '&'           Operator
 '$input'      Name
 ';'           Punctuation
-' \n   '      Text
+' '           Text
+'\n'          Text
+
+'   '         Text
 'if'          Keyword
 ' '           Text
 '('           Punctuation
@@ -8031,7 +8301,10 @@
 '&'           Operator
 '$input'      Name
 ';'           Punctuation
-' \n   '      Text
+' '           Text
+'\n'          Text
+
+'   '         Text
 'if'          Keyword
 ' '           Text
 '('           Punctuation
@@ -8090,7 +8363,8 @@
 ')'           Punctuation
 ' '           Text
 'SWIGTYPE'    Name
-' \n'         Text
+' '           Text
+'\n'          Text
 
 '#'           Comment.Preproc
 'ifdef __cplusplus' Comment.Preproc
@@ -8210,7 +8484,8 @@
 ')'           Punctuation
 ' '           Text
 'SWIGTYPE'    Name
-' \n'         Text
+' '           Text
+'\n'          Text
 
 '%'           Operator
 '{'           Punctuation
@@ -8323,7 +8598,10 @@
 ')'           Punctuation
 ' '           Text
 '{'           Punctuation
-' \n  '       Text
+' '           Text
+'\n'          Text
+
+'  '          Text
 'const'       Keyword
 ' '           Text
 'char'        Keyword.Type
@@ -8505,7 +8783,8 @@
 ' '           Text
 '%'           Operator
 '}'           Punctuation
-' \n'         Text
+' '           Text
+'\n'          Text
 
 '%typemap'    Name.Function
 '('           Punctuation
@@ -8636,7 +8915,10 @@
 ' '           Text
 '%'           Operator
 '}'           Punctuation
-' \n\n'       Text
+' '           Text
+'\n'          Text
+
+'\n'          Text
 
 '%typemap'    Name.Function
 '('           Punctuation
@@ -9044,7 +9326,8 @@
 ' '           Text
 '%'           Operator
 '}'           Punctuation
-' \n'         Text
+' '           Text
+'\n'          Text
 
 '%typemap'    Name.Function
 '('           Punctuation
@@ -10482,7 +10765,10 @@
 '    '        Text
 'char'        Keyword.Type
 ','           Punctuation
-' \n    '     Text
+' '           Text
+'\n'          Text
+
+'    '        Text
 'const'       Keyword
 ' '           Text
 'char'        Keyword.Type
@@ -10552,10 +10838,16 @@
 ' '           Text
 'char'        Keyword.Type
 ','           Punctuation
-' \n    '     Text
+' '           Text
+'\n'          Text
+
+'    '        Text
 'short'       Keyword.Type
 ','           Punctuation
-' \n    '     Text
+' '           Text
+'\n'          Text
+
+'    '        Text
 'const'       Keyword
 ' '           Text
 'unsigned'    Keyword.Type
@@ -10564,7 +10856,10 @@
 ' '           Text
 '&'           Operator
 ','           Punctuation
-' \n    '     Text
+' '           Text
+'\n'          Text
+
+'    '        Text
 'const'       Keyword
 ' '           Text
 'short'       Keyword.Type
@@ -10597,13 +10892,22 @@
 ' '           Text
 'short'       Keyword.Type
 ','           Punctuation
-' \n    '     Text
+' '           Text
+'\n'          Text
+
+'    '        Text
 'int'         Keyword.Type
 ','           Punctuation
-' \n    '     Text
+' '           Text
+'\n'          Text
+
+'    '        Text
 'long'        Keyword.Type
 ','           Punctuation
-' \n    '     Text
+' '           Text
+'\n'          Text
+
+'    '        Text
 'const'       Keyword
 ' '           Text
 'unsigned'    Keyword.Type
@@ -10612,14 +10916,20 @@
 ' '           Text
 '&'           Operator
 ','           Punctuation
-' \n    '     Text
+' '           Text
+'\n'          Text
+
+'    '        Text
 'const'       Keyword
 ' '           Text
 'int'         Keyword.Type
 ' '           Text
 '&'           Operator
 ','           Punctuation
-' \n    '     Text
+' '           Text
+'\n'          Text
+
+'    '        Text
 'const'       Keyword
 ' '           Text
 'long'        Keyword.Type
@@ -10652,17 +10962,26 @@
 ' '           Text
 'int'         Keyword.Type
 ','           Punctuation
-' \n    '     Text
+' '           Text
+'\n'          Text
+
+'    '        Text
 'unsigned'    Keyword.Type
 ' '           Text
 'long'        Keyword.Type
 ','           Punctuation
-' \n    '     Text
+' '           Text
+'\n'          Text
+
+'    '        Text
 'long'        Keyword.Type
 ' '           Text
 'long'        Keyword.Type
 ','           Punctuation
-' \n    '     Text
+' '           Text
+'\n'          Text
+
+'    '        Text
 'const'       Keyword
 ' '           Text
 'unsigned'    Keyword.Type
@@ -10671,7 +10990,10 @@
 ' '           Text
 '&'           Operator
 ','           Punctuation
-' \n    '     Text
+' '           Text
+'\n'          Text
+
+'    '        Text
 'const'       Keyword
 ' '           Text
 'unsigned'    Keyword.Type
@@ -10680,7 +11002,10 @@
 ' '           Text
 '&'           Operator
 ','           Punctuation
-' \n    '     Text
+' '           Text
+'\n'          Text
+
+'    '        Text
 'const'       Keyword
 ' '           Text
 'long'        Keyword.Type
@@ -11031,24 +11356,36 @@
 '    '        Text
 'SWIGTYPE'    Name
 ','           Punctuation
-' \n    '     Text
+' '           Text
+'\n'          Text
+
+'    '        Text
 'SWIGTYPE'    Name
 ' '           Text
 '*'           Operator
 ','           Punctuation
-' \n    '     Text
+' '           Text
+'\n'          Text
+
+'    '        Text
 'SWIGTYPE'    Name
 ' '           Text
 '&'           Operator
 ','           Punctuation
-' \n    '     Text
+' '           Text
+'\n'          Text
+
+'    '        Text
 'SWIGTYPE'    Name
 ' '           Text
 '*'           Operator
 'const'       Keyword
 '&'           Operator
 ','           Punctuation
-' \n    '     Text
+' '           Text
+'\n'          Text
+
+'    '        Text
 'SWIGTYPE'    Name
 ' '           Text
 '['           Punctuation
@@ -11088,23 +11425,38 @@
 ' '           Text
 'int'         Keyword.Type
 ','           Punctuation
-' \n                 ' Text
+' '           Text
+'\n'          Text
+
+'                 ' Text
 'long'        Keyword.Type
 ','           Punctuation
-' \n                 ' Text
+' '           Text
+'\n'          Text
+
+'                 ' Text
 'short'       Keyword.Type
 ','           Punctuation
-' \n                 ' Text
+' '           Text
+'\n'          Text
+
+'                 ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'int'         Keyword.Type
 ','           Punctuation
-' \n                 ' Text
+' '           Text
+'\n'          Text
+
+'                 ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'long'        Keyword.Type
 ','           Punctuation
-' \n                 ' Text
+' '           Text
+'\n'          Text
+
+'                 ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'short'       Keyword.Type
@@ -12469,7 +12821,10 @@
 ' '           Text
 '%'           Operator
 '}'           Punctuation
-' \n\n'       Text
+' '           Text
+'\n'          Text
+
+'\n'          Text
 
 '/* Typemaps used for the generation of proxy and type wrapper class code */' Comment.Multiline
 '\n'          Text
diff --git a/tests/examplefiles/swig/swig_std_vector.i.output b/tests/examplefiles/swig/swig_std_vector.i.output
index 3c3dc702..6d383eb0 100644
--- a/tests/examplefiles/swig/swig_std_vector.i.output
+++ b/tests/examplefiles/swig/swig_std_vector.i.output
@@ -39,7 +39,10 @@
 ')'           Punctuation
 '\n'          Text
 
-'  \n  '      Text
+'  '          Text
+'\n'          Text
+
+'  '          Text
 'void'        Keyword.Type
 ' '           Text
 'reserve'     Name
@@ -87,7 +90,10 @@
 ')'           Punctuation
 '\n'          Text
 
-'  \n  '      Text
+'  '          Text
+'\n'          Text
+
+'  '          Text
 'void'        Keyword.Type
 ' '           Text
 'reserve'     Name
@@ -179,7 +185,10 @@
 
 '%'           Operator
 '}'           Punctuation
-'    \n\n'    Text
+'    '        Text
+'\n'          Text
+
+'\n'          Text
 
 '// exported classes\n' Comment.Single
 
@@ -530,7 +539,10 @@
 'endif'       Comment.Preproc
 '\n'          Comment.Preproc
 
-'  \n    '    Text
+'  '          Text
+'\n'          Text
+
+'    '        Text
 '%std_vector_methods' Name.Function
 '('           Punctuation
 'vector'      Name
@@ -600,7 +612,10 @@
 ' '           Text
 'size_type'   Name
 ';'           Punctuation
-'    \n    '  Text
+'    '        Text
+'\n'          Text
+
+'    '        Text
 'typedef'     Keyword
 ' '           Text
 'ptrdiff_t'   Keyword.Type
@@ -953,7 +968,10 @@
 ' '           Text
 'size_type'   Name
 ';'           Punctuation
-'    \n    '  Text
+'    '        Text
+'\n'          Text
+
+'    '        Text
 'typedef'     Keyword
 ' '           Text
 'ptrdiff_t'   Keyword.Type
@@ -1281,7 +1299,10 @@
 '_Alloc'      Name.Class
 ' '           Text
 '>'           Operator
-' \n  '       Text
+' '           Text
+'\n'          Text
+
+'  '          Text
 'class'       Keyword
 ' '           Text
 'vector'      Name.Class
@@ -1307,7 +1328,10 @@
 ' '           Text
 'size_type'   Name
 ';'           Punctuation
-'    \n    '  Text
+'    '        Text
+'\n'          Text
+
+'    '        Text
 'typedef'     Keyword
 ' '           Text
 'ptrdiff_t'   Keyword.Type
diff --git a/tests/snippets/c/test_preproc_file3.txt b/tests/snippets/c/test_preproc_file3.txt
new file mode 100644
index 00000000..65702a12
--- /dev/null
+++ b/tests/snippets/c/test_preproc_file3.txt
@@ -0,0 +1,18 @@
+Space around line break before macro is valid C, but failed to parse previously.
+
+---input---
+foo(); 
+ #define FOO 0
+
+---tokens---
+'foo'         Name
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+' '           Text
+'\n'          Text
+
+' '           Text
+'#'           Comment.Preproc
+'define FOO 0' Comment.Preproc
+'\n'          Comment.Preproc
diff --git a/tests/snippets/c/test_preproc_file4.txt b/tests/snippets/c/test_preproc_file4.txt
new file mode 100644
index 00000000..66dd47b3
--- /dev/null
+++ b/tests/snippets/c/test_preproc_file4.txt
@@ -0,0 +1,13 @@
+Comments can precede preprocessor macros
+
+---input---
+/* Comment */ /* Another */ #define FOO 0
+
+---tokens---
+'/* Comment */' Comment.Multiline
+' '           Text
+'/* Another */' Comment.Multiline
+' '           Text
+'#'           Comment.Preproc
+'define FOO 0' Comment.Preproc
+'\n'          Comment.Preproc
diff --git a/tests/snippets/c/test_preproc_file5.txt b/tests/snippets/c/test_preproc_file5.txt
new file mode 100644
index 00000000..e178ccae
--- /dev/null
+++ b/tests/snippets/c/test_preproc_file5.txt
@@ -0,0 +1,19 @@
+Preprocessor macros should appear only at the beginning of the line.
+Otherwise we should produce an error token.
+
+---input---
+foo(); #define FOO 0
+
+---tokens---
+'foo'         Name
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+' '           Text
+'#'           Error
+'define'      Name
+' '           Text
+'FOO'         Name
+' '           Text
+'0'           Literal.Number.Integer
+'\n'          Text