Merged in timgilbert/pygments-main/clj-keyword-fix (pull request #326)

Tweaking clojure keyword lexing

1 files changed, 226 insertions, 1 deletions

diff --git a/tests/test_clexer.py b/tests/test_clexer.py
index c995bb2b..188569dc 100644
--- a/tests/test_clexer.py
+++ b/tests/test_clexer.py
@@ -9,8 +9,9 @@
 
 import unittest
 import os
+import textwrap
 
-from pygments.token import Text, Number
+from pygments.token import Text, Number, Token
 from pygments.lexers import CLexer
 
 
@@ -29,3 +30,227 @@ class CLexerTest(unittest.TestCase):
             wanted.append((Text, ' '))
         wanted = [(Text, '')] + wanted[:-1] + [(Text, '\n')]
         self.assertEqual(list(self.lexer.get_tokens(code)), wanted)
+
+    def testSwitch(self):
+        fragment = u'''\
+        int main()
+        {
+            switch (0)
+            {
+                case 0:
+                default:
+                    ;
+            }
+        }
+        '''
+        tokens = [
+            (Token.Text, u''),
+            (Token.Keyword.Type, u'int'),
+            (Token.Text, u' '),
+            (Token.Name.Function, u'main'),
+            (Token.Text, u''),
+            (Token.Punctuation, u'('),
+            (Token.Punctuation, u')'),
+            (Token.Text, u'\n'),
+            (Token.Text, u''),
+            (Token.Punctuation, u'{'),
+            (Token.Text, u'\n'),
+            (Token.Text, u'    '),
+            (Token.Keyword, u'switch'),
+            (Token.Text, u' '),
+            (Token.Punctuation, u'('),
+            (Token.Literal.Number.Integer, u'0'),
+            (Token.Punctuation, u')'),
+            (Token.Text, u'\n'),
+            (Token.Text, u'    '),
+            (Token.Punctuation, u'{'),
+            (Token.Text, u'\n'),
+            (Token.Text, u'        '),
+            (Token.Keyword, u'case'),
+            (Token.Text, u' '),
+            (Token.Literal.Number.Integer, u'0'),
+            (Token.Operator, u':'),
+            (Token.Text, u'\n'),
+            (Token.Text, u'        '),
+            (Token.Keyword, u'default'),
+            (Token.Operator, u':'),
+            (Token.Text, u'\n'),
+            (Token.Text, u'            '),
+            (Token.Punctuation, u';'),
+            (Token.Text, u'\n'),
+            (Token.Text, u'    '),
+            (Token.Punctuation, u'}'),
+            (Token.Text, u'\n'),
+            (Token.Punctuation, u'}'),
+            (Token.Text, u'\n'),
+            (Token.Text, u''),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment))))
+
+    def testSwitchSpaceBeforeColon(self):
+        fragment = u'''\
+        int main()
+        {
+            switch (0)
+            {
+                case 0 :
+                default :
+                    ;
+            }
+        }
+        '''
+        tokens = [
+            (Token.Text, u''),
+            (Token.Keyword.Type, u'int'),
+            (Token.Text, u' '),
+            (Token.Name.Function, u'main'),
+            (Token.Text, u''),
+            (Token.Punctuation, u'('),
+            (Token.Punctuation, u')'),
+            (Token.Text, u'\n'),
+            (Token.Text, u''),
+            (Token.Punctuation, u'{'),
+            (Token.Text, u'\n'),
+            (Token.Text, u'    '),
+            (Token.Keyword, u'switch'),
+            (Token.Text, u' '),
+            (Token.Punctuation, u'('),
+            (Token.Literal.Number.Integer, u'0'),
+            (Token.Punctuation, u')'),
+            (Token.Text, u'\n'),
+            (Token.Text, u'    '),
+            (Token.Punctuation, u'{'),
+            (Token.Text, u'\n'),
+            (Token.Text, u'        '),
+            (Token.Keyword, u'case'),
+            (Token.Text, u' '),
+            (Token.Literal.Number.Integer, u'0'),
+            (Token.Text, u' '),
+            (Token.Operator, u':'),
+            (Token.Text, u'\n'),
+            (Token.Text, u'        '),
+            (Token.Keyword, u'default'),
+            (Token.Text, u' '),
+            (Token.Operator, u':'),
+            (Token.Text, u'\n'),
+            (Token.Text, u'            '),
+            (Token.Punctuation, u';'),
+            (Token.Text, u'\n'),
+            (Token.Text, u'    '),
+            (Token.Punctuation, u'}'),
+            (Token.Text, u'\n'),
+            (Token.Punctuation, u'}'),
+            (Token.Text, u'\n'),
+            (Token.Text, u''),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment))))
+
+    def testLabel(self):
+        fragment = u'''\
+        int main()
+        {
+        foo:
+          goto foo;
+        }
+        '''
+        tokens = [
+            (Token.Text, u''),
+            (Token.Keyword.Type, u'int'),
+            (Token.Text, u' '),
+            (Token.Name.Function, u'main'),
+            (Token.Text, u''),
+            (Token.Punctuation, u'('),
+            (Token.Punctuation, u')'),
+            (Token.Text, u'\n'),
+            (Token.Text, u''),
+            (Token.Punctuation, u'{'),
+            (Token.Text, u'\n'),
+            (Token.Name.Label, u'foo'),
+            (Token.Punctuation, u':'),
+            (Token.Text, u'\n'),
+            (Token.Text, u'  '),
+            (Token.Keyword, u'goto'),
+            (Token.Text, u' '),
+            (Token.Name, u'foo'),
+            (Token.Punctuation, u';'),
+            (Token.Text, u'\n'),
+            (Token.Punctuation, u'}'),
+            (Token.Text, u'\n'),
+            (Token.Text, u''),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment))))
+
+    def testLabelSpaceBeforeColon(self):
+        fragment = u'''\
+        int main()
+        {
+        foo :
+          goto foo;
+        }
+        '''
+        tokens = [
+            (Token.Text, u''),
+            (Token.Keyword.Type, u'int'),
+            (Token.Text, u' '),
+            (Token.Name.Function, u'main'),
+            (Token.Text, u''),
+            (Token.Punctuation, u'('),
+            (Token.Punctuation, u')'),
+            (Token.Text, u'\n'),
+            (Token.Text, u''),
+            (Token.Punctuation, u'{'),
+            (Token.Text, u'\n'),
+            (Token.Name.Label, u'foo'),
+            (Token.Text, u' '),
+            (Token.Punctuation, u':'),
+            (Token.Text, u'\n'),
+            (Token.Text, u'  '),
+            (Token.Keyword, u'goto'),
+            (Token.Text, u' '),
+            (Token.Name, u'foo'),
+            (Token.Punctuation, u';'),
+            (Token.Text, u'\n'),
+            (Token.Punctuation, u'}'),
+            (Token.Text, u'\n'),
+            (Token.Text, u''),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment))))
+
+    def testLabelFollowedByStatement(self):
+        fragment = u'''\
+        int main()
+        {
+        foo:return 0;
+          goto foo;
+        }
+        '''
+        tokens = [
+            (Token.Text, u''),
+            (Token.Keyword.Type, u'int'),
+            (Token.Text, u' '),
+            (Token.Name.Function, u'main'),
+            (Token.Text, u''),
+            (Token.Punctuation, u'('),
+            (Token.Punctuation, u')'),
+            (Token.Text, u'\n'),
+            (Token.Text, u''),
+            (Token.Punctuation, u'{'),
+            (Token.Text, u'\n'),
+            (Token.Name.Label, u'foo'),
+            (Token.Punctuation, u':'),
+            (Token.Keyword, u'return'),
+            (Token.Text, u' '),
+            (Token.Literal.Number.Integer, u'0'),
+            (Token.Punctuation, u';'),
+            (Token.Text, u'\n'),
+            (Token.Text, u'  '),
+            (Token.Keyword, u'goto'),
+            (Token.Text, u' '),
+            (Token.Name, u'foo'),
+            (Token.Punctuation, u';'),
+            (Token.Text, u'\n'),
+            (Token.Punctuation, u'}'),
+            (Token.Text, u'\n'),
+            (Token.Text, u''),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment))))