[svn] Use the Punctuation token type everywhere.

8 files changed, 112 insertions, 90 deletions

diff --git a/TODO b/TODO
index 74dc5847..dfb946fe 100644
--- a/TODO
+++ b/TODO
@@ -37,9 +37,6 @@ for 0.6
 - tell the PHP and DelphiLexer how to differ between Operators and
   text.
 
-- add a `Punctuation` token type for symbols that are not text
-  but also not a symbol (blocks in ruby etc)
-
 - add support for function name highlighting to c++ lexer
 
 - readd property support for C# lexer
diff --git a/pygments/cmdline.py b/pygments/cmdline.py
index 844f84d3..cd78899f 100644
--- a/pygments/cmdline.py
+++ b/pygments/cmdline.py
@@ -18,12 +18,7 @@ from pygments.formatters import FORMATTERS, get_formatter_by_name, \
      get_formatter_for_filename, TerminalFormatter
 
 
-def main(args):
-    """
-    Main command line entry point.
-    """
-
-    USAGE = """\
+USAGE = """\
 Usage: %s [-l <lexer>] [-f <formatter>] [-O <options>] [-o <outfile>] [<infile>]
        %s -S <style> -f <formatter> [-a <arg>] [-O <options>]
        %s -L | -h | -V
@@ -50,91 +45,110 @@ dependent.
 The -L option lists all available lexers and formatters.
 The -h option prints this help.
 The -V option prints the package version.
-""" % ((args[0],)*3)
+"""
+
+
+def _parse_options(o_str):
+    opts = {}
+    if not o_str:
+        return opts
+    o_args = o_str.split(',')
+    for o_arg in o_args:
+        o_arg = o_arg.strip()
+        try:
+            o_key, o_val = o_arg.split('=')
+            o_key = o_key.strip()
+            o_val = o_val.strip()
+        except ValueError:
+            O_opts[o_arg] = True
+        else:
+            O_opts[o_key] = o_val
+    return opts
+
+
+def _print_lflist():
+    # print version
+    main(['', '-V'])
+
+    print
+    print "Lexers:"
+    print "~~~~~~~"
+
+    info = []
+    maxlen = 0
+    for _, fullname, names, exts, _ in LEXERS.itervalues():
+        tup = (', '.join(names)+':', fullname,
+               exts and '(extensions ' + ', '.join(exts) + ')' or '')
+        info.append(tup)
+        if len(tup[0]) > maxlen: maxlen = len(tup[0])
+    info.sort()
+    for i in info:
+        print ('%-'+str(maxlen)+'s %s %s') % i
+
+    print
+    print "Formatters:"
+    print "~~~~~~~~~~~"
+
+    info = []
+    maxlen = 0
+    for fullname, names, exts, doc in FORMATTERS.itervalues():
+        tup = (', '.join(names)+':', doc,
+               exts and '(extensions ' + ', '.join(exts) + ')' or '')
+        info.append(tup)
+        if len(tup[0]) > maxlen: maxlen = len(tup[0])
+    info.sort()
+    for i in info:
+        print ('%-'+str(maxlen)+'s %s %s') % i
+
+
+def main(args):
+    """
+    Main command line entry point.
+    """
+    usage = USAGE % ((args[0],) * 3)
 
     try:
         opts, args = getopt.getopt(args[1:], "l:f:o:O:LhVS:a:")
     except getopt.GetoptError:
-        print >>sys.stderr, USAGE
+        print >>sys.stderr, usage
         return 2
     opts = dict(opts)
 
     if not opts and not args:
-        print USAGE
+        print usage
         return 0
 
     if opts.pop('-h', None) is not None:
-        print USAGE
+        print usage
         return 0
 
     if opts.pop('-V', None) is not None:
         print 'Pygments version %s, (c) 2006 by %s.' % (__version__, __author__)
         return 0
 
+    # handle ``pygmentize -L``
     L_opt = opts.pop('-L', None)
     if L_opt is not None:
         if opts or args:
-            print >>sys.stderr, USAGE
+            print >>sys.stderr, usage
             return 2
 
-        # print version
-        main(['', '-V'])
-        print
-        print "Lexers:"
-        print "~~~~~~~"
-
-        info = []
-        maxlen = 0
-        for _, fullname, names, exts, _ in LEXERS.itervalues():
-            tup = (', '.join(names)+':', fullname,
-                   exts and '(extensions ' + ', '.join(exts) + ')' or '')
-            info.append(tup)
-            if len(tup[0]) > maxlen: maxlen = len(tup[0])
-        info.sort()
-        for i in info:
-            print ('%-'+str(maxlen)+'s %s %s') % i
-
-        print
-        print "Formatters:"
-        print "~~~~~~~~~~~"
-
-        info = []
-        maxlen = 0
-        for fullname, names, exts, doc in FORMATTERS.itervalues():
-            tup = (', '.join(names)+':', doc,
-                   exts and '(extensions ' + ', '.join(exts) + ')' or '')
-            info.append(tup)
-            if len(tup[0]) > maxlen: maxlen = len(tup[0])
-        info.sort()
-        for i in info:
-            print ('%-'+str(maxlen)+'s %s %s') % i
+        _print_lflist()
         return 0
 
-    O_opts = {}
-    o_str = opts.pop('-O', None)
-    if o_str:
-        try:
-            o_args = o_str.split(',')
-            for o_arg in o_args:
-                try:
-                    o_key, o_val = o_arg.split('=')
-                except ValueError:
-                    O_opts[o_arg] = True
-                else:
-                    O_opts[o_key] = o_val
-        except ValueError:
-            print >>sys.stderr, 'Error in -O specification.'
-            return 2
+    # parse -O options
+    O_opts = _parse_options(opts.pop('-O', None))
 
+    # handle ``pygmentize -S``
     S_opt = opts.pop('-S', None)
     a_opt = opts.pop('-a', None)
     if S_opt is not None:
         f_opt = opts.pop('-f', None)
         if not f_opt:
-            print >>sys.stderr, USAGE
+            print >>sys.stderr, usage
             return 2
         if opts or args:
-            print >>sys.stderr, USAGE
+            print >>sys.stderr, usage
             return 2
 
         try:
@@ -148,10 +162,12 @@ The -V option prints the package version.
         print fmter.get_style_defs(arg)
         return 0
 
+    # if no -S is given, -a is not allowed
     if a_opt is not None:
-        print >>sys.stderr, USAGE
+        print >>sys.stderr, usage
         return 2
 
+    # select formatter
     outfn = opts.pop('-o', None)
     fmter = opts.pop('-f', None)
     if fmter:
@@ -178,6 +194,7 @@ The -V option prints the package version.
             fmter = TerminalFormatter(**O_opts)
         outfile = sys.stdout
 
+    # select lexer
     lexer = opts.pop('-l', None)
     if lexer:
         try:
@@ -187,6 +204,10 @@ The -V option prints the package version.
             return 1
 
     if args:
+        if len(args) > 1:
+            print >>sys.stderr, usage
+            return 2
+
         infn = args[0]
         if not lexer:
             try:
@@ -206,6 +227,7 @@ The -V option prints the package version.
             return 2
         code = sys.stdin.read()
 
+    # ... and do it!
     try:
         highlight(code, lexer, fmter, outfile)
     except Exception, err:
@@ -213,4 +235,5 @@ The -V option prints the package version.
         print >>sys.stderr, 'Error while highlighting:'
         print >>sys.stderr, traceback.format_exc(0).splitlines()[-1]
         return 1
+
     return 0
diff --git a/pygments/lexers/agile.py b/pygments/lexers/agile.py
index 03e81624..3c63cbda 100644
--- a/pygments/lexers/agile.py
+++ b/pygments/lexers/agile.py
@@ -623,13 +623,14 @@ class PerlLexer(RegexLexer):
             (r'(\[\]|\*\*|::|<<|>>|>=|<=|<=>|={3}|!=|=~|'
              r'!~|&&?|\|\||\.{1,3})', Operator),
             (r'[-+/*%=<>&^|!\\~]=?', Operator),
-            (r'[\(\)\[\]:;,<>/\?\{\}]', Text),
+            (r'[\(\)\[\]:;,<>/\?\{\}]', Punctuation), # yes, there's no shortage
+                                                      # of punctuation in Perl!
             (r'(?=\w)', Name, 'name'),
         ],
         'varname': [
             (r'\s+', Text),
-            (r'\{', Text, '#pop'), # hash syntax?
-            (r'\)|,', Text, '#pop'), # argument specifier
+            (r'\{', Punctuation, '#pop'), # hash syntax?
+            (r'\)|,', Punctuation, '#pop'), # argument specifier
             (r'[a-zA-Z0-9_]+::', Name.Namespace),
             (r'[a-zA-Z0-9_:]+', Name.Variable, '#pop'),
         ],
@@ -646,8 +647,8 @@ class PerlLexer(RegexLexer):
             (r'[a-zA-Z_][\w_]*[\!\?]?', Name.Function),
             (r'\s+', Text),
             # argument declaration
-            (r'\([$@%]*\)\s*', Text),
-            (r'.*?{', Text, '#pop'),
+            (r'(\([$@%]*\))(\s*)', bygroups(Punctuation, Text)),
+            (r'.*?{', Punctuation, '#pop'),
         ],
         'cb-string': [
             (r'\\[\{\}\\]', String.Other),
@@ -703,7 +704,7 @@ class LuaLexer(RegexLexer):
 
             (r'\n', Text),
             (r'[^\S\n]', Text),
-            (r'[\[\]\{\}\(\)\.,:;]', Text),
+            (r'[\[\]\{\}\(\)\.,:;]', Punctuation),
 
             (r'(==|~=|<=|>=|\.\.|\.\.\.|[=+\-*/%^<>#])', Operator),
             (r'(and|or|not)\b', Operator.Word),
@@ -727,7 +728,7 @@ class LuaLexer(RegexLexer):
         'funcname': [
             ('[A-Za-z_][A-Za-z0-9_]*', Name.Function, '#pop'),
             # inline function
-            ('\(', Text, '#pop'),
+            ('\(', Punctuation, '#pop'),
         ],
 
         'classname': [
@@ -778,11 +779,12 @@ class LuaLexer(RegexLexer):
                 elif '.' in value:
                     a, b = value.split('.')
                     yield index, Name, a
-                    yield index + len(a), Text, u'.'
+                    yield index + len(a), Punctuation, u'.'
                     yield index + len(a) + 1, Name, b
                     continue
             yield index, token, value
 
+
 class SchemeLexer(RegexLexer):
     """
     A Scheme lexer, parsing a stream and outputting the tokens
diff --git a/pygments/lexers/compiled.py b/pygments/lexers/compiled.py
index 491c234c..37a5983a 100644
--- a/pygments/lexers/compiled.py
+++ b/pygments/lexers/compiled.py
@@ -55,7 +55,7 @@ class CLexer(RegexLexer):
             (r'0[0-7]+[Ll]?', Number.Oct),
             (r'(\d+\.\d*|\.\d+)', Number.Float),
             (r'\d+', Number.Integer),
-            (r'[~!%^&*()+=|\[\]:,.<>/?-]', Text),
+            (r'[~!%^&*()+=|\[\]:,.<>/?-]', Punctuation), # missing: Operators
             (r'(auto|break|case|const|continue|default|do|else|enum|extern|'
              r'for|goto|if|register|restricted|return|sizeof|static|struct|'
              r'switch|typedef|union|volatile|virtual|while)\b', Keyword),
@@ -82,19 +82,19 @@ class CLexer(RegexLexer):
              r'([a-zA-Z_][a-zA-Z0-9_]*)'             # method name
              r'(\s*\([^;]*?\))'                      # signature
              r'(' + _ws + r')(;)',
-             bygroups(using(this), Name.Function, using(this), Text, Text)),
+             bygroups(using(this), Name.Function, using(this), Text, Punctuation)),
             ('', Text, 'statement'),
         ],
         'statement' : [
             include('whitespace'),
             include('statements'),
             ('[{}]', Keyword),
-            (';', Text, '#pop'),
+            (';', Punctuation, '#pop'),
         ],
         'function': [
             include('whitespace'),
             include('statements'),
-            (';', Text),
+            (';', Punctuation),
             ('{', Keyword, '#push'),
             ('}', Keyword, '#pop'),
         ],
@@ -145,7 +145,7 @@ class CppLexer(RegexLexer):
             (r'0[0-7]+[Ll]?', Number.Oct),
             (r'(\d+\.\d*|\.\d+)', Number.Float),
             (r'\d+', Number.Integer),
-            (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Text),
+            (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation),
             (r'(asm|auto|break|case|catch|const|const_cast|continue|'
              r'default|delete|do|dynamic_cast|else|enum|explicit|export|'
              r'extern|for|friend|goto|if|mutable|namespace|new|operator|'
diff --git a/pygments/lexers/dotnet.py b/pygments/lexers/dotnet.py
index c240d17a..c8b278d8 100644
--- a/pygments/lexers/dotnet.py
+++ b/pygments/lexers/dotnet.py
@@ -11,7 +11,7 @@
 import re
 
 from pygments.lexer import RegexLexer, bygroups, using, this
-from pygments.token import \
+from pygments.token import Punctuation, \
      Text, Comment, Operator, Keyword, Name, String, Number, Literal
 
 __all__ = ['CSharpLexer', 'BooLexer', 'VbNetLexer']
@@ -39,7 +39,7 @@ class CSharpLexer(RegexLexer):
             (r'//.*?\n', Comment),
             (r'/[*](.|\n)*?[*]/', Comment),
             (r'\n', Text),
-            (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Text),
+            (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation),
             (r'[{}]', Keyword),
             (r'@"(\\\\|\\"|[^"])*"', String),
             (r'"(\\\\|\\"|[^"\n])*["\n]', String),
@@ -85,7 +85,7 @@ class BooLexer(RegexLexer):
             (r'\s+', Text),
             (r'(#|//).*$', Comment),
             (r'/[*]', Comment, 'comment'),
-            (r'[]{}:(),.;[]', Text),
+            (r'[]{}:(),.;[]', Punctuation),
             (r'\\\n', Text),
             (r'\\', Text),
             (r'(in|is|and|or|not)\b', Operator.Word),
@@ -155,7 +155,7 @@ class VbNetLexer(RegexLexer):
              r'#ExternalSource.*?\n|#End\s+ExternalSource|'
              r'#Region.*?\n|#End\s+Region|#ExternalChecksum',
              Comment.Preproc),
-            (r'[\(\){}!#,.:]', Text),
+            (r'[\(\){}!#,.:]', Punctuation),
             (r'Option\s+(Strict|Explicit|Compare)\s+'
              r'(On|Off|Binary|Text)', Keyword.Declaration),
             (r'(?<!\.)(AddHandler|Alias|'
diff --git a/pygments/lexers/other.py b/pygments/lexers/other.py
index 94d3c3b6..707544d8 100644
--- a/pygments/lexers/other.py
+++ b/pygments/lexers/other.py
@@ -12,7 +12,7 @@
 import re
 
 from pygments.lexer import RegexLexer, include, bygroups, using
-from pygments.token import Error, \
+from pygments.token import Error, Punctuation, \
      Text, Comment, Operator, Keyword, Name, String, Number
 from pygments.util import shebang_matches
 
@@ -115,7 +115,7 @@ class SqlLexer(RegexLexer):
             (r'[0-9]+', Number.Integer),
             (r"'(''|[^'])*'", String),
             (r'[a-zA-Z_][a-zA-Z0-9_]*', Name),
-            (r'[;:()\[\],\.]', Text)
+            (r'[;:()\[\],\.]', Punctuation)
         ],
         'multiline-comments': [
             (r'/\*', Comment.Multiline, 'multiline-comments'),
@@ -191,12 +191,12 @@ class BashLexer(RegexLexer):
         'curly': [
             (r'}', Keyword, '#pop'),
             (r':-', Keyword),
-            (r'[^}:]+', Text),
-            (r':', Text),
+            (r'[^}:]+', Punctuation),
+            (r':', Punctuation),
         ],
         'paren': [
             (r'\)', Keyword, '#pop'),
-            (r'[^)]*', Text),
+            (r'[^)]*', Punctuation),
         ],
     }
 
diff --git a/pygments/lexers/templates.py b/pygments/lexers/templates.py
index df942b4b..2075c886 100644
--- a/pygments/lexers/templates.py
+++ b/pygments/lexers/templates.py
@@ -20,7 +20,7 @@ from pygments.lexers.web import \
 from pygments.lexers.agile import PythonLexer
 from pygments.lexer import Lexer, DelegatingLexer, RegexLexer, bygroups, \
      include, using, this
-from pygments.token import Error, \
+from pygments.token import Error, Punctuation, \
      Text, Comment, Operator, Keyword, Name, String, Number, Other
 from pygments.util import html_doctype_matches, looks_like_xml
 
@@ -208,7 +208,7 @@ class DjangoLexer(RegexLexer):
              Keyword),
             include('varnames'),
             (r'\%\}', Comment.Preproc, '#pop'),
-            (r'.', Text)
+            (r'.', Punctuation)
         ]
     }
 
diff --git a/pygments/lexers/text.py b/pygments/lexers/text.py
index e4623662..5c5d761f 100644
--- a/pygments/lexers/text.py
+++ b/pygments/lexers/text.py
@@ -13,7 +13,7 @@
 import re
 
 from pygments.lexer import RegexLexer, bygroups, include
-from pygments.token import \
+from pygments.token import Punctuation, \
     Text, Comment, Keyword, Name, String, Generic, Operator, Number
 
 
@@ -70,7 +70,7 @@ class MakefileLexer(RegexLexer):
         ],
         'block-header': [
             (r'[^,\n]', String),
-            (r',', Text),
+            (r',', Punctuation),
             (r'\n[\t ]+', Text, 'block'),
             (r'\n', Text, '#pop')
         ],
@@ -220,7 +220,7 @@ class GroffLexer(RegexLexer):
     tokens = {
         'root': [
             (r'(?i)(\.)(\w+)', bygroups(Text, Keyword), 'request'),
-            (r'\.', Text, 'request'),
+            (r'\.', Punctuation, 'request'),
             # Regular characters, slurp till we find a backslash or newline
             (r'[^\\\n]*', Text, 'textline'),
         ],