Added options for rendering LaTeX in source code comments in the LaTeX formatter (#461).

4 files changed, 131 insertions, 13 deletions

diff --git a/AUTHORS b/AUTHORS
index 580bde46..1aba5c3a 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -45,6 +45,7 @@ Other contributors, listed alphabetically, are:
 * Benjamin Peterson -- Test suite refactoring
 * Justin Reidy -- MXML lexer
 * Andre Roberge -- Tango style
+* Konrad Rudolph -- LaTeX formatter enhancements
 * Mario Ruggier -- Evoque lexers
 * Stou Sandalski -- NumPy, FORTRAN, tcsh and XSLT lexers
 * Matteo Sasso -- Common Lisp lexer
diff --git a/CHANGES b/CHANGES
index bc870883..0ab55845 100644
--- a/CHANGES
+++ b/CHANGES
@@ -16,6 +16,9 @@ Version 1.2
   * CMake
   * OOC
 
+- Added options for rendering LaTeX in source code comments in the
+  LaTeX formatter (#461).
+
 - Updated the Logtalk lexer.
 
 - Added `line_number_start` option to image formatter (#456).
diff --git a/pygments/filters/__init__.py b/pygments/filters/__init__.py
index d7aaa657..13a2a936 100644
--- a/pygments/filters/__init__.py
+++ b/pygments/filters/__init__.py
@@ -283,10 +283,78 @@ class VisibleWhitespaceFilter(Filter):
                 yield ttype, value
 
 
+class GobbleFilter(Filter):
+    """
+    Gobbles source code lines (eats initial characters).
+
+    This filter drops the first ``n`` characters off every line of code.  This
+    may be useful when the source code fed to the lexer is indented by a fixed
+    amount of space that isn't desired in the output.
+
+    Options accepted:
+
+    `n` : int
+       The number of characters to gobble.
+
+    *New in Pygments 1.2.*
+    """
+    def __init__(self, **options):
+        Filter.__init__(self, **options)
+        self.n = get_int_opt(options, 'n', 0)
+
+    def gobble(self, value, left):
+        if left < len(value):
+            return value[left:], 0
+        else:
+            return '', left - len(value)
+
+    def filter(self, lexer, stream):
+        n = self.n
+        left = n # How many characters left to gobble.
+        for ttype, value in stream:
+            # Remove ``left`` tokens from first line, ``n`` from all others.
+            parts = value.split('\n')
+            (parts[0], left) = self.gobble(parts[0], left)
+            for i in range(1, len(parts)):
+                (parts[i], left) = self.gobble(parts[i], n)
+            value = '\n'.join(parts)
+
+            if value != '':
+                yield ttype, value
+
+
+class TokenMergeFilter(Filter):
+    """
+    Merges consecutive tokens with the same token type in the output stream of a
+    lexer.
+
+    *New in Pygments 1.2.*
+    """
+    def __init__(self, **options):
+        Filter.__init__(self, **options)
+
+    def filter(self, lexer, stream):
+        output = []
+        current_type = None
+        current_value = None
+        for ttype, value in stream:
+            if ttype is current_type:
+                current_value += value
+            else:
+                if not current_type is None:
+                    yield current_type, current_value
+                current_type = ttype
+                current_value = value
+        if not current_type is None:
+            yield current_type, current_value
+
+
 FILTERS = {
     'codetagify':     CodeTagFilter,
     'keywordcase':    KeywordCaseFilter,
     'highlight':      NameHighlightFilter,
     'raiseonerror':   RaiseOnErrorTokenFilter,
     'whitespace':     VisibleWhitespaceFilter,
+    'gobble':         GobbleFilter,
+    'tokenmerge':     TokenMergeFilter,
 }
diff --git a/pygments/formatters/latex.py b/pygments/formatters/latex.py
index 8ba8bb4b..a9059b14 100644
--- a/pygments/formatters/latex.py
+++ b/pygments/formatters/latex.py
@@ -18,12 +18,16 @@ __all__ = ['LatexFormatter']
 
 
 def escape_tex(text, commandprefix):
-    return text.replace('@', '\x00').    \
-                replace('[', '\x01').    \
-                replace(']', '\x02').    \
-                replace('\x00', '@%sZat[]' % commandprefix).\
-                replace('\x01', '@%sZlb[]' % commandprefix).\
-                replace('\x02', '@%sZrb[]' % commandprefix)
+    return text.replace('\\', '\x00'). \
+                replace('{', '\x01'). \
+                replace('}', '\x02'). \
+                replace('^', '\x03'). \
+                replace('_', '\x04'). \
+                replace('\x00', r'\%sZbs{}' % commandprefix). \
+                replace('\x01', r'\%sZob{}' % commandprefix). \
+                replace('\x02', r'\%sZcb{}' % commandprefix). \
+                replace('\x03', r'\%sZca{}' % commandprefix). \
+                replace('\x04', r'\%sZus{}' % commandprefix)
 
 
 DOC_TEMPLATE = r'''
@@ -92,9 +96,11 @@ STYLE_TEMPLATE = r'''
 
 %(styles)s
 
-\def\%(cp)sZat{@}
-\def\%(cp)sZlb{[}
-\def\%(cp)sZrb{]}
+\def\%(cp)sZbs{\char`\\}
+\def\%(cp)sZus{\char`\_}
+\def\%(cp)sZob{\char`\{}
+\def\%(cp)sZcb{\char`\}}
+\def\%(cp)sZca{\char`\^}
 \makeatother
 '''
 
@@ -177,6 +183,16 @@ class LatexFormatter(Formatter):
         *New in Pygments 0.7.*
 
         *New in Pygments 0.10:* the default is now ``'PY'`` instead of ``'C'``.
+
+    `texcomments`
+        If set to ``True``, enables LaTeX comment lines.  That is, LaTex markup
+        in comment tokens is not escaped so that LaTeX can render it (default:
+        ``False``).  *New in Pygments 1.2.*
+
+    `mathescape`
+        If set to ``True``, enables LaTeX math mode escape in comments. That
+        is, ``'$...$'`` inside a comment will trigger math mode (default:
+        ``False``).  *New in Pygments 1.2.*
     """
     name = 'LaTeX'
     aliases = ['latex', 'tex']
@@ -192,6 +208,8 @@ class LatexFormatter(Formatter):
         self.verboptions = options.get('verboptions', '')
         self.nobackground = get_bool_opt(options, 'nobackground', False)
         self.commandprefix = options.get('commandprefix', 'PY')
+        self.texcomments = get_bool_opt(options, 'texcomments', False)
+        self.mathescape = get_bool_opt(options, 'mathescape', False)
 
         self._create_stylesheet()
 
@@ -260,18 +278,46 @@ class LatexFormatter(Formatter):
             realoutfile = outfile
             outfile = StringIO()
 
-        outfile.write(r'\begin{Verbatim}[commandchars=@\[\]')
+        outfile.write(r'\begin{Verbatim}[commandchars=\\\{\}')
         if self.linenos:
             start, step = self.linenostart, self.linenostep
             outfile.write(',numbers=left' +
                           (start and ',firstnumber=%d' % start or '') +
                           (step and ',stepnumber=%d' % step or ''))
+        if self.mathescape or self.texcomments:
+            outfile.write(r',codes={\catcode`\$=3\catcode`\^=7\catcode`\_=8}')
         if self.verboptions:
             outfile.write(',' + self.verboptions)
         outfile.write(']\n')
 
         for ttype, value in tokensource:
-            value = escape_tex(value, self.commandprefix)
+            if ttype in Token.Comment:
+                if self.texcomments:
+                    # Try to guess comment starting lexeme and escape it ...
+                    start = value[0:1]
+                    for i in xrange(1, len(value)):
+                        if start[0] != value[i]:
+                            break
+                        start += value[i]
+
+                    value = value[len(start):]
+                    start = escape_tex(start, self.commandprefix)
+
+                    # ... but do not escape inside comment.
+                    value = start + value
+                elif self.mathescape:
+                    # Only escape parts not inside a math environment.
+                    parts = value.split('$')
+                    in_math = False
+                    for i, part in enumerate(parts):
+                        if not in_math:
+                            parts[i] = escape_tex(part, self.commandprefix)
+                        in_math = not in_math
+                    value = '$'.join(parts)
+                else:
+                    value = escape_tex(value, self.commandprefix)
+            else:
+                value = escape_tex(value, self.commandprefix)
             styles = []
             while ttype is not Token:
                 try:
@@ -285,10 +331,10 @@ class LatexFormatter(Formatter):
                 spl = value.split('\n')
                 for line in spl[:-1]:
                     if line:
-                        outfile.write("@%s[%s][%s]" % (cp, styleval, line))
+                        outfile.write("\\%s{%s}{%s}" % (cp, styleval, line))
                     outfile.write('\n')
                 if spl[-1]:
-                    outfile.write("@%s[%s][%s]" % (cp, styleval, spl[-1]))
+                    outfile.write("\\%s{%s}{%s}" % (cp, styleval, spl[-1]))
             else:
                 outfile.write(value)