Merged in timgilbert/pygments-main/clj-keyword-fix (pull request #326)

Tweaking clojure keyword lexing

5 files changed, 72 insertions, 20 deletions

diff --git a/pygments/formatters/_mapping.py b/pygments/formatters/_mapping.py
index 79f592b3..4c05ac8e 100755
--- a/pygments/formatters/_mapping.py
+++ b/pygments/formatters/_mapping.py
@@ -25,6 +25,7 @@ from pygments.formatters.img import JpgImageFormatter
 from pygments.formatters.latex import LatexFormatter
 from pygments.formatters.other import NullFormatter
 from pygments.formatters.other import RawTokenFormatter
+from pygments.formatters.other import TestcaseFormatter
 from pygments.formatters.rtf import RtfFormatter
 from pygments.formatters.svg import SvgFormatter
 from pygments.formatters.terminal import TerminalFormatter
@@ -43,7 +44,8 @@ FORMATTERS = {
     RtfFormatter: ('RTF', ('rtf',), ('*.rtf',), 'Format tokens as RTF markup. This formatter automatically outputs full RTF documents with color information and other useful stuff. Perfect for Copy and Paste into Microsoft\xc2\xae Word\xc2\xae documents.'),
     SvgFormatter: ('SVG', ('svg',), ('*.svg',), 'Format tokens as an SVG graphics file.  This formatter is still experimental. Each line of code is a ``<text>`` element with explicit ``x`` and ``y`` coordinates containing ``<tspan>`` elements with the individual token styles.'),
     Terminal256Formatter: ('Terminal256', ('terminal256', 'console256', '256'), (), 'Format tokens with ANSI color sequences, for output in a 256-color terminal or console. Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'),
-    TerminalFormatter: ('Terminal', ('terminal', 'console'), (), 'Format tokens with ANSI color sequences, for output in a text console. Color sequences are terminated at newlines, so that paging the output works correctly.')
+    TerminalFormatter: ('Terminal', ('terminal', 'console'), (), 'Format tokens with ANSI color sequences, for output in a text console. Color sequences are terminated at newlines, so that paging the output works correctly.'),
+    TestcaseFormatter: ('Testcase', ('testcase',), (), 'Format tokens as appropriate for a new testcase.')
 }
 
 if __name__ == '__main__':
diff --git a/pygments/formatters/html.py b/pygments/formatters/html.py
index 3bc60e8a..970e595b 100644
--- a/pygments/formatters/html.py
+++ b/pygments/formatters/html.py
@@ -523,7 +523,8 @@ class HtmlFormatter(Formatter):
                                                self.cssfile)
                 except AttributeError:
                     print('Note: Cannot determine output file name, ' \
-                          'using current directory as base for the CSS file name', file=sys.stderr)
+                          'using current directory as base for the CSS file name',
+                          file=sys.stderr)
                     cssfilename = self.cssfile
             # write CSS file only if noclobber_cssfile isn't given as an option.
             try:
diff --git a/pygments/formatters/latex.py b/pygments/formatters/latex.py
index 413cca63..352684b0 100644
--- a/pygments/formatters/latex.py
+++ b/pygments/formatters/latex.py
@@ -375,9 +375,9 @@ class LatexFormatter(Formatter):
                         if len(sep1) > 0:
                             b,sep2,text = text.partition(self.right)
                             if len(sep2) > 0:
-                                value = value + escape_tex(a, self.commandprefix) + b
+                                value += escape_tex(a, self.commandprefix) + b
                             else:
-                                value = value + escape_tex(a + sep1 + b, self.commandprefix)
+                                value += escape_tex(a + sep1 + b, self.commandprefix)
                         else:
                             value = value + escape_tex(a, self.commandprefix)
                 else:
diff --git a/pygments/formatters/other.py b/pygments/formatters/other.py
index 7368a642..c8269b19 100644
--- a/pygments/formatters/other.py
+++ b/pygments/formatters/other.py
@@ -14,7 +14,7 @@ from pygments.util import OptionError, get_choice_opt
 from pygments.token import Token
 from pygments.console import colorize
 
-__all__ = ['NullFormatter', 'RawTokenFormatter']
+__all__ = ['NullFormatter', 'RawTokenFormatter', 'TestcaseFormatter']
 
 
 class NullFormatter(Formatter):
@@ -114,3 +114,49 @@ class RawTokenFormatter(Formatter):
             for ttype, value in tokensource:
                 write("%s\t%r\n" % (ttype, value))
         flush()
+
+TESTCASE_BEFORE = u'''\
+    def testNeedsName(self):
+        fragment = %r
+        tokens = [
+'''
+TESTCASE_AFTER = u'''\
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+'''
+
+
+class TestcaseFormatter(Formatter):
+    """
+    Format tokens as appropriate for a new testcase.
+
+    .. versionadded:: 2.0
+    """
+    name = 'Testcase'
+    aliases = ['testcase']
+
+    def __init__(self, **options):
+        Formatter.__init__(self, **options)
+        #if self.encoding != 'utf-8':
+        #    print >>sys.stderr, "NOTICE: Forcing encoding to utf-8, as all Pygments source is"
+        if self.encoding is not None and self.encoding != 'utf-8':
+            raise ValueError("Only None and utf-u are allowed encodings.")
+
+    def format(self, tokensource, outfile):
+        indentation = ' ' * 12
+        rawbuf = []
+        outbuf = []
+        for ttype, value in tokensource:
+            rawbuf.append(value)
+            outbuf.append('%s(%s, %r),\n' % (indentation, ttype, value))
+
+        before = TESTCASE_BEFORE % (u''.join(rawbuf),)
+        during = u''.join(outbuf)
+        after = TESTCASE_AFTER
+        if self.encoding is None:
+            outfile.write(before + during + after)
+        else:
+            outfile.write(before.encode('utf-8'))
+            outfile.write(during.encode('utf-8'))
+            outfile.write(after.encode('utf-8'))
+        outfile.flush()
diff --git a/pygments/formatters/rtf.py b/pygments/formatters/rtf.py
index 9d87e8f1..cf65a927 100644
--- a/pygments/formatters/rtf.py
+++ b/pygments/formatters/rtf.py
@@ -10,7 +10,7 @@
 """
 
 from pygments.formatter import Formatter
-from pygments.util import get_int_opt
+from pygments.util import get_int_opt, _surrogatepair
 
 
 __all__ = ['RtfFormatter']
@@ -22,6 +22,10 @@ class RtfFormatter(Formatter):
     documents with color information and other useful stuff. Perfect for Copy and
     Paste into Microsoft® Word® documents.
 
+    Please note that ``encoding`` and ``outencoding`` options are ignored.
+    The RTF format is ASCII natively, but handles unicode characters correctly
+    thanks to escape sequences.
+
     .. versionadded:: 0.6
 
     Additional options accepted:
@@ -74,28 +78,27 @@ class RtfFormatter(Formatter):
 
         # escape text
         text = self._escape(text)
-        if self.encoding in ('utf-8', 'utf-16', 'utf-32'):
-            encoding = 'iso-8859-15'
-        else:
-            encoding = self.encoding or 'iso-8859-15'
 
         buf = []
         for c in text:
-            if ord(c) > 128:
-                ansic = c.encode(encoding, 'ignore') or '?'
-                if ord(ansic) > 128:
-                    ansic = '\\\'%x' % ord(ansic)
-                else:
-                    ansic = c
-                buf.append(r'\ud{\u%d%s}' % (ord(c), ansic))
-            else:
+            cn = ord(c)
+            if cn < (2**7):
+                # ASCII character
                 buf.append(str(c))
+            elif (2**7) <= cn < (2**16):
+                # single unicode escape sequence
+                buf.append(r'{\u%d}' % cn)
+            elif (2**16) <= cn:
+                # RTF limits unicode to 16 bits.
+                # Force surrogate pairs
+                h,l = _surrogatepair(cn)
+                buf.append(r'{\u%d}{\u%d}' % (h,l))
 
         return ''.join(buf).replace('\n', '\\par\n')
 
     def format_unencoded(self, tokensource, outfile):
         # rtf 1.8 header
-        outfile.write(r'{\rtf1\ansi\deff0'
+        outfile.write(r'{\rtf1\ansi\uc0\deff0'
                       r'{\fonttbl{\f0\fmodern\fprq1\fcharset0%s;}}'
                       r'{\colortbl;' % (self.fontface and
                                         ' ' + self._escape(self.fontface) or
@@ -114,7 +117,7 @@ class RtfFormatter(Formatter):
                         int(color[4:6], 16)
                     ))
                     offset += 1
-        outfile.write(r'}\f0')
+        outfile.write(r'}\f0 ')
         if self.fontsize:
             outfile.write(r'\fs%d' % (self.fontsize))