diff options
| -rw-r--r-- | CHANGES | 4 | ||||
| -rw-r--r-- | docs/src/tokens.txt | 8 | ||||
| -rw-r--r-- | pygments/filters/__init__.py | 169 | ||||
| -rw-r--r-- | pygments/formatters/terminal.py | 3 | ||||
| -rw-r--r-- | pygments/lexers/functional.py | 6 | ||||
| -rw-r--r-- | pygments/styles/autumn.py | 80 | ||||
| -rw-r--r-- | pygments/styles/borland.py | 4 | ||||
| -rw-r--r-- | pygments/styles/colorful.py | 4 | ||||
| -rw-r--r-- | pygments/styles/default.py | 3 | ||||
| -rw-r--r-- | pygments/styles/emacs.py | 3 | ||||
| -rw-r--r-- | pygments/styles/friendly.py | 3 | ||||
| -rw-r--r-- | pygments/styles/fruity.py | 3 | ||||
| -rw-r--r-- | pygments/styles/manni.py | 3 | ||||
| -rw-r--r-- | pygments/styles/murphy.py | 3 | ||||
| -rw-r--r-- | pygments/styles/native.py | 3 | ||||
| -rw-r--r-- | pygments/styles/pastie.py | 3 | ||||
| -rw-r--r-- | pygments/styles/perldoc.py | 3 | ||||
| -rw-r--r-- | pygments/styles/trac.py | 3 | ||||
| -rw-r--r-- | pygments/token.py | 2 | ||||
| -rw-r--r-- | tests/run.py | 4 | ||||
| -rw-r--r-- | tests/test_basic_api.py | 36 |
21 files changed, 262 insertions, 88 deletions
@@ -5,6 +5,10 @@ Version 0.8 (in development) ---------------------------- (codename to be selected, released May XX, 2007) +- Added a `RaiseOnErrorTokenFilter` that raises an exception when the + lexer generates an error token, and a `VisibleWhitespaceFilter` that + converts whitespace (spaces, tabs, newlines) into visible characters. + - Fixed the `do_insertions()` helper function to yield correct indices. - The ReST lexer now automatically highlights source code blocks in diff --git a/docs/src/tokens.txt b/docs/src/tokens.txt index f98b4d6e..3a1cd4af 100644 --- a/docs/src/tokens.txt +++ b/docs/src/tokens.txt @@ -57,8 +57,9 @@ For some tokens aliases are already defined: Inside the `pygments.token` module the following aliases are defined: -============= ============================ ================================== +============= ============================ ==================================== `Text` `Token.Text` for any type of text data +`Whitespace` `Token.Text.Whitespace` for specially highlighted whitespace `Error` `Token.Error` represents lexer errors `Other` `Token.Other` special token for data not matched by a parser (e.g. HTML @@ -73,7 +74,10 @@ Inside the `pygments.token` module the following aliases are defined: `Comment` `Token.Comment` any kind of comments `Generic` `Token.Generic` generic tokens (have a look at the explanation below) -============= ============================ ================================== +============= ============================ ==================================== + +The `Whitespace` token type is new in Pygments 0.8. It is used only by the +`VisibleWhitespaceFilter` currently. Normally you just create token types using the already defined aliases. For each of those token aliases, a number of subtypes exists (excluding the special tokens diff --git a/pygments/filters/__init__.py b/pygments/filters/__init__.py index 4b4bd28d..3df10a4f 100644 --- a/pygments/filters/__init__.py +++ b/pygments/filters/__init__.py @@ -15,10 +15,11 @@ except NameError: from sets import Set as set import re -from pygments.token import String, Comment, Keyword, Name, Error, \ +from pygments.token import String, Comment, Keyword, Name, Error, Whitespace, \ string_to_tokentype from pygments.filter import Filter -from pygments.util import get_list_opt, ClassNotFound +from pygments.util import get_list_opt, get_int_opt, get_bool_opt, \ + ClassNotFound, OptionError from pygments.plugin import find_plugin_filters @@ -56,14 +57,30 @@ def get_all_filters(): yield name +def _replace_special(ttype, value, regex, specialttype, + replacefunc=lambda x: x): + last = 0 + for match in regex.finditer(value): + start, end = match.start(), match.end() + if start != last: + yield ttype, value[last:start] + yield specialttype, replacefunc(value[start:end]) + last = end + if last != len(value): + yield ttype, value[last:] + + class CodeTagFilter(Filter): """ Highlight special code tags in comments and docstrings. - Per default, the list of highlighted tags is ``XXX``, ``TODO``, ``BUG`` and - ``NOTE``. You can override this list by specifying a `codetags` parameter - that takes a list of words. + Options accepted: + + `codetags` : list of strings + A list of strings that are flagged as code tags. The default is to + highlight ``XXX``, ``TODO``, ``BUG`` and ``NOTE``. """ + def __init__(self, **options): Filter.__init__(self, **options) tags = get_list_opt(options, 'codetags', @@ -73,39 +90,38 @@ class CodeTagFilter(Filter): ])) def filter(self, lexer, stream): + regex = self.tag_re for ttype, value in stream: if ttype in String.Doc or \ ttype in Comment and \ ttype not in Comment.Preproc: - last = 0 - for match in self.tag_re.finditer(value): - start = match.start() - end = match.end() - if start != last: - yield ttype, value[last:start] - yield Comment.Special, value[start:end] - last = end - if last != len(value): - yield ttype, value[last:] - continue - yield ttype, value + for sttype, svalue in _replace_special(ttype, value, regex, + Comment.Special): + yield sttype, svalue + else: + yield ttype, value class KeywordCaseFilter(Filter): """ - Convert keywords to ``lower``, ``upper`` or ``capitalize`` which means - first letter uppercase, rest lowercase. + Convert keywords to lowercase or uppercase or capitalize them, which + means first letter uppercase, rest lowercase. This can be useful e.g. if you highlight Pascal code and want to adapt the - code to your styleguide. The default is ``lower``, override that by - providing the `case` parameter. + code to your styleguide. + + Options accepted: + + `case` : string + The casing to convert keywords to. Must be one of ``'lower'``, + ``'upper'`` or ``'capitalize'``. The default is ``'lower'``. """ def __init__(self, **options): Filter.__init__(self, **options) case = options.get('case', 'lower') if case not in ('lower', 'upper', 'capitalize'): - raise TypeError('unknown conversion method %r' % case) + raise OptionError('unknown conversion method %r' % case) self.convert = getattr(unicode, case) def filter(self, lexer, stream): @@ -129,6 +145,16 @@ class NameHighlightFilter(Filter): This would highlight the names "foo", "bar" and "baz" as functions. `Name.Function` is the default token type. + + Options accepted: + + `names` : list of strings + A list of names that should be given the different token type. + There is no default. + `tokentype` : TokenType or string + A token type or a string containing a token type name that is + used for highlighting the strings in `names`. The default is + `Name.Function`. """ def __init__(self, **options): @@ -153,22 +179,115 @@ class ErrorToken(Exception): class RaiseOnErrorTokenFilter(Filter): """ - Raise an ``pygments.filters.ErrorToken`` exception when the lexer - generates an error token. + Raise an exception when the lexer generates an error token. + + Options accepted: + + `excclass` : Exception class + The exception class to raise. + The default is `pygments.filters.ErrorToken`. *New in Pygments 0.8.* """ + def __init__(self, **options): + Filter.__init__(self, **options) + self.exception = options.get('excclass', ErrorToken) + try: + # issubclass() will raise TypeError if first argument is not a class + if not issubclass(self.exception, Exception): + raise TypeError + except TypeError: + raise OptionError('excclass option is not an exception class') + def filter(self, lexer, stream): for ttype, value in stream: if ttype is Error: - raise ErrorToken(value) + raise self.exception(value) yield ttype, value +class VisibleWhitespaceFilter(Filter): + """ + Convert tabs, newlines and/or spaces to visible characters. + + Options accepted: + + `spaces` : string or bool + If this is a one-character string, spaces will be replaces by this string. + If it is another true value, spaces will be replaced by ``·`` (unicode + MIDDLE DOT). If it is a false value, spaces will not be replaced. The + default is ``False``. + `tabs` : string or bool + The same as for `spaces`, but the default replacement character is ``»`` + (unicode RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK). The default value + is ``False``. Note: this will not work if the `tabsize` option for the + lexer is nonzero, as tabs will already have been expanded then. + `tabsize` : int + If tabs are to be replaced by this filter, this is the total number of + characters that a tab should be expanded to. The default is ``8``. + `newlines` : string or bool + The same as for `spaces`, but the default replacement character is ``¶`` + (unicode PILCROW SIGN). The default value is ``False``. + `wstokentype` : bool + If true, give whitespace the special `Whitespace` token type. This allows + styling the visible whitespace differently (e.g. greyed out), but it can + disrupt background colors. The default is ``True`` + + *New in Pygments 0.8.* + """ + + def __init__(self, **options): + Filter.__init__(self, **options) + for name, default in {'spaces': u'·', 'tabs': u'»', 'newlines': u'¶'}.items(): + opt = options.get(name, False) + if isinstance(opt, basestring) and len(opt) == 1: + setattr(self, name, opt) + else: + setattr(self, name, (opt and default or '')) + tabsize = get_int_opt(options, 'tabsize', 8) + if self.tabs: + self.tabs += ' '*(tabsize-1) + if self.newlines: + self.newlines += '\n' + self.wstt = get_bool_opt(options, 'wstokentype', True) + + def filter(self, lexer, stream): + if self.wstt: + spaces = self.spaces or ' ' + tabs = self.tabs or '\t' + newlines = self.newlines or '\n' + regex = re.compile(r'\s') + def replacefunc(wschar): + if wschar == ' ': + return spaces + elif wschar == '\t': + return tabs + elif wschar == '\n': + return newlines + return wschar + + for ttype, value in stream: + for sttype, svalue in _replace_special(ttype, value, regex, + Whitespace, replacefunc): + yield sttype, svalue + else: + spaces, tabs, newlines = self.spaces, self.tabs, self.newlines + # simpler processing + for ttype, value in stream: + if spaces: + value = value.replace(' ', spaces) + if tabs: + value = value.replace('\t', tabs) + if newlines: + value = value.replace('\n', newlines) + yield ttype, value + + FILTERS = { 'codetagify': CodeTagFilter, 'keywordcase': KeywordCaseFilter, 'highlight': NameHighlightFilter, 'raiseonerror': RaiseOnErrorTokenFilter, + 'whitespace': VisibleWhitespaceFilter, } diff --git a/pygments/formatters/terminal.py b/pygments/formatters/terminal.py index cad37c7d..adac90df 100644 --- a/pygments/formatters/terminal.py +++ b/pygments/formatters/terminal.py @@ -11,7 +11,7 @@ from pygments.formatter import Formatter from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic, Token + Number, Operator, Generic, Token, Whitespace from pygments.console import ansiformat @@ -23,6 +23,7 @@ __all__ = ['TerminalFormatter'] TERMINAL_COLORS = { Token: ('', ''), + Whitespace: ('lightgray', 'darkgray'), Comment: ('lightgray', 'darkgray'), Keyword: ('darkblue', 'blue'), Keyword.Type: ('teal', 'turquoise'), diff --git a/pygments/lexers/functional.py b/pygments/lexers/functional.py index e1e442e2..a248b476 100644 --- a/pygments/lexers/functional.py +++ b/pygments/lexers/functional.py @@ -16,9 +16,9 @@ try: except NameError: from sets import Set as set -from pygments.lexer import Lexer, RegexLexer -from pygments.token import Error, Text, \ - Comment, Operator, Keyword, Name, String, Number, Generic, Punctuation +from pygments.lexer import RegexLexer +from pygments.token import Text, Comment, Operator, Keyword, Name, \ + String, Number, Punctuation __all__ = ['SchemeLexer', 'HaskellLexer'] diff --git a/pygments/styles/autumn.py b/pygments/styles/autumn.py index 4508db17..b0862e41 100644 --- a/pygments/styles/autumn.py +++ b/pygments/styles/autumn.py @@ -11,7 +11,7 @@ from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic + Number, Operator, Generic, Whitespace class AutumnStyle(Style): @@ -22,42 +22,44 @@ class AutumnStyle(Style): default_style = "" styles = { - Comment: "italic #aaaaaa", - Comment.Preproc: "noitalic #4c8317", - Comment.Special: "italic #0000aa", - - Keyword: "#0000aa", - Keyword.Type: "#00aaaa", - - Operator.Word: "#0000aa", - - Name.Builtin: "#00aaaa", - Name.Function: "#00aa00", - Name.Class: "underline #00aa00", - Name.Namespace: "underline #00aaaa", - Name.Variable: "#aa0000", - Name.Constant: "#aa0000", - Name.Entity: "bold #800", - Name.Attribute: "#1e90ff", - Name.Tag: "bold #1e90ff", - Name.Decorator: "#888888", - - String: "#aa5500", - String.Symbol: "#0000aa", - String.Regex: "#009999", - - Number: "#009999", - - Generic.Heading: "bold #000080", - Generic.Subheading: "bold #800080", - Generic.Deleted: "#aa0000", - Generic.Inserted: "#00aa00", - Generic.Error: "#aa0000", - Generic.Emph: "italic", - Generic.Strong: "bold", - Generic.Prompt: "#555555", - Generic.Output: "#888888", - Generic.Traceback: "#aa0000", - - Error: "#F00 bg:#FAA" + Whitespace: '#bbbbbb', + + Comment: 'italic #aaaaaa', + Comment.Preproc: 'noitalic #4c8317', + Comment.Special: 'italic #0000aa', + + Keyword: '#0000aa', + Keyword.Type: '#00aaaa', + + Operator.Word: '#0000aa', + + Name.Builtin: '#00aaaa', + Name.Function: '#00aa00', + Name.Class: 'underline #00aa00', + Name.Namespace: 'underline #00aaaa', + Name.Variable: '#aa0000', + Name.Constant: '#aa0000', + Name.Entity: 'bold #800', + Name.Attribute: '#1e90ff', + Name.Tag: 'bold #1e90ff', + Name.Decorator: '#888888', + + String: '#aa5500', + String.Symbol: '#0000aa', + String.Regex: '#009999', + + Number: '#009999', + + Generic.Heading: 'bold #000080', + Generic.Subheading: 'bold #800080', + Generic.Deleted: '#aa0000', + Generic.Inserted: '#00aa00', + Generic.Error: '#aa0000', + Generic.Emph: 'italic', + Generic.Strong: 'bold', + Generic.Prompt: '#555555', + Generic.Output: '#888888', + Generic.Traceback: '#aa0000', + + Error: '#F00 bg:#FAA' } diff --git a/pygments/styles/borland.py b/pygments/styles/borland.py index 696178d3..5763073b 100644 --- a/pygments/styles/borland.py +++ b/pygments/styles/borland.py @@ -11,7 +11,7 @@ from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic + Number, Operator, Generic, Whitespace class BorlandStyle(Style): @@ -22,6 +22,8 @@ class BorlandStyle(Style): default_style = '' styles = { + Whitespace: '#bbbbbb', + Comment: 'italic #008800', Comment.Preproc: 'noitalic', Comment.Special: 'noitalic bold', diff --git a/pygments/styles/colorful.py b/pygments/styles/colorful.py index 16f084e6..83988291 100644 --- a/pygments/styles/colorful.py +++ b/pygments/styles/colorful.py @@ -11,7 +11,7 @@ from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic + Number, Operator, Generic, Whitespace class ColorfulStyle(Style): @@ -22,6 +22,8 @@ class ColorfulStyle(Style): default_style = "" styles = { + Whitespace: "#bbbbbb", + Comment: "#888", Comment.Preproc: "#579", Comment.Special: "bold #cc0000", diff --git a/pygments/styles/default.py b/pygments/styles/default.py index bfa40099..b66d4c95 100644 --- a/pygments/styles/default.py +++ b/pygments/styles/default.py @@ -12,7 +12,7 @@ from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic + Number, Operator, Generic, Whitespace class DefaultStyle(Style): @@ -24,6 +24,7 @@ class DefaultStyle(Style): default_style = "" styles = { + Whitespace: "#bbbbbb", Comment: "italic #408080", Comment.Preproc: "noitalic #BC7A00", diff --git a/pygments/styles/emacs.py b/pygments/styles/emacs.py index cf7b2dc8..c27268b8 100644 --- a/pygments/styles/emacs.py +++ b/pygments/styles/emacs.py @@ -11,7 +11,7 @@ from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic + Number, Operator, Generic, Whitespace class EmacsStyle(Style): @@ -23,6 +23,7 @@ class EmacsStyle(Style): default_style = "" styles = { + Whitespace: "#bbbbbb", Comment: "italic #008800", Comment.Preproc: "noitalic", Comment.Special: "noitalic bold", diff --git a/pygments/styles/friendly.py b/pygments/styles/friendly.py index b1e95037..95ee3b37 100644 --- a/pygments/styles/friendly.py +++ b/pygments/styles/friendly.py @@ -11,7 +11,7 @@ from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic + Number, Operator, Generic, Whitespace class FriendlyStyle(Style): @@ -23,6 +23,7 @@ class FriendlyStyle(Style): default_style = "" styles = { + Whitespace: "#bbbbbb", Comment: "italic #60a0b0", Comment.Preproc: "noitalic #007020", Comment.Special: "noitalic bg:#fff0f0", diff --git a/pygments/styles/fruity.py b/pygments/styles/fruity.py index ba1f388a..95fbcc3e 100644 --- a/pygments/styles/fruity.py +++ b/pygments/styles/fruity.py @@ -11,7 +11,7 @@ from pygments.style import Style from pygments.token import Token, Comment, Name, Keyword, \ - Generic, Number, String + Generic, Number, String, Whitespace class FruityStyle(Style): """ @@ -21,6 +21,7 @@ class FruityStyle(Style): background_color = '#111111' styles = { + Whitespace: '#888888', Token: '#ffffff', Generic.Output: '#444444 bg:#222222', Keyword: '#fb660a bold', diff --git a/pygments/styles/manni.py b/pygments/styles/manni.py index bdc979e2..c160f873 100644 --- a/pygments/styles/manni.py +++ b/pygments/styles/manni.py @@ -14,7 +14,7 @@ from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic + Number, Operator, Generic, Whitespace class ManniStyle(Style): @@ -25,6 +25,7 @@ class ManniStyle(Style): background_color = '#f0f3f3' styles = { + Whitespace: '#bbbbbb', Comment: 'italic #0099FF', Comment.Preproc: 'noitalic #009999', Comment.Special: 'bold', diff --git a/pygments/styles/murphy.py b/pygments/styles/murphy.py index 06cc795b..5152fe68 100644 --- a/pygments/styles/murphy.py +++ b/pygments/styles/murphy.py @@ -11,7 +11,7 @@ from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic + Number, Operator, Generic, Whitespace class MurphyStyle(Style): @@ -22,6 +22,7 @@ class MurphyStyle(Style): default_style = "" styles = { + Whitespace: "#bbbbbb", Comment: "#666 italic", Comment.Preproc: "#579 noitalic", Comment.Special: "#c00 bold", diff --git a/pygments/styles/native.py b/pygments/styles/native.py index c9b290b0..76021a53 100644 --- a/pygments/styles/native.py +++ b/pygments/styles/native.py @@ -11,7 +11,7 @@ from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic, Token + Number, Operator, Generic, Token, Whitespace class NativeStyle(Style): @@ -23,6 +23,7 @@ class NativeStyle(Style): styles = { Token: '#d0d0d0', + Whitespace: '#666666', Comment: 'italic #999999', Comment.Preproc: 'noitalic bold #cd2828', diff --git a/pygments/styles/pastie.py b/pygments/styles/pastie.py index 49ab9cee..b0332252 100644 --- a/pygments/styles/pastie.py +++ b/pygments/styles/pastie.py @@ -13,7 +13,7 @@ from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic + Number, Operator, Generic, Whitespace class PastieStyle(Style): @@ -24,6 +24,7 @@ class PastieStyle(Style): default_style = '' styles = { + Whitespace: '#bbbbbb', Comment: '#888888', Comment.Preproc: 'bold #cc0000', Comment.Special: 'bg:#fff0f0 bold #cc0000', diff --git a/pygments/styles/perldoc.py b/pygments/styles/perldoc.py index 6ce06624..91e63074 100644 --- a/pygments/styles/perldoc.py +++ b/pygments/styles/perldoc.py @@ -13,7 +13,7 @@ from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic + Number, Operator, Generic, Whitespace class PerldocStyle(Style): @@ -25,6 +25,7 @@ class PerldocStyle(Style): default_style = '' styles = { + Whitespace: '#bbbbbb', Comment: '#228B22', Comment.Preproc: '#1e889b', Comment.Special: '#8B008B bold', diff --git a/pygments/styles/trac.py b/pygments/styles/trac.py index 0e32986f..d6bd4cf1 100644 --- a/pygments/styles/trac.py +++ b/pygments/styles/trac.py @@ -11,7 +11,7 @@ from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic + Number, Operator, Generic, Whitespace class TracStyle(Style): @@ -22,6 +22,7 @@ class TracStyle(Style): default_style = '' styles = { + Whitespace: '#bbbbbb', Comment: 'italic #999988', Comment.Preproc: 'bold noitalic #999999', Comment.Special: 'bold #999999', diff --git a/pygments/token.py b/pygments/token.py index e4b81e55..66183955 100644 --- a/pygments/token.py +++ b/pygments/token.py @@ -56,6 +56,7 @@ Token = _TokenType() # Special token types Text = Token.Text +Whitespace = Text.Whitespace Error = Token.Error # Text that doesn't belong to this lexer (e.g. HTML in PHP) Other = Token.Other @@ -122,6 +123,7 @@ STANDARD_TYPES = { Token: '', Text: '', + Whitespace: 'w', Error: 'err', Other: 'x', diff --git a/tests/run.py b/tests/run.py index bd285c2b..8a78af5c 100644 --- a/tests/run.py +++ b/tests/run.py @@ -77,8 +77,8 @@ def run_tests(with_coverage=False): WIDTH = 70 print >>sys.stderr, \ - ('Pygments Test Suite running %s, stand by...' % - (with_coverage and "with coverage analysis" or "")).center(WIDTH) + ('Pygments Test Suite running%s, stand by...' % + (with_coverage and " with coverage analysis" or "")).center(WIDTH) print >>sys.stderr, ('(using Python %s)' % sys.version.split()[0]).center(WIDTH) print >>sys.stderr, '='*WIDTH diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index d05a29c5..805f3104 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -71,15 +71,43 @@ class LexersTest(unittest.TestCase): a(isinstance(x, lexers.PythonLexer)) ae(x.options["opt"], "val") - def test_filters(self): + +class FiltersTest(unittest.TestCase): + + def test_basic(self): + filter_args = { + 'whitespace': {'spaces': True, 'tabs': True, 'newlines': True}, + 'highlight': {'names': ['isinstance', 'lexers', 'x']}, + } for x in filters.FILTERS.keys(): lx = lexers.PythonLexer() - lx.add_filter(x) + lx.add_filter(x, **filter_args.get(x, {})) text = file(os.path.join(testdir, testfile)).read().decode('utf-8') tokens = list(lx.get_tokens(text)) roundtext = ''.join([t[1] for t in tokens]) - self.assertEquals(roundtext, text, - "lexer roundtrip with %s filter failed" % x) + if x not in ('whitespace', 'keywordcase'): + # these filters change the text + self.assertEquals(roundtext, text, + "lexer roundtrip with %s filter failed" % x) + + def test_raiseonerror(self): + lx = lexers.PythonLexer() + lx.add_filter('raiseonerror', excclass=RuntimeError) + self.assertRaises(RuntimeError, list, lx.get_tokens('$')) + + def test_whitespace(self): + lx = lexers.PythonLexer() + lx.add_filter('whitespace', spaces='%') + text = file(os.path.join(testdir, testfile)).read().decode('utf-8') + lxtext = ''.join([t[1] for t in list(lx.get_tokens(text))]) + self.failIf(' ' in lxtext) + + def test_keywordcase(self): + lx = lexers.PythonLexer() + lx.add_filter('keywordcase', case='capitalize') + text = file(os.path.join(testdir, testfile)).read().decode('utf-8') + lxtext = ''.join([t[1] for t in list(lx.get_tokens(text))]) + self.assert_('Def' in lxtext and 'Class' in lxtext) class FormattersTest(unittest.TestCase): |