diff options
Diffstat (limited to 'scripts/find_error.py')
| -rwxr-xr-x | scripts/find_error.py | 62 |
1 files changed, 35 insertions, 27 deletions
diff --git a/scripts/find_error.py b/scripts/find_error.py index cb22ae89..7aaa9bee 100755 --- a/scripts/find_error.py +++ b/scripts/find_error.py @@ -8,22 +8,25 @@ the text where Error tokens are being generated, along with some context. - :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -import sys, os +from __future__ import print_function -try: - import pygments -except ImportError: - # try parent path - sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +import os +import sys + +# always prefer Pygments from source if exists +srcpath = os.path.join(os.path.dirname(__file__), '..') +if os.path.isdir(os.path.join(srcpath, 'pygments')): + sys.path.insert(0, srcpath) from pygments.lexer import RegexLexer from pygments.lexers import get_lexer_for_filename, get_lexer_by_name from pygments.token import Error, Text, _TokenType +from pygments.cmdline import _parse_options class DebuggingRegexLexer(RegexLexer): @@ -83,16 +86,16 @@ class DebuggingRegexLexer(RegexLexer): break -def main(fn, lexer=None): +def main(fn, lexer=None, options={}): if lexer is not None: lx = get_lexer_by_name(lexer) else: try: - lx = get_lexer_for_filename(os.path.basename(fn)) + lx = get_lexer_for_filename(os.path.basename(fn), **options) except ValueError: try: name, rest = fn.split('_', 1) - lx = get_lexer_by_name(name) + lx = get_lexer_by_name(name, **options) except ValueError: raise AssertionError('no lexer found for file %r' % fn) debug_lexer = False @@ -100,38 +103,40 @@ def main(fn, lexer=None): if lx.__class__.__bases__ == (RegexLexer,): lx.__class__.__bases__ = (DebuggingRegexLexer,) debug_lexer = True + elif lx.__class__.__bases__ == (DebuggingRegexLexer,): + # already debugged before + debug_lexer = True lno = 1 - text = file(fn, 'U').read() + text = open(fn, 'U').read() text = text.strip('\n') + '\n' - text = text.decode('latin1') tokens = [] states = [] def show_token(tok, state): reprs = map(repr, tok) - print ' ' + reprs[1] + ' ' + ' ' * (29-len(reprs[1])) + reprs[0], + print(' ' + reprs[1] + ' ' + ' ' * (29-len(reprs[1])) + reprs[0], end=' ') if debug_lexer: - print ' ' + ' ' * (29-len(reprs[0])) + repr(state), - print + print(' ' + ' ' * (29-len(reprs[0])) + repr(state), end=' ') + print() for type, val in lx.get_tokens(text): lno += val.count('\n') if type == Error: - print 'Error parsing', fn, 'on line', lno - print 'Previous tokens' + (debug_lexer and ' and states' or '') + ':' + print('Error parsing', fn, 'on line', lno) + print('Previous tokens' + (debug_lexer and ' and states' or '') + ':') if showall: - for tok, state in zip(tokens, states): + for tok, state in map(None, tokens, states): show_token(tok, state) else: - for i in range(len(tokens) - num, len(tokens)): + for i in range(max(len(tokens) - num, 0), len(tokens)): show_token(tokens[i], states[i]) - print 'Error token:' + print('Error token:') l = len(repr(val)) - print ' ' + repr(val), + print(' ' + repr(val), end=' ') if debug_lexer and hasattr(lx, 'statestack'): - print ' ' * (60-l) + repr(lx.statestack), - print - print + print(' ' * (60-l) + repr(lx.statestack), end=' ') + print() + print() return 1 tokens.append((type, val)) if debug_lexer: @@ -140,7 +145,7 @@ def main(fn, lexer=None): else: states.append(None) if showall: - for tok, state in zip(tokens, states): + for tok, state in map(None, tokens, states): show_token(tok, state) return 0 @@ -148,10 +153,11 @@ def main(fn, lexer=None): num = 10 showall = False lexer = None +options = {} if __name__ == '__main__': import getopt - opts, args = getopt.getopt(sys.argv[1:], 'n:l:a') + opts, args = getopt.getopt(sys.argv[1:], 'n:l:aO:') for opt, val in opts: if opt == '-n': num = int(val) @@ -159,7 +165,9 @@ if __name__ == '__main__': showall = True elif opt == '-l': lexer = val + elif opt == '-O': + options = _parse_options([val]) ret = 0 for f in args: - ret += main(f, lexer) + ret += main(f, lexer, options) sys.exit(bool(ret)) |