diff options
Diffstat (limited to 'pygments/util.py')
| -rw-r--r-- | pygments/util.py | 214 |
1 files changed, 162 insertions, 52 deletions
diff --git a/pygments/util.py b/pygments/util.py index caac1144..0859c05d 100644 --- a/pygments/util.py +++ b/pygments/util.py @@ -5,31 +5,31 @@ Utility functions. - :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re import sys -import codecs split_path_re = re.compile(r'[/\\ ]') doctype_lookup_re = re.compile(r'''(?smx) (<\?.*?\?>)?\s* <!DOCTYPE\s+( + [a-zA-Z_][a-zA-Z0-9]* + (?: \s+ # optional in HTML5 [a-zA-Z_][a-zA-Z0-9]*\s+ - [a-zA-Z_][a-zA-Z0-9]*\s+ - "[^"]*") + "[^"]*")? + ) [^>]*> ''') tag_re = re.compile(r'<(.+?)(\s.*?)?>.*?</.+?>(?uism)') +xml_decl_re = re.compile(r'\s*<\?xml[^>]*\?>', re.I) class ClassNotFound(ValueError): - """ - If one of the get_*_by_* functions didn't find a matching class. - """ + """Raised if one of the lookup functions didn't find a matching class.""" class OptionError(Exception): @@ -52,10 +52,10 @@ def get_bool_opt(options, optname, default=None): return string elif isinstance(string, int): return bool(string) - elif not isinstance(string, basestring): + elif not isinstance(string, string_types): raise OptionError('Invalid type %r for option %s; use ' '1/0, yes/no, true/false, on/off' % ( - string, optname)) + string, optname)) elif string.lower() in ('1', 'yes', 'true', 'on'): return True elif string.lower() in ('0', 'no', 'false', 'off'): @@ -63,7 +63,7 @@ def get_bool_opt(options, optname, default=None): else: raise OptionError('Invalid value %r for option %s; use ' '1/0, yes/no, true/false, on/off' % ( - string, optname)) + string, optname)) def get_int_opt(options, optname, default=None): @@ -73,23 +73,23 @@ def get_int_opt(options, optname, default=None): except TypeError: raise OptionError('Invalid type %r for option %s; you ' 'must give an integer value' % ( - string, optname)) + string, optname)) except ValueError: raise OptionError('Invalid value %r for option %s; you ' 'must give an integer value' % ( - string, optname)) + string, optname)) def get_list_opt(options, optname, default=None): val = options.get(optname, default) - if isinstance(val, basestring): + if isinstance(val, string_types): return val.split() elif isinstance(val, (list, tuple)): return list(val) else: raise OptionError('Invalid type %r for option %s; you ' 'must give a list value' % ( - val, optname)) + val, optname)) def docstring_headline(obj): @@ -105,10 +105,7 @@ def docstring_headline(obj): def make_analysator(f): - """ - Return a static text analysation function that - returns float values. - """ + """Return a static text analyser function that returns float values.""" def text_analyse(text): try: rv = f(text) @@ -125,8 +122,7 @@ def make_analysator(f): def shebang_matches(text, regex): - """ - Check if the given regular expression matches the last part of the + r"""Check if the given regular expression matches the last part of the shebang if one exists. >>> from pygments.util import shebang_matches @@ -164,15 +160,15 @@ def shebang_matches(text, regex): if x and not x.startswith('-')][-1] except IndexError: return False - regex = re.compile('^%s(\.(exe|cmd|bat|bin))?$' % regex, re.IGNORECASE) + regex = re.compile(r'^%s(\.(exe|cmd|bat|bin))?$' % regex, re.IGNORECASE) if regex.search(found) is not None: return True return False def doctype_matches(text, regex): - """ - Check if the doctype matches a regular expression (if present). + """Check if the doctype matches a regular expression (if present). + Note that this method only checks the first part of a DOCTYPE. eg: 'html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"' """ @@ -180,21 +176,21 @@ def doctype_matches(text, regex): if m is None: return False doctype = m.group(2) - return re.compile(regex).match(doctype.strip()) is not None + return re.compile(regex, re.I).match(doctype.strip()) is not None def html_doctype_matches(text): - """ - Check if the file looks like it has a html doctype. - """ - return doctype_matches(text, r'html\s+PUBLIC\s+"-//W3C//DTD X?HTML.*') + """Check if the file looks like it has a html doctype.""" + return doctype_matches(text, r'html') _looks_like_xml_cache = {} + + def looks_like_xml(text): - """ - Check if a doctype exists or if we have some tags. - """ + """Check if a doctype exists or if we have some tags.""" + if xml_decl_re.match(text): + return True key = hash(text) try: return _looks_like_xml_cache[key] @@ -206,15 +202,20 @@ def looks_like_xml(text): _looks_like_xml_cache[key] = rv return rv + # Python narrow build compatibility def _surrogatepair(c): + # Given a unicode character code + # with length greater than 16 bits, + # return the two 16 bit surrogate pair. + # From example D28 of: + # http://www.unicode.org/book/ch03.pdf return (0xd7c0 + (c >> 10), (0xdc00 + (c & 0x3ff))) + def unirange(a, b): - """ - Returns a regular expression string to match the given non-BMP range. - """ + """Returns a regular expression string to match the given non-BMP range.""" if b < a: raise ValueError("Bad character range") if a < 0x10000 or b < 0x10000: @@ -251,27 +252,136 @@ def unirange(a, b): return u'(?:' + u'|'.join(buf) + u')' + +def format_lines(var_name, seq, raw=False, indent_level=0): + """Formats a sequence of strings for output.""" + lines = [] + base_indent = ' ' * indent_level * 4 + inner_indent = ' ' * (indent_level + 1) * 4 + lines.append(base_indent + var_name + ' = (') + if raw: + # These should be preformatted reprs of, say, tuples. + for i in seq: + lines.append(inner_indent + i + ',') + else: + for i in seq: + # Force use of single quotes + r = repr(i + '"') + lines.append(inner_indent + r[:-2] + r[-1] + ',') + lines.append(base_indent + ')') + return '\n'.join(lines) + + +def duplicates_removed(it, already_seen=()): + """ + Returns a list with duplicates removed from the iterable `it`. + + Order is preserved. + """ + lst = [] + seen = set() + for i in it: + if i in seen or i in already_seen: + continue + lst.append(i) + seen.add(i) + return lst + + +class Future(object): + """Generic class to defer some work. + + Handled specially in RegexLexerMeta, to support regex string construction at + first use. + """ + def get(self): + raise NotImplementedError + + +def guess_decode(text): + """Decode *text* with guessed encoding. + + First try UTF-8; this should fail for non-UTF-8 encodings. + Then try the preferred locale encoding. + Fall back to latin-1, which always works. + """ + try: + text = text.decode('utf-8') + return text, 'utf-8' + except UnicodeDecodeError: + try: + import locale + prefencoding = locale.getpreferredencoding() + text = text.decode() + return text, prefencoding + except (UnicodeDecodeError, LookupError): + text = text.decode('latin1') + return text, 'latin1' + + +def guess_decode_from_terminal(text, term): + """Decode *text* coming from terminal *term*. + + First try the terminal encoding, if given. + Then try UTF-8. Then try the preferred locale encoding. + Fall back to latin-1, which always works. + """ + if getattr(term, 'encoding', None): + try: + text = text.decode(term.encoding) + except UnicodeDecodeError: + pass + else: + return text, term.encoding + return guess_decode(text) + + +def terminal_encoding(term): + """Return our best guess of encoding for the given *term*.""" + if getattr(term, 'encoding', None): + return term.encoding + import locale + return locale.getpreferredencoding() + + # Python 2/3 compatibility -if sys.version_info < (3,0): - b = bytes = str +if sys.version_info < (3, 0): + unichr = unichr + xrange = xrange + string_types = (str, unicode) + text_type = unicode u_prefix = 'u' - import StringIO, cStringIO - BytesIO = cStringIO.StringIO + iteritems = dict.iteritems + itervalues = dict.itervalues + import StringIO + import cStringIO + # unfortunately, io.StringIO in Python 2 doesn't accept str at all StringIO = StringIO.StringIO - uni_open = codecs.open + BytesIO = cStringIO.StringIO else: - import builtins - bytes = builtins.bytes + unichr = chr + xrange = range + string_types = (str,) + text_type = str u_prefix = '' - def b(s): - if isinstance(s, str): - return bytes(map(ord, s)) - elif isinstance(s, bytes): - return s - else: - raise TypeError("Invalid argument %r for b()" % (s,)) - import io - BytesIO = io.BytesIO - StringIO = io.StringIO - uni_open = builtins.open + iteritems = dict.items + itervalues = dict.values + from io import StringIO, BytesIO, TextIOWrapper + + class UnclosingTextIOWrapper(TextIOWrapper): + # Don't close underlying buffer on destruction. + def close(self): + pass + + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + for slots_var in orig_vars.get('__slots__', ()): + orig_vars.pop(slots_var) + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper |