diff options
Diffstat (limited to 'pygments/util.py')
| -rw-r--r-- | pygments/util.py | 161 |
1 files changed, 120 insertions, 41 deletions
diff --git a/pygments/util.py b/pygments/util.py index caac1144..8376a67f 100644 --- a/pygments/util.py +++ b/pygments/util.py @@ -5,13 +5,12 @@ Utility functions. - :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re import sys -import codecs split_path_re = re.compile(r'[/\\ ]') @@ -27,9 +26,7 @@ tag_re = re.compile(r'<(.+?)(\s.*?)?>.*?</.+?>(?uism)') class ClassNotFound(ValueError): - """ - If one of the get_*_by_* functions didn't find a matching class. - """ + """Raised if one of the lookup functions didn't find a matching class.""" class OptionError(Exception): @@ -52,7 +49,7 @@ def get_bool_opt(options, optname, default=None): return string elif isinstance(string, int): return bool(string) - elif not isinstance(string, basestring): + elif not isinstance(string, string_types): raise OptionError('Invalid type %r for option %s; use ' '1/0, yes/no, true/false, on/off' % ( string, optname)) @@ -82,7 +79,7 @@ def get_int_opt(options, optname, default=None): def get_list_opt(options, optname, default=None): val = options.get(optname, default) - if isinstance(val, basestring): + if isinstance(val, string_types): return val.split() elif isinstance(val, (list, tuple)): return list(val) @@ -105,10 +102,7 @@ def docstring_headline(obj): def make_analysator(f): - """ - Return a static text analysation function that - returns float values. - """ + """Return a static text analyser function that returns float values.""" def text_analyse(text): try: rv = f(text) @@ -125,8 +119,7 @@ def make_analysator(f): def shebang_matches(text, regex): - """ - Check if the given regular expression matches the last part of the + """Check if the given regular expression matches the last part of the shebang if one exists. >>> from pygments.util import shebang_matches @@ -171,8 +164,8 @@ def shebang_matches(text, regex): def doctype_matches(text, regex): - """ - Check if the doctype matches a regular expression (if present). + """Check if the doctype matches a regular expression (if present). + Note that this method only checks the first part of a DOCTYPE. eg: 'html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"' """ @@ -184,17 +177,13 @@ def doctype_matches(text, regex): def html_doctype_matches(text): - """ - Check if the file looks like it has a html doctype. - """ + """Check if the file looks like it has a html doctype.""" return doctype_matches(text, r'html\s+PUBLIC\s+"-//W3C//DTD X?HTML.*') _looks_like_xml_cache = {} def looks_like_xml(text): - """ - Check if a doctype exists or if we have some tags. - """ + """Check if a doctype exists or if we have some tags.""" key = hash(text) try: return _looks_like_xml_cache[key] @@ -209,12 +198,15 @@ def looks_like_xml(text): # Python narrow build compatibility def _surrogatepair(c): + # Given a unicode character code + # with length greater than 16 bits, + # return the two 16 bit surrogate pair. + # From example D28 of: + # http://www.unicode.org/book/ch03.pdf return (0xd7c0 + (c >> 10), (0xdc00 + (c & 0x3ff))) def unirange(a, b): - """ - Returns a regular expression string to match the given non-BMP range. - """ + """Returns a regular expression string to match the given non-BMP range.""" if b < a: raise ValueError("Bad character range") if a < 0x10000 or b < 0x10000: @@ -251,27 +243,114 @@ def unirange(a, b): return u'(?:' + u'|'.join(buf) + u')' + +def format_lines(var_name, seq, raw=False, indent_level=0): + """Formats a sequence of strings for output.""" + lines = [] + base_indent = ' ' * indent_level * 4 + inner_indent = ' ' * (indent_level + 1) * 4 + lines.append(base_indent + var_name + ' = (') + if raw: + # These should be preformatted reprs of, say, tuples. + for i in seq: + lines.append(inner_indent + i + ',') + else: + for i in seq: + # Force use of single quotes + r = repr(i + '"') + lines.append(inner_indent + r[:-2] + r[-1] + ',') + lines.append(base_indent + ')') + return '\n'.join(lines) + + +class Future(object): + """Generic class to defer some work. + + Handled specially in RegexLexerMeta, to support regex string construction at + first use. + """ + def get(self): + raise NotImplementedError + + +def guess_decode(text): + """Decode *text* with guessed encoding. + + First try UTF-8; this should fail for non-UTF-8 encodings. + Then try the preferred locale encoding. + Fall back to latin-1, which always works. + """ + try: + text = text.decode('utf-8') + return text, 'utf-8' + except UnicodeDecodeError: + try: + import locale + prefencoding = locale.getpreferredencoding() + text = text.decode() + return text, prefencoding + except (UnicodeDecodeError, LookupError): + text = text.decode('latin1') + return text, 'latin1' + + +def guess_decode_from_terminal(text, term): + """Decode *text* coming from terminal *term*. + + First try the terminal encoding, if given. + Then try UTF-8. Then try the preferred locale encoding. + Fall back to latin-1, which always works. + """ + if getattr(term, 'encoding', None): + try: + text = text.decode(term.encoding) + except UnicodeDecodeError: + pass + else: + return text, term.encoding + return guess_decode(text) + + +def terminal_encoding(term): + """Return our best guess of encoding for the given *term*.""" + if getattr(term, 'encoding', None): + return term.encoding + import locale + return locale.getpreferredencoding() + + # Python 2/3 compatibility -if sys.version_info < (3,0): - b = bytes = str +if sys.version_info < (3, 0): + unichr = unichr + xrange = xrange + string_types = (str, unicode) + text_type = unicode u_prefix = 'u' + iteritems = dict.iteritems + itervalues = dict.itervalues import StringIO, cStringIO - BytesIO = cStringIO.StringIO + # unfortunately, io.StringIO in Python 2 doesn't accept str at all StringIO = StringIO.StringIO - uni_open = codecs.open + BytesIO = cStringIO.StringIO else: - import builtins - bytes = builtins.bytes + unichr = chr + xrange = range + string_types = (str,) + text_type = str u_prefix = '' - def b(s): - if isinstance(s, str): - return bytes(map(ord, s)) - elif isinstance(s, bytes): - return s - else: - raise TypeError("Invalid argument %r for b()" % (s,)) - import io - BytesIO = io.BytesIO - StringIO = io.StringIO - uni_open = builtins.open + iteritems = dict.items + itervalues = dict.values + from io import StringIO, BytesIO + + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + for slots_var in orig_vars.get('__slots__', ()): + orig_vars.pop(slots_var) + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper |