diff options
Diffstat (limited to 'pygments/lexer.py')
| -rw-r--r-- | pygments/lexer.py | 21 |
1 files changed, 19 insertions, 2 deletions
diff --git a/pygments/lexer.py b/pygments/lexer.py index 46f86076..82f09318 100644 --- a/pygments/lexer.py +++ b/pygments/lexer.py @@ -21,6 +21,12 @@ __all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer', 'LexerContext', 'include', 'bygroups', 'using', 'this'] +_encoding_map = [('\xef\xbb\xbf', 'utf-8'), + ('\xff\xfe\0\0', 'utf-32'), + ('\0\0\xfe\xff', 'utf-32be'), + ('\xff\xfe', 'utf-16'), + ('\xfe\xff', 'utf-16be')] + _default_analyse = staticmethod(lambda x: 0.0) @@ -142,8 +148,19 @@ class Lexer(object): raise ImportError('To enable chardet encoding guessing, ' 'please install the chardet library ' 'from http://chardet.feedparser.org/') - enc = chardet.detect(text) - text = text.decode(enc['encoding']) + # check for BOM first + decoded = None + for bom, encoding in _encoding_map: + if text.startswith(bom): + decoded = unicode(text[len(bom):], encoding, + errors='replace') + break + # no BOM found, so use chardet + if decoded is None: + enc = chardet.detect(text[:1024]) # Guess using first 1KB + decoded = unicode(text, enc.get('encoding') or 'utf-8', + errors='replace') + text = decoded else: text = text.decode(self.encoding) # text now *is* a unicode string |