diff options
Diffstat (limited to 'tests/test_examplefiles.py')
-rw-r--r-- | tests/test_examplefiles.py | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/tests/test_examplefiles.py b/tests/test_examplefiles.py index 1d3515df..a938ebaa 100644 --- a/tests/test_examplefiles.py +++ b/tests/test_examplefiles.py @@ -54,11 +54,12 @@ def check_lexer(lx, absfn, outfn): text = fp.read() finally: fp.close() - text = text.lstrip(u'\xef\xbb\xbf\ufeff') #remove BOM text = text.replace(b('\r\n'), b('\n')) text = text.strip(b('\n')) + b('\n') try: text = text.decode('utf-8') + if text.startswith(u'\ufeff'): + text = text[len(u'\ufeff'):] except UnicodeError: text = text.decode('latin1') ntext = [] |