Remove BOM when the input is unicode.

1 files changed, 2 insertions, 1 deletions

diff --git a/tests/test_examplefiles.py b/tests/test_examplefiles.py
index 1d3515df..a938ebaa 100644
--- a/tests/test_examplefiles.py
+++ b/tests/test_examplefiles.py
@@ -54,11 +54,12 @@ def check_lexer(lx, absfn, outfn):
         text = fp.read()
     finally:
         fp.close()
-    text = text.lstrip(u'\xef\xbb\xbf\ufeff') #remove BOM
     text = text.replace(b('\r\n'), b('\n'))
     text = text.strip(b('\n')) + b('\n')
     try:
         text = text.decode('utf-8')
+        if text.startswith(u'\ufeff'):
+            text = text[len(u'\ufeff'):]
     except UnicodeError:
         text = text.decode('latin1')
     ntext = []