[svn] Add encoding support. All processing is now done with unicode strings.

2 files changed, 9 insertions, 6 deletions

diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py
index e5ba623d..41aa01dc 100644
--- a/tests/test_basic_api.py
+++ b/tests/test_basic_api.py
@@ -22,8 +22,8 @@ class LexersTest(unittest.TestCase):
 
     def test_import_all(self):
         # instantiate every lexer, to see if the token type defs are correct
-        for x in pygments.lexers.LEXERS.keys():
-            c = getattr(pygments.lexers, x)()
+        for x in lexers.LEXERS.keys():
+            c = getattr(lexers, x)()
 
     def test_lexer_classes(self):
         a = self.assert_
@@ -41,7 +41,9 @@ class LexersTest(unittest.TestCase):
             for token in tokens:
                 a(isinstance(token, tuple))
                 a(isinstance(token[0], _TokenType))
-                a(isinstance(token[1], str))
+                if isinstance(token[1], str):
+                    print repr(token[1])
+                a(isinstance(token[1], unicode))
                 txt += token[1]
             ae(txt, test_content, "%s lexer roundtrip failed: %r != %r" %
                     (lexer.name, test_content, txt))
diff --git a/tests/test_examplefiles.py b/tests/test_examplefiles.py
index 6347ab88..247f986d 100644
--- a/tests/test_examplefiles.py
+++ b/tests/test_examplefiles.py
@@ -38,11 +38,12 @@ for fn in os.listdir(os.path.join(testdir, 'examplefiles')):
     def test(self, lx=lx, absfn=absfn):
         text = file(absfn, 'U').read()
         text = text.strip('\n') + '\n'
-        ntext = ''
+        text = text.decode('latin1')
+        ntext = []
         for type, val in lx.get_tokens(text):
-            ntext += val
+            ntext.append(val)
             self.failIf(type == Error, 'lexer generated error token for '+absfn)
-        if ntext != text:
+        if u''.join(ntext) != text:
             self.fail('round trip failed for '+absfn)
 
     setattr(ExampleFileTest, 'test_%i' % lfd, test)