[svn] Make the C# lexer unicode aware.

Warning, major hack ahead: the code to support multiple
token defs is not the prettiest.

1 files changed, 8 insertions, 1 deletions

diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py
index 805f3104..abdeb86d 100644
--- a/tests/test_basic_api.py
+++ b/tests/test_basic_api.py
@@ -41,7 +41,14 @@ class LexersTest(unittest.TestCase):
             a(isinstance(result, float) and 0.0 <= result <= 1.0)
 
             if issubclass(lexer, RegexLexer):
-                a('root' in lexer._tokens, '%s has no root state' % lexer)
+                if not hasattr(lexer, '_tokens'):
+                    # if there's no "_tokens", the lexer has to be one with
+                    # multiple tokendef variants
+                    a(lexer.token_variants)
+                    for variant in lexer.tokens:
+                        a('root' in lexer.tokens[variant])
+                else:
+                    a('root' in lexer._tokens, '%s has no root state' % lexer)
 
             inst = lexer(opt1="val1", opt2="val2")
             tokens = list(inst.get_tokens(test_content))