Make sure source_token_lines always returns Unicode strings for the token text.

2 files changed, 11 insertions, 10 deletions

diff --git a/coverage/phystokens.py b/coverage/phystokens.py
index 92da8d32..7092d39e 100644
--- a/coverage/phystokens.py
+++ b/coverage/phystokens.py
@@ -109,7 +109,7 @@ def source_token_lines(source):
                 mark_end = False
             else:
                 if mark_start and scol > col:
-                    line.append(("ws", " " * (scol - col)))
+                    line.append(("ws", u" " * (scol - col)))
                     mark_start = False
                 tok_class = tokenize.tok_name.get(ttype, 'xx').lower()[:3]
                 if ttype == token.NAME and keyword.iskeyword(ttext):
diff --git a/coverage/plugin.py b/coverage/plugin.py
index 5d61e7f8..b88f2dfe 100644
--- a/coverage/plugin.py
+++ b/coverage/plugin.py
@@ -4,7 +4,7 @@
 """Plugin interfaces for coverage.py"""
 
 from coverage import files
-from coverage.misc import _needs_to_implement
+from coverage.misc import contract, _needs_to_implement
 
 
 class CoveragePlugin(object):
@@ -182,26 +182,27 @@ class FileReporter(object):
     def excluded_lines(self):
         return set()
 
+    def arcs(self):
+        return []
+
+    def no_branch_lines(self):
+        return set()
+
     def translate_lines(self, lines):
         return set(lines)
 
     def translate_arcs(self, arcs):
         return arcs
 
-    def no_branch_lines(self):
-        return set()
-
     def exit_counts(self):
         return {}
 
-    def arcs(self):
-        return []
-
+    @contract(returns='unicode')
     def source(self):
         """Return the source for the code, a Unicode string."""
         # A generic implementation: read the text of self.filename
-        with open(self.filename) as f:
-            return f.read()
+        with open(self.filename, "rb") as f:
+            return f.read().decode("utf8")
 
     def source_token_lines(self):
         """Generate a series of tokenized lines, one for each line in `source`.