1 files changed, 12 insertions, 9 deletions
diff --git a/utils.py b/utils.py
index fcc7904..6ce4bf0 100644
--- a/utils.py
+++ b/utils.py
@@ -128,16 +128,19 @@ def category_id(cid):
     return MSG_TYPES_LONG.get(cid)
 
 
+def _decoding_readline(stream, module):
+    return lambda: stream.readline().decode(module.file_encoding,
+                                           'replace')
+
+
 def tokenize_module(module):
-    stream = module.file_stream
-    stream.seek(0)
-    readline = stream.readline
-    if sys.version_info < (3, 0):
-        if module.file_encoding is not None:
-            readline = lambda: stream.readline().decode(module.file_encoding,
-                                                        'replace')
-        return list(tokenize.generate_tokens(readline))
-    return list(tokenize.tokenize(readline))
+    with module.stream() as stream:
+        readline = stream.readline
+        if sys.version_info < (3, 0):
+            if module.file_encoding is not None:
+                readline = _decoding_readline(stream, module)
+            return list(tokenize.generate_tokens(readline))
+        return list(tokenize.tokenize(readline))
 
 def build_message_def(checker, msgid, msg_tuple):
     if implements(checker, (IRawChecker, ITokenChecker)):