Avoid warnings under Python 3.10

Fixed Python deprecation issues related to module importing, as well as
file access within the Lingua plugin, for deprecated APIs that began to
emit warnings under Python 3.10.  Pull request courtesy Petr Viktorin.

This modernizes the code to avoid `DeprecationWarning` and `ResourceWarning` encountered in the test suite under Python 3.10a4:
- [load_module](https://docs.python.org/3/library/importlib.html#importlib.abc.Loader.load_module) is deprecated
- Some files weren't being closed

This changes the semantics of the `compat.load_module` function: on Python 3.5+, the module is no longer inserted in `sys.modules`. All non-test calls did `del sys.modules[self.module_id]` right after the call, anyway. On older Python, the module is inserted and then deleted.

(Some additional `DeprecationWarning` come from Setuptools: https://github.com/pypa/setuptools/pull/2517)

Closes: #328
Pull-request: https://github.com/sqlalchemy/mako/pull/328
Pull-request-sha: 87c1d09e5f39de7b19e749568c437928580f7553

Change-Id: I9cb3772f6812ef14297792344caf2f9aa5337adf

6 files changed, 41 insertions, 11 deletions

diff --git a/doc/build/unreleased/328.rst b/doc/build/unreleased/328.rst
new file mode 100644
index 0000000..a3fba59
--- /dev/null
+++ b/doc/build/unreleased/328.rst
@@ -0,0 +1,7 @@
+.. change::
+    :tags: bug, py3k
+    :tickets: 328
+
+    Fixed Python deprecation issues related to module importing, as well as
+    file access within the Lingua plugin, for deprecated APIs that began to
+    emit warnings under Python 3.10.  Pull request courtesy Petr Viktorin.
diff --git a/mako/compat.py b/mako/compat.py
index 9aac98c..06bb8d9 100644
--- a/mako/compat.py
+++ b/mako/compat.py
@@ -99,11 +99,20 @@ else:
 
 
 if py3k:
-    from importlib import machinery
-
-    def load_module(module_id, path):
-        return machinery.SourceFileLoader(module_id, path).load_module()
-
+    from importlib import machinery, util
+
+    if hasattr(util, 'module_from_spec'):
+        # Python 3.5+
+        def load_module(module_id, path):
+            spec = util.spec_from_file_location(module_id, path)
+            module = util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+            return module
+    else:
+        def load_module(module_id, path):
+            module = machinery.SourceFileLoader(module_id, path).load_module()
+            del sys.modules[module_id]
+            return module
 
 else:
     import imp
@@ -111,7 +120,9 @@ else:
     def load_module(module_id, path):
         fp = open(path, "rb")
         try:
-            return imp.load_source(module_id, path, fp)
+            module = imp.load_source(module_id, path, fp)
+            del sys.modules[module_id]
+            return module
         finally:
             fp.close()
 
diff --git a/mako/ext/linguaplugin.py b/mako/ext/linguaplugin.py
index 0f6d165..c40fa74 100644
--- a/mako/ext/linguaplugin.py
+++ b/mako/ext/linguaplugin.py
@@ -27,7 +27,15 @@ class LinguaMakoExtractor(Extractor, MessageExtractor):
         self.python_extractor = get_extractor("x.py")
         if fileobj is None:
             fileobj = open(filename, "rb")
-        return self.process_file(fileobj)
+            must_close = True
+        else:
+            must_close = False
+        try:
+            for message in self.process_file(fileobj):
+                yield message
+        finally:
+            if must_close:
+                fileobj.close()
 
     def process_python(self, code, code_lineno, translator_strings):
         source = code.getvalue().strip()
diff --git a/mako/template.py b/mako/template.py
index 3fd0871..5ed2320 100644
--- a/mako/template.py
+++ b/mako/template.py
@@ -12,7 +12,6 @@ import os
 import re
 import shutil
 import stat
-import sys
 import tempfile
 import types
 import weakref
@@ -414,14 +413,12 @@ class Template(object):
                     self, data, filename, path, self.module_writer
                 )
             module = compat.load_module(self.module_id, path)
-            del sys.modules[self.module_id]
             if module._magic_number != codegen.MAGIC_NUMBER:
                 data = util.read_file(filename)
                 _compile_module_file(
                     self, data, filename, path, self.module_writer
                 )
                 module = compat.load_module(self.module_id, path)
-                del sys.modules[self.module_id]
             ModuleInfo(module, path, self, filename, None, None, None)
         else:
             # template filename and no module directory, compile code
diff --git a/test/ext/test_babelplugin.py b/test/ext/test_babelplugin.py
index 5f25a6a..ca12444 100644
--- a/test/ext/test_babelplugin.py
+++ b/test/ext/test_babelplugin.py
@@ -63,6 +63,7 @@ class ExtractMakoTestCase(TemplateTest):
     @skip()
     def test_extract(self):
         mako_tmpl = open(os.path.join(template_base, "gettext.mako"))
+        self.addCleanup(mako_tmpl.close)
         messages = list(
             extract(
                 mako_tmpl,
@@ -103,6 +104,7 @@ class ExtractMakoTestCase(TemplateTest):
         mako_tmpl = open(
             os.path.join(template_base, "gettext_utf8.mako"), "rb"
         )
+        self.addCleanup(mako_tmpl.close)
         message = next(
             extract(mako_tmpl, set(["_", None]), [], {"encoding": "utf-8"})
         )
@@ -113,6 +115,7 @@ class ExtractMakoTestCase(TemplateTest):
         mako_tmpl = open(
             os.path.join(template_base, "gettext_cp1251.mako"), "rb"
         )
+        self.addCleanup(mako_tmpl.close)
         message = next(
             extract(mako_tmpl, set(["_", None]), [], {"encoding": "cp1251"})
         )
diff --git a/test/test_util.py b/test/test_util.py
index f3f3edb..e40390f 100644
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
 import os
+import sys
 import unittest
 
 from mako import compat
@@ -43,10 +44,13 @@ class UtilTest(unittest.TestCase):
     @skip_if(lambda: compat.pypy, "Pypy does this differently")
     def test_load_module(self):
         fn = os.path.join(os.path.dirname(__file__), "test_util.py")
+        sys.modules.pop("mako.template")
         module = compat.load_module("mako.template", fn)
+        self.assertNotIn("mako.template", sys.modules)
+        self.assertIn("UtilTest", dir(module))
         import mako.template
 
-        self.assertEqual(module, mako.template)
+        self.assertNotEqual(module, mako.template)
 
     def test_load_plugin_failure(self):
         loader = util.PluginLoader("fakegroup")