Fix exception causes in lookup.py

Mako now performs exception chaining using ``raise from``, correctly
identifying underlying exception conditions when it raises its own
exceptions. Pull request courtesy Ram Rachum.

Additionally includes cleanup of the test suite to include
better exception fixtures.

Closes: #319
Pull-request: https://github.com/sqlalchemy/mako/pull/319
Pull-request-sha: d06526ac3f80ca9d24cd8143d8afde254f80b094

Additionally:
Fixes: #348

Change-Id: Ibb2864de822bf4b63adf22a6bb32cf0758d296bd

34 files changed, 554 insertions, 302 deletions

diff --git a/doc/build/unreleased/raise_from.rst b/doc/build/unreleased/raise_from.rst
new file mode 100644
index 0000000..bec705d
--- /dev/null
+++ b/doc/build/unreleased/raise_from.rst
@@ -0,0 +1,6 @@
+.. change::
+    :tags: bug, py3k
+
+    Mako now performs exception chaining using ``raise from``, correctly
+    identifying underlying exception conditions when it raises its own
+    exceptions. Pull request courtesy Ram Rachum.
diff --git a/mako/lookup.py b/mako/lookup.py
index 17cef8e..7afe242 100644
--- a/mako/lookup.py
+++ b/mako/lookup.py
@@ -241,7 +241,7 @@ class TemplateLookup(TemplateCollection):
                 return self._check(uri, self._collection[uri])
             else:
                 return self._collection[uri]
-        except KeyError:
+        except KeyError as e:
             u = re.sub(r"^\/+", "", uri)
             for dir_ in self.directories:
                 # make sure the path seperators are posix - os.altsep is empty
@@ -252,8 +252,8 @@ class TemplateLookup(TemplateCollection):
                     return self._load(srcfile, uri)
             else:
                 raise exceptions.TopLevelLookupException(
-                    "Cant locate template for uri %r" % uri
-                )
+                    "Can't locate template for uri %r" % uri
+                ) from e
 
     def adjust_uri(self, uri, relativeto):
         """Adjust the given ``uri`` based on the given relative URI."""
@@ -337,11 +337,11 @@ class TemplateLookup(TemplateCollection):
                 return template
             self._collection.pop(uri, None)
             return self._load(template.filename, uri)
-        except OSError:
+        except OSError as e:
             self._collection.pop(uri, None)
             raise exceptions.TemplateLookupException(
-                "Cant locate template for uri %r" % uri
-            )
+                "Can't locate template for uri %r" % uri
+            ) from e
 
     def put_string(self, uri, text):
         """Place a new :class:`.Template` object into this
diff --git a/mako/pyparser.py b/mako/pyparser.py
index 2a1ba5f..5c55505 100644
--- a/mako/pyparser.py
+++ b/mako/pyparser.py
@@ -34,7 +34,7 @@ def parse(code, mode="exec", **exception_kwargs):
 
     try:
         return _ast_util.parse(code, "<unknown>", mode)
-    except Exception:
+    except Exception as e:
         raise exceptions.SyntaxException(
             "(%s) %s (%r)"
             % (
@@ -43,7 +43,7 @@ def parse(code, mode="exec", **exception_kwargs):
                 code[0:50],
             ),
             **exception_kwargs,
-        )
+        ) from e
 
 
 class FindIdentifiers(_ast_util.NodeVisitor):
diff --git a/mako/runtime.py b/mako/runtime.py
index f5dbe1a..6d7fa68 100644
--- a/mako/runtime.py
+++ b/mako/runtime.py
@@ -835,8 +835,10 @@ def _lookup_template(context, uri, relativeto):
     uri = lookup.adjust_uri(uri, relativeto)
     try:
         return lookup.get_template(uri)
-    except exceptions.TopLevelLookupException:
-        raise exceptions.TemplateLookupException(str(compat.exception_as()))
+    except exceptions.TopLevelLookupException as e:
+        raise exceptions.TemplateLookupException(
+            str(compat.exception_as())
+        ) from e
 
 
 def _populate_self_namespace(context, template, self_ns=None):
diff --git a/setup.cfg b/setup.cfg
index 46ee8f7..d4fb9eb 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -44,7 +44,7 @@ exclude =
 [options.extras_require]
 babel =
     Babel
-lingua = 
+lingua =
     lingua
 
 [options.entry_points]
diff --git a/test/__init__.py b/test/__init__.py
index 9b85cdf..e69de29 100644
--- a/test/__init__.py
+++ b/test/__init__.py
@@ -1,195 +0,0 @@
-import contextlib
-import os
-import re
-import unittest
-from unittest import mock  # noqa
-
-from mako.cache import CacheImpl
-from mako.cache import register_plugin
-from mako.template import Template
-from mako.util import update_wrapper
-
-template_base = os.path.join(os.path.dirname(__file__), "templates")
-module_base = os.path.join(template_base, "modules")
-
-
-class TemplateTest(unittest.TestCase):
-    def _file_template(self, filename, **kw):
-        filepath = self._file_path(filename)
-        return Template(
-            uri=filename, filename=filepath, module_directory=module_base, **kw
-        )
-
-    def _file_path(self, filename):
-        name, ext = os.path.splitext(filename)
-        py3k_path = os.path.join(template_base, name + "_py3k" + ext)
-        if os.path.exists(py3k_path):
-            return py3k_path
-
-        return os.path.join(template_base, filename)
-
-    def _do_file_test(
-        self,
-        filename,
-        expected,
-        filters=None,
-        unicode_=True,
-        template_args=None,
-        **kw,
-    ):
-        t1 = self._file_template(filename, **kw)
-        self._do_test(
-            t1,
-            expected,
-            filters=filters,
-            unicode_=unicode_,
-            template_args=template_args,
-        )
-
-    def _do_memory_test(
-        self,
-        source,
-        expected,
-        filters=None,
-        unicode_=True,
-        template_args=None,
-        **kw,
-    ):
-        t1 = Template(text=source, **kw)
-        self._do_test(
-            t1,
-            expected,
-            filters=filters,
-            unicode_=unicode_,
-            template_args=template_args,
-        )
-
-    def _do_test(
-        self,
-        template,
-        expected,
-        filters=None,
-        template_args=None,
-        unicode_=True,
-    ):
-        if template_args is None:
-            template_args = {}
-        if unicode_:
-            output = template.render_unicode(**template_args)
-        else:
-            output = template.render(**template_args)
-
-        if filters:
-            output = filters(output)
-        eq_(output, expected)
-
-
-def eq_(a, b, msg=None):
-    """Assert a == b, with repr messaging on failure."""
-    assert a == b, msg or "%r != %r" % (a, b)
-
-
-def teardown():
-    import shutil
-
-    shutil.rmtree(module_base, True)
-
-
-@contextlib.contextmanager
-def raises(except_cls, message=None):
-    try:
-        yield
-        success = False
-    except except_cls as e:
-        if message:
-            assert re.search(message, str(e), re.UNICODE), "%r !~ %s" % (
-                message,
-                e,
-            )
-            print(str(e).encode("utf-8"))
-        success = True
-
-    # assert outside the block so it works for AssertionError too !
-    assert success, "Callable did not raise an exception"
-
-
-def assert_raises(except_cls, callable_, *args, **kw):
-    with raises(except_cls):
-        return callable_(*args, **kw)
-
-
-def assert_raises_message(except_cls, msg, callable_, *args, **kwargs):
-    with raises(except_cls, msg):
-        return callable_(*args, **kwargs)
-
-
-def skip_if(predicate, reason=None):
-    """Skip a test if predicate is true."""
-    reason = reason or predicate.__name__
-
-    def decorate(fn):
-        fn_name = fn.__name__
-
-        def maybe(*args, **kw):
-            if predicate():
-                msg = "'%s' skipped: %s" % (fn_name, reason)
-                raise unittest.SkipTest(msg)
-            else:
-                return fn(*args, **kw)
-
-        return update_wrapper(maybe, fn)
-
-    return decorate
-
-
-def requires_pygments_14(fn):
-    try:
-        import pygments
-
-        version = pygments.__version__
-    except:
-        version = "0"
-    return skip_if(
-        lambda: version < "1.4", "Requires pygments 1.4 or greater"
-    )(fn)
-
-
-def requires_no_pygments_exceptions(fn):
-    def go(*arg, **kw):
-        from mako import exceptions
-
-        exceptions._install_fallback()
-        try:
-            return fn(*arg, **kw)
-        finally:
-            exceptions._install_highlighting()
-
-    return update_wrapper(go, fn)
-
-
-class PlainCacheImpl(CacheImpl):
-    """Simple memory cache impl so that tests which
-    use caching can run without beaker."""
-
-    def __init__(self, cache):
-        self.cache = cache
-        self.data = {}
-
-    def get_or_create(self, key, creation_function, **kw):
-        if key in self.data:
-            return self.data[key]
-        else:
-            self.data[key] = data = creation_function(**kw)
-            return data
-
-    def put(self, key, value, **kw):
-        self.data[key] = value
-
-    def get(self, key, **kw):
-        return self.data[key]
-
-    def invalidate(self, key, **kw):
-        del self.data[key]
-
-
-register_plugin("plain", __name__, "PlainCacheImpl")
diff --git a/test/ext/test_babelplugin.py b/test/ext/test_babelplugin.py
index a69d884..9ef2daf 100644
--- a/test/ext/test_babelplugin.py
+++ b/test/ext/test_babelplugin.py
@@ -2,9 +2,9 @@ import io
 import os
 import unittest
 
-from .. import skip_if
-from .. import template_base
-from .. import TemplateTest
+from ..util.exclusions import skip_if
+from ..util.fixtures import template_base
+from ..util.fixtures import TemplateTest
 
 try:
     import babel.messages.extract as babel
diff --git a/test/ext/test_linguaplugin.py b/test/ext/test_linguaplugin.py
index 53d7073..fa5f76d 100644
--- a/test/ext/test_linguaplugin.py
+++ b/test/ext/test_linguaplugin.py
@@ -1,8 +1,8 @@
 import os
 
-from .. import skip_if
-from .. import template_base
-from .. import TemplateTest
+from ..util.exclusions import skip_if
+from ..util.fixtures import template_base
+from ..util.fixtures import TemplateTest
 
 try:
     import lingua
diff --git a/test/test_ast.py b/test/test_ast.py
index 0f30c5a..3f86a86 100644
--- a/test/test_ast.py
+++ b/test/test_ast.py
@@ -3,7 +3,7 @@ import unittest
 from mako import ast
 from mako import exceptions
 from mako import pyparser
-from test import eq_
+from .util.assertions import eq_
 
 exception_kwargs = {"source": "", "lineno": 0, "pos": 0, "filename": ""}
 
diff --git a/test/test_block.py b/test/test_block.py
index 0cbe347..a55ca89 100644
--- a/test/test_block.py
+++ b/test/test_block.py
@@ -1,9 +1,9 @@
 from mako import exceptions
 from mako.lookup import TemplateLookup
 from mako.template import Template
-from test import assert_raises_message
-from test import TemplateTest
-from test.util import result_lines
+from .util.assertions import assert_raises_message
+from .util.fixtures import TemplateTest
+from .util.helpers import result_lines
 
 
 class BlockTest(TemplateTest):
diff --git a/test/test_cache.py b/test/test_cache.py
index 7c6e3a5..b48f7fb 100644
--- a/test/test_cache.py
+++ b/test/test_cache.py
@@ -7,10 +7,10 @@ from mako.cache import register_plugin
 from mako.ext import beaker_cache
 from mako.lookup import TemplateLookup
 from mako.template import Template
-from test import eq_
-from test import module_base
-from test import TemplateTest
-from test.util import result_lines
+from .util.assertions import eq_
+from .util.fixtures import module_base
+from .util.fixtures import TemplateTest
+from .util.helpers import result_lines
 
 if beaker_cache.has_beaker:
     import beaker
diff --git a/test/test_call.py b/test/test_call.py
index 36d15dc..82d48da 100644
--- a/test/test_call.py
+++ b/test/test_call.py
@@ -1,8 +1,8 @@
 from mako.template import Template
-from test import eq_
-from test import TemplateTest
-from test.util import flatten_result
-from test.util import result_lines
+from .util.assertions import eq_
+from .util.fixtures import TemplateTest
+from .util.helpers import flatten_result
+from .util.helpers import result_lines
 
 
 class CallTest(TemplateTest):
diff --git a/test/test_cmd.py b/test/test_cmd.py
index 0dfa378..8fc155a 100644
--- a/test/test_cmd.py
+++ b/test/test_cmd.py
@@ -1,12 +1,12 @@
 from contextlib import contextmanager
 import os
+from unittest import mock
 
 from mako.cmd import cmdline
-from test import eq_
-from test import mock
-from test import raises
-from test import template_base
-from test import TemplateTest
+from .util.assertions import eq_
+from .util.assertions import raises
+from .util.fixtures import template_base
+from .util.fixtures import TemplateTest
 
 
 class CmdTest(TemplateTest):
diff --git a/test/test_decorators.py b/test/test_decorators.py
index 195a636..6153371 100644
--- a/test/test_decorators.py
+++ b/test/test_decorators.py
@@ -1,7 +1,7 @@
 import unittest
 
 from mako.template import Template
-from test.util import flatten_result
+from .util.helpers import flatten_result
 
 
 class DecoratorTest(unittest.TestCase):
diff --git a/test/test_def.py b/test/test_def.py
index 6505c1f..5f99192 100644
--- a/test/test_def.py
+++ b/test/test_def.py
@@ -1,10 +1,10 @@
 from mako import lookup
 from mako.template import Template
-from test import assert_raises
-from test import eq_
-from test import TemplateTest
-from test.util import flatten_result
-from test.util import result_lines
+from .util.assertions import assert_raises
+from .util.assertions import eq_
+from .util.fixtures import TemplateTest
+from .util.helpers import flatten_result
+from .util.helpers import result_lines
 
 
 class DefTest(TemplateTest):
diff --git a/test/test_exceptions.py b/test/test_exceptions.py
index dc3b735..b4246bd 100644
--- a/test/test_exceptions.py
+++ b/test/test_exceptions.py
@@ -4,10 +4,10 @@ import sys
 from mako import exceptions
 from mako.lookup import TemplateLookup
 from mako.template import Template
-from test import requires_no_pygments_exceptions
-from test import requires_pygments_14
-from test import TemplateTest
-from test.util import result_lines
+from .util.exclusions import requires_no_pygments_exceptions
+from .util.exclusions import requires_pygments_14
+from .util.fixtures import TemplateTest
+from .util.helpers import result_lines
 
 
 class ExceptionsTest(TemplateTest):
diff --git a/test/test_filters.py b/test/test_filters.py
index 7aa7662..abe19c9 100644
--- a/test/test_filters.py
+++ b/test/test_filters.py
@@ -3,10 +3,10 @@
 import unittest
 
 from mako.template import Template
-from test import eq_
-from test import TemplateTest
-from test.util import flatten_result
-from test.util import result_lines
+from .util.assertions import eq_
+from .util.fixtures import TemplateTest
+from .util.helpers import flatten_result
+from .util.helpers import result_lines
 
 
 class FilterTest(TemplateTest):
diff --git a/test/test_inheritance.py b/test/test_inheritance.py
index 7217e33..7824241 100644
--- a/test/test_inheritance.py
+++ b/test/test_inheritance.py
@@ -1,7 +1,7 @@
 import unittest
 
 from mako import lookup
-from test.util import result_lines
+from .util.helpers import result_lines
 
 
 class InheritanceTest(unittest.TestCase):
diff --git a/test/test_lexer.py b/test/test_lexer.py
index 2dbd924..08201b2 100644
--- a/test/test_lexer.py
+++ b/test/test_lexer.py
@@ -6,10 +6,10 @@ from mako import parsetree
 from mako import util
 from mako.lexer import Lexer
 from mako.template import Template
-from test import assert_raises_message
-from test import eq_
-from test import TemplateTest
-from test.util import flatten_result
+from .util.assertions import assert_raises_message
+from .util.assertions import eq_
+from .util.fixtures import TemplateTest
+from .util.helpers import flatten_result
 
 # create fake parsetree classes which are constructed
 # exactly as the repr() of a real parsetree object.
diff --git a/test/test_lookup.py b/test/test_lookup.py
index 0dacfbb..eebb97b 100644
--- a/test/test_lookup.py
+++ b/test/test_lookup.py
@@ -1,16 +1,19 @@
 import os
+import tempfile
 import unittest
 
-from mako import compat
 from mako import exceptions
 from mako import lookup
 from mako import runtime
 from mako.template import Template
 from mako.util import FastEncodingBuffer
-from test import assert_raises_message
-from test import eq_
-from test import template_base
-from test.util import result_lines
+from .util.assertions import assert_raises_message
+from .util.assertions import assert_raises_with_given_cause
+from .util.fixtures import template_base
+from .util.helpers import file_with_template_code
+from .util.helpers import replace_file_with_dir
+from .util.helpers import result_lines
+from .util.helpers import rewind_compile_time
 
 tl = lookup.TemplateLookup(directories=[template_base])
 
@@ -43,21 +46,22 @@ class LookupTest(unittest.TestCase):
         """test that hitting an existent directory still raises
         LookupError."""
 
-        self.assertRaises(
-            exceptions.TopLevelLookupException, tl.get_template, "/subdir"
+        assert_raises_with_given_cause(
+            exceptions.TopLevelLookupException,
+            KeyError,
+            tl.get_template,
+            "/subdir",
         )
 
     def test_no_lookup(self):
         t = Template("hi <%include file='foo.html'/>")
-        try:
-            t.render()
-            assert False
-        except exceptions.TemplateLookupException:
-            eq_(
-                str(compat.exception_as()),
-                "Template 'memory:%s' has no TemplateLookup associated"
-                % hex(id(t)),
-            )
+
+        assert_raises_message(
+            exceptions.TemplateLookupException,
+            "Template 'memory:%s' has no TemplateLookup associated"
+            % hex(id(t)),
+            t.render,
+        )
 
     def test_uri_adjust(self):
         tl = lookup.TemplateLookup(directories=["/foo/bar"])
@@ -82,8 +86,11 @@ class LookupTest(unittest.TestCase):
         f = tl.get_template("foo")
         assert f.uri in tl._collection
         f.filename = "nonexistent"
-        self.assertRaises(
-            exceptions.TemplateLookupException, tl.get_template, "foo"
+        assert_raises_with_given_cause(
+            exceptions.TemplateLookupException,
+            FileNotFoundError,
+            tl.get_template,
+            "foo",
         )
         assert f.uri not in tl._collection
 
@@ -120,3 +127,25 @@ class LookupTest(unittest.TestCase):
 
         # this is OK since the .. cancels out
         runtime._lookup_template(ctx, "foo/../index.html", index.uri)
+
+    def test_checking_against_bad_filetype(self):
+        with tempfile.TemporaryDirectory() as tempdir:
+            tl = lookup.TemplateLookup(directories=[tempdir])
+            index_file = file_with_template_code(
+                os.path.join(tempdir, "index.html")
+            )
+
+            with rewind_compile_time():
+                tmpl = Template(filename=index_file)
+
+            tl.put_template("index.html", tmpl)
+
+            replace_file_with_dir(index_file)
+
+            assert_raises_with_given_cause(
+                exceptions.TemplateLookupException,
+                OSError,
+                tl._check,
+                "index.html",
+                tl._collection["index.html"],
+            )
diff --git a/test/test_loop.py b/test/test_loop.py
index 8104e49..709ec97 100644
--- a/test/test_loop.py
+++ b/test/test_loop.py
@@ -7,9 +7,9 @@ from mako.lookup import TemplateLookup
 from mako.runtime import LoopContext
 from mako.runtime import LoopStack
 from mako.template import Template
-from test import assert_raises_message
-from test import TemplateTest
-from test.util import flatten_result
+from .util.assertions import assert_raises_message
+from .util.fixtures import TemplateTest
+from .util.helpers import flatten_result
 
 
 class TestLoop(unittest.TestCase):
diff --git a/test/test_namespace.py b/test/test_namespace.py
index 8d223d5..bdd1641 100644
--- a/test/test_namespace.py
+++ b/test/test_namespace.py
@@ -1,9 +1,11 @@
+from mako import exceptions
 from mako import lookup
 from mako.template import Template
-from test import eq_
-from test import TemplateTest
-from test.util import flatten_result
-from test.util import result_lines
+from .util.assertions import assert_raises_message_with_given_cause
+from .util.assertions import eq_
+from .util.fixtures import TemplateTest
+from .util.helpers import flatten_result
+from .util.helpers import result_lines
 
 
 class NamespaceTest(TemplateTest):
@@ -994,3 +996,35 @@ class NamespaceTest(TemplateTest):
             "this is lala",
             "this is foo",
         ]
+
+    def test_nonexistent_namespace_uri(self):
+        collection = lookup.TemplateLookup()
+        collection.put_string(
+            "main.html",
+            """
+            <%namespace name="defs" file="eefs.html"/>
+
+            this is main.  ${defs.def1("hi")}
+            ${defs.def2("there")}
+""",
+        )
+
+        collection.put_string(
+            "defs.html",
+            """
+        <%def name="def1(s)">
+            def1: ${s}
+        </%def>
+
+        <%def name="def2(x)">
+            def2: ${x}
+        </%def>
+""",
+        )
+
+        assert_raises_message_with_given_cause(
+            exceptions.TemplateLookupException,
+            "Can't locate template for uri 'eefs.html",
+            exceptions.TopLevelLookupException,
+            collection.get_template("main.html").render,
+        )
diff --git a/test/test_pygen.py b/test/test_pygen.py
index b8c11db..5200e3e 100644
--- a/test/test_pygen.py
+++ b/test/test_pygen.py
@@ -3,7 +3,7 @@ import unittest
 
 from mako.pygen import adjust_whitespace
 from mako.pygen import PythonPrinter
-from test import eq_
+from .util.assertions import eq_
 
 
 class GeneratePythonTest(unittest.TestCase):
diff --git a/test/test_runtime.py b/test/test_runtime.py
index d87d264..07802f9 100644
--- a/test/test_runtime.py
+++ b/test/test_runtime.py
@@ -3,7 +3,7 @@
 import unittest
 
 from mako import runtime
-from test import eq_
+from .util.assertions import eq_
 
 
 class ContextTest(unittest.TestCase):
diff --git a/test/test_template.py b/test/test_template.py
index 557603c..ad7f59d 100644
--- a/test/test_template.py
+++ b/test/test_template.py
@@ -11,14 +11,14 @@ from mako.lookup import TemplateLookup
 from mako.template import ModuleInfo
 from mako.template import ModuleTemplate
 from mako.template import Template
-from test import assert_raises
-from test import assert_raises_message
-from test import eq_
-from test import module_base
-from test import template_base
-from test import TemplateTest
-from test.util import flatten_result
-from test.util import result_lines
+from .util.assertions import assert_raises
+from .util.assertions import assert_raises_message
+from .util.assertions import eq_
+from .util.fixtures import module_base
+from .util.fixtures import template_base
+from .util.fixtures import TemplateTest
+from .util.helpers import flatten_result
+from .util.helpers import result_lines
 
 
 class ctx:
diff --git a/test/test_tgplugin.py b/test/test_tgplugin.py
index 9b8055f..9d5799b 100644
--- a/test/test_tgplugin.py
+++ b/test/test_tgplugin.py
@@ -1,7 +1,7 @@
 from mako.ext.turbogears import TGPlugin
-from test import template_base
-from test import TemplateTest
-from test.util import result_lines
+from .util.fixtures import template_base
+from .util.fixtures import TemplateTest
+from .util.helpers import result_lines
 
 tl = TGPlugin(options=dict(directories=[template_base]), extension="html")
 
diff --git a/test/test_util.py b/test/test_util.py
index 2646300..be987fe 100644
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -7,9 +7,9 @@ import unittest
 from mako import compat
 from mako import exceptions
 from mako import util
-from test import assert_raises_message
-from test import eq_
-from test import skip_if
+from .util.assertions import assert_raises_message
+from .util.assertions import eq_
+from .util.exclusions import skip_if
 
 
 class UtilTest(unittest.TestCase):
diff --git a/test/util.py b/test/util.py
deleted file mode 100644
index 29225e2..0000000
--- a/test/util.py
+++ /dev/null
@@ -1,13 +0,0 @@
-import re
-
-
-def flatten_result(result):
-    return re.sub(r"[\s\r\n]+", " ", result).strip()
-
-
-def result_lines(result):
-    return [
-        x.strip()
-        for x in re.split(r"\r?\n", re.sub(r" +", " ", result))
-        if x.strip() != ""
-    ]
diff --git a/test/util/__init__.py b/test/util/__init__.py
new file mode 100644
index 0000000..f8cb359
--- /dev/null
+++ b/test/util/__init__.py
@@ -0,0 +1 @@
+from unittest import mock  # noqa
diff --git a/test/util/assertions.py b/test/util/assertions.py
new file mode 100644
index 0000000..a6c5d4e
--- /dev/null
+++ b/test/util/assertions.py
@@ -0,0 +1,160 @@
+import contextlib
+import re
+import sys
+
+
+def eq_(a, b, msg=None):
+    """Assert a == b, with repr messaging on failure."""
+    assert a == b, msg or "%r != %r" % (a, b)
+
+
+@contextlib.contextmanager
+def raises(except_cls, message=None):
+    try:
+        yield
+        success = False
+    except except_cls as e:
+        if message:
+            assert re.search(message, str(e), re.UNICODE), "%r !~ %s" % (
+                message,
+                e,
+            )
+            print(str(e).encode("utf-8"))
+        success = True
+
+    # assert outside the block so it works for AssertionError too !
+    assert success, "Callable did not raise an exception"
+
+
+def _assert_proper_exception_context(exception):
+    """assert that any exception we're catching does not have a __context__
+    without a __cause__, and that __suppress_context__ is never set.
+
+    Python 3 will report nested as exceptions as "during the handling of
+    error X, error Y occurred". That's not what we want to do. We want
+    these exceptions in a cause chain.
+
+    """
+
+    if (
+        exception.__context__ is not exception.__cause__
+        and not exception.__suppress_context__
+    ):
+        assert False, (
+            "Exception %r was correctly raised but did not set a cause, "
+            "within context %r as its cause."
+            % (exception, exception.__context__)
+        )
+
+
+def _assert_proper_cause_cls(exception, cause_cls):
+    """assert that any exception we're catching does not have a __context__
+    without a __cause__, and that __suppress_context__ is never set.
+
+    Python 3 will report nested as exceptions as "during the handling of
+    error X, error Y occurred". That's not what we want to do. We want
+    these exceptions in a cause chain.
+
+    """
+    assert isinstance(exception.__cause__, cause_cls), (
+        "Exception %r was correctly raised but has cause %r, which does not "
+        "have the expected cause type %r."
+        % (exception, exception.__cause__, cause_cls)
+    )
+
+
+def assert_raises(except_cls, callable_, *args, **kw):
+    return _assert_raises(except_cls, callable_, args, kw)
+
+
+def assert_raises_with_proper_context(except_cls, callable_, *args, **kw):
+    return _assert_raises(except_cls, callable_, args, kw, check_context=True)
+
+
+def assert_raises_with_given_cause(
+    except_cls, cause_cls, callable_, *args, **kw
+):
+    return _assert_raises(except_cls, callable_, args, kw, cause_cls=cause_cls)
+
+
+def assert_raises_message(except_cls, msg, callable_, *args, **kwargs):
+    return _assert_raises(except_cls, callable_, args, kwargs, msg=msg)
+
+
+def assert_raises_message_with_proper_context(
+    except_cls, msg, callable_, *args, **kwargs
+):
+    return _assert_raises(
+        except_cls, callable_, args, kwargs, msg=msg, check_context=True
+    )
+
+
+def assert_raises_message_with_given_cause(
+    except_cls, msg, cause_cls, callable_, *args, **kwargs
+):
+    return _assert_raises(
+        except_cls, callable_, args, kwargs, msg=msg, cause_cls=cause_cls
+    )
+
+
+def _assert_raises(
+    except_cls,
+    callable_,
+    args,
+    kwargs,
+    msg=None,
+    check_context=False,
+    cause_cls=None,
+):
+
+    with _expect_raises(except_cls, msg, check_context, cause_cls) as ec:
+        callable_(*args, **kwargs)
+    return ec.error
+
+
+class _ErrorContainer:
+    error = None
+
+
+@contextlib.contextmanager
+def _expect_raises(except_cls, msg=None, check_context=False, cause_cls=None):
+    ec = _ErrorContainer()
+    if check_context:
+        are_we_already_in_a_traceback = sys.exc_info()[0]
+    try:
+        yield ec
+        success = False
+    except except_cls as err:
+        ec.error = err
+        success = True
+        if msg is not None:
+            # I'm often pdbing here, and "err" above isn't
+            # in scope, so assign the string explicitly
+            error_as_string = str(err)
+            assert re.search(msg, error_as_string, re.UNICODE), "%r !~ %s" % (
+                msg,
+                error_as_string,
+            )
+        if cause_cls is not None:
+            _assert_proper_cause_cls(err, cause_cls)
+        if check_context and not are_we_already_in_a_traceback:
+            _assert_proper_exception_context(err)
+        print(str(err).encode("utf-8"))
+
+    # it's generally a good idea to not carry traceback objects outside
+    # of the except: block, but in this case especially we seem to have
+    # hit some bug in either python 3.10.0b2 or greenlet or both which
+    # this seems to fix:
+    # https://github.com/python-greenlet/greenlet/issues/242
+    del ec
+
+    # assert outside the block so it works for AssertionError too !
+    assert success, "Callable did not raise an exception"
+
+
+def expect_raises(except_cls, check_context=True):
+    return _expect_raises(except_cls, check_context=check_context)
+
+
+def expect_raises_message(except_cls, msg, check_context=True):
+    return _expect_raises(except_cls, msg=msg, check_context=check_context)
diff --git a/test/util/exclusions.py b/test/util/exclusions.py
new file mode 100644
index 0000000..8eb596e
--- /dev/null
+++ b/test/util/exclusions.py
@@ -0,0 +1,47 @@
+import unittest
+
+from mako.util import update_wrapper
+
+
+def skip_if(predicate, reason=None):
+    """Skip a test if predicate is true."""
+    reason = reason or predicate.__name__
+
+    def decorate(fn):
+        fn_name = fn.__name__
+
+        def maybe(*args, **kw):
+            if predicate():
+                msg = "'%s' skipped: %s" % (fn_name, reason)
+                raise unittest.SkipTest(msg)
+            else:
+                return fn(*args, **kw)
+
+        return update_wrapper(maybe, fn)
+
+    return decorate
+
+
+def requires_pygments_14(fn):
+    try:
+        import pygments
+
+        version = pygments.__version__
+    except:
+        version = "0"
+    return skip_if(
+        lambda: version < "1.4", "Requires pygments 1.4 or greater"
+    )(fn)
+
+
+def requires_no_pygments_exceptions(fn):
+    def go(*arg, **kw):
+        from mako import exceptions
+
+        exceptions._install_fallback()
+        try:
+            return fn(*arg, **kw)
+        finally:
+            exceptions._install_highlighting()
+
+    return update_wrapper(go, fn)
diff --git a/test/util/fixtures.py b/test/util/fixtures.py
new file mode 100644
index 0000000..d4b28db
--- /dev/null
+++ b/test/util/fixtures.py
@@ -0,0 +1,129 @@
+import os
+import unittest
+
+from mako.cache import CacheImpl
+from mako.cache import register_plugin
+from mako.template import Template
+from .assertions import eq_
+
+
+def _ensure_environment_variable(key, fallback):
+    env_var = os.getenv(key)
+    if env_var is None:
+        return fallback
+    return env_var
+
+
+def _get_module_base():
+    return _ensure_environment_variable(
+        "TEST_MODULE_BASE", os.path.abspath("./test/templates/modules")
+    )
+
+
+def _get_template_base():
+    return _ensure_environment_variable(
+        "TEST_TEMPLATE_BASE", os.path.abspath("./test/templates/")
+    )
+
+
+module_base = _get_module_base()
+template_base = _get_template_base()
+
+
+class TemplateTest(unittest.TestCase):
+    def _file_template(self, filename, **kw):
+        filepath = self._file_path(filename)
+        return Template(
+            uri=filename, filename=filepath, module_directory=module_base, **kw
+        )
+
+    def _file_path(self, filename):
+        name, ext = os.path.splitext(filename)
+        py3k_path = os.path.join(template_base, name + "_py3k" + ext)
+        if os.path.exists(py3k_path):
+            return py3k_path
+
+        return os.path.join(template_base, filename)
+
+    def _do_file_test(
+        self,
+        filename,
+        expected,
+        filters=None,
+        unicode_=True,
+        template_args=None,
+        **kw,
+    ):
+        t1 = self._file_template(filename, **kw)
+        self._do_test(
+            t1,
+            expected,
+            filters=filters,
+            unicode_=unicode_,
+            template_args=template_args,
+        )
+
+    def _do_memory_test(
+        self,
+        source,
+        expected,
+        filters=None,
+        unicode_=True,
+        template_args=None,
+        **kw,
+    ):
+        t1 = Template(text=source, **kw)
+        self._do_test(
+            t1,
+            expected,
+            filters=filters,
+            unicode_=unicode_,
+            template_args=template_args,
+        )
+
+    def _do_test(
+        self,
+        template,
+        expected,
+        filters=None,
+        template_args=None,
+        unicode_=True,
+    ):
+        if template_args is None:
+            template_args = {}
+        if unicode_:
+            output = template.render_unicode(**template_args)
+        else:
+            output = template.render(**template_args)
+
+        if filters:
+            output = filters(output)
+        eq_(output, expected)
+
+
+class PlainCacheImpl(CacheImpl):
+    """Simple memory cache impl so that tests which
+    use caching can run without beaker."""
+
+    def __init__(self, cache):
+        self.cache = cache
+        self.data = {}
+
+    def get_or_create(self, key, creation_function, **kw):
+        if key in self.data:
+            return self.data[key]
+        else:
+            self.data[key] = data = creation_function(**kw)
+            return data
+
+    def put(self, key, value, **kw):
+        self.data[key] = value
+
+    def get(self, key, **kw):
+        return self.data[key]
+
+    def invalidate(self, key, **kw):
+        del self.data[key]
+
+
+register_plugin("plain", __name__, "PlainCacheImpl")
diff --git a/test/util/helpers.py b/test/util/helpers.py
new file mode 100644
index 0000000..0ca4c2d
--- /dev/null
+++ b/test/util/helpers.py
@@ -0,0 +1,50 @@
+import contextlib
+import pathlib
+import re
+import time
+from unittest import mock
+
+from test.util.fixtures import module_base
+
+
+def flatten_result(result):
+    return re.sub(r"[\s\r\n]+", " ", result).strip()
+
+
+def result_lines(result):
+    return [
+        x.strip()
+        for x in re.split(r"\r?\n", re.sub(r" +", " ", result))
+        if x.strip() != ""
+    ]
+
+
+def replace_file_with_dir(pathspec):
+    path = pathlib.Path(pathspec)
+    path.unlink(missing_ok=True)
+    path.mkdir(exist_ok=True)
+    return path
+
+
+def file_with_template_code(filespec):
+    with open(filespec, "w") as f:
+        f.write(
+            """
+i am an artificial template just for you
+"""
+        )
+    return filespec
+
+
+@contextlib.contextmanager
+def rewind_compile_time(hours=1):
+    rewound = time.time() - (hours * 3_600)
+    with mock.patch("mako.codegen.time") as codegen_time:
+        codegen_time.time.return_value = rewound
+        yield
+
+
+def teardown():
+    import shutil
+
+    shutil.rmtree(module_base, True)
diff --git a/tox.ini b/tox.ini
index d8edd3c..dc0530b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -15,6 +15,8 @@ deps=pytest>=3.1.0
 
 setenv=
     cov: COVERAGE={[testenv]cov_args}
+    TEST_TEMPLATE_BASE={toxinidir}/test/templates
+    TEST_MODULE_BASE={env:TEST_TEMPLATE_BASE}/modules
 
 commands=pytest {env:COVERAGE:} {posargs}