Refactored the Loader class to use the Renderer class's unicode.

As a result, it is no longer necessary to pass decode_errors to
the Loader class.

7 files changed, 185 insertions, 95 deletions

diff --git a/pystache/custom_template.py b/pystache/custom_template.py
index c3f232d..5f6a79b 100644
--- a/pystache/custom_template.py
+++ b/pystache/custom_template.py
@@ -120,6 +120,9 @@ class View(CustomizedTemplate):
         return renderer.render(template, self.context)
 
 
+# TODO: finalize this class's name.
+# TODO: get this class fully working with test cases, and then refactor
+#   and replace the View class.
 class CustomLoader(object):
 
     """
diff --git a/pystache/defaults.py b/pystache/defaults.py
index b9156d6..69c4995 100644
--- a/pystache/defaults.py
+++ b/pystache/defaults.py
@@ -3,8 +3,15 @@
 """
 This module provides a central location for defining default behavior.
 
+Throughout the package, these defaults take effect only when the user
+does not otherwise specify a value.
+
 """
 
+import cgi
+import sys
+
+
 # How to handle encoding errors when decoding strings from str to unicode.
 #
 # This value is passed as the "errors" argument to Python's built-in
@@ -14,5 +21,25 @@ This module provides a central location for defining default behavior.
 #
 DECODE_ERRORS = 'strict'
 
+# The name of the encoding to use when converting to unicode any strings of
+# type str encountered during the rendering process.
+STRING_ENCODING = sys.getdefaultencoding()
+
+# The name of the encoding to use when converting file contents to unicode.
+# This default takes precedence over the STRING_ENCODING default for
+# strings that arise from files.
+FILE_ENCODING = sys.getdefaultencoding()
+
+# The escape function to apply to strings that require escaping when
+# rendering templates (e.g. for tags enclosed in double braces).
+# Only unicode strings will be passed to this function.
+#
+# The quote=True argument causes double quotes to be escaped,
+# but not single quotes:
+#
+#   http://docs.python.org/library/cgi.html#cgi.escape
+#
+TAG_ESCAPE = lambda u: cgi.escape(u, quote=True)
+
 # The default template extension.
 TEMPLATE_EXTENSION = 'mustache'
diff --git a/pystache/loader.py b/pystache/loader.py
index fb30857..d7cdca1 100644
--- a/pystache/loader.py
+++ b/pystache/loader.py
@@ -1,7 +1,7 @@
 # coding: utf-8
 
 """
-This module provides a Reader class to read a template given a path.
+This module provides a Loader class for locating and reading templates.
 
 """
 
@@ -14,62 +14,82 @@ from . import defaults
 from .locator import Locator
 
 
+def _to_unicode(s, encoding=None):
+    """
+    Raises a TypeError exception if the given string is already unicode.
+
+    """
+    if encoding is None:
+        encoding = defaults.STRING_ENCODING
+    return unicode(s, encoding, defaults.DECODE_ERRORS)
+
+
 class Loader(object):
 
-    def __init__(self, encoding=None, decode_errors=None, extension=None):
-        """
-        Construct a template reader.
+    """
+    Loads the template associated to a name or user-defined object.
 
-        Arguments:
+    """
 
-          decode_errors: the string to pass as the errors argument to the
-            built-in function unicode() when converting str strings to
-            unicode.  Defaults to the package default.
+    def __init__(self, file_encoding=None, extension=None, to_unicode=None):
+        """
+        Construct a template loader instance.
 
-          encoding: the file encoding.  This is the name of the encoding to
-            use when converting file contents to unicode.  This name is
-            passed as the encoding argument to Python's built-in function
-            unicode().  Defaults to the encoding name returned by
-            sys.getdefaultencoding().
+        Arguments:
 
           extension: the template file extension.  Pass False for no
             extension (i.e. to use extensionless template files).
             Defaults to the package default.
 
-        """
-        if decode_errors is None:
-            decode_errors = defaults.DECODE_ERRORS
+          file_encoding: the name of the encoding to use when converting file
+            contents to unicode.  Defaults to the package default.
 
-        if encoding is None:
-            encoding = sys.getdefaultencoding()
+          to_unicode: the function to use when converting strings of type
+            str to unicode.  The function should have the signature:
+
+              to_unicode(s, encoding=None)
+
+            It should accept a string of type str and an optional encoding
+            name and return a string of type unicode.  Defaults to calling
+            Python's built-in function unicode() using the package encoding
+            and decode-errors defaults.
 
+        """
         if extension is None:
             extension = defaults.TEMPLATE_EXTENSION
 
-        self.decode_errors = decode_errors
-        self.encoding = encoding
+        if file_encoding is None:
+            file_encoding = defaults.FILE_ENCODING
+
+        if to_unicode is None:
+            to_unicode = _to_unicode
+
         self.extension = extension
+        self.file_encoding = file_encoding
+        self.to_unicode = to_unicode
 
-    # TODO: eliminate redundancy with the Renderer class's unicode code.
     def unicode(self, s, encoding=None):
         """
-        Call Python's built-in function unicode(), and return the result.
+        Convert a string to unicode using the given encoding, and return it.
+
+        This function uses the underlying to_unicode attribute.
+
+        Arguments:
 
-        For unicode strings (or unicode subclasses), this function calls
-        Python's unicode() without the encoding and errors arguments.
-        Thus, unlike Python's built-in unicode(), it is okay to pass unicode
-        strings to this function.  (Passing a unicode string to Python's
-        unicode() with the encoding argument throws the following
-        error: "TypeError: decoding Unicode is not supported.")
+          s: a basestring instance to convert to unicode.  Unlike Python's
+            built-in unicode() function, it is okay to pass unicode strings
+            to this function.  (Passing a unicode string to Python's unicode()
+            with the encoding argument throws the error, "TypeError: decoding
+            Unicode is not supported.")
+
+          encoding: the encoding to pass to the to_unicode attribute.
+            Defaults to None.
 
         """
         if isinstance(s, unicode):
             return unicode(s)
 
-        if encoding is None:
-            encoding = self.encoding
-
-        return unicode(s, encoding, self.decode_errors)
+        return self.to_unicode(s, encoding)
 
     def read(self, path, encoding=None):
         """
@@ -79,6 +99,9 @@ class Loader(object):
         with open(path, 'r') as f:
             text = f.read()
 
+        if encoding is None:
+            encoding = self.file_encoding
+
         return self.unicode(text, encoding)
 
     # TODO: unit-test this method.
diff --git a/pystache/renderer.py b/pystache/renderer.py
index ffc3194..9ed0949 100644
--- a/pystache/renderer.py
+++ b/pystache/renderer.py
@@ -5,9 +5,7 @@ This module provides a Renderer class to render templates.
 
 """
 
-import cgi
 import os
-import sys
 
 from . import defaults
 from .context import Context
@@ -15,12 +13,6 @@ from .loader import Loader
 from .renderengine import RenderEngine
 
 
-# The quote=True argument causes double quotes to be escaped,
-# but not single quotes:
-#   http://docs.python.org/library/cgi.html#cgi.escape
-DEFAULT_ESCAPE = lambda s: cgi.escape(s, quote=True)
-
-
 class Renderer(object):
 
     """
@@ -40,6 +32,7 @@ class Renderer(object):
 
     """
 
+    # TODO: rename default_encoding to string_encoding.
     def __init__(self, file_encoding=None, default_encoding=None,
                  decode_errors=None, search_dirs=None, file_extension=None,
                  escape=None, partials=None):
@@ -82,7 +75,7 @@ class Renderer(object):
             to unicode any strings of type str encountered during the
             rendering process.  The name will be passed as the encoding
             argument to the built-in function unicode().  Defaults to the
-            encoding name returned by sys.getdefaultencoding().
+            package default.
 
           decode_errors: the string to pass as the errors argument to the
             built-in function unicode() when converting str strings to
@@ -102,10 +95,10 @@ class Renderer(object):
             decode_errors = defaults.DECODE_ERRORS
 
         if default_encoding is None:
-            default_encoding = sys.getdefaultencoding()
+            default_encoding = defaults.STRING_ENCODING
 
         if escape is None:
-            escape = DEFAULT_ESCAPE
+            escape = defaults.TAG_ESCAPE
 
         # This needs to be after we set the default default_encoding.
         if file_encoding is None:
@@ -121,6 +114,7 @@ class Renderer(object):
             search_dirs = [search_dirs]
 
         self.decode_errors = decode_errors
+        # TODO: rename this attribute to string_encoding.
         self.default_encoding = default_encoding
         self.escape = escape
         self.file_encoding = file_encoding
@@ -152,7 +146,7 @@ class Renderer(object):
         """
         return unicode(self.escape(self._to_unicode_soft(s)))
 
-    def unicode(self, s):
+    def unicode(self, s, encoding=None):
         """
         Convert a string to unicode, using default_encoding and decode_errors.
 
@@ -165,17 +159,20 @@ class Renderer(object):
               TypeError: decoding Unicode is not supported
 
         """
+        if encoding is None:
+            encoding = self.default_encoding
+
         # TODO: Wrap UnicodeDecodeErrors with a message about setting
         # the default_encoding and decode_errors attributes.
-        return unicode(s, self.default_encoding, self.decode_errors)
+        return unicode(s, encoding, self.decode_errors)
 
     def _make_loader(self):
         """
         Create a Loader instance using current attributes.
 
         """
-        return Loader(encoding=self.file_encoding, decode_errors=self.decode_errors,
-                      extension=self.file_extension)
+        return Loader(file_encoding=self.file_encoding, extension=self.file_extension,
+                      to_unicode=self.unicode)
 
     def _make_load_template(self):
         """
diff --git a/tests/test_custom_template.py b/tests/test_custom_template.py
index d46b5bf..eedf431 100644
--- a/tests/test_custom_template.py
+++ b/tests/test_custom_template.py
@@ -146,10 +146,11 @@ class CustomLoaderTests(unittest.TestCase, AssertIsMixin, AssertStringMixin):
     def test_init__defaults(self):
         custom = CustomLoader()
 
-        # Check the reader attribute.
+        # Check the loader attribute.
         loader = custom.loader
-        self.assertEquals(loader.decode_errors, 'strict')
-        self.assertEquals(loader.encoding, sys.getdefaultencoding())
+        self.assertEquals(loader.extension, 'mustache')
+        self.assertEquals(loader.file_encoding, sys.getdefaultencoding())
+        to_unicode = loader.to_unicode
 
         # Check search_dirs.
         self.assertEquals(custom.search_dirs, [])
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 1836466..a285e68 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -10,40 +10,71 @@ import sys
 import unittest
 
 from .common import AssertStringMixin
+from pystache import defaults
 from pystache.loader import Loader
 
 
 DATA_DIR = 'tests/data'
 
 
-class LoaderTestCase(unittest.TestCase, AssertStringMixin):
-
-    def _get_path(self, filename):
-        return os.path.join(DATA_DIR, filename)
-
-    def test_init__decode_errors(self):
-        # Test the default value.
-        reader = Loader()
-        self.assertEquals(reader.decode_errors, 'strict')
-
-        reader = Loader(decode_errors='replace')
-        self.assertEquals(reader.decode_errors, 'replace')
-
-    def test_init__encoding(self):
-        # Test the default value.
-        reader = Loader()
-        self.assertEquals(reader.encoding, sys.getdefaultencoding())
-
-        reader = Loader(encoding='foo')
-        self.assertEquals(reader.encoding, 'foo')
+class LoaderTests(unittest.TestCase, AssertStringMixin):
 
     def test_init__extension(self):
+        loader = Loader(extension='foo')
+        self.assertEquals(loader.extension, 'foo')
+
+    def test_init__extension__default(self):
         # Test the default value.
-        reader = Loader()
-        self.assertEquals(reader.extension, 'mustache')
+        loader = Loader()
+        self.assertEquals(loader.extension, 'mustache')
+
+    def test_init__file_encoding(self):
+        loader = Loader(file_encoding='bar')
+        self.assertEquals(loader.file_encoding, 'bar')
+
+    def test_init__file_encoding__default(self):
+        file_encoding = defaults.FILE_ENCODING
+        try:
+            defaults.FILE_ENCODING = 'foo'
+            loader = Loader()
+            self.assertEquals(loader.file_encoding, 'foo')
+        finally:
+            defaults.FILE_ENCODING = file_encoding
+
+    def test_init__to_unicode(self):
+        to_unicode = lambda x: x
+        loader = Loader(to_unicode=to_unicode)
+        self.assertEquals(loader.to_unicode, to_unicode)
+
+    def test_init__to_unicode__default(self):
+        loader = Loader()
+        self.assertRaises(TypeError, loader.to_unicode, u"abc")
+
+        decode_errors = defaults.DECODE_ERRORS
+        string_encoding = defaults.STRING_ENCODING
+
+        nonascii = 'abcdé'
+
+        try:
+            defaults.DECODE_ERRORS = 'strict'
+            defaults.STRING_ENCODING = 'ascii'
+            loader = Loader()
+            self.assertRaises(UnicodeDecodeError, loader.to_unicode, nonascii)
+
+            defaults.DECODE_ERRORS = 'ignore'
+            loader = Loader()
+            self.assertString(loader.to_unicode(nonascii), u'abcd')
+
+            defaults.STRING_ENCODING = 'utf-8'
+            loader = Loader()
+            self.assertString(loader.to_unicode(nonascii), u'abcdé')
+
+        finally:
+            defaults.DECODE_ERRORS = decode_errors
+            defaults.STRING_ENCODING = string_encoding
 
-        reader = Loader(extension='foo')
-        self.assertEquals(reader.extension, 'foo')
+    def _get_path(self, filename):
+        return os.path.join(DATA_DIR, filename)
 
     def test_unicode__basic__input_str(self):
         """
@@ -80,19 +111,24 @@ class LoaderTestCase(unittest.TestCase, AssertStringMixin):
 
         self.assertString(actual, u"foo")
 
-    def test_unicode__encoding_attribute(self):
+    def test_unicode__to_unicode__attribute(self):
         """
         Test unicode(): encoding attribute.
 
         """
         reader = Loader()
 
-        non_ascii = u'é'.encode('utf-8')
+        non_ascii = u'abcdé'.encode('utf-8')
 
         self.assertRaises(UnicodeDecodeError, reader.unicode, non_ascii)
 
-        reader.encoding = 'utf-8'
-        self.assertEquals(reader.unicode(non_ascii), u"é")
+        def to_unicode(s, encoding=None):
+            if encoding is None:
+                encoding = 'utf-8'
+            return unicode(s, encoding)
+
+        reader.to_unicode = to_unicode
+        self.assertString(reader.unicode(non_ascii), u"abcdé")
 
     def test_unicode__encoding_argument(self):
         """
@@ -101,13 +137,14 @@ class LoaderTestCase(unittest.TestCase, AssertStringMixin):
         """
         reader = Loader()
 
-        non_ascii = u'é'.encode('utf-8')
+        non_ascii = u'abcdé'.encode('utf-8')
 
         self.assertRaises(UnicodeDecodeError, reader.unicode, non_ascii)
 
         actual = reader.unicode(non_ascii, encoding='utf-8')
-        self.assertEquals(actual, u'é')
+        self.assertString(actual, u'abcdé')
 
+    # TODO: check the read() unit tests.
     def test_read(self):
         """
         Test read().
@@ -118,18 +155,18 @@ class LoaderTestCase(unittest.TestCase, AssertStringMixin):
         actual = reader.read(path)
         self.assertString(actual, u'ascii: abc')
 
-    def test_read__encoding__attribute(self):
+    def test_read__file_encoding__attribute(self):
         """
-        Test read(): encoding attribute respected.
+        Test read(): file_encoding attribute respected.
 
         """
-        reader = Loader()
+        loader = Loader()
         path = self._get_path('non_ascii.mustache')
 
-        self.assertRaises(UnicodeDecodeError, reader.read, path)
+        self.assertRaises(UnicodeDecodeError, loader.read, path)
 
-        reader.encoding = 'utf-8'
-        actual = reader.read(path)
+        loader.file_encoding = 'utf-8'
+        actual = loader.read(path)
         self.assertString(actual, u'non-ascii: é')
 
     def test_read__encoding__argument(self):
@@ -145,9 +182,9 @@ class LoaderTestCase(unittest.TestCase, AssertStringMixin):
         actual = reader.read(path, encoding='utf-8')
         self.assertString(actual, u'non-ascii: é')
 
-    def test_get__decode_errors(self):
+    def test_reader__to_unicode__attribute(self):
         """
-        Test get(): decode_errors attribute.
+        Test read(): to_unicode attribute respected.
 
         """
         reader = Loader()
@@ -155,7 +192,7 @@ class LoaderTestCase(unittest.TestCase, AssertStringMixin):
 
         self.assertRaises(UnicodeDecodeError, reader.read, path)
 
-        reader.decode_errors = 'ignore'
-        actual = reader.read(path)
-        self.assertString(actual, u'non-ascii: ')
+        #reader.decode_errors = 'ignore'
+        #actual = reader.read(path)
+        #self.assertString(actual, u'non-ascii: ')
 
diff --git a/tests/test_renderer.py b/tests/test_renderer.py
index 5c702d3..49f3999 100644
--- a/tests/test_renderer.py
+++ b/tests/test_renderer.py
@@ -196,19 +196,21 @@ class RendererTestCase(unittest.TestCase):
 
     def test__make_loader__attributes(self):
         """
-        Test that _make_locator() sets all attributes correctly..
+        Test that _make_loader() sets all attributes correctly..
 
         """
+        unicode_ = lambda x: x
+
         renderer = Renderer()
-        renderer.decode_errors = 'dec'
         renderer.file_encoding = 'enc'
         renderer.file_extension = 'ext'
+        renderer.unicode = unicode_
 
         loader = renderer._make_loader()
 
-        self.assertEquals(loader.decode_errors, 'dec')
-        self.assertEquals(loader.encoding, 'enc')
         self.assertEquals(loader.extension, 'ext')
+        self.assertEquals(loader.file_encoding, 'enc')
+        self.assertEquals(loader.to_unicode, unicode_)
 
     ## Test the render() method.