Merge remote branch 'origin/stpl_bugfix-0.8'

3 files changed, 72 insertions, 9 deletions

diff --git a/apidoc/stpl.rst b/apidoc/stpl.rst
index 843791a..0ab2ab8 100644..100755
--- a/apidoc/stpl.rst
+++ b/apidoc/stpl.rst
@@ -179,3 +179,10 @@ Lets see how ``block_content.tpl`` renders:
 .. autoclass:: SimpleTemplate
    :members:
 
+Known bugs
+==============================
+
+Some syntax construcs allowed in python are problematic within a template. The following syntaxes won't work with SimpleTemplate:
+
+  * Multi-line statements must end with a backslash (``\``) and a comment, if presend, must not contain any additional ``#`` characters.
+  * Multi-line strings are not supported yet.
diff --git a/bottle.py b/bottle.py
index 197871f..05b349e 100755
--- a/bottle.py
+++ b/bottle.py
@@ -80,6 +80,7 @@ import sys
 import thread
 import threading
 import time
+import tokenize
 
 from Cookie import SimpleCookie
 from tempfile import TemporaryFile
@@ -1575,14 +1576,28 @@ class SimpleTemplate(BaseTemplate):
         ptrbuffer = [] # Buffer for printable strings and token tuple instances
         codebuffer = [] # Buffer for generated python code
         touni = functools.partial(unicode, encoding=self.encoding)
+        multiline = dedent = False
 
-        def tokenize(line):
+        def yield_tokens(line):
             for i, part in enumerate(re.split(r'\{\{(.*?)\}\}', line)):
                 if i % 2:
                     if part.startswith('!'): yield 'RAW', part[1:]
                     else: yield 'CMD', part
                 else: yield 'TXT', part
 
+        def split_comment(codeline):
+            """ Removes comments from a line of code. """
+            line = codeline.splitlines()[0]
+            try:
+                tokens = list(tokenize.generate_tokens(iter(line).next))
+            except tokenize.TokenError:
+                return line.rsplit('#',1) if '#' in line else (line, '')
+            for token in tokens:
+                if token[0] == tokenize.COMMENT:
+                    start, end = token[2][1], token[3][1]
+                    return codeline[:start] + codeline[end:], codeline[start:end]
+            return line, ''
+
         def flush(): # Flush the ptrbuffer
             if not ptrbuffer: return
             cline = ''
@@ -1596,7 +1611,7 @@ class SimpleTemplate(BaseTemplate):
             cline = cline[:-2]
             if cline[:-1].endswith('\\\\\\\\\\n'):
                 cline = cline[:-7] + cline[-1] # 'nobr\\\\\n' --> 'nobr'
-            cline = '_printlist((' + cline + '))'
+            cline = '_printlist([' + cline + '])'
             del ptrbuffer[:] # Do this before calling code() again
             code(cline)
 
@@ -1614,15 +1629,19 @@ class SimpleTemplate(BaseTemplate):
                 if m: line = line.replace('coding','coding (removed)')
             if line.strip()[:2].count('%') == 1:
                 line = line.split('%',1)[1].lstrip() # Full line following the %
-                cline = line.split('#')[0].strip() # Line without commends (TODO: fails for 'a="#"')
-                cmd = re.split(r'[^a-zA-Z0-9_]', line)[0]
+                cline = split_comment(line)[0].strip()
+                cmd = re.split(r'[^a-zA-Z0-9_]', cline)[0]
                 flush() ##encodig (TODO: why?)
-                if cmd in self.blocks:
+                if cmd in self.blocks or multiline:
+                    cmd = multiline or cmd
                     dedent = cmd in self.dedent_blocks # "else:"
-                    oneline = not cline.endswith(':') # "if 1: pass"
-                    if dedent and not oneline: cmd = stack.pop()
+                    if dedent and not oneline and not multiline:
+                        cmd = stack.pop()
                     code(line)
-                    if not oneline: stack.append(cmd)
+                    oneline = not cline.endswith(':') # "if 1: pass"
+                    multiline = cmd if cline.endswith('\\') else False
+                    if not oneline and not multiline:
+                        stack.append(cmd)
                 elif cmd == 'end' and stack:
                     code('#end(%s) %s' % (stack.pop(), line.strip()[3:]))
                 elif cmd == 'include':
@@ -1644,7 +1663,7 @@ class SimpleTemplate(BaseTemplate):
             else: # Line starting with text (not '%') or '%%' (escaped)
                 if line.strip().startswith('%%'):
                     line = line.replace('%%', '%', 1)
-                ptrbuffer.append(tokenize(line))
+                ptrbuffer.append(yield_tokens(line))
         flush()
         return '\n'.join(codebuffer) + '\n'
 
diff --git a/test/test_stpl.py b/test/test_stpl.py
index 0f5b38f..cb4e48e 100644..100755
--- a/test/test_stpl.py
+++ b/test/test_stpl.py
@@ -80,14 +80,51 @@ class TestSimpleTemplate(unittest.TestCase):
         t = SimpleTemplate("%if 1:\nyes\n%else     :\nno\n%end\n")
         self.assertEqual(u"yes\n", ''.join(t.render()))
 
+    def test_commentbug(self):
+        ''' A "#" sign within an string is not a comment '''
+        t = SimpleTemplate("%if '#':\nyes\n%end\n")
+        self.assertEqual(u"yes\n", ''.join(t.render()))
+
+    def test_multiline(self):
+        ''' Block statements with non-terminating newlines '''
+        t = SimpleTemplate("%if 1\\\n%and 1:\nyes\n%end\n")
+        self.assertEqual(u"yes\n", ''.join(t.render()))
+
+    def test_newline_in_parameterlist(self):
+        ''' Block statements with non-terminating newlines in list '''
+        t = SimpleTemplate("%a=[1,\n%2]\n{{len(a)}}")
+        self.assertEqual(u"2", ''.join(t.render()))
+
     def test_dedentbug(self):
         ''' One-Line dednet blocks should not change indention '''
         t = SimpleTemplate('%if x: a="if"\n%else: a="else"\n{{a}}')
         self.assertEqual(u"if", ''.join(t.render(x=True)))
         self.assertEqual(u"else", ''.join(t.render(x=False)))
+        t = SimpleTemplate('%if x:\n%a="if"\n%else: a="else"\n{{a}}')
+        self.assertEqual(u"if", ''.join(t.render(x=True)))
+        self.assertEqual(u"else", ''.join(t.render(x=False)))
         t = SimpleTemplate('%if x: a="if"\n%else: a="else"\n%end')
         self.assertRaises(NameError, t.render)
 
+    def test_onelinebugs(self):
+        ''' One-Line blocks should not change indention '''
+        t = SimpleTemplate('%if x:\n%a=1\n%end\n{{a}}')
+        self.assertEqual(u"1", ''.join(t.render(x=True)))
+        t = SimpleTemplate('%if x: a=1\n{{a}}')
+        self.assertEqual(u"1", ''.join(t.render(x=True)))
+        t = SimpleTemplate('%if x:\n%a=1\n%else:\n%a=2\n%end\n{{a}}')
+        self.assertEqual(u"1", ''.join(t.render(x=True)))
+        self.assertEqual(u"2", ''.join(t.render(x=False)))
+        t = SimpleTemplate('%if x:   a=1\n%else:\n%a=2\n%end\n{{a}}')
+        self.assertEqual(u"1", ''.join(t.render(x=True)))
+        self.assertEqual(u"2", ''.join(t.render(x=False)))
+        t = SimpleTemplate('%if x:\n%a=1\n%else:   a=2\n{{a}}')
+        self.assertEqual(u"1", ''.join(t.render(x=True)))
+        self.assertEqual(u"2", ''.join(t.render(x=False)))
+        t = SimpleTemplate('%if x:   a=1\n%else:   a=2\n{{a}}')
+        self.assertEqual(u"1", ''.join(t.render(x=True)))
+        self.assertEqual(u"2", ''.join(t.render(x=False)))
+
     def test_onelineblocks(self):
         """ Templates: one line code blocks """
         t = SimpleTemplate("start\n%a=''\n%for i in l: a += str(i)\n{{a}}\nend")